Re-enable IP address usage hiding (#4327)

api/src/main/java/org/apache/cloudstack/usage/Usage.java

@@ -63,4 +63,6 @@ public interface Usage {
     public Date getEndDate();
 
     public Long getVirtualSize();
+
+    public boolean isHidden();
 }

engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java

@@ -116,8 +116,8 @@ public class UsageEventUtils {
     }
 
     public static void publishUsageEvent(String usageType, long accountId, long zoneId, long ipAddressId, String ipAddress, boolean isSourceNat, String guestType,
-        boolean isSystem, String entityType, String entityUUID) {
+        boolean isSystem, boolean usageHidden, String entityType, String entityUUID) {
-        saveUsageEvent(usageType, accountId, zoneId, ipAddressId, ipAddress, isSourceNat, guestType, isSystem);
+        saveUsageEvent(usageType, accountId, zoneId, ipAddressId, ipAddress, isSourceNat, guestType, isSystem, usageHidden);
         publishUsageEvent(usageType, accountId, zoneId, entityType, entityUUID);
     }
 
@@ -182,8 +182,12 @@ public class UsageEventUtils {
     }
 
     public static void saveUsageEvent(String usageType, long accountId, long zoneId, long ipAddressId, String ipAddress, boolean isSourceNat, String guestType,
-        boolean isSystem) {
+        boolean isSystem, boolean usageHidden) {
-        s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, ipAddressId, ipAddress, isSourceNat, guestType, isSystem));
+        final UsageEventVO usageEventVO = new UsageEventVO(usageType, accountId, zoneId, ipAddressId, ipAddress, isSourceNat, guestType, isSystem);
+        s_usageEventDao.persist(usageEventVO);
+        if (usageHidden) {
+            s_usageEventDao.saveDetails(usageEventVO.getId(), Map.of("hidden", "true"));
+        }
     }
 
     public static void saveUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId,

engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java

@@ -206,5 +206,7 @@ public interface IpAddressManager {
     public boolean isIpEqualsGatewayOrNetworkOfferingsEmpty(Network network, String requestedIp);
 
     void releasePodIp(Long id) throws CloudRuntimeException;
+
+    boolean isUsageHidden(IPAddressVO address);
 }
 

engine/schema/src/main/java/com/cloud/usage/UsageIPAddressVO.java

@@ -58,10 +58,13 @@ public class UsageIPAddressVO implements InternalIdentity {
     @Temporal(value = TemporalType.TIMESTAMP)
     private Date released = null;
 
+    @Column(name = "is_hidden")
+    private boolean isHidden = false;
+
     protected UsageIPAddressVO() {
     }
 
-    public UsageIPAddressVO(long id, long accountId, long domainId, long zoneId, String address, boolean isSourceNat, boolean isSystem, Date assigned, Date released) {
+    public UsageIPAddressVO(long id, long accountId, long domainId, long zoneId, String address, boolean isSourceNat, boolean isSystem, Date assigned, Date released, boolean isHidden) {
         this.id = id;
         this.accountId = accountId;
         this.domainId = domainId;
@@ -71,6 +74,7 @@ public class UsageIPAddressVO implements InternalIdentity {
         this.isSystem = isSystem;
         this.assigned = assigned;
         this.released = released;
+        this.isHidden = isHidden;
     }
 
     public UsageIPAddressVO(long accountId, String address, Date assigned, Date released) {
@@ -120,4 +124,8 @@ public class UsageIPAddressVO implements InternalIdentity {
     public void setReleased(Date released) {
         this.released = released;
     }
+
+    public boolean isHidden() {
+        return isHidden;
+    }
 }

engine/schema/src/main/java/com/cloud/usage/UsageVO.java

@@ -106,6 +106,9 @@ public class UsageVO implements Usage, InternalIdentity {
     @Column(name = "quota_calculated")
     private Integer quotaCalculated = 0;
 
+    @Column(name = "is_hidden")
+    private boolean isHidden = false;
+
     public Integer getQuotaCalculated() {
         return quotaCalculated;
     }
@@ -215,7 +218,7 @@ public class UsageVO implements Usage, InternalIdentity {
 
     //IPAddress Usage
     public UsageVO(Long zoneId, Long accountId, Long domainId, String description, String usageDisplay, int usageType, Double rawUsage, Long usageId, long size,
-            String type, Date startDate, Date endDate) {
+            String type, Date startDate, Date endDate, boolean isHidden) {
         this.zoneId = zoneId;
         this.accountId = accountId;
         this.domainId = domainId;
@@ -228,6 +231,7 @@ public class UsageVO implements Usage, InternalIdentity {
         this.type = type;
         this.startDate = startDate  == null ? null : new Date(startDate.getTime());
         this.endDate = endDate  == null ? null : new Date(endDate.getTime());
+        this.isHidden = isHidden;
     }
 
     @Override
@@ -340,6 +344,11 @@ public class UsageVO implements Usage, InternalIdentity {
         return endDate  == null ? null : new Date(endDate.getTime());
     }
 
+    @Override
+    public boolean isHidden() {
+        return isHidden;
+    }
+
     public void setId(Long id) {
         this.id = id;
     }
@@ -383,4 +392,8 @@ public class UsageVO implements Usage, InternalIdentity {
     public void setVirtualSize(Long virtualSize) {
         this.virtualSize = virtualSize;
     }
+
+    public void setHidden(boolean hidden) {
+        this.isHidden = hidden;
+    }
 }

engine/schema/src/main/java/com/cloud/usage/dao/UsageIPAddressDaoImpl.java

@@ -40,16 +40,20 @@ public class UsageIPAddressDaoImpl extends GenericDaoBase<UsageIPAddressVO, Long
 
     protected static final String UPDATE_RELEASED = "UPDATE usage_ip_address SET released = ? WHERE account_id = ? AND public_ip_address = ? and released IS NULL";
     protected static final String GET_USAGE_RECORDS_BY_ACCOUNT =
-        "SELECT id, account_id, domain_id, zone_id, public_ip_address, is_source_nat, is_system, assigned, released " + "FROM usage_ip_address "
+        "SELECT id, account_id, domain_id, zone_id, public_ip_address, is_source_nat, is_system, assigned, released, is_hidden "
+            + "FROM usage_ip_address "
             + "WHERE account_id = ? AND ((released IS NULL AND assigned <= ?) OR (assigned BETWEEN ? AND ?) OR "
             + "      (released BETWEEN ? AND ?) OR ((assigned <= ?) AND (released >= ?)))";
     protected static final String GET_USAGE_RECORDS_BY_DOMAIN =
-        "SELECT id, account_id, domain_id, zone_id, public_ip_address, is_source_nat, is_system, assigned, released " + "FROM usage_ip_address "
+        "SELECT id, account_id, domain_id, zone_id, public_ip_address, is_source_nat, is_system, assigned, released, is_hidden "
+            + "FROM usage_ip_address "
             + "WHERE domain_id = ? AND ((released IS NULL AND assigned <= ?) OR (assigned BETWEEN ? AND ?) OR "
             + "      (released BETWEEN ? AND ?) OR ((assigned <= ?) AND (released >= ?)))";
-    protected static final String GET_ALL_USAGE_RECORDS = "SELECT id, account_id, domain_id, zone_id, public_ip_address, is_source_nat, is_system, assigned, released "
+    protected static final String GET_ALL_USAGE_RECORDS =
-        + "FROM usage_ip_address " + "WHERE (released IS NULL AND assigned <= ?) OR (assigned BETWEEN ? AND ?) OR "
+        "SELECT id, account_id, domain_id, zone_id, public_ip_address, is_source_nat, is_system, assigned, released, is_hidden "
-        + "      (released BETWEEN ? AND ?) OR ((assigned <= ?) AND (released >= ?))";
+            + "FROM usage_ip_address "
+            + "WHERE (released IS NULL AND assigned <= ?) OR (assigned BETWEEN ? AND ?) OR "
+            + "      (released BETWEEN ? AND ?) OR ((assigned <= ?) AND (released >= ?))";
 
     public UsageIPAddressDaoImpl() {
     }
@@ -128,6 +132,7 @@ public class UsageIPAddressDaoImpl extends GenericDaoBase<UsageIPAddressVO, Long
                 Date releasedDate = null;
                 String assignedTS = rs.getString(8);
                 String releasedTS = rs.getString(9);
+                Boolean isHidden = Boolean.valueOf(rs.getBoolean(10));
 
                 if (assignedTS != null) {
                     assignedDate = DateUtil.parseDateString(s_gmtTimeZone, assignedTS);
@@ -136,7 +141,7 @@ public class UsageIPAddressDaoImpl extends GenericDaoBase<UsageIPAddressVO, Long
                     releasedDate = DateUtil.parseDateString(s_gmtTimeZone, releasedTS);
                 }
 
-                usageRecords.add(new UsageIPAddressVO(id, acctId, dId, zId, addr, isSourceNat, isSystem, assignedDate, releasedDate));
+                usageRecords.add(new UsageIPAddressVO(id, acctId, dId, zId, addr, isSourceNat, isSystem, assignedDate, releasedDate, isHidden));
             }
         } catch (Exception e) {
             txn.rollback();

engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql

@@ -833,3 +833,7 @@ INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_vers
 
 -- Fix OS category for Guest OS 'Other PV Virtio-SCSI (64-bit)'
 UPDATE `cloud`.`guest_os` SET category_id = 7 WHERE id = 275 AND display_name = 'Other PV Virtio-SCSI (64-bit)';
+
+-- Add flag 'hidden' in tables usage_ip_address and cloud_usage
+ALTER TABLE `cloud_usage`.`usage_ip_address` ADD COLUMN `is_hidden` smallint(1) NOT NULL DEFAULT '0' COMMENT 'is usage hidden';
+ALTER TABLE `cloud_usage`.`cloud_usage` ADD COLUMN `is_hidden` smallint(1) NOT NULL DEFAULT '0' COMMENT 'is usage hidden';

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -4123,8 +4123,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                     // range
                     final List<IPAddressVO> ips = _publicIpAddressDao.listByVlanId(vlan.getId());
                     for (final IPAddressVO ip : ips) {
+                        final boolean usageHidden = _ipAddrMgr.isUsageHidden(ip);
                         UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_IP_ASSIGN, vlanOwner.getId(), ip.getDataCenterId(), ip.getId(), ip.getAddress().toString(),
-                                ip.isSourceNat(), vlan.getVlanType().toString(), ip.getSystem(), ip.getClass().getName(), ip.getUuid());
+                                ip.isSourceNat(), vlan.getVlanType().toString(), ip.getSystem(), usageHidden, ip.getClass().getName(), ip.getUuid());
                     }
                     // increment resource count for dedicated public ip's
                     _resourceLimitMgr.incrementResourceCount(vlanOwner.getId(), ResourceType.public_ip, new Long(ips.size()));
@@ -4204,8 +4205,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                         s_logger.warn("Some ip addresses failed to be released as a part of vlan " + vlanDbId + " removal");
                     } else {
                         resourceCountToBeDecrement++;
+                        final boolean usageHidden = _ipAddrMgr.isUsageHidden(ip);
                         UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_IP_RELEASE, acctVln.get(0).getAccountId(), ip.getDataCenterId(), ip.getId(),
-                                ip.getAddress().toString(), ip.isSourceNat(), vlanRange.getVlanType().toString(), ip.getSystem(), ip.getClass().getName(), ip.getUuid());
+                                ip.getAddress().toString(), ip.isSourceNat(), vlanRange.getVlanType().toString(), ip.getSystem(), usageHidden, ip.getClass().getName(), ip.getUuid());
                     }
                 }
             } finally {
@@ -4347,8 +4349,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
 
            // generate usage event for dedication of every ip address in the range
             for (final IPAddressVO ip : ips) {
+                final boolean usageHidden = _ipAddrMgr.isUsageHidden(ip);
                 UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_IP_ASSIGN, vlanOwner.getId(), ip.getDataCenterId(), ip.getId(), ip.getAddress().toString(), ip.isSourceNat(),
-                        vlan.getVlanType().toString(), ip.getSystem(), ip.getClass().getName(), ip.getUuid());
+                        vlan.getVlanType().toString(), ip.getSystem(), usageHidden, ip.getClass().getName(), ip.getUuid());
             }
         } else if (domain != null && !forSystemVms) {
             // Create an DomainVlanMapVO entry
@@ -4441,8 +4444,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             // generate usage events to remove dedication for every ip in the range that has been disassociated
             for (final IPAddressVO ip : ips) {
                 if (!ipsInUse.contains(ip)) {
+                    final boolean usageHidden = _ipAddrMgr.isUsageHidden(ip);
                     UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_IP_RELEASE, acctVln.get(0).getAccountId(), ip.getDataCenterId(), ip.getId(), ip.getAddress().toString(),
-                            ip.isSourceNat(), vlan.getVlanType().toString(), ip.getSystem(), ip.getClass().getName(), ip.getUuid());
+                            ip.isSourceNat(), vlan.getVlanType().toString(), ip.getSystem(), usageHidden, ip.getClass().getName(), ip.getUuid());
                 }
             }
             // decrement resource count for dedicated public ip's

server/src/main/java/com/cloud/network/IpAddressManagerImpl.java

@@ -101,6 +101,8 @@ import com.cloud.network.dao.IPAddressVO;
 import com.cloud.network.dao.LoadBalancerDao;
 import com.cloud.network.dao.NetworkAccountDao;
 import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkDetailsDao;
+import com.cloud.network.dao.NetworkDetailVO;
 import com.cloud.network.dao.NetworkDomainDao;
 import com.cloud.network.dao.NetworkServiceMapDao;
 import com.cloud.network.dao.PhysicalNetworkDao;
@@ -212,6 +214,8 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
     @Inject
     NetworkDao _networksDao;
     @Inject
+    NetworkDetailsDao _networkDetailsDao;
+    @Inject
     NicDao _nicDao;
     @Inject
     RulesManager _rulesMgr;
@@ -924,9 +928,10 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
                                     VlanVO vlan = _vlanDao.findById(addr.getVlanId());
                                     String guestType = vlan.getVlanType().toString();
                                     if (!isIpDedicated(addr)) {
+                                        final boolean usageHidden = isUsageHidden(addr);
                                         UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_IP_ASSIGN, owner.getId(), addr.getDataCenterId(), addr.getId(),
-                                                addr.getAddress().toString(),
+                                                addr.getAddress().toString(), addr.isSourceNat(), guestType, addr.getSystem(), usageHidden,
-                                                addr.isSourceNat(), guestType, addr.getSystem(), addr.getClass().getName(), addr.getUuid());
+                                                addr.getClass().getName(), addr.getUuid());
                                     }
                                     if (updateIpResourceCount(addr)) {
                                         _resourceLimitMgr.incrementResourceCount(owner.getId(), ResourceType.public_ip);
@@ -1277,9 +1282,10 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
                     ipaddr.setAllocatedInDomainId(ipOwner.getDomainId());
                     ipaddr.setAllocatedToAccountId(ipOwner.getId());
                     ipaddr = _ipAddressDao.persist(ipaddr);
+                    final boolean usageHidden = isUsageHidden(ipaddr);
 
                     UsageEventUtils.publishUsageEvent(EventTypes.EVENT_PORTABLE_IP_ASSIGN, ipaddr.getId(), ipaddr.getDataCenterId(), ipaddr.getId(),
-                            ipaddr.getAddress().toString(), ipaddr.isSourceNat(), null, ipaddr.getSystem(), ipaddr.getClass().getName(), ipaddr.getUuid());
+                            ipaddr.getAddress().toString(), ipaddr.isSourceNat(), null, ipaddr.getSystem(), usageHidden, ipaddr.getClass().getName(), ipaddr.getUuid());
 
                     return ipaddr;
                 }
@@ -1829,8 +1835,9 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
                         String guestType = vlan.getVlanType().toString();
                         if (!isIpDedicated(ip)) {
                             String eventType = ip.isPortable() ? EventTypes.EVENT_PORTABLE_IP_RELEASE : EventTypes.EVENT_NET_IP_RELEASE;
+                            final boolean usageHidden = isUsageHidden(ip);
                             UsageEventUtils.publishUsageEvent(eventType, ip.getAllocatedToAccountId(), ip.getDataCenterId(), addrId, ip.getAddress().addr(), ip.isSourceNat(),
-                                    guestType, ip.getSystem(), ip.getClass().getName(), ip.getUuid());
+                                    guestType, ip.getSystem(), usageHidden, ip.getClass().getName(), ip.getUuid());
                         }
                     }
 
@@ -2238,4 +2245,17 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
         }
         return false;
     }
+
+    @Override
+    public boolean isUsageHidden(IPAddressVO ip) {
+        Long networkId = ip.getAssociatedWithNetworkId();
+        if (networkId == null) {
+            networkId = ip.getSourceNetworkId();
+        }
+        if (networkId == null) {
+            throw new CloudRuntimeException("No network for IP " + ip.getId());
+        }
+        NetworkDetailVO networkDetail = _networkDetailsDao.findDetail(networkId, Network.hideIpAddressUsage);
+        return networkDetail != null && "true".equals(networkDetail.getValue());
+    }
 }

server/src/main/java/com/cloud/usage/UsageServiceImpl.java

@@ -347,6 +347,9 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag
             }
         }
 
+        // Filter out hidden usages
+        sc.addAnd("isHidden", SearchCriteria.Op.EQ, false);
+
         if ((adjustedStartDate != null) && (adjustedEndDate != null) && adjustedStartDate.before(adjustedEndDate)) {
             sc.addAnd("startDate", SearchCriteria.Op.BETWEEN, adjustedStartDate, adjustedEndDate);
             sc.addAnd("endDate", SearchCriteria.Op.BETWEEN, adjustedStartDate, adjustedEndDate);

usage/src/main/java/com/cloud/usage/UsageManagerImpl.java

@@ -1417,8 +1417,10 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
             long sourceNat = event.getSize();
             boolean isSourceNat = (sourceNat == 1) ? true : false;
             boolean isSystem = (event.getTemplateId() == null || event.getTemplateId() == 0) ? false : true;
+            final UsageEventDetailsVO hiddenDetail = _usageEventDetailsDao.findDetail(event.getId(), "hidden");
+            final boolean isHidden = hiddenDetail != null && "true".equals(hiddenDetail.getValue());
             UsageIPAddressVO ipAddressVO =
-                    new UsageIPAddressVO(id, event.getAccountId(), acct.getDomainId(), zoneId, ipAddress, isSourceNat, isSystem, event.getCreateDate(), null);
+                    new UsageIPAddressVO(id, event.getAccountId(), acct.getDomainId(), zoneId, ipAddress, isSourceNat, isSystem, event.getCreateDate(), null, isHidden);
             _usageIPAddressDao.persist(ipAddressVO);
         } else if (EventTypes.EVENT_NET_IP_RELEASE.equals(event.getType())) {
             SearchCriteria<UsageIPAddressVO> sc = _usageIPAddressDao.createSearchCriteria();

usage/src/main/java/com/cloud/usage/parser/IPAddressUsageParser.java

@@ -87,7 +87,7 @@ public class IPAddressUsageParser {
             String key = "" + IpId;
 
             // store the info in the IP map
-            IPMap.put(key, new IpInfo(usageIp.getZoneId(), IpId, usageIp.getAddress(), usageIp.isSourceNat(), usageIp.isSystem()));
+            IPMap.put(key, new IpInfo(usageIp.getZoneId(), IpId, usageIp.getAddress(), usageIp.isSourceNat(), usageIp.isSystem(), usageIp.isHidden()));
 
             Date IpAssignDate = usageIp.getAssigned();
             Date IpReleaseDeleteDate = usageIp.getReleased();
@@ -118,7 +118,7 @@ public class IPAddressUsageParser {
             // Only create a usage record if we have a runningTime of bigger than zero.
             if (useTime > 0L) {
                 IpInfo info = IPMap.get(ipIdKey);
-                createUsageRecord(info.getZoneId(), useTime, startDate, endDate, account, info.getIpId(), info.getIPAddress(), info.isSourceNat(), info.isSystem);
+                createUsageRecord(info.getZoneId(), useTime, startDate, endDate, account, info.getIpId(), info.getIPAddress(), info.isSourceNat(), info.isSystem, info.isHidden());
             }
         }
 
@@ -138,7 +138,7 @@ public class IPAddressUsageParser {
     }
 
     private static void createUsageRecord(long zoneId, long runningTime, Date startDate, Date endDate, AccountVO account, long ipId, String ipAddress,
-        boolean isSourceNat, boolean isSystem) {
+        boolean isSourceNat, boolean isSystem, boolean isHidden) {
         if (s_logger.isDebugEnabled()) {
             s_logger.debug("Total usage time " + runningTime + "ms");
         }
@@ -159,7 +159,7 @@ public class IPAddressUsageParser {
 
         UsageVO usageRecord =
             new UsageVO(zoneId, account.getAccountId(), account.getDomainId(), usageDesc, usageDisplay + " Hrs", UsageTypes.IP_ADDRESS, new Double(usage), ipId,
-                (isSystem ? 1 : 0), (isSourceNat ? "SourceNat" : ""), startDate, endDate);
+                (isSystem ? 1 : 0), (isSourceNat ? "SourceNat" : ""), startDate, endDate, isHidden);
         s_usageDao.persist(usageRecord);
     }
 
@@ -169,13 +169,15 @@ public class IPAddressUsageParser {
         private String ipAddress;
         private boolean isSourceNat;
         private boolean isSystem;
+        private boolean isHidden;
 
-        public IpInfo(long zoneId, long ipId, String ipAddress, boolean isSourceNat, boolean isSystem) {
+        public IpInfo(long zoneId, long ipId, String ipAddress, boolean isSourceNat, boolean isSystem, boolean isHidden) {
             this.zoneId = zoneId;
             this.ipId = ipId;
             this.ipAddress = ipAddress;
             this.isSourceNat = isSourceNat;
             this.isSystem = isSystem;
+            this.isHidden = isHidden;
         }
 
         public long getZoneId() {
@@ -193,5 +195,9 @@ public class IPAddressUsageParser {
         public boolean isSourceNat() {
             return isSourceNat;
         }
+
+        public boolean isHidden() {
+            return isHidden;
+        }
     }
 }