diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 96b3299e707..01ad12a71e0 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -1193,6 +1193,10 @@ public class EventTypes { entityEventDetails.put(EVENT_QUOTA_TARIFF_UPDATE, QuotaTariff.class); } + public static boolean isNetworkEvent(String eventType) { + return EVENT_NETWORK_CREATE.equals(eventType) || EVENT_NETWORK_DELETE.equals(eventType) || + EVENT_NETWORK_UPDATE.equals(eventType); + } public static String getEntityForEvent(String eventName) { Object entityClass = entityEventDetails.get(eventName); if (entityClass == null) { diff --git a/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java b/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java index 78af844c0a8..32ae34056ec 100644 --- a/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java +++ b/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java @@ -46,6 +46,7 @@ public class UsageTypes { public static final int VM_SNAPSHOT_ON_PRIMARY = 27; public static final int BACKUP = 28; public static final int BUCKET = 29; + public static final int NETWORK = 30; public static final int VPC = 31; public static List listUsageTypes() { @@ -73,6 +74,7 @@ public class UsageTypes { responseList.add(new UsageTypeResponse(VM_SNAPSHOT_ON_PRIMARY, "VM Snapshot on primary storage usage")); responseList.add(new UsageTypeResponse(BACKUP, "Backup storage usage")); responseList.add(new UsageTypeResponse(BUCKET, "Bucket storage usage")); + responseList.add(new UsageTypeResponse(NETWORK, "Network usage")); responseList.add(new UsageTypeResponse(VPC, "VPC usage")); return responseList; } diff --git a/client/pom.xml b/client/pom.xml index a73aed7d032..1d11fa74650 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,10 +28,6 @@ 4.20.0.0-SNAPSHOT - - juniper-contrail - https://juniper.github.io/contrail-maven/snapshots - juniper-tungsten-api https://github.com/radu-todirica/tungsten-api/raw/master @@ -271,11 +267,6 @@ cloud-plugin-network-nvp ${project.version} - - org.apache.cloudstack - cloud-plugin-network-contrail - ${project.version} - org.apache.cloudstack cloud-plugin-network-palo-alto @@ -1107,6 +1098,11 @@ cloud-plugin-api-vmware-sioc ${project.version} + + org.apache.cloudstack + cloud-plugin-network-contrail + ${project.version} + org.apache.cloudstack cloud-plugin-backup-veeam diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index ba9e5646bb5..d07fee32276 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -1466,6 +1466,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra if (isNetworkImplemented(network)) { logger.debug("Network id={} is already implemented", networkId); implemented.set(guru, network); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_UPDATE, network.getAccountId(), network.getDataCenterId(), network.getId(), + network.getName(), network.getNetworkOfferingId(), null, network.getState().name(), Network.class.getName(), network.getUuid(), true); return implemented; } @@ -1522,6 +1524,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra network.setRestartRequired(false); _networksDao.update(network.getId(), network); implemented.set(guru, network); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_CREATE, network.getAccountId(), network.getDataCenterId(), network.getId(), + network.getName(), network.getNetworkOfferingId(), null, null, null, network.getState().name(), network.getUuid()); return implemented; } catch (final NoTransitionException e) { logger.error(e.getMessage()); @@ -3370,6 +3374,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final Pair, Long> networkMsg = new Pair, Long>(Network.class, networkFinal.getId()); _messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, networkMsg); } + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_DELETE, network.getAccountId(), network.getDataCenterId(), network.getId(), + network.getName(), network.getNetworkOfferingId(), null, null, null, Network.class.getName(), network.getUuid()); return true; } catch (final CloudRuntimeException e) { logger.error("Failed to delete network", e); diff --git a/engine/schema/src/main/java/com/cloud/usage/UsageNetworksVO.java b/engine/schema/src/main/java/com/cloud/usage/UsageNetworksVO.java new file mode 100644 index 00000000000..1385e69da03 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/UsageNetworksVO.java @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +@Entity +@Table(name = "usage_networks") +public class UsageNetworksVO implements InternalIdentity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "network_id") + private long networkId; + + @Column(name = "network_offering_id") + private long networkOfferingId; + + @Column(name = "zone_id") + private long zoneId; + + @Column(name = "account_id") + private long accountId; + + @Column(name = "domain_id") + private long domainId; + + @Column(name = "state") + private String state; + + @Column(name = "created") + @Temporal(value = TemporalType.TIMESTAMP) + private Date created = null; + + @Column(name = "removed") + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed = null; + + protected UsageNetworksVO() { + } + + public UsageNetworksVO(long id, long networkId, long networkOfferingId, long zoneId, long accountId, long domainId, String state, Date created, Date removed) { + this.id = id; + this.networkId = networkId; + this.networkOfferingId = networkOfferingId; + this.zoneId = zoneId; + this.domainId = domainId; + this.accountId = accountId; + this.state = state; + this.created = created; + this.removed = removed; + } + + public UsageNetworksVO(long networkId, long networkOfferingId, long zoneId, long accountId, long domainId, String state, Date created, Date removed) { + this.networkId = networkId; + this.networkOfferingId = networkOfferingId; + this.zoneId = zoneId; + this.domainId = domainId; + this.accountId = accountId; + this.state = state; + this.created = created; + this.removed = removed; + } + + @Override + public long getId() { + return id; + } + + public long getZoneId() { + return zoneId; + } + + public long getAccountId() { + return accountId; + } + + public long getDomainId() { + return domainId; + } + + public long getNetworkId() { + return networkId; + } + + public void setNetworkId(long networkId) { + this.networkId = networkId; + } + + public long getNetworkOfferingId() { + return networkOfferingId; + } + + public void setNetworkOfferingId(long networkOfferingId) { + this.networkOfferingId = networkOfferingId; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } +} diff --git a/engine/schema/src/main/java/com/cloud/usage/UsageVO.java b/engine/schema/src/main/java/com/cloud/usage/UsageVO.java index 50884e3c012..b1f6d295fdd 100644 --- a/engine/schema/src/main/java/com/cloud/usage/UsageVO.java +++ b/engine/schema/src/main/java/com/cloud/usage/UsageVO.java @@ -113,6 +113,9 @@ public class UsageVO implements Usage, InternalIdentity { @Column(name = "is_hidden") private boolean isHidden = false; + @Column(name = "state") + private String state; + public Integer getQuotaCalculated() { return quotaCalculated; } @@ -401,6 +404,14 @@ public class UsageVO implements Usage, InternalIdentity { this.isHidden = hidden; } + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + @Override public String toString() { return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "usageId", "usageType"); diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDao.java new file mode 100644 index 00000000000..9be4c49b55e --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import com.cloud.usage.UsageNetworksVO; +import com.cloud.utils.db.GenericDao; + +import java.util.Date; +import java.util.List; + +public interface UsageNetworksDao extends GenericDao { + void update(long networkId, long newNetworkOffering, String state); + + void remove(long networkId, Date removed); + + List getUsageRecords(Long accountId, Date startDate, Date endDate); +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDaoImpl.java new file mode 100644 index 00000000000..f8ffbf74f85 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDaoImpl.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import com.cloud.network.Network; +import com.cloud.usage.UsageNetworksVO; +import com.cloud.utils.DateUtil; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.TransactionLegacy; + +import org.springframework.stereotype.Component; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +@Component +public class UsageNetworksDaoImpl extends GenericDaoBase implements UsageNetworksDao { + private static final Logger LOGGER = LogManager.getLogger(UsageNetworksDaoImpl.class); + protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT id, network_id, network_offering_id, zone_id, account_id, domain_id, state, created, removed FROM usage_networks WHERE " + + " account_id = ? AND ((removed IS NULL AND created <= ?) OR (created BETWEEN ? AND ?) OR (removed BETWEEN ? AND ?) " + + " OR ((created <= ?) AND (removed >= ?)))"; + + + @Override + public void update(long networkId, long newNetworkOffering, String state) { + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + try { + SearchCriteria sc = this.createSearchCriteria(); + sc.addAnd("networkId", SearchCriteria.Op.EQ, networkId); + sc.addAnd("removed", SearchCriteria.Op.NULL); + UsageNetworksVO vo = findOneBy(sc); + if (vo != null) { + vo.setNetworkOfferingId(newNetworkOffering); + vo.setState(state); + update(vo.getId(), vo); + } + } catch (final Exception e) { + txn.rollback(); + LOGGER.error(String.format("Error updating usage of network due to [%s].", e.getMessage()), e); + } finally { + txn.close(); + } + } + + @Override + public void remove(long networkId, Date removed) { + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + try { + SearchCriteria sc = this.createSearchCriteria(); + sc.addAnd("networkId", SearchCriteria.Op.EQ, networkId); + sc.addAnd("removed", SearchCriteria.Op.NULL); + UsageNetworksVO vo = findOneBy(sc); + if (vo != null) { + vo.setRemoved(removed); + vo.setState(Network.State.Destroy.name()); + update(vo.getId(), vo); + } + } catch (final Exception e) { + txn.rollback(); + LOGGER.error(String.format("Error updating usage of network due to [%s].", e.getMessage()), e); + } finally { + txn.close(); + } + } + + @Override + public List getUsageRecords(Long accountId, Date startDate, Date endDate) { + List usageRecords = new ArrayList<>(); + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + PreparedStatement pstmt; + try { + int i = 1; + pstmt = txn.prepareAutoCloseStatement(GET_USAGE_RECORDS_BY_ACCOUNT); + pstmt.setLong(i++, accountId); + + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), startDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), startDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), startDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate)); + + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + long id = rs.getLong(1); + long networkId = rs.getLong(2); + long networkOfferingId = rs.getLong(3); + long zoneId = rs.getLong(4); + long acctId = rs.getLong(5); + long domId = rs.getLong(6); + String stateTS = rs.getString(7); + Date createdDate = null; + Date removedDate = null; + String createdTS = rs.getString(8); + String removedTS = rs.getString(9); + + if (createdTS != null) { + createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS); + } + if (removedTS != null) { + removedDate = DateUtil.parseDateString(s_gmtTimeZone, removedTS); + } + usageRecords.add(new UsageNetworksVO(id, networkId, networkOfferingId, zoneId, acctId, domId, stateTS, createdDate, removedDate)); + } + } catch (Exception e) { + txn.rollback(); + LOGGER.warn("Error getting networks usage records", e); + } finally { + txn.close(); + } + + return usageRecords; + } +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml index fc455f872fa..d6d72f9228e 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml @@ -67,6 +67,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 372e0903b43..c70c6d4334e 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -197,6 +197,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql index de9c62258c8..c05f5503c3e 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql @@ -31,6 +31,20 @@ SET usage_unit = 'IOPS', updated_on = NOW() WHERE effective_on = '2010-05-04 00:00:00' AND name IN ('VM_DISK_IO_READ', 'VM_DISK_IO_WRITE'); +-- PR #7236 - [Usage] Create network billing +CREATE TABLE IF NOT EXISTS `cloud_usage`.`usage_networks` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `network_offering_id` bigint(20) unsigned NOT NULL, + `zone_id` bigint(20) unsigned NOT NULL, + `network_id` bigint(20) unsigned NOT NULL, + `account_id` bigint(20) unsigned NOT NULL, + `domain_id` bigint(20) unsigned NOT NULL, + `state` varchar(100) DEFAULT NULL, + `removed` datetime DEFAULT NULL, + `created` datetime NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB CHARSET=utf8; + -- allow for bigger urls ALTER TABLE `cloud`.`vm_template` MODIFY COLUMN `url` VARCHAR(1024) DEFAULT NULL COMMENT 'the url where the template exists externally'; diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index 2f571221f1a..d5df3ae8a91 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.stream.Collectors; import com.cloud.host.HostTagVO; +import com.cloud.network.dao.NetworkVO; import com.cloud.network.vpc.VpcVO; import javax.inject.Inject; @@ -37,6 +38,7 @@ import org.apache.cloudstack.backup.BackupOfferingVO; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.quota.constant.QuotaTypes; +import org.apache.cloudstack.quota.dao.NetworkDao; import org.apache.cloudstack.quota.dao.VmTemplateDao; import org.apache.cloudstack.quota.dao.VpcDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; @@ -173,6 +175,9 @@ public class PresetVariableHelper { @Inject BackupOfferingDao backupOfferingDao; + @Inject + NetworkDao networkDao; + @Inject VpcDao vpcDao; @@ -280,6 +285,7 @@ public class PresetVariableHelper { loadPresetVariableValueForNetworkOffering(usageRecord, value); loadPresetVariableValueForVmSnapshot(usageRecord, value); loadPresetVariableValueForBackup(usageRecord, value); + loadPresetVariableValueForNetwork(usageRecord, value); loadPresetVariableValueForVpc(usageRecord, value); return value; @@ -697,6 +703,22 @@ public class PresetVariableHelper { return backupOffering; } + protected void loadPresetVariableValueForNetwork(UsageVO usageRecord, Value value) { + int usageType = usageRecord.getUsageType(); + if (usageType != QuotaTypes.NETWORK) { + logNotLoadingMessageInTrace("Network", usageType); + return; + } + + Long networkId = usageRecord.getUsageId(); + NetworkVO network = networkDao.findByIdIncludingRemoved(networkId); + validateIfObjectIsNull(network, networkId, "Network"); + + value.setId(network.getUuid()); + value.setName(network.getName()); + value.setState(usageRecord.getState()); + } + protected void loadPresetVariableValueForVpc(UsageVO usageRecord, Value value) { int usageType = usageRecord.getUsageType(); if (usageType != QuotaTypes.VPC) { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index f3accd83be8..a1dc7b3c1bb 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -43,6 +43,7 @@ public class Value extends GenericPresetVariable { private BackupOffering backupOffering; private String hypervisorType; private String volumeFormat; + private String state; public Host getHost() { return host; @@ -205,4 +206,13 @@ public class Value extends GenericPresetVariable { public String getVolumeFormat() { return volumeFormat; } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + fieldNamesToIncludeInToString.add("state"); + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java index 0c6ff82a4f5..3ed162b2ba1 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java @@ -56,6 +56,7 @@ public class QuotaTypes extends UsageTypes { quotaTypeList.put(VM_SNAPSHOT_ON_PRIMARY, new QuotaTypes(VM_SNAPSHOT_ON_PRIMARY, "VM_SNAPSHOT_ON_PRIMARY", UsageUnitTypes.GB_MONTH.toString(), "VM Snapshot primary storage usage")); quotaTypeList.put(BACKUP, new QuotaTypes(BACKUP, "BACKUP", UsageUnitTypes.GB_MONTH.toString(), "Backup storage usage")); quotaTypeList.put(BUCKET, new QuotaTypes(BUCKET, "BUCKET", UsageUnitTypes.GB_MONTH.toString(), "Object Store bucket usage")); + quotaTypeList.put(NETWORK, new QuotaTypes(NETWORK, "NETWORK", UsageUnitTypes.COMPUTE_MONTH.toString(), "Network usage")); quotaTypeList.put(VPC, new QuotaTypes(VPC, "VPC", UsageUnitTypes.COMPUTE_MONTH.toString(), "VPC usage")); quotaTypeMap = Collections.unmodifiableMap(quotaTypeList); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/NetworkDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/NetworkDao.java new file mode 100644 index 00000000000..57b1981e998 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/NetworkDao.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.network.dao.NetworkVO; +import com.cloud.utils.db.GenericDao; + +public interface NetworkDao extends GenericDao { +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/NetworkDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/NetworkDaoImpl.java new file mode 100644 index 00000000000..bc581238d51 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/NetworkDaoImpl.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.network.dao.NetworkVO; +import com.cloud.utils.db.GenericDaoBase; + +/** + * This class was created to specifically use in {@link org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableHelper}.

+ * It was not possible to inject {@link com.cloud.network.dao.NetworkDao} due to its complex injection hierarchy. + */ +public class NetworkDaoImpl extends GenericDaoBase implements NetworkDao { +} diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index f7492590196..45af4b8a29a 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -471,6 +471,7 @@ public class PresetVariableHelperTest { Mockito.doNothing().when(presetVariableHelperSpy).loadPresetVariableValueForNetworkOffering(Mockito.any(UsageVO.class), Mockito.any(Value.class)); Mockito.doNothing().when(presetVariableHelperSpy).loadPresetVariableValueForVmSnapshot(Mockito.any(UsageVO.class), Mockito.any(Value.class)); Mockito.doNothing().when(presetVariableHelperSpy).loadPresetVariableValueForBackup(Mockito.any(UsageVO.class), Mockito.any(Value.class)); + Mockito.doNothing().when(presetVariableHelperSpy).loadPresetVariableValueForNetwork(Mockito.any(UsageVO.class), Mockito.any(Value.class)); Mockito.doNothing().when(presetVariableHelperSpy).loadPresetVariableValueForVpc(Mockito.any(UsageVO.class), Mockito.any(Value.class)); Value result = presetVariableHelperSpy.getPresetVariableValue(usageVoMock); diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java index 9e65de754c8..4d0162b33c9 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java @@ -150,4 +150,12 @@ public class ValueTest { variable.setVolumeFormat(null); Assert.assertTrue(variable.fieldNamesToIncludeInToString.contains("volumeFormat")); } + + @Test + public void setStateTestAddFieldStateToCollection() { + Value variable = new Value(); + variable.setState(null); + Assert.assertTrue(variable.fieldNamesToIncludeInToString.contains("state")); + } + } diff --git a/plugins/network-elements/juniper-contrail/pom.xml b/plugins/network-elements/juniper-contrail/pom.xml index a6a0df9f9d5..3f71146d580 100644 --- a/plugins/network-elements/juniper-contrail/pom.xml +++ b/plugins/network-elements/juniper-contrail/pom.xml @@ -27,12 +27,6 @@ 4.20.0.0-SNAPSHOT ../../pom.xml - - - juniper-contrail - https://juniper.github.io/contrail-maven/snapshots - - org.apache.cloudstack diff --git a/plugins/pom.xml b/plugins/pom.xml index bfe3060addf..e29f385dc37 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -100,7 +100,6 @@ network-elements/bigswitch network-elements/dns-notifier - network-elements/juniper-contrail network-elements/elastic-loadbalancer network-elements/globodns network-elements/internal-loadbalancer @@ -231,6 +230,7 @@ hypervisors/vmware network-elements/cisco-vnmc network-elements/nsx + network-elements/juniper-contrail diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 6168de1ef63..1314d7dd574 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -2061,7 +2061,12 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C if (implementedNetwork == null || implementedNetwork.first() == null) { logger.warn("Failed to provision the network " + network); } - return implementedNetwork.second(); + Network implemented = implementedNetwork.second(); + if (implemented != null) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_CREATE, implemented.getAccountId(), implemented.getDataCenterId(), implemented.getId(), + implemented.getName(), implemented.getNetworkOfferingId(), null, null, null, Network.class.getName(), implemented.getUuid()); + } + return implemented; } catch (ResourceUnavailableException ex) { logger.warn("Failed to implement persistent guest network " + network + "due to ", ex); CloudRuntimeException e = new CloudRuntimeException("Failed to implement persistent guest network"); @@ -3458,7 +3463,10 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C } } } - return getNetwork(network.getId()); + Network updatedNetwork = getNetwork(network.getId()); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_UPDATE, updatedNetwork.getAccountId(), updatedNetwork.getDataCenterId(), updatedNetwork.getId(), + updatedNetwork.getName(), updatedNetwork.getNetworkOfferingId(), null, updatedNetwork.getState().name(), Network.class.getName(), updatedNetwork.getUuid(), true); + return updatedNetwork; } protected Pair validateMtuOnUpdate(NetworkVO network, Long zoneId, Integer publicMtu, Integer privateMtu) { diff --git a/server/src/test/java/com/cloud/network/NetworkModelImplTest.java b/server/src/test/java/com/cloud/network/NetworkModelImplTest.java index 6eb3e5d186b..889031f2b50 100644 --- a/server/src/test/java/com/cloud/network/NetworkModelImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkModelImplTest.java @@ -41,6 +41,7 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; import java.util.ArrayList; @@ -49,7 +50,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) public class NetworkModelImplTest { diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 6c824c1fe7d..aaffa63978a 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -1035,7 +1035,7 @@ class TestSecuredVmMigration(cloudstackTestCase): time.sleep(interval) restarted_host = Host.list( cls.apiclient, - hostid=host.id, + id=host.id, type='Routing' )[0] if restarted_host.state == "Up": @@ -1100,7 +1100,7 @@ class TestSecuredVmMigration(cloudstackTestCase): time.sleep(interval) host = Host.list( self.apiclient, - hostid=hostId, + id=hostId, type='Routing' )[0] if host.state != state: @@ -1160,7 +1160,7 @@ class TestSecuredVmMigration(cloudstackTestCase): host = Host.list( self.apiclient, zoneid=self.zone.id, - hostid=host.id, + id=host.id, type='Routing' )[0] if host.details.secured != secured: diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 2b436bb440c..8cd0a4560ab 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -32,6 +32,9 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import com.cloud.network.Network; +import com.cloud.usage.dao.UsageNetworksDao; +import com.cloud.usage.parser.NetworksUsageParser; import com.cloud.network.vpc.Vpc; import com.cloud.usage.dao.UsageVpcDao; import com.cloud.usage.parser.VpcUsageParser; @@ -168,6 +171,8 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna private QuotaAlertManager _alertManager; @Inject private QuotaStatement _quotaStatement; + @Inject + private UsageNetworksDao usageNetworksDao; @Inject private BucketStatisticsDao _bucketStatisticsDao; @@ -1043,6 +1048,11 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna logger.debug("Bucket usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")"); } } + parsed = NetworksUsageParser.parse(account, currentStartDate, currentEndDate); + if (!parsed) { + logger.debug("Networks usage not parsed for account [{}}].", account); + } + parsed = VpcUsageParser.parse(account, currentStartDate, currentEndDate); if (!parsed) { logger.debug(String.format("VPC usage failed to parse for account [%s].", account)); @@ -1081,6 +1091,8 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna createVmSnapshotOnPrimaryEvent(event); } else if (isBackupEvent(eventType)) { createBackupEvent(event); + } else if (EventTypes.isNetworkEvent(eventType)) { + handleNetworkEvent(event); } else if (EventTypes.isVpcEvent(eventType)) { handleVpcEvent(event); } @@ -2131,6 +2143,21 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna } } + private void handleNetworkEvent(UsageEventVO event) { + Account account = _accountDao.findByIdIncludingRemoved(event.getAccountId()); + long domainId = account.getDomainId(); + if (EventTypes.EVENT_NETWORK_DELETE.equals(event.getType())) { + usageNetworksDao.remove(event.getResourceId(), event.getCreateDate()); + } else if (EventTypes.EVENT_NETWORK_CREATE.equals(event.getType())) { + UsageNetworksVO usageNetworksVO = new UsageNetworksVO(event.getResourceId(), event.getOfferingId(), event.getZoneId(), event.getAccountId(), domainId, Network.State.Allocated.name(), event.getCreateDate(), null); + usageNetworksDao.persist(usageNetworksVO); + } else if (EventTypes.EVENT_NETWORK_UPDATE.equals(event.getType())) { + usageNetworksDao.update(event.getResourceId(), event.getOfferingId(), event.getResourceType()); + } else { + logger.error("Unknown event type [{}] in Networks event parser. Skipping it.", event.getType()); + } + } + private void handleVpcEvent(UsageEventVO event) { Account account = _accountDao.findByIdIncludingRemoved(event.getAccountId()); long domainId = account.getDomainId(); diff --git a/usage/src/main/java/com/cloud/usage/parser/NetworksUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/NetworksUsageParser.java new file mode 100644 index 00000000000..a27e2ba226e --- /dev/null +++ b/usage/src/main/java/com/cloud/usage/parser/NetworksUsageParser.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.parser; + +import com.cloud.usage.UsageNetworksVO; +import com.cloud.usage.UsageVO; +import com.cloud.usage.dao.UsageDao; +import com.cloud.usage.dao.UsageNetworksDao; +import com.cloud.user.AccountVO; +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +import org.apache.cloudstack.usage.UsageTypes; +import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.stereotype.Component; + +import java.text.DecimalFormat; +import java.util.Date; +import java.util.List; + +@Component +public class NetworksUsageParser { + private static final Logger LOGGER = LogManager.getLogger(NetworksUsageParser.class.getName()); + + @Inject + private UsageNetworksDao networksDao; + @Inject + private UsageDao usageDao; + + private static UsageDao staticUsageDao; + private static UsageNetworksDao staticNetworksDao; + + @PostConstruct + void init() { + staticUsageDao = usageDao; + staticNetworksDao = networksDao; + } + + public static boolean parse(AccountVO account, Date startDate, Date endDate) { + LOGGER.debug(String.format("Parsing all networks usage events for account [%s].", account.getId())); + if ((endDate == null) || endDate.after(new Date())) { + endDate = new Date(); + } + + final List usageNetworksVO = staticNetworksDao.getUsageRecords(account.getId(), startDate, endDate); + if (CollectionUtils.isEmpty(usageNetworksVO)) { + LOGGER.debug(String.format("Cannot find any VPC usage for account [%s] in period between [%s] and [%s].", account, startDate, endDate)); + return true; + } + + for (final UsageNetworksVO usageNetwork : usageNetworksVO) { + Long zoneId = usageNetwork.getZoneId(); + Date createdDate = usageNetwork.getCreated(); + Date removedDate = usageNetwork.getRemoved(); + if (createdDate.before(startDate)) { + createdDate = startDate; + } + + if (removedDate == null || removedDate.after(endDate)) { + removedDate = endDate; + } + + final long duration = (removedDate.getTime() - createdDate.getTime()) + 1; + final float usage = duration / 1000f / 60f / 60f; + DecimalFormat dFormat = new DecimalFormat("#.######"); + String usageDisplay = dFormat.format(usage); + + long networkId = usageNetwork.getNetworkId(); + long networkOfferingId = usageNetwork.getNetworkOfferingId(); + LOGGER.debug(String.format("Creating network usage record with id [%s], network offering [%s], usage [%s], startDate [%s], and endDate [%s], for account [%s].", + networkId, networkOfferingId, usageDisplay, startDate, endDate, account.getId())); + + String description = String.format("Network usage for network ID: %d, network offering: %d", usageNetwork.getNetworkId(), usageNetwork.getNetworkOfferingId()); + UsageVO usageRecord = + new UsageVO(zoneId, account.getAccountId(), account.getDomainId(), description, usageDisplay + " Hrs", + UsageTypes.NETWORK, (double) usage, null, null, usageNetwork.getNetworkOfferingId(), null, usageNetwork.getNetworkId(), + (long)0, null, startDate, endDate); + usageRecord.setState(usageNetwork.getState()); + staticUsageDao.persist(usageRecord); + } + + return true; + } +}