From 60af31c9c0ba9919b518ae0a18d622cef876ab4d Mon Sep 17 00:00:00 2001 From: Harikrishna Date: Mon, 3 Feb 2025 18:00:57 +0530 Subject: [PATCH 01/21] Decrypt zone, cluster, storage details for configuration values (#10237) Co-authored-by: dahn Co-authored-by: Bryan Lima <42067040+BryanMLima@users.noreply.github.com> --- .../java/com/cloud/dc/ClusterDetailsDao.java | 3 ++- .../com/cloud/dc/ClusterDetailsDaoImpl.java | 16 ++++++++---- .../java/com/cloud/dc/ClusterDetailsVO.java | 18 ++++++++----- .../dc/dao/DataCenterDetailsDaoImpl.java | 5 ++-- .../java/com/cloud/domain/DomainDetailVO.java | 18 ++++++++----- .../cloud/domain/dao/DomainDetailsDao.java | 5 ++-- .../domain/dao/DomainDetailsDaoImpl.java | 26 +++++++------------ .../dao/StoragePoolDetailsDaoImpl.java | 2 +- .../java/com/cloud/user/AccountDetailVO.java | 18 ++++++++----- .../com/cloud/user/AccountDetailsDao.java | 5 ++-- .../com/cloud/user/AccountDetailsDaoImpl.java | 26 +++++++------------ .../resourcedetail/ResourceDetailsDao.java | 2 ++ .../ResourceDetailsDaoBase.java | 18 +++++++++++++ .../db/ImageStoreDetailsDaoImpl.java | 4 +-- 14 files changed, 97 insertions(+), 69 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java index 06c9c525504..b54cc8b3c21 100644 --- a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java @@ -19,8 +19,9 @@ package com.cloud.dc; import java.util.Map; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; -public interface ClusterDetailsDao extends GenericDao { +public interface ClusterDetailsDao extends GenericDao, ResourceDetailsDao { Map findDetails(long clusterId); void persist(long clusterId, Map details); diff --git a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java index c2058ad5644..37e10910978 100644 --- a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java @@ -26,12 +26,13 @@ import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; import com.cloud.utils.crypt.DBEncryptionUtil; -import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; + +public class ClusterDetailsDaoImpl extends ResourceDetailsDaoBase implements ClusterDetailsDao, ScopedConfigStorage { -public class ClusterDetailsDaoImpl extends GenericDaoBase implements ClusterDetailsDao, ScopedConfigStorage { protected final SearchBuilder ClusterSearch; protected final SearchBuilder DetailSearch; @@ -42,11 +43,11 @@ public class ClusterDetailsDaoImpl extends GenericDaoBase findDetails(long clusterId) { SearchCriteria sc = ClusterSearch.create(); @@ -138,7 +144,7 @@ public class ClusterDetailsDaoImpl extends GenericDaoBase key) { ClusterDetailsVO vo = findDetail(id, key.key()); - return vo == null ? null : vo.getValue(); + return vo == null ? null : getActualValue(vo); } @Override diff --git a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsVO.java b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsVO.java index 6eb9e7466a7..b213f8f2594 100644 --- a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsVO.java +++ b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsVO.java @@ -23,11 +23,11 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; -import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.api.ResourceDetail; @Entity @Table(name = "cluster_details") -public class ClusterDetailsVO implements InternalIdentity { +public class ClusterDetailsVO implements ResourceDetail { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -35,7 +35,7 @@ public class ClusterDetailsVO implements InternalIdentity { private long id; @Column(name = "cluster_id") - private long clusterId; + private long resourceId; @Column(name = "name") private String name; @@ -47,13 +47,14 @@ public class ClusterDetailsVO implements InternalIdentity { } public ClusterDetailsVO(long clusterId, String name, String value) { - this.clusterId = clusterId; + this.resourceId = clusterId; this.name = name; this.value = value; } - public long getClusterId() { - return clusterId; + @Override + public long getResourceId() { + return resourceId; } public String getName() { @@ -64,6 +65,11 @@ public class ClusterDetailsVO implements InternalIdentity { return value; } + @Override + public boolean isDisplay() { + return true; + } + public void setValue(String value) { this.value = value; } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java index e36c8ebd6c7..27210dfcf0d 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java @@ -16,7 +16,6 @@ // under the License. package com.cloud.dc.dao; -import org.apache.cloudstack.api.ResourceDetail; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; @@ -45,8 +44,8 @@ public class DataCenterDetailsDaoImpl extends ResourceDetailsDaoBase key) { - ResourceDetail vo = findDetail(id, key.key()); - return vo == null ? null : vo.getValue(); + DataCenterDetailVO vo = findDetail(id, key.key()); + return vo == null ? null : getActualValue(vo); } @Override diff --git a/engine/schema/src/main/java/com/cloud/domain/DomainDetailVO.java b/engine/schema/src/main/java/com/cloud/domain/DomainDetailVO.java index df5a2283baa..6f803cc9f2f 100644 --- a/engine/schema/src/main/java/com/cloud/domain/DomainDetailVO.java +++ b/engine/schema/src/main/java/com/cloud/domain/DomainDetailVO.java @@ -23,18 +23,18 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; -import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.api.ResourceDetail; @Entity @Table(name = "domain_details") -public class DomainDetailVO implements InternalIdentity { +public class DomainDetailVO implements ResourceDetail { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id") private long id; @Column(name = "domain_id") - private long domainId; + private long resourceId; @Column(name = "name") private String name; @@ -46,13 +46,14 @@ public class DomainDetailVO implements InternalIdentity { } public DomainDetailVO(long domainId, String name, String value) { - this.domainId = domainId; + this.resourceId = domainId; this.name = name; this.value = value; } - public long getDomainId() { - return domainId; + @Override + public long getResourceId() { + return resourceId; } public String getName() { @@ -63,6 +64,11 @@ public class DomainDetailVO implements InternalIdentity { return value; } + @Override + public boolean isDisplay() { + return true; + } + public void setValue(String value) { this.value = value; } diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java index 6b53e49764e..ae149ff4381 100644 --- a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java @@ -20,8 +20,9 @@ import java.util.Map; import com.cloud.domain.DomainDetailVO; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; -public interface DomainDetailsDao extends GenericDao { +public interface DomainDetailsDao extends GenericDao, ResourceDetailsDao { Map findDetails(long domainId); void persist(long domainId, Map details); @@ -31,6 +32,4 @@ public interface DomainDetailsDao extends GenericDao { void deleteDetails(long domainId); void update(long domainId, Map details); - - String getActualValue(DomainDetailVO domainDetailVO); } diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java index 50097d154f5..72532f4ea26 100644 --- a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java @@ -24,8 +24,6 @@ import javax.inject.Inject; import com.cloud.domain.DomainDetailVO; import com.cloud.domain.DomainVO; -import com.cloud.utils.crypt.DBEncryptionUtil; -import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -35,9 +33,9 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; -public class DomainDetailsDaoImpl extends GenericDaoBase implements DomainDetailsDao, ScopedConfigStorage { +public class DomainDetailsDaoImpl extends ResourceDetailsDaoBase implements DomainDetailsDao, ScopedConfigStorage { protected final SearchBuilder domainSearch; @Inject @@ -47,14 +45,14 @@ public class DomainDetailsDaoImpl extends GenericDaoBase i protected DomainDetailsDaoImpl() { domainSearch = createSearchBuilder(); - domainSearch.and("domainId", domainSearch.entity().getDomainId(), Op.EQ); + domainSearch.and("domainId", domainSearch.entity().getResourceId(), Op.EQ); domainSearch.done(); } @Override public Map findDetails(long domainId) { QueryBuilder sc = QueryBuilder.create(DomainDetailVO.class); - sc.and(sc.entity().getDomainId(), Op.EQ, domainId); + sc.and(sc.entity().getResourceId(), Op.EQ, domainId); List results = sc.list(); Map details = new HashMap(results.size()); for (DomainDetailVO r : results) { @@ -80,11 +78,16 @@ public class DomainDetailsDaoImpl extends GenericDaoBase i @Override public DomainDetailVO findDetail(long domainId, String name) { QueryBuilder sc = QueryBuilder.create(DomainDetailVO.class); - sc.and(sc.entity().getDomainId(), Op.EQ, domainId); + sc.and(sc.entity().getResourceId(), Op.EQ, domainId); sc.and(sc.entity().getName(), Op.EQ, name); return sc.find(); } + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new DomainDetailVO(resourceId, key, value)); + } + @Override public void deleteDetails(long domainId) { SearchCriteria sc = domainSearch.create(); @@ -129,13 +132,4 @@ public class DomainDetailsDaoImpl extends GenericDaoBase i } return vo == null ? null : getActualValue(vo); } - - @Override - public String getActualValue(DomainDetailVO domainDetailVO) { - ConfigurationVO configurationVO = _configDao.findByName(domainDetailVO.getName()); - if (configurationVO != null && configurationVO.isEncrypted()) { - return DBEncryptionUtil.decrypt(domainDetailVO.getValue()); - } - return domainDetailVO.getValue(); - } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java index 0c39a8c581a..559978ef284 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java @@ -45,7 +45,7 @@ public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase key) { StoragePoolDetailVO vo = findDetail(id, key.key()); - return vo == null ? null : vo.getValue(); + return vo == null ? null : getActualValue(vo); } @Override diff --git a/engine/schema/src/main/java/com/cloud/user/AccountDetailVO.java b/engine/schema/src/main/java/com/cloud/user/AccountDetailVO.java index 863f6c96008..aa6e49666dd 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountDetailVO.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountDetailVO.java @@ -23,18 +23,18 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; -import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.api.ResourceDetail; @Entity @Table(name = "account_details") -public class AccountDetailVO implements InternalIdentity { +public class AccountDetailVO implements ResourceDetail { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id") private long id; @Column(name = "account_id") - private long accountId; + private long resourceId; @Column(name = "name") private String name; @@ -46,13 +46,14 @@ public class AccountDetailVO implements InternalIdentity { } public AccountDetailVO(long accountId, String name, String value) { - this.accountId = accountId; + this.resourceId = accountId; this.name = name; this.value = value; } - public long getAccountId() { - return accountId; + @Override + public long getResourceId() { + return resourceId; } public String getName() { @@ -63,6 +64,11 @@ public class AccountDetailVO implements InternalIdentity { return value; } + @Override + public boolean isDisplay() { + return true; + } + public void setValue(String value) { this.value = value; } diff --git a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java index 514433e8068..65bbe1670a8 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java @@ -19,8 +19,9 @@ package com.cloud.user; import java.util.Map; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; -public interface AccountDetailsDao extends GenericDao { +public interface AccountDetailsDao extends GenericDao, ResourceDetailsDao { Map findDetails(long accountId); void persist(long accountId, Map details); @@ -34,6 +35,4 @@ public interface AccountDetailsDao extends GenericDao { * they will get created */ void update(long accountId, Map details); - - String getActualValue(AccountDetailVO accountDetailVO); } diff --git a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java index de562e27f9e..8cea616b97d 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java @@ -23,7 +23,6 @@ import java.util.Optional; import javax.inject.Inject; -import com.cloud.utils.crypt.DBEncryptionUtil; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; @@ -34,16 +33,15 @@ import com.cloud.domain.dao.DomainDetailsDao; import com.cloud.domain.dao.DomainDao; import com.cloud.user.dao.AccountDao; -import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; -public class AccountDetailsDaoImpl extends GenericDaoBase implements AccountDetailsDao, ScopedConfigStorage { +public class AccountDetailsDaoImpl extends ResourceDetailsDaoBase implements AccountDetailsDao, ScopedConfigStorage { protected final SearchBuilder accountSearch; @Inject @@ -57,14 +55,14 @@ public class AccountDetailsDaoImpl extends GenericDaoBase protected AccountDetailsDaoImpl() { accountSearch = createSearchBuilder(); - accountSearch.and("accountId", accountSearch.entity().getAccountId(), Op.EQ); + accountSearch.and("accountId", accountSearch.entity().getResourceId(), Op.EQ); accountSearch.done(); } @Override public Map findDetails(long accountId) { QueryBuilder sc = QueryBuilder.create(AccountDetailVO.class); - sc.and(sc.entity().getAccountId(), Op.EQ, accountId); + sc.and(sc.entity().getResourceId(), Op.EQ, accountId); List results = sc.list(); Map details = new HashMap(results.size()); for (AccountDetailVO r : results) { @@ -90,11 +88,16 @@ public class AccountDetailsDaoImpl extends GenericDaoBase @Override public AccountDetailVO findDetail(long accountId, String name) { QueryBuilder sc = QueryBuilder.create(AccountDetailVO.class); - sc.and(sc.entity().getAccountId(), Op.EQ, accountId); + sc.and(sc.entity().getResourceId(), Op.EQ, accountId); sc.and(sc.entity().getName(), Op.EQ, name); return sc.find(); } + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new AccountDetailVO(resourceId, key, value)); + } + @Override public void deleteDetails(long accountId) { SearchCriteria sc = accountSearch.create(); @@ -154,13 +157,4 @@ public class AccountDetailsDaoImpl extends GenericDaoBase } return value; } - - @Override - public String getActualValue(AccountDetailVO accountDetailVO) { - ConfigurationVO configurationVO = _configDao.findByName(accountDetailVO.getName()); - if (configurationVO != null && configurationVO.isEncrypted()) { - return DBEncryptionUtil.decrypt(accountDetailVO.getValue()); - } - return accountDetailVO.getValue(); - } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java index 5a173191be1..6daf8f02231 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java @@ -97,4 +97,6 @@ public interface ResourceDetailsDao extends GenericDao public void addDetail(long resourceId, String key, String value, boolean display); public List findResourceIdsByNameAndValueIn(String name, Object[] values); + + String getActualValue(ResourceDetail resourceDetail); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java index 37ebfebf5dd..556c832e991 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.cloud.utils.crypt.DBEncryptionUtil; import org.apache.cloudstack.api.ResourceDetail; import com.cloud.utils.db.GenericDaoBase; @@ -28,8 +29,16 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.SearchCriteria.Op; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; + +import javax.inject.Inject; public abstract class ResourceDetailsDaoBase extends GenericDaoBase implements ResourceDetailsDao { + + @Inject + private ConfigurationDao configDao; + private SearchBuilder AllFieldsSearch; public ResourceDetailsDaoBase() { @@ -201,4 +210,13 @@ public abstract class ResourceDetailsDaoBase extends G return customSearch(sc, null); } + + @Override + public String getActualValue(ResourceDetail resourceDetail) { + ConfigurationVO configurationVO = configDao.findByName(resourceDetail.getName()); + if (configurationVO != null && configurationVO.isEncrypted()) { + return DBEncryptionUtil.decrypt(resourceDetail.getValue()); + } + return resourceDetail.getValue(); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java index 8e5ce770f45..1b0644820c5 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java @@ -37,7 +37,6 @@ import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; @Component public class ImageStoreDetailsDaoImpl extends ResourceDetailsDaoBase implements ImageStoreDetailsDao, ScopedConfigStorage { - protected final SearchBuilder storeSearch; public ImageStoreDetailsDaoImpl() { @@ -108,12 +107,11 @@ public class ImageStoreDetailsDaoImpl extends ResourceDetailsDaoBase key) { ImageStoreDetailVO vo = findDetail(id, key.key()); - return vo == null ? null : vo.getValue(); + return vo == null ? null : getActualValue(vo); } @Override public void addDetail(long resourceId, String key, String value, boolean display) { super.addDetail(new ImageStoreDetailVO(resourceId, key, value, display)); } - } From fa5c11e6b2eecc82c16b4c1f67ecfdd7e616fd37 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 3 Feb 2025 13:31:38 +0100 Subject: [PATCH 02/21] UI: list backup offerings by zoneid when assign vm to backup offering (#10217) --- ui/src/config/section/compute.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 37bccf86758..52567450f86 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -219,6 +219,10 @@ export default { args: ['virtualmachineid', 'backupofferingid'], show: (record) => { return !record.backupofferingid }, mapping: { + backupofferingid: { + api: 'listBackupOfferings', + params: (record) => { return { zoneid: record.zoneid } } + }, virtualmachineid: { value: (record, params) => { return record.id } } From c1bc57b844b64bb9f7602567156cfb25f41a8cee Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 3 Feb 2025 07:59:46 -0500 Subject: [PATCH 03/21] List default network offerings when multiple physical networks for guest traffic type exists (#10222) --- .../configuration/ConfigurationManagerImpl.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 9df33b47257..94a0fd2ea0d 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -6955,10 +6955,6 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati sc.addAnd("id", SearchCriteria.Op.EQ, id); } - if (tags != null) { - sc.addAnd("tags", SearchCriteria.Op.EQ, tags); - } - if (isTagged != null) { if (isTagged) { sc.addAnd("tags", SearchCriteria.Op.NNULL); @@ -6967,6 +6963,17 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } + if (tags != null) { + if (GuestType.Shared.name().equalsIgnoreCase(guestIpType)) { + SearchCriteria tagsSc = networkOfferingJoinDao.createSearchCriteria(); + tagsSc.addAnd("tags", SearchCriteria.Op.EQ, tags); + tagsSc.addOr("isDefault", SearchCriteria.Op.EQ, true); + sc.addAnd("tags", SearchCriteria.Op.SC, tagsSc); + } else { + sc.addAnd("tags", SearchCriteria.Op.EQ, tags); + } + } + if (zoneId != null) { SearchBuilder sb = networkOfferingJoinDao.createSearchBuilder(); sb.and("zoneId", sb.entity().getZoneId(), SearchCriteria.Op.FIND_IN_SET); @@ -7027,7 +7034,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati boolean addOffering = true; List checkForProviders = new ArrayList(); - if (checkForTags && ! checkNetworkOfferingTags(pNtwkTags, allowNullTag, offering.getTags())) { + if (checkForTags && !checkNetworkOfferingTags(pNtwkTags, allowNullTag, offering.getTags())) { continue; } From 238d0c5e30a8445082050c9f35ff00f1cc9d096f Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Tue, 4 Feb 2025 14:29:20 +0530 Subject: [PATCH 04/21] Fix NPE while checking for user data provider (#10255) --- .../com/cloud/network/NetworkModelImpl.java | 6 +++++- .../element/ConfigDriveNetworkElement.java | 19 ++++++++++++++++--- .../network/element/VirtualRouterElement.java | 7 ++++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index 23018ab72fd..2f904a3275f 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -935,8 +935,12 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi @Override public UserDataServiceProvider getUserDataUpdateProvider(Network network) { - String userDataProvider = _ntwkSrvcDao.getProviderForServiceInNetwork(network.getId(), Service.UserData); + if (network == null) { + s_logger.warn("No network details, can't fetch user data provider"); + return null; + } + String userDataProvider = _ntwkSrvcDao.getProviderForServiceInNetwork(network.getId(), Service.UserData); if (userDataProvider == null) { s_logger.debug("Network " + network + " doesn't support service " + Service.UserData.getName()); return null; diff --git a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java index 38d71b9c507..15238dc1c3b 100644 --- a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java @@ -320,8 +320,12 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle try { final Network network = _networkMgr.getNetwork(nic.getNetworkId()); final UserDataServiceProvider userDataUpdateProvider = _networkModel.getUserDataUpdateProvider(network); + if (userDataUpdateProvider == null) { + LOG.warn("Failed to get user data provider"); + return false; + } final Provider provider = userDataUpdateProvider.getProvider(); - if (provider.equals(Provider.ConfigDrive)) { + if (Provider.ConfigDrive.equals(provider)) { try { return deleteConfigDriveIso(vm); } catch (ResourceUnavailableException e) { @@ -336,7 +340,12 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle @Override public boolean prepareMigration(NicProfile nic, Network network, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context) { - if (_networkModel.getUserDataUpdateProvider(network).getProvider().equals(Provider.ConfigDrive)) { + final UserDataServiceProvider userDataUpdateProvider = _networkModel.getUserDataUpdateProvider(network); + if (userDataUpdateProvider == null) { + LOG.warn("Failed to prepare for migration, can't get user data provider"); + return false; + } + if (Provider.ConfigDrive.equals(userDataUpdateProvider.getProvider())) { LOG.trace(String.format("[prepareMigration] for vm: %s", vm.getInstanceName())); try { if (isConfigDriveIsoOnHostCache(vm.getId())) { @@ -384,7 +393,11 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle } private void recreateConfigDriveIso(NicProfile nic, Network network, VirtualMachineProfile vm, DeployDestination dest) throws ResourceUnavailableException { - if (nic.isDefaultNic() && _networkModel.getUserDataUpdateProvider(network).getProvider().equals(Provider.ConfigDrive)) { + final UserDataServiceProvider userDataUpdateProvider = _networkModel.getUserDataUpdateProvider(network); + if (userDataUpdateProvider == null) { + return; + } + if (nic.isDefaultNic() && Provider.ConfigDrive.equals(userDataUpdateProvider.getProvider())) { DiskTO diskToUse = null; for (DiskTO disk : vm.getDisks()) { if (disk.getType() == Volume.Type.ISO && disk.getPath() != null && disk.getPath().contains("configdrive")) { diff --git a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java index 5680111ca1a..334cfb59073 100644 --- a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java @@ -767,7 +767,12 @@ NetworkMigrationResponder, AggregatedCommandExecutor, RedundantResource, DnsServ @Override public boolean saveHypervisorHostname(NicProfile nicProfile, Network network, VirtualMachineProfile vm, DeployDestination dest) throws ResourceUnavailableException { - if (_networkModel.getUserDataUpdateProvider(network).getProvider().equals(Provider.VirtualRouter) && vm.getVirtualMachine().getType() == VirtualMachine.Type.User) { + final UserDataServiceProvider userDataUpdateProvider = _networkModel.getUserDataUpdateProvider(network); + if (userDataUpdateProvider == null) { + s_logger.warn("Failed to update hypervisor host details, can't get user data provider"); + return false; + } + if (Provider.VirtualRouter.equals(userDataUpdateProvider.getProvider()) && vm.getVirtualMachine().getType() == VirtualMachine.Type.User) { VirtualMachine uvm = vm.getVirtualMachine(); UserVmVO destVm = _userVmDao.findById(uvm.getId()); VirtualMachineProfile profile = null; From 37c29f82eda886e87c0dcb342a1a03e699c480d7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 4 Feb 2025 15:54:41 +0530 Subject: [PATCH 05/21] server: fix snapshot physical size (#10216) Signed-off-by: Abhishek Kumar Co-authored-by: Wei Zhou --- .../com/cloud/api/query/QueryManagerImpl.java | 76 ++++++------ .../cloud/api/query/dao/SnapshotJoinDao.java | 6 +- .../api/query/dao/SnapshotJoinDaoImpl.java | 34 ++++-- .../query/dao/SnapshotJoinDaoImplTest.java | 109 ++++++++++++++++++ 4 files changed, 171 insertions(+), 54 deletions(-) create mode 100644 server/src/test/java/com/cloud/api/query/dao/SnapshotJoinDaoImplTest.java diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 42128525782..3063b0d49a2 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.api.query; +import static com.cloud.vm.VmDetailConstants.SSH_PUBLIC_KEY; + import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; @@ -34,39 +36,6 @@ import java.util.stream.Stream; import javax.inject.Inject; -import com.cloud.network.dao.IPAddressDao; -import com.cloud.network.dao.IPAddressVO; -import com.cloud.storage.StoragePool; -import com.cloud.storage.StoragePoolHostVO; -import com.cloud.event.EventVO; -import com.cloud.event.dao.EventDao; -import com.cloud.host.HostVO; -import com.cloud.offering.ServiceOffering; -import com.cloud.service.ServiceOfferingDetailsVO; -import com.cloud.storage.VMTemplateStoragePoolVO; -import com.cloud.storage.dao.StoragePoolHostDao; -import com.cloud.storage.dao.VMTemplatePoolDao; -import com.cloud.host.Host; -import com.cloud.host.dao.HostDao; -import com.cloud.network.as.AutoScaleVmGroupVmMapVO; -import com.cloud.network.as.dao.AutoScaleVmGroupDao; -import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; -import com.cloud.network.dao.NetworkDao; -import com.cloud.network.dao.NetworkVO; -import com.cloud.network.dao.PublicIpQuarantineDao; -import com.cloud.network.PublicIpQuarantine; -import com.cloud.network.vo.PublicIpQuarantineVO; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.user.AccountVO; -import com.cloud.user.SSHKeyPairVO; -import com.cloud.user.dao.SSHKeyPairDao; -import com.cloud.vm.InstanceGroupVMMapVO; -import com.cloud.vm.NicVO; -import com.cloud.vm.UserVmDetailVO; -import com.cloud.vm.dao.InstanceGroupVMMapDao; -import com.cloud.vm.dao.NicDao; -import com.cloud.vm.dao.UserVmDetailsDao; -import com.cloud.storage.VolumeVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker; @@ -108,6 +77,7 @@ import org.apache.cloudstack.api.command.user.account.ListAccountsCmd; import org.apache.cloudstack.api.command.user.account.ListProjectAccountsCmd; import org.apache.cloudstack.api.command.user.address.ListQuarantinedIpsCmd; import org.apache.cloudstack.api.command.user.affinitygroup.ListAffinityGroupsCmd; +import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; import org.apache.cloudstack.api.command.user.event.ListEventsCmd; import org.apache.cloudstack.api.command.user.iso.ListIsosCmd; import org.apache.cloudstack.api.command.user.job.ListAsyncJobsCmd; @@ -129,6 +99,7 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.BucketResponse; import org.apache.cloudstack.api.response.DetailOptionsResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; @@ -255,22 +226,38 @@ import com.cloud.dc.dao.DedicatedResourceDao; import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; +import com.cloud.event.EventVO; +import com.cloud.event.dao.EventDao; import com.cloud.event.dao.EventJoinDao; import com.cloud.exception.CloudAuthenticationException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.ha.HighAvailabilityManager; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.network.PublicIpQuarantine; import com.cloud.network.RouterHealthCheckResult; import com.cloud.network.VNF; import com.cloud.network.VpcVirtualNetworkApplianceService; +import com.cloud.network.as.AutoScaleVmGroupVmMapVO; +import com.cloud.network.as.dao.AutoScaleVmGroupDao; +import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PublicIpQuarantineDao; import com.cloud.network.dao.RouterHealthCheckResultDao; import com.cloud.network.dao.RouterHealthCheckResultVO; import com.cloud.network.router.VirtualNetworkApplianceManager; import com.cloud.network.security.SecurityGroupVMMapVO; import com.cloud.network.security.dao.SecurityGroupVMMapDao; +import com.cloud.network.vo.PublicIpQuarantineVO; import com.cloud.offering.DiskOffering; +import com.cloud.offering.ServiceOffering; import com.cloud.org.Grouping; import com.cloud.projects.Project; import com.cloud.projects.Project.ListProjectResourcesCriteria; @@ -286,6 +273,7 @@ import com.cloud.server.ResourceManagerUtil; import com.cloud.server.ResourceMetaDataService; import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; +import com.cloud.service.ServiceOfferingDetailsVO; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.service.dao.ServiceOfferingDetailsDao; @@ -298,24 +286,34 @@ import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; +import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiServiceImpl; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VolumeDao; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.template.VirtualMachineTemplate.State; import com.cloud.template.VirtualMachineTemplate.TemplateFilter; import com.cloud.user.Account; import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; import com.cloud.user.DomainManager; +import com.cloud.user.SSHKeyPairVO; import com.cloud.user.User; import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.SSHKeyPairDao; import com.cloud.user.dao.UserDao; import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; @@ -331,18 +329,20 @@ import com.cloud.utils.db.SearchCriteria.Func; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.DomainRouterVO; +import com.cloud.vm.InstanceGroupVMMapVO; +import com.cloud.vm.NicVO; +import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.DomainRouterDao; +import com.cloud.vm.dao.InstanceGroupVMMapDao; +import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; -import org.apache.cloudstack.api.response.BucketResponse; - -import static com.cloud.vm.VmDetailConstants.SSH_PUBLIC_KEY; @Component public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements QueryService, Configurable { @@ -5627,7 +5627,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Integer count = snapshotDataPair.second(); if (count == 0) { - // empty result return snapshotDataPair; } List snapshotData = snapshotDataPair.first(); @@ -5637,7 +5636,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } else { snapshots = snapshotJoinDao.searchBySnapshotStorePair(snapshotData.stream().map(SnapshotJoinVO::getSnapshotStorePair).toArray(String[]::new)); } - return new Pair<>(snapshots, count); } diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java index 4e916e66ae7..25dfbfe6714 100644 --- a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java @@ -23,10 +23,7 @@ import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.response.SnapshotResponse; import com.cloud.api.query.vo.SnapshotJoinVO; -import com.cloud.utils.Pair; -import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; -import com.cloud.utils.db.SearchCriteria; public interface SnapshotJoinDao extends GenericDao { @@ -34,8 +31,7 @@ public interface SnapshotJoinDao extends GenericDao { SnapshotResponse setSnapshotResponse(SnapshotResponse snapshotResponse, SnapshotJoinVO snapshot); - Pair, Integer> searchIncludingRemovedAndCount(final SearchCriteria sc, final Filter filter); - List searchBySnapshotStorePair(String... pairs); + List findByDistinctIds(Long zoneId, Long... ids); } diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java index 8b951c174f4..fe462859310 100644 --- a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java @@ -18,6 +18,8 @@ package com.cloud.api.query.dao; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -33,6 +35,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.query.QueryService; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; @@ -45,7 +48,6 @@ import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeVO; import com.cloud.user.Account; import com.cloud.user.AccountService; -import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -195,13 +197,6 @@ public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation, Integer> searchIncludingRemovedAndCount(final SearchCriteria sc, final Filter filter) { - List objects = searchIncludingRemoved(sc, filter, null, false); - Integer count = getDistinctCount(sc); - return new Pair<>(objects, count); - } - @Override public List searchBySnapshotStorePair(String... pairs) { // set detail batch query size @@ -246,14 +241,33 @@ public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation findById(Long zoneId, long id) { + SearchBuilder sb = createSearchBuilder(); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("id", id); + if (zoneId != null) { + sc.setParameters("zoneId", zoneId); + } + List snapshotJoinVOS = search(sc, null); + if (CollectionUtils.isEmpty(snapshotJoinVOS)) { + return null; + } + snapshotJoinVOS.sort(Comparator.comparing(SnapshotJoinVO::getSnapshotStorePair)); + return Collections.singletonList(snapshotJoinVOS.get(0)); + } + @Override public List findByDistinctIds(Long zoneId, Long... ids) { if (ids == null || ids.length == 0) { return new ArrayList<>(); } - + if (ids.length == 1) { + return findById(zoneId, ids[0]); + } Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotStorePair", QueryService.SortKeyAscending.value(), null, null); - SearchCriteria sc = snapshotIdsSearch.create(); if (zoneId != null) { sc.setParameters("zoneId", zoneId); diff --git a/server/src/test/java/com/cloud/api/query/dao/SnapshotJoinDaoImplTest.java b/server/src/test/java/com/cloud/api/query/dao/SnapshotJoinDaoImplTest.java new file mode 100644 index 00000000000..c1159d3aa88 --- /dev/null +++ b/server/src/test/java/com/cloud/api/query/dao/SnapshotJoinDaoImplTest.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.api.query.dao; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.api.query.vo.SnapshotJoinVO; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class SnapshotJoinDaoImplTest { + + @Spy + @InjectMocks + SnapshotJoinDaoImpl snapshotJoinDao = new SnapshotJoinDaoImpl(); + + @Mock + SearchCriteria mockSearchCriteria; + + @Before + public void setUp() { + SnapshotJoinVO mockSnap = mock(SnapshotJoinVO.class); + SearchBuilder mockSearchBuilder = mock(SearchBuilder.class); + when(mockSearchBuilder.entity()).thenReturn(mockSnap); + doReturn(mockSearchBuilder).when(snapshotJoinDao).createSearchBuilder(); + when(mockSearchBuilder.create()).thenReturn(mockSearchCriteria); + } + + @Test + public void testFindById_WithNullZoneId_EmptyResult() { + Long zoneId = null; + long id = 1L; + doReturn(Collections.emptyList()).when(snapshotJoinDao).search(mockSearchCriteria, null); + List result = snapshotJoinDao.findById(zoneId, id); + assertNull(result); + verify(mockSearchCriteria).setParameters("id", id); + verify(mockSearchCriteria, never()).setParameters("zoneId", zoneId); + } + + @Test + public void testFindById_WithValidZoneId_EmptyResult() { + Long zoneId = 1L; + long id = 1L; + doReturn(Collections.emptyList()).when(snapshotJoinDao).search(mockSearchCriteria, null); + List result = snapshotJoinDao.findById(zoneId, id); + assertNull(result); + verify(mockSearchCriteria).setParameters("id", id); + verify(mockSearchCriteria).setParameters("zoneId", zoneId); + } + + @Test + public void testFindById_WithValidResults() { + Long zoneId = 1L; + long id = 1L; + SnapshotJoinVO snapshot1 = mock(SnapshotJoinVO.class); + when(snapshot1.getSnapshotStorePair()).thenReturn("Primary_1"); + SnapshotJoinVO snapshot2 = mock(SnapshotJoinVO.class); + when(snapshot2.getSnapshotStorePair()).thenReturn("Image_1"); + doReturn(Arrays.asList(snapshot1, snapshot2)).when(snapshotJoinDao).search(mockSearchCriteria, null); + List result = snapshotJoinDao.findById(zoneId, id); + assertNotNull(result); + assertEquals(1, result.size()); + assertEquals("Image_1", result.get(0).getSnapshotStorePair()); + verify(mockSearchCriteria).setParameters("id", id); + verify(mockSearchCriteria).setParameters("zoneId", zoneId); + } + + @Test + public void testFindById_WithNullResults() { + long id = 1L; + doReturn(null).when(snapshotJoinDao).search(mockSearchCriteria, null); + List result = snapshotJoinDao.findById(null, id); + assertNull(result); + } +} From 1b2f6c999851d9482eb49ae44ba3066c9721dee4 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 4 Feb 2025 17:03:07 +0530 Subject: [PATCH 06/21] Hide register template, create/upload volume and create vpc buttons when zone is not created. (#10243) --- ui/src/config/section/compute.js | 5 +++++ ui/src/config/section/image.js | 8 ++++++-- ui/src/config/section/network.js | 17 +++++++++-------- ui/src/config/section/storage.js | 5 ++++- ui/src/utils/zone.js | 25 +++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 ui/src/utils/zone.js diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 52567450f86..5ebcde5602d 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -17,6 +17,7 @@ import { shallowRef, defineAsyncComponent } from 'vue' import store from '@/store' +import { isZoneCreated } from '@/utils/zone' export default { name: 'compute', @@ -99,6 +100,7 @@ export default { label: 'label.vm.add', docHelp: 'adminguide/virtual_machines.html#creating-vms', listView: true, + show: () => { isZoneCreated() }, component: () => import('@/views/compute/DeployVM.vue') }, { @@ -567,6 +569,7 @@ export default { docHelp: 'plugins/cloudstack-kubernetes-service.html#creating-a-new-kubernetes-cluster', listView: true, popup: true, + show: () => { isZoneCreated() }, component: shallowRef(defineAsyncComponent(() => import('@/views/compute/CreateKubernetesCluster.vue'))) }, { @@ -695,6 +698,7 @@ export default { icon: 'plus-outlined', label: 'label.new.autoscale.vmgroup', listView: true, + show: () => { isZoneCreated() }, component: () => import('@/views/compute/CreateAutoScaleVmGroup.vue') }, { @@ -785,6 +789,7 @@ export default { icon: 'plus-outlined', label: 'label.new.instance.group', listView: true, + show: () => { isZoneCreated() }, args: ['name'] }, { diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index aeeec2c3758..2b5153c5865 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -17,6 +17,7 @@ import { shallowRef, defineAsyncComponent } from 'vue' import store from '@/store' +import { isZoneCreated } from '@/utils/zone' export default { name: 'image', @@ -110,16 +111,17 @@ export default { docHelp: 'adminguide/templates.html#uploading-templates-from-a-remote-http-server', listView: true, popup: true, + show: () => { isZoneCreated() }, component: shallowRef(defineAsyncComponent(() => import('@/views/image/RegisterOrUploadTemplate.vue'))) }, { api: 'registerTemplate', icon: 'cloud-upload-outlined', label: 'label.upload.template.from.local', - show: () => { return 'getUploadParamsForTemplate' in store.getters.apis }, docHelp: 'adminguide/templates.html#uploading-templates-and-isos-from-a-local-computer', listView: true, popup: true, + show: () => { return isZoneCreated() && 'getUploadParamsForTemplate' in store.getters.apis }, component: shallowRef(defineAsyncComponent(() => import('@/views/image/RegisterOrUploadTemplate.vue'))) }, { @@ -270,13 +272,14 @@ export default { docHelp: 'adminguide/templates.html#id10', listView: true, popup: true, + show: () => { isZoneCreated() }, component: shallowRef(defineAsyncComponent(() => import('@/views/image/RegisterOrUploadIso.vue'))) }, { api: 'registerIso', icon: 'cloud-upload-outlined', label: 'label.upload.iso.from.local', - show: () => { return 'getUploadParamsForIso' in store.getters.apis }, + show: () => { return isZoneCreated() && 'getUploadParamsForIso' in store.getters.apis }, docHelp: 'adminguide/templates.html#id10', listView: true, popup: true, @@ -389,6 +392,7 @@ export default { label: 'label.kubernetes.version.add', listView: true, popup: true, + show: () => { isZoneCreated() }, component: shallowRef(defineAsyncComponent(() => import('@/views/image/AddKubernetesSupportedVersion.vue'))) }, { diff --git a/ui/src/config/section/network.js b/ui/src/config/section/network.js index edbe4bb37b7..f0415a783a5 100644 --- a/ui/src/config/section/network.js +++ b/ui/src/config/section/network.js @@ -19,6 +19,7 @@ import { shallowRef, defineAsyncComponent } from 'vue' import store from '@/store' import tungsten from '@/assets/icons/tungsten.svg?inline' import { isAdmin } from '@/role' +import { isZoneCreated } from '@/utils/zone' export default { name: 'network', @@ -123,7 +124,7 @@ export default { listView: true, popup: true, show: () => { - if (!store.getters.zones || store.getters.zones.length === 0) { + if (!isZoneCreated()) { return false } const AdvancedZones = store.getters.zones.filter(zone => zone.networktype === 'Advanced') @@ -245,6 +246,7 @@ export default { icon: 'plus-outlined', label: 'label.add.vpc', docHelp: 'adminguide/networking_and_traffic.html#adding-a-virtual-private-cloud', + show: () => { isZoneCreated() }, listView: true, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/network/CreateVpc.vue'))) @@ -306,7 +308,7 @@ export default { component: shallowRef(defineAsyncComponent(() => import('@/views/network/IngressEgressRuleConfigure.vue'))) }], show: () => { - if (!store.getters.zones || store.getters.zones.length === 0) { + if (!isZoneCreated()) { return false } const listZoneHaveSGEnabled = store.getters.zones.filter(zone => zone.securitygroupsenabled === true) @@ -394,6 +396,7 @@ export default { label: 'label.vnf.appliance.add', docHelp: 'adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances', listView: true, + show: () => { isZoneCreated() }, component: () => import('@/views/compute/DeployVnfAppliance.vue') }, { @@ -941,6 +944,7 @@ export default { label: 'label.add.vpn.gateway', docHelp: 'adminguide/networking_and_traffic.html#creating-a-vpn-gateway-for-the-vpc', listView: true, + show: () => { isZoneCreated() }, args: ['vpcid'] }, { @@ -1116,6 +1120,7 @@ export default { icon: 'plus-outlined', label: 'label.add.vpn.user', listView: true, + show: () => { isZoneCreated() }, args: (record, store) => { if (store.userInfo.roletype === 'User') { return ['username', 'password'] @@ -1195,6 +1200,7 @@ export default { docHelp: 'adminguide/networking_and_traffic.html#creating-and-updating-a-vpn-customer-gateway', listView: true, popup: true, + show: () => { isZoneCreated() }, component: shallowRef(defineAsyncComponent(() => import('@/views/network/CreateVpnCustomerGateway.vue'))) }, { @@ -1384,12 +1390,7 @@ export default { component: shallowRef(defineAsyncComponent(() => import('@/views/network/GuestVlanNetworksTab.vue'))), show: (record) => { return (record.allocationstate === 'Allocated') } }], - show: () => { - if (!store.getters.zones || store.getters.zones.length === 0) { - return false - } - return true - } + show: () => { isZoneCreated() } } ] } diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 28c451105a1..95894b33429 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -17,6 +17,7 @@ import { shallowRef, defineAsyncComponent } from 'vue' import store from '@/store' +import { isZoneCreated } from '@/utils/zone' export default { name: 'storage', @@ -103,6 +104,7 @@ export default { icon: 'plus-outlined', docHelp: 'adminguide/storage.html#creating-a-new-volume', label: 'label.action.create.volume', + show: () => { isZoneCreated() }, listView: true, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/storage/CreateVolume.vue'))) @@ -112,7 +114,7 @@ export default { icon: 'cloud-upload-outlined', docHelp: 'adminguide/storage.html#uploading-an-existing-volume-to-a-virtual-machine', label: 'label.upload.volume.from.local', - show: () => { return 'getUploadParamsForVolume' in store.getters.apis }, + show: () => { return isZoneCreated() && 'getUploadParamsForVolume' in store.getters.apis }, listView: true, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/storage/UploadLocalVolume.vue'))) @@ -122,6 +124,7 @@ export default { icon: 'link-outlined', docHelp: 'adminguide/storage.html#uploading-an-existing-volume-to-a-virtual-machine', label: 'label.upload.volume.from.url', + show: () => { isZoneCreated() }, listView: true, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/storage/UploadVolume.vue'))) diff --git a/ui/src/utils/zone.js b/ui/src/utils/zone.js new file mode 100644 index 00000000000..266bf9df445 --- /dev/null +++ b/ui/src/utils/zone.js @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import store from '@/store' + +export function isZoneCreated () { + if (!store.getters.zones || store.getters.zones.length === 0) { + return false + } + return true +} From 55e8eaab89cad7053b3bf0cd2d82808d70f8ef70 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Tue, 4 Feb 2025 15:18:49 +0100 Subject: [PATCH 07/21] Linstor: encryption support (#10126) This introduces a new encryption mode, instead of a simple bool. Now also storage driver can just provide encrypted volumes to CloudStack. --- .../main/java/com/cloud/storage/Storage.java | 65 +++++---- .../resource/LibvirtComputingResource.java | 3 +- .../kvm/storage/KVMStorageProcessor.java | 4 +- plugins/storage/volume/linstor/CHANGELOG.md | 5 + .../LinstorBackupSnapshotCommandWrapper.java | 18 ++- .../kvm/storage/LinstorStorageAdaptor.java | 2 +- .../LinstorPrimaryDataStoreDriverImpl.java | 68 +++++++++- ...LinstorPrimaryDataStoreDriverImplTest.java | 87 ++++++++++++ .../datastore/util/LinstorUtilTest.java | 127 ++++++++++++++++++ pom.xml | 2 +- 10 files changed, 346 insertions(+), 35 deletions(-) create mode 100644 plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java create mode 100644 plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 1163fcc892f..0d7a3ed90c0 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -135,34 +135,49 @@ public class Storage { ISODISK /* Template corresponding to a iso (non root disk) present in an OVA */ } + public enum EncryptionSupport { + /** + * Encryption not supported. + */ + Unsupported, + /** + * Will use hypervisor encryption driver (qemu -> luks) + */ + Hypervisor, + /** + * Storage pool handles encryption and just provides an encrypted volume + */ + Storage + } + public static enum StoragePoolType { - Filesystem(false, true, true), // local directory - NetworkFilesystem(true, true, true), // NFS - IscsiLUN(true, false, false), // shared LUN, with a clusterfs overlay - Iscsi(true, false, false), // for e.g., ZFS Comstar - ISO(false, false, false), // for iso image - LVM(false, false, false), // XenServer local LVM SR - CLVM(true, false, false), - RBD(true, true, false), // http://libvirt.org/storage.html#StorageBackendRBD - SharedMountPoint(true, true, true), - VMFS(true, true, false), // VMware VMFS storage - PreSetup(true, true, false), // for XenServer, Storage Pool is set up by customers. - EXT(false, true, false), // XenServer local EXT SR - OCFS2(true, false, false), - SMB(true, false, false), - Gluster(true, false, false), - PowerFlex(true, true, true), // Dell EMC PowerFlex/ScaleIO (formerly VxFlexOS) - ManagedNFS(true, false, false), - Linstor(true, true, false), - DatastoreCluster(true, true, false), // for VMware, to abstract pool of clusters - StorPool(true, true, true), - FiberChannel(true, true, false); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) + Filesystem(false, true, EncryptionSupport.Hypervisor), // local directory + NetworkFilesystem(true, true, EncryptionSupport.Hypervisor), // NFS + IscsiLUN(true, false, EncryptionSupport.Unsupported), // shared LUN, with a clusterfs overlay + Iscsi(true, false, EncryptionSupport.Unsupported), // for e.g., ZFS Comstar + ISO(false, false, EncryptionSupport.Unsupported), // for iso image + LVM(false, false, EncryptionSupport.Unsupported), // XenServer local LVM SR + CLVM(true, false, EncryptionSupport.Unsupported), + RBD(true, true, EncryptionSupport.Unsupported), // http://libvirt.org/storage.html#StorageBackendRBD + SharedMountPoint(true, true, EncryptionSupport.Hypervisor), + VMFS(true, true, EncryptionSupport.Unsupported), // VMware VMFS storage + PreSetup(true, true, EncryptionSupport.Unsupported), // for XenServer, Storage Pool is set up by customers. + EXT(false, true, EncryptionSupport.Unsupported), // XenServer local EXT SR + OCFS2(true, false, EncryptionSupport.Unsupported), + SMB(true, false, EncryptionSupport.Unsupported), + Gluster(true, false, EncryptionSupport.Unsupported), + PowerFlex(true, true, EncryptionSupport.Hypervisor), // Dell EMC PowerFlex/ScaleIO (formerly VxFlexOS) + ManagedNFS(true, false, EncryptionSupport.Unsupported), + Linstor(true, true, EncryptionSupport.Storage), + DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters + StorPool(true, true, EncryptionSupport.Hypervisor), + FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) private final boolean shared; private final boolean overProvisioning; - private final boolean encryption; + private final EncryptionSupport encryption; - StoragePoolType(boolean shared, boolean overProvisioning, boolean encryption) { + StoragePoolType(boolean shared, boolean overProvisioning, EncryptionSupport encryption) { this.shared = shared; this.overProvisioning = overProvisioning; this.encryption = encryption; @@ -177,6 +192,10 @@ public class Storage { } public boolean supportsEncryption() { + return encryption == EncryptionSupport.Hypervisor || encryption == EncryptionSupport.Storage; + } + + public EncryptionSupport encryptionSupportMode() { return encryption; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 7df170cd361..71f33d9be57 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3180,7 +3180,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv disk.setCacheMode(DiskDef.DiskCacheMode.valueOf(volumeObjectTO.getCacheMode().toString().toUpperCase())); } - if (volumeObjectTO.requiresEncryption()) { + if (volumeObjectTO.requiresEncryption() && + pool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor ) { String secretUuid = createLibvirtVolumeSecret(conn, volumeObjectTO.getPath(), volumeObjectTO.getPassphrase()); DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat())); disk.setLibvirtDiskEncryptDetails(encryptDetails); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 8cee8434b5e..d58cef8c79d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -50,6 +50,7 @@ import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtUtilitiesHelper; import com.cloud.storage.JavaStorageLayer; import com.cloud.storage.MigrationOptions; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; @@ -1452,7 +1453,8 @@ public class KVMStorageProcessor implements StorageProcessor { } } - if (encryptDetails != null) { + if (encryptDetails != null && + attachingPool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor) { diskdef.setLibvirtDiskEncryptDetails(encryptDetails); } diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index 419a7f983ee..98fc8f69512 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Volume snapshots on zfs used the wrong dataset path to hide/unhide snapdev +## [2024-12-19] + +### Added +- Native CloudStack encryption support + ## [2024-12-13] ### Fixed diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java index a572759c35a..fac2ccd2589 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java @@ -96,18 +96,23 @@ public final class LinstorBackupSnapshotCommandWrapper // NOTE: the qemu img will also contain the drbd metadata at the end final QemuImg qemu = new QemuImg(waitMilliSeconds); qemu.convert(srcFile, dstFile); - s_logger.info("Backup snapshot " + srcFile + " to " + dstPath); + s_logger.info(String.format("Backup snapshot '%s' to '%s'", srcPath, dstPath)); return dstPath; } private SnapshotObjectTO setCorrectSnapshotSize(final SnapshotObjectTO dst, final String dstPath) { final File snapFile = new File(dstPath); - final long size = snapFile.exists() ? snapFile.length() : 0; + long size; + if (snapFile.exists()) { + size = snapFile.length(); + } else { + s_logger.warn(String.format("Snapshot file %s does not exist. Reporting size 0", dstPath)); + size = 0; + } - final SnapshotObjectTO snapshot = new SnapshotObjectTO(); - snapshot.setPath(dst.getPath() + File.separator + dst.getName()); - snapshot.setPhysicalSize(size); - return snapshot; + dst.setPath(dst.getPath() + File.separator + dst.getName()); + dst.setPhysicalSize(size); + return dst; } @Override @@ -157,6 +162,7 @@ public final class LinstorBackupSnapshotCommandWrapper s_logger.info("Backup shrunk " + dstPath + " to actual size " + src.getVolume().getSize()); SnapshotObjectTO snapshot = setCorrectSnapshotSize(dst, dstPath); + s_logger.info(String.format("Actual file size for '%s' is %d", dstPath, snapshot.getPhysicalSize())); return new CopyCmdAnswer(snapshot); } catch (final Exception e) { final String error = String.format("Failed to backup snapshot with id [%s] with a pool %s, due to %s", diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 9b7a376e8f2..6a4d6c7a349 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -407,7 +407,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor { if (rsc.getFlags() != null && rsc.getFlags().contains(ApiConsts.FLAG_DRBD_DISKLESS) && !rsc.getFlags().contains(ApiConsts.FLAG_TIE_BREAKER)) { - ApiCallRcList delAnswers = api.resourceDelete(rsc.getName(), localNodeName); + ApiCallRcList delAnswers = api.resourceDelete(rsc.getName(), localNodeName, true); logLinstorAnswers(delAnswers); } } catch (ApiException apiEx) { diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 27904ed441b..4132fbd278a 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -21,11 +21,14 @@ import com.linbit.linstor.api.CloneWaiter; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.ApiCallRc; import com.linbit.linstor.api.model.ApiCallRcList; +import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.LayerType; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionCloneRequest; import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted; import com.linbit.linstor.api.model.ResourceDefinitionCreate; +import com.linbit.linstor.api.model.ResourceGroup; import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceMakeAvailable; import com.linbit.linstor.api.model.Snapshot; @@ -34,6 +37,7 @@ import com.linbit.linstor.api.model.VolumeDefinition; import com.linbit.linstor.api.model.VolumeDefinitionModify; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.inject.Inject; import java.util.Arrays; @@ -43,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.ResizeVolumeAnswer; @@ -103,8 +108,11 @@ import org.apache.cloudstack.storage.snapshot.SnapshotObject; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.storage.volume.VolumeObject; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; +import java.nio.charset.StandardCharsets; + public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver { private static final Logger s_logger = Logger.getLogger(LinstorPrimaryDataStoreDriverImpl.class); @Inject private PrimaryDataStoreDao _storagePoolDao; @@ -393,11 +401,56 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver storagePoolVO.getUserInfo() : "DfltRscGrp"; } + /** + * Returns the layerlist of the resourceGroup with encryption(LUKS) added above STORAGE. + * If the resourceGroup layer list already contains LUKS this layer list will be returned. + * @param api Linstor developers API + * @param resourceGroup Resource group to get the encryption layer list + * @return layer list with LUKS added + */ + public List getEncryptedLayerList(DevelopersApi api, String resourceGroup) { + try { + List rscGrps = api.resourceGroupList( + Collections.singletonList(resourceGroup), Collections.emptyList(), null, null); + + if (CollectionUtils.isEmpty(rscGrps)) { + throw new CloudRuntimeException( + String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); + } + + final ResourceGroup rscGrp = rscGrps.get(0); + List layers = Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE); + List curLayerStack = rscGrp.getSelectFilter() != null ? + rscGrp.getSelectFilter().getLayerStack() : Collections.emptyList(); + if (CollectionUtils.isNotEmpty(curLayerStack)) { + layers = curLayerStack.stream().map(LayerType::valueOf).collect(Collectors.toList()); + if (!layers.contains(LayerType.LUKS)) { + layers.add(layers.size() - 1, LayerType.LUKS); // lowest layer is STORAGE + } + } + return layers; + } catch (ApiException e) { + throw new CloudRuntimeException( + String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); + } + } + private String createResourceBase( - String rscName, long sizeInBytes, String volName, String vmName, DevelopersApi api, String rscGrp) { + String rscName, long sizeInBytes, String volName, String vmName, + @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, String rscGrp) { ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn(); rscGrpSpawn.setResourceDefinitionName(rscName); rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024); + if (passPhraseId != null) { + AutoSelectFilter asf = new AutoSelectFilter(); + List luksLayers = getEncryptedLayerList(api, rscGrp); + asf.setLayerStack(luksLayers.stream().map(LayerType::toString).collect(Collectors.toList())); + rscGrpSpawn.setSelectFilter(asf); + if (passPhrase != null) { + String utf8Passphrase = new String(passPhrase, StandardCharsets.UTF_8); + rscGrpSpawn.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); + } + } try { @@ -422,7 +475,8 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid(); String deviceName = createResourceBase( - rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), linstorApi, rscGrp); + rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), vol.getPassphraseId(), vol.getPassphrase(), + linstorApi, rscGrp); try { @@ -463,6 +517,14 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver s_logger.info("Clone resource definition " + cloneRes + " to " + rscName); ResourceDefinitionCloneRequest cloneRequest = new ResourceDefinitionCloneRequest(); cloneRequest.setName(rscName); + if (volumeInfo.getPassphraseId() != null) { + List encryptionLayer = getEncryptedLayerList(linstorApi, getRscGrp(storagePoolVO)); + cloneRequest.setLayerList(encryptionLayer); + if (volumeInfo.getPassphrase() != null) { + String utf8Passphrase = new String(volumeInfo.getPassphrase(), StandardCharsets.UTF_8); + cloneRequest.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); + } + } ResourceDefinitionCloneStarted cloneStarted = linstorApi.resourceDefinitionClone( cloneRes, cloneRequest); @@ -915,6 +977,8 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver tInfo.getSize(), tInfo.getName(), "", + null, + null, api, getRscGrp(pool)); diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java new file mode 100644 index 00000000000..75276739468 --- /dev/null +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.driver; + +import com.linbit.linstor.api.ApiException; +import com.linbit.linstor.api.DevelopersApi; +import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.LayerType; +import com.linbit.linstor.api.model.ResourceGroup; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LinstorPrimaryDataStoreDriverImplTest { + + private DevelopersApi api; + + @InjectMocks + private LinstorPrimaryDataStoreDriverImpl linstorPrimaryDataStoreDriver; + + @Before + public void setUp() { + api = mock(DevelopersApi.class); + } + + @Test + public void testGetEncryptedLayerList() throws ApiException { + ResourceGroup dfltRscGrp = new ResourceGroup(); + dfltRscGrp.setName("DfltRscGrp"); + + ResourceGroup bCacheRscGrp = new ResourceGroup(); + bCacheRscGrp.setName("BcacheGrp"); + AutoSelectFilter asf = new AutoSelectFilter(); + asf.setLayerStack(Arrays.asList(LayerType.DRBD.name(), LayerType.BCACHE.name(), LayerType.STORAGE.name())); + asf.setStoragePool("nvmePool"); + bCacheRscGrp.setSelectFilter(asf); + + ResourceGroup encryptedGrp = new ResourceGroup(); + encryptedGrp.setName("EncryptedGrp"); + AutoSelectFilter asf2 = new AutoSelectFilter(); + asf2.setLayerStack(Arrays.asList(LayerType.DRBD.name(), LayerType.LUKS.name(), LayerType.STORAGE.name())); + asf2.setStoragePool("ssdPool"); + encryptedGrp.setSelectFilter(asf2); + + when(api.resourceGroupList(Collections.singletonList("DfltRscGrp"), Collections.emptyList(), null, null)) + .thenReturn(Collections.singletonList(dfltRscGrp)); + when(api.resourceGroupList(Collections.singletonList("BcacheGrp"), Collections.emptyList(), null, null)) + .thenReturn(Collections.singletonList(bCacheRscGrp)); + when(api.resourceGroupList(Collections.singletonList("EncryptedGrp"), Collections.emptyList(), null, null)) + .thenReturn(Collections.singletonList(encryptedGrp)); + + List layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "DfltRscGrp"); + Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); + + layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "BcacheGrp"); + Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.BCACHE, LayerType.LUKS, LayerType.STORAGE), layers); + + layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "EncryptedGrp"); + Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); + } +} diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java new file mode 100644 index 00000000000..55f0c6ebe6d --- /dev/null +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.util; + +import com.linbit.linstor.api.ApiException; +import com.linbit.linstor.api.DevelopersApi; +import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.Node; +import com.linbit.linstor.api.model.Properties; +import com.linbit.linstor.api.model.ProviderKind; +import com.linbit.linstor.api.model.ResourceGroup; +import com.linbit.linstor.api.model.StoragePool; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LinstorUtilTest { + + private static final String LINSTOR_URL_TEST = "devnull.com:3370"; + private DevelopersApi api; + + private Node mockNode(String name) { + Node nodeMock = new Node(); + nodeMock.setName(name); + + return nodeMock; + } + + private StoragePool mockStoragePool(String name, String node, ProviderKind kind) { + StoragePool sp = new StoragePool(); + sp.setStoragePoolName(name); + sp.setNodeName(node); + sp.setProviderKind(kind); + return sp; + } + + @Before + public void setUp() throws ApiException { + api = mock(DevelopersApi.class); + + when(api.nodeList(Collections.emptyList(), Collections.emptyList(), null, null)) + .thenReturn(Arrays.asList(mockNode("nodeA"), mockNode("nodeB"), mockNode("nodeC"))); + + ResourceGroup csGroup = new ResourceGroup(); + csGroup.setName("cloudstack"); + AutoSelectFilter asf = new AutoSelectFilter(); + asf.setPlaceCount(2); + csGroup.setSelectFilter(asf); + when(api.resourceGroupList(Collections.singletonList("cloudstack"), null, null, null)) + .thenReturn(Collections.singletonList(csGroup)); + + when(api.viewStoragePools(Collections.emptyList(), null, null, null, null, true)) + .thenReturn(Arrays.asList( + mockStoragePool("thinpool", "nodeA", ProviderKind.LVM_THIN), + mockStoragePool("thinpool", "nodeB", ProviderKind.LVM_THIN), + mockStoragePool("thinpool", "nodeC", ProviderKind.LVM_THIN) + )); + +// when(LinstorUtil.getLinstorAPI(LINSTOR_URL_TEST)).thenReturn(api); + } + + @Test + public void testGetLinstorNodeNames() throws ApiException { + List linstorNodes = LinstorUtil.getLinstorNodeNames(api); + Assert.assertEquals(Arrays.asList("nodeA", "nodeB", "nodeC"), linstorNodes); + } + + @Test + public void testGetSnapshotPath() { + { + StoragePool spLVMThin = new StoragePool(); + Properties lvmThinProps = new Properties(); + lvmThinProps.put("StorDriver/StorPoolName", "storage/storage-thin"); + spLVMThin.setProps(lvmThinProps); + spLVMThin.setProviderKind(ProviderKind.LVM_THIN); + String snapPath = LinstorUtil.getSnapshotPath(spLVMThin, "cs-cb32532a-dd8f-47e0-a81c-8a75573d3545", "snap3"); + Assert.assertEquals("/dev/mapper/storage-cs--cb32532a--dd8f--47e0--a81c--8a75573d3545_00000_snap3", snapPath); + } + + { + StoragePool spZFS = new StoragePool(); + Properties zfsProps = new Properties(); + zfsProps.put("StorDriver/StorPoolName", "linstorPool"); + spZFS.setProps(zfsProps); + spZFS.setProviderKind(ProviderKind.ZFS); + + String snapPath = LinstorUtil.getSnapshotPath(spZFS, "cs-cb32532a-dd8f-47e0-a81c-8a75573d3545", "snap2"); + Assert.assertEquals("zfs://linstorPool/cs-cb32532a-dd8f-47e0-a81c-8a75573d3545_00000@snap2", snapPath); + } + } + + @Test + public void testGetRscGroupStoragePools() throws ApiException { + List storagePools = LinstorUtil.getRscGroupStoragePools(api, "cloudstack"); + + List names = storagePools.stream() + .map(sp -> String.format("%s::%s", sp.getNodeName(), sp.getStoragePoolName())) + .collect(Collectors.toList()); + Assert.assertEquals(names, Arrays.asList("nodeA::thinpool", "nodeB::thinpool", "nodeC::thinpool")); + } +} diff --git a/pom.xml b/pom.xml index da0252acbb4..0771a124db2 100644 --- a/pom.xml +++ b/pom.xml @@ -169,7 +169,7 @@ 10.1 2.6.6 0.6.0 - 0.5.2 + 0.6.0 0.10.2 3.4.4_1 4.0.1 From 90c960eeed9f7067961cd581064f7ca12459ad78 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 4 Feb 2025 16:00:58 +0100 Subject: [PATCH 08/21] VPC VR: fix ACL between tier and private gateway (#10268) --- systemvm/debian/opt/cloud/bin/cs/CsAddress.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index 3cb782daf7a..3d6d1f6f722 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -542,8 +542,10 @@ class CsIP: (self.dev, guestNetworkCidr, self.address['gateway'], self.dev)]) if self.is_private_gateway(): - self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % + self.fw.append(["filter", "front", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % (self.address['network'], self.dev, self.dev)]) + self.fw.append(["filter", "front", "-A FORWARD -d %s -o %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % + (self.address['network'], self.dev)]) self.fw.append(["filter", "", "-A ACL_INBOUND_%s -j DROP" % self.dev]) self.fw.append(["mangle", "", "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" % From df99a294832692cd90dc2f408b83f76390bf0729 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Thu, 6 Feb 2025 10:18:04 +0100 Subject: [PATCH 09/21] linstor: Fix using multiple primary storage with same linstor-controller (#10280) --- plugins/storage/volume/linstor/CHANGELOG.md | 6 + .../kvm/storage/LinstorStorageAdaptor.java | 121 +++++++- .../LinstorPrimaryDataStoreDriverImpl.java | 263 +++++++++++++++--- .../storage/datastore/util/LinstorUtil.java | 115 ++++++++ 4 files changed, 457 insertions(+), 48 deletions(-) diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index 98fc8f69512..fb247eef5df 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2025-01-27] + +### Fixed + +- Use of multiple primary storages on the same linstor controller + ## [2025-01-20] ### Fixed diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 6a4d6c7a349..9de2479b20b 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -60,6 +60,11 @@ import com.linbit.linstor.api.model.Volume; import com.linbit.linstor.api.model.VolumeDefinition; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; @StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor) public class LinstorStorageAdaptor implements StorageAdaptor { @@ -198,10 +203,10 @@ public class LinstorStorageAdaptor implements StorageAdaptor { final DevelopersApi api = getLinstorAPI(pool); try { - List definitionList = api.resourceDefinitionList( - Collections.singletonList(rscName), null, null, null); + ResourceDefinition resourceDefinition = LinstorUtil.findResourceDefinition( + api, rscName, lpool.getResourceGroup()); - if (definitionList.isEmpty()) { + if (resourceDefinition == null) { ResourceGroupSpawn rgSpawn = new ResourceGroupSpawn(); rgSpawn.setResourceDefinitionName(rscName); rgSpawn.addVolumeSizesItem(size / 1024); // linstor uses KiB @@ -211,22 +216,28 @@ public class LinstorStorageAdaptor implements StorageAdaptor { handleLinstorApiAnswers(answers, "Linstor: Unable to spawn resource."); } + String foundRscName = resourceDefinition != null ? resourceDefinition.getName() : rscName; + // query linstor for the device path List resources = api.viewResources( Collections.emptyList(), - Collections.singletonList(rscName), + Collections.singletonList(foundRscName), Collections.emptyList(), null, null, null); - makeResourceAvailable(api, rscName, false); + makeResourceAvailable(api, foundRscName, false); if (!resources.isEmpty() && !resources.get(0).getVolumes().isEmpty()) { final String devPath = resources.get(0).getVolumes().get(0).getDevicePath(); s_logger.info("Linstor: Created drbd device: " + devPath); final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath, name, pool); kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + long allocatedKib = resources.get(0).getVolumes().get(0).getAllocatedSizeKib() != null ? + resources.get(0).getVolumes().get(0).getAllocatedSizeKib() : 0; + kvmDisk.setSize(allocatedKib >= 0 ? allocatedKib * 1024 : 0); + kvmDisk.setVirtualSize(size); return kvmDisk; } else { s_logger.error("Linstor: viewResources didn't return resources or volumes."); @@ -470,21 +481,56 @@ public class LinstorStorageAdaptor implements StorageAdaptor { return false; } + /** + * Decrements the aux property key for template resource and deletes or just deletes if not template resource. + * @param api + * @param rscName + * @param rscGrpName + * @return + * @throws ApiException + */ + private boolean deRefOrDeleteResource(DevelopersApi api, String rscName, String rscGrpName) throws ApiException { + boolean deleted = false; + List existingRDs = LinstorUtil.getRDListStartingWith(api, rscName); + for (ResourceDefinition rd : existingRDs) { + int expectedProps = 0; // if it is a non template resource, we don't expect any _cs-template-for- prop + String propKey = LinstorUtil.getTemplateForAuxPropKey(rscGrpName); + if (rd.getProps().containsKey(propKey)) { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + rdm.deleteProps(Collections.singletonList(propKey)); + api.resourceDefinitionModify(rd.getName(), rdm); + expectedProps = 1; + } + + // if there is only one template-for property left for templates, the template isn't needed anymore + // or if it isn't a template anyway, it will not have this Aux property + // _cs-template-for- poperties work like a ref-count. + if (rd.getProps().keySet().stream() + .filter(key -> key.startsWith("Aux/" + LinstorUtil.CS_TEMPLATE_FOR_PREFIX)) + .count() == expectedProps) { + ApiCallRcList answers = api.resourceDefinitionDelete(rd.getName()); + checkLinstorAnswersThrow(answers); + deleted = true; + } + } + return deleted; + } + @Override public boolean deletePhysicalDisk(String name, KVMStoragePool pool, Storage.ImageFormat format) { s_logger.debug("Linstor: deletePhysicalDisk " + name); final DevelopersApi api = getLinstorAPI(pool); + final String rscName = getLinstorRscName(name); + final LinstorStoragePool linstorPool = (LinstorStoragePool) pool; + String rscGrpName = linstorPool.getResourceGroup(); try { - final String rscName = getLinstorRscName(name); - s_logger.debug("Linstor: delete resource definition " + rscName); - ApiCallRcList answers = api.resourceDefinitionDelete(rscName); - handleLinstorApiAnswers(answers, "Linstor: Unable to delete resource definition " + rscName); + return deRefOrDeleteResource(api, rscName, rscGrpName); } catch (ApiException apiEx) { + s_logger.error("Linstor: ApiEx - " + apiEx.getMessage()); throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); } - return true; } @Override @@ -558,6 +604,56 @@ public class LinstorStorageAdaptor implements StorageAdaptor { return false; } + /** + * Checks if the given disk is the SystemVM template, by checking its properties file in the same directory. + * The initial systemvm template resource isn't created on the management server, but + * we now need to know if the systemvm template is used, while copying. + * @param disk + * @return True if it is the systemvm template disk, else false. + */ + private static boolean isSystemTemplate(KVMPhysicalDisk disk) { + Path diskPath = Paths.get(disk.getPath()); + Path propFile = diskPath.getParent().resolve("template.properties"); + if (Files.exists(propFile)) { + java.util.Properties templateProps = new java.util.Properties(); + try { + templateProps.load(new FileInputStream(propFile.toFile())); + String desc = templateProps.getProperty("description"); + if (desc.startsWith("SystemVM Template")) { + return true; + } + } catch (IOException e) { + return false; + } + } + return false; + } + + /** + * Conditionally sets the correct aux properties for templates or basic resources. + * @param api + * @param srcDisk + * @param destPool + * @param name + */ + private void setRscDfnAuxProperties( + DevelopersApi api, KVMPhysicalDisk srcDisk, KVMStoragePool destPool, String name) { + // if it is the initial systemvm disk copy, we need to apply the _cs-template-for property. + if (isSystemTemplate(srcDisk)) { + applyAuxProps(api, name, "SystemVM Template", null); + LinstorStoragePool linPool = (LinstorStoragePool) destPool; + final String rscName = getLinstorRscName(name); + try { + LinstorUtil.setAuxTemplateForProperty(api, rscName, linPool.getResourceGroup()); + } catch (ApiException apiExc) { + s_logger.error(String.format("Error setting aux template for property for %s", rscName)); + logLinstorAnswers(apiExc.getApiCallRcList()); + } + } else { + applyAuxProps(api, name, srcDisk.getDispName(), srcDisk.getVmName()); + } + } + @Override public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPools, int timeout, byte[] srcPassphrase, byte[] destPassphrase, Storage.ProvisioningType provisioningType) { @@ -571,15 +667,14 @@ public class LinstorStorageAdaptor implements StorageAdaptor { name, QemuImg.PhysicalDiskFormat.RAW, provisioningType, disk.getVirtualSize(), null); final DevelopersApi api = getLinstorAPI(destPools); - applyAuxProps(api, name, disk.getDispName(), disk.getVmName()); + setRscDfnAuxProperties(api, disk, destPools, name); s_logger.debug(String.format("Linstor.copyPhysicalDisk: dstPath: %s", dstDisk.getPath())); final QemuImgFile destFile = new QemuImgFile(dstDisk.getPath()); destFile.setFormat(dstDisk.getFormat()); destFile.setSize(disk.getVirtualSize()); - boolean zeroedDevice = resourceSupportZeroBlocks(destPools, LinstorUtil.RSC_PREFIX + name); - + boolean zeroedDevice = resourceSupportZeroBlocks(destPools, getLinstorRscName(name)); try { final QemuImg qemu = new QemuImg(timeout, zeroedDevice, true); qemu.convert(srcFile, destFile); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 4132fbd278a..22cb4eb8911 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -28,6 +28,8 @@ import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionCloneRequest; import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted; import com.linbit.linstor.api.model.ResourceDefinitionCreate; + +import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroup; import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceMakeAvailable; @@ -71,6 +73,7 @@ import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.Volume; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; @@ -90,6 +93,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; @@ -98,6 +102,7 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.RemoteHostEndPoint; import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -435,15 +440,27 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } - private String createResourceBase( - String rscName, long sizeInBytes, String volName, String vmName, - @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, String rscGrp) { + /** + * Spawns a new Linstor resource with the given arguments. + * @param api + * @param newRscName + * @param sizeInBytes + * @param isTemplate + * @param rscGrpName + * @param volName + * @param vmName + * @throws ApiException + */ + private void spawnResource( + DevelopersApi api, String newRscName, long sizeInBytes, boolean isTemplate, String rscGrpName, + String volName, String vmName, @Nullable Long passPhraseId, @Nullable byte[] passPhrase) throws ApiException + { ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn(); - rscGrpSpawn.setResourceDefinitionName(rscName); + rscGrpSpawn.setResourceDefinitionName(newRscName); rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024); if (passPhraseId != null) { AutoSelectFilter asf = new AutoSelectFilter(); - List luksLayers = getEncryptedLayerList(api, rscGrp); + List luksLayers = getEncryptedLayerList(api, rscGrpName); asf.setLayerStack(luksLayers.stream().map(LayerType::toString).collect(Collectors.toList())); rscGrpSpawn.setSelectFilter(asf); if (passPhrase != null) { @@ -452,16 +469,103 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } + if (isTemplate) { + Properties props = new Properties(); + props.put(LinstorUtil.getTemplateForAuxPropKey(rscGrpName), "true"); + rscGrpSpawn.setResourceDefinitionProps(props); + } + + s_logger.info("Linstor: Spawn resource " + newRscName); + ApiCallRcList answers = api.resourceGroupSpawn(rscGrpName, rscGrpSpawn); + checkLinstorAnswersThrow(answers); + + answers = LinstorUtil.applyAuxProps(api, newRscName, volName, vmName); + checkLinstorAnswersThrow(answers); + } + + /** + * Condition if a template resource can be shared with the given resource group. + * @param tgtRscGrp + * @param tgtLayerStack + * @param rg + * @return True if the template resource can be shared, else false. + */ + private boolean canShareTemplateForResourceGroup( + ResourceGroup tgtRscGrp, List tgtLayerStack, ResourceGroup rg) { + List rgLayerStack = rg.getSelectFilter() != null ? + rg.getSelectFilter().getLayerStack() : null; + return Objects.equals(tgtLayerStack, rgLayerStack) && + Objects.equals(tgtRscGrp.getSelectFilter().getStoragePoolList(), + rg.getSelectFilter().getStoragePoolList()); + } + + /** + * Searches for a shareable template for this rscGrpName and sets the aux template property. + * @param api + * @param rscName + * @param rscGrpName + * @param existingRDs + * @return + * @throws ApiException + */ + private boolean foundShareableTemplate( + DevelopersApi api, String rscName, String rscGrpName, + List> existingRDs) throws ApiException { + if (!existingRDs.isEmpty()) { + ResourceGroup tgtRscGrp = api.resourceGroupList( + Collections.singletonList(rscGrpName), null, null, null).get(0); + List tgtLayerStack = tgtRscGrp.getSelectFilter() != null ? + tgtRscGrp.getSelectFilter().getLayerStack() : null; + + // check if there is already a template copy, that we could reuse + // this means if select filters are similar enough to allow cloning from + for (Pair rdPair : existingRDs) { + ResourceGroup rg = rdPair.second(); + if (canShareTemplateForResourceGroup(tgtRscGrp, tgtLayerStack, rg)) { + LinstorUtil.setAuxTemplateForProperty(api, rscName, rscGrpName); + return true; + } + } + } + return false; + } + + /** + * Creates a new Linstor resource. + * @param rscName + * @param sizeInBytes + * @param volName + * @param vmName + * @param api + * @param rscGrp + * @param poolId + * @param isTemplate indicates if the resource is a template + * @return true if a new resource was created, false if it already existed or was reused. + */ + private boolean createResourceBase( + String rscName, long sizeInBytes, String volName, String vmName, + @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, + String rscGrp, long poolId, boolean isTemplate) + { try { - s_logger.info("Linstor: Spawn resource " + rscName); - ApiCallRcList answers = api.resourceGroupSpawn(rscGrp, rscGrpSpawn); - checkLinstorAnswersThrow(answers); + s_logger.debug(String.format("createRscBase: %s :: %s :: %b", rscName, rscGrp, isTemplate)); + List> existingRDs = LinstorUtil.getRDAndRGListStartingWith(api, rscName); - answers = LinstorUtil.applyAuxProps(api, rscName, volName, vmName); - checkLinstorAnswersThrow(answers); - - return LinstorUtil.getDevicePath(api, rscName); + String fullRscName = String.format("%s-%d", rscName, poolId); + boolean alreadyCreated = existingRDs.stream() + .anyMatch(p -> p.first().getName().equalsIgnoreCase(fullRscName)) || + existingRDs.stream().anyMatch(p -> p.first().getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrp))); + if (!alreadyCreated) { + boolean createNewRsc = !foundShareableTemplate(api, rscName, rscGrp, existingRDs); + if (createNewRsc) { + String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; + spawnResource(api, newRscName, sizeInBytes, isTemplate, rscGrp, + volName, vmName, passPhraseId, passPhrase); + } + return createNewRsc; + } + return false; } catch (ApiException apiEx) { s_logger.error("Linstor: ApiEx - " + apiEx.getMessage()); @@ -474,9 +578,9 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver final String rscGrp = getRscGrp(storagePoolVO); final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid(); - String deviceName = createResourceBase( + createResourceBase( rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), vol.getPassphraseId(), vol.getPassphrase(), - linstorApi, rscGrp); + linstorApi, rscGrp, storagePoolVO.getId(), false); try { @@ -503,17 +607,72 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } + /** + * Update resource-definitions resource-group to the correct one if it isn't already the intended. + * @param api Linstor api + * @param rscName resource name to check the resource group + * @param tgtRscGrp resource group name to set + * @throws ApiException exception if any api error occurred + */ + private void updateRscGrpIfNecessary(DevelopersApi api, String rscName, String tgtRscGrp) throws ApiException { + List rscDfns = api.resourceDefinitionList( + Collections.singletonList(rscName), null, null, null); + if (rscDfns != null && !rscDfns.isEmpty()) { + ResourceDefinition rscDfn = rscDfns.get(0); + + if (!rscDfn.getResourceGroupName().equalsIgnoreCase(tgtRscGrp)) { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + rdm.setResourceGroup(tgtRscGrp); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + + if (answers.hasError()) { + String bestError = LinstorUtil.getBestErrorMessage(answers); + s_logger.error(String.format("Update resource group on %s error: %s", rscName, bestError)); + throw new CloudRuntimeException(bestError); + } else { + s_logger.info(String.format("Successfully changed resource group to %s on %s", tgtRscGrp, rscName)); + } + } + } + } + + /** + * If a resource is cloned, all properties are cloned too, but the _cs-template-for properties, + * should only stay on the template resource, so delete them in this method. + * @param api + * @param rscName + * @throws ApiException + */ + private void deleteTemplateForProps( + DevelopersApi api, String rscName) throws ApiException { + List rdList = api.resourceDefinitionList( + Collections.singletonList(rscName), null, null, null); + + if (CollectionUtils.isNotEmpty(rdList)) { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + List deleteProps = rdList.get(0).getProps().keySet().stream() + .filter(key -> key.startsWith("Aux/" + LinstorUtil.CS_TEMPLATE_FOR_PREFIX)) + .collect(Collectors.toList()); + rdm.deleteProps(deleteProps); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + checkLinstorAnswers(answers); + } + } + private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolVO storagePoolVO) { // get the cached template on this storage VMTemplateStoragePoolVO tmplPoolRef = _vmTemplatePoolDao.findByPoolTemplate( storagePoolVO.getId(), csCloneId, null); if (tmplPoolRef != null) { - final String cloneRes = LinstorUtil.RSC_PREFIX + tmplPoolRef.getLocalDownloadPath(); + final String templateRscName = LinstorUtil.RSC_PREFIX + tmplPoolRef.getLocalDownloadPath(); final String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getUuid(); final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); try { + ResourceDefinition templateRD = LinstorUtil.findResourceDefinition( + linstorApi, templateRscName, getRscGrp(storagePoolVO)); + final String cloneRes = templateRD != null ? templateRD.getName() : templateRscName; s_logger.info("Clone resource definition " + cloneRes + " to " + rscName); ResourceDefinitionCloneRequest cloneRequest = new ResourceDefinitionCloneRequest(); cloneRequest.setName(rscName); @@ -540,6 +699,9 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver resizeResource(linstorApi, rscName, volumeInfo.getSize()); } + updateRscGrpIfNecessary(linstorApi, rscName, getRscGrp(storagePoolVO)); + + deleteTemplateForProps(linstorApi, rscName); LinstorUtil.applyAuxProps(linstorApi, rscName, volumeInfo.getName(), volumeInfo.getAttachedVmName()); applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeInfo.getMaxIops()); @@ -967,12 +1129,37 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver return LinstorUtil.getDevicePath(api, restoredName); } + /** + * Updates the template_spool_ref DB entry to indicate that this template was fully downloaded and is ready. + * @param templateId + * @param destTemplateInfoUuid + * @param destDataStoreId + * @param templateSize + */ + private void updateTemplateSpoolRef( + long templateId, String destTemplateInfoUuid, long destDataStoreId, long templateSize) { + VMTemplateStoragePoolVO destVolumeTemplateStoragePoolVO = _vmTemplatePoolDao.findByPoolTemplate( + destDataStoreId, templateId, null); + if (destVolumeTemplateStoragePoolVO == null) { + throw new CloudRuntimeException( + String.format("Unable to find template_spool_ref entry for pool_id %d and template_id %d", + destDataStoreId, templateId)); + } + destVolumeTemplateStoragePoolVO.setDownloadPercent(100); + destVolumeTemplateStoragePoolVO.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + destVolumeTemplateStoragePoolVO.setState(ObjectInDataStoreStateMachine.State.Ready); + destVolumeTemplateStoragePoolVO.setTemplateSize(templateSize); + destVolumeTemplateStoragePoolVO.setLocalDownloadPath(destTemplateInfoUuid); + destVolumeTemplateStoragePoolVO.setInstallPath(destTemplateInfoUuid); + _vmTemplatePoolDao.persist(destVolumeTemplateStoragePoolVO); + } + private Answer copyTemplate(DataObject srcData, DataObject dstData) { TemplateInfo tInfo = (TemplateInfo) dstData; final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId()); final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress()); final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid(); - createResourceBase( + boolean newCreated = createResourceBase( LinstorUtil.RSC_PREFIX + dstData.getUuid(), tInfo.getSize(), tInfo.getName(), @@ -980,30 +1167,36 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver null, null, api, - getRscGrp(pool)); + getRscGrp(pool), + pool.getId(), + true); - int nMaxExecutionMinutes = NumbersUtil.parseInt( - _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); - CopyCommand cmd = new CopyCommand( - srcData.getTO(), - dstData.getTO(), - nMaxExecutionMinutes * 60 * 1000, - VirtualMachineManager.ExecuteInSequence.value()); Answer answer; + if (newCreated) { + int nMaxExecutionMinutes = NumbersUtil.parseInt( + _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); + CopyCommand cmd = new CopyCommand( + srcData.getTO(), + dstData.getTO(), + nMaxExecutionMinutes * 60 * 1000, + VirtualMachineManager.ExecuteInSequence.value()); - try { - Optional optEP = getLinstorEP(api, rscName); - if (optEP.isPresent()) { - answer = optEP.get().sendMessage(cmd); - } - else { - answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint."); + try { + Optional optEP = getLinstorEP(api, rscName); + if (optEP.isPresent()) { + answer = optEP.get().sendMessage(cmd); + } else { + answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint."); + deleteResourceDefinition(pool, rscName); + } + } catch (ApiException exc) { + s_logger.error("copy template failed: ", exc); deleteResourceDefinition(pool, rscName); + throw new CloudRuntimeException(exc.getBestMessage()); } - } catch (ApiException exc) { - s_logger.error("copy template failed: ", exc); - deleteResourceDefinition(pool, rscName); - throw new CloudRuntimeException(exc.getBestMessage()); + } else { + updateTemplateSpoolRef(dstData.getId(), tInfo.getUuid(), dstData.getDataStore().getId(), srcData.getSize()); + answer = new Answer(new CopyCmdAnswer(dstData.getTO())); } return answer; } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 2dbfdea6d11..e252753502c 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -26,6 +26,7 @@ import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ProviderKind; import com.linbit.linstor.api.model.Resource; +import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroup; import com.linbit.linstor.api.model.ResourceWithVolumes; @@ -37,8 +38,11 @@ import javax.annotation.Nonnull; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.log4j.Logger; @@ -48,6 +52,7 @@ public class LinstorUtil { public final static String PROVIDER_NAME = "Linstor"; public static final String RSC_PREFIX = "cs-"; public static final String RSC_GROUP = "resourceGroup"; + public static final String CS_TEMPLATE_FOR_PREFIX = "_cs-template-for-"; public static final String TEMP_VOLUME_ID = "tempVolumeId"; @@ -287,4 +292,114 @@ public class LinstorUtil { } return answers; } + + /** + * Returns all resource definitions that start with the given `startWith` name. + * @param api + * @param startWith startWith String + * @return a List with all ResourceDefinition starting with `startWith` + * @throws ApiException + */ + public static List getRDListStartingWith(DevelopersApi api, String startWith) + throws ApiException + { + List rscDfns = api.resourceDefinitionList(null, null, null, null); + + return rscDfns.stream() + .filter(rscDfn -> rscDfn.getName().toLowerCase().startsWith(startWith.toLowerCase())) + .collect(Collectors.toList()); + } + + /** + * Returns a pair list of resource-definitions with ther 1:1 mapped resource-group objects that start with the + * resource name `startWith` + * @param api + * @param startWith + * @return + * @throws ApiException + */ + public static List> getRDAndRGListStartingWith(DevelopersApi api, String startWith) + throws ApiException + { + List foundRDs = getRDListStartingWith(api, startWith); + + List rscGrpStrings = foundRDs.stream() + .map(ResourceDefinition::getResourceGroupName) + .collect(Collectors.toList()); + + Map rscGrps = api.resourceGroupList(rscGrpStrings, null, null, null).stream() + .collect(Collectors.toMap(ResourceGroup::getName, rscGrp -> rscGrp)); + + return foundRDs.stream() + .map(rd -> new Pair<>(rd, rscGrps.get(rd.getResourceGroupName()))) + .collect(Collectors.toList()); + } + + /** + * The full name of template-for aux property key. + * @param rscGrpName + * @return + */ + public static String getTemplateForAuxPropKey(String rscGrpName) { + return String.format("Aux/%s%s", CS_TEMPLATE_FOR_PREFIX, rscGrpName); + } + + /** + * Template resource should have a _cs-template-for-... property, that indicates to which resource-group + * this template belongs, it works like a refcount to keep it alive if there are still such properties on the + * template resource. That methods set the correct property on the given resource. + * @param api + * @param rscName Resource name to set the property. + * @param rscGrpName Resource group this template should belong too. + * @throws ApiException + */ + public static void setAuxTemplateForProperty(DevelopersApi api, String rscName, String rscGrpName) + throws ApiException + { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + Properties props = new Properties(); + String propKey = LinstorUtil.getTemplateForAuxPropKey(rscGrpName); + props.put(propKey, "true"); + rdm.setOverrideProps(props); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + + if (answers.hasError()) { + String bestError = LinstorUtil.getBestErrorMessage(answers); + s_logger.error(String.format("Set %s on %s error: %s", propKey, rscName, bestError)); + throw new CloudRuntimeException(bestError); + } else { + s_logger.info(String.format("Set %s property on %s", propKey, rscName)); + } + } + + /** + * Find the correct resource definition to clone from. + * There could be multiple resource definitions for the same template, with the same prefix. + * This method searches for which resource group the resource definition was intended and returns that. + * If no exact resource definition could be found, we return the first with a similar name as a fallback. + * If there is not even one with the correct prefix, we return null. + * @param api + * @param rscName + * @param rscGrpName + * @return The resource-definition to clone from, if no template and no match, return null. + * @throws ApiException + */ + public static ResourceDefinition findResourceDefinition(DevelopersApi api, String rscName, String rscGrpName) + throws ApiException { + List rscDfns = api.resourceDefinitionList(null, null, null, null); + + List rdsStartingWith = rscDfns.stream() + .filter(rscDfn -> rscDfn.getName().toLowerCase().startsWith(rscName.toLowerCase())) + .collect(Collectors.toList()); + + if (rdsStartingWith.isEmpty()) { + return null; + } + + Optional rd = rdsStartingWith.stream() + .filter(rscDfn -> rscDfn.getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrpName))) + .findFirst(); + + return rd.orElseGet(() -> rdsStartingWith.get(0)); + } } From 802bf5fce7c57b38810308af6b8f75c3a37bab01 Mon Sep 17 00:00:00 2001 From: dahn Date: Thu, 6 Feb 2025 11:33:26 +0100 Subject: [PATCH 10/21] Revert "server: fix attach uploaded volume (#10267)" (#10323) This reverts commit 1c84ce4e23e3fd243022c6c533fc14c10439c6f3. --- .../cloud/storage/VolumeApiServiceImpl.java | 169 +++++------- .../storage/VolumeApiServiceImplTest.java | 249 ------------------ 2 files changed, 65 insertions(+), 353 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 68546f185b7..7f867eb01a9 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -133,9 +133,7 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.Pod; -import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.dc.dao.HostPodDao; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEvent; @@ -155,7 +153,6 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorCapabilitiesVO; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.offering.DiskOffering; -import com.cloud.org.Cluster; import com.cloud.org.Grouping; import com.cloud.projects.Project; import com.cloud.projects.ProjectManager; @@ -326,8 +323,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject private VmWorkJobDao _workJobDao; @Inject - ClusterDao clusterDao; - @Inject private ClusterDetailsDao _clusterDetailsDao; @Inject private StorageManager storageMgr; @@ -351,8 +346,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic protected ProjectManager projectManager; @Inject protected StoragePoolDetailsDao storagePoolDetailsDao; - @Inject - HostPodDao podDao; protected Gson _gson; @@ -2387,98 +2380,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return attachVolumeToVM(command.getVirtualMachineId(), command.getId(), command.getDeviceId()); } - protected VolumeVO getVmExistingVolumeForVolumeAttach(UserVmVO vm, VolumeInfo volumeToAttach) { - VolumeVO existingVolumeOfVm = null; - VMTemplateVO template = _templateDao.findById(vm.getTemplateId()); - List rootVolumesOfVm = _volsDao.findByInstanceAndType(vm.getId(), Volume.Type.ROOT); - if (rootVolumesOfVm.size() > 1 && template != null && !template.isDeployAsIs()) { - throw new CloudRuntimeException("The VM " + vm.getHostName() + " has more than one ROOT volume and is in an invalid state."); - } else { - if (!rootVolumesOfVm.isEmpty()) { - existingVolumeOfVm = rootVolumesOfVm.get(0); - } else { - // locate data volume of the vm - List diskVolumesOfVm = _volsDao.findByInstanceAndType(vm.getId(), Volume.Type.DATADISK); - for (VolumeVO diskVolume : diskVolumesOfVm) { - if (diskVolume.getState() != Volume.State.Allocated) { - existingVolumeOfVm = diskVolume; - break; - } - } - } - } - if (existingVolumeOfVm == null) { - if (s_logger.isTraceEnabled()) { - s_logger.trace(String.format("No existing volume found for VM (%s/%s) to attach volume %s/%s", - vm.getName(), vm.getUuid(), - volumeToAttach.getName(), volumeToAttach.getUuid())); - } - return null; - } - if (s_logger.isTraceEnabled()) { - String msg = "attaching volume %s/%s to a VM (%s/%s) with an existing volume %s/%s on primary storage %s"; - s_logger.trace(String.format(msg, - volumeToAttach.getName(), volumeToAttach.getUuid(), - vm.getName(), vm.getUuid(), - existingVolumeOfVm.getName(), existingVolumeOfVm.getUuid(), - existingVolumeOfVm.getPoolId())); - } - return existingVolumeOfVm; - } - - protected StoragePool getPoolForAllocatedOrUploadedVolumeForAttach(final VolumeInfo volumeToAttach, final UserVmVO vm) { - DataCenter zone = _dcDao.findById(vm.getDataCenterId()); - Pair clusterHostId = virtualMachineManager.findClusterAndHostIdForVm(vm, false); - long podId = vm.getPodIdToDeployIn(); - if (clusterHostId.first() != null) { - Cluster cluster = clusterDao.findById(clusterHostId.first()); - podId = cluster.getPodId(); - } - Pod pod = podDao.findById(podId); - DiskOfferingVO offering = _diskOfferingDao.findById(volumeToAttach.getDiskOfferingId()); - DiskProfile diskProfile = new DiskProfile(volumeToAttach.getId(), volumeToAttach.getVolumeType(), - volumeToAttach.getName(), volumeToAttach.getId(), volumeToAttach.getSize(), offering.getTagsArray(), - offering.isUseLocalStorage(), offering.isRecreatable(), - volumeToAttach.getTemplateId()); - diskProfile.setHyperType(vm.getHypervisorType()); - StoragePool pool = _volumeMgr.findStoragePool(diskProfile, zone, pod, clusterHostId.first(), - clusterHostId.second(), vm, Collections.emptySet()); - if (pool == null) { - throw new CloudRuntimeException(String.format("Failed to find a primary storage for volume in state: %s", volumeToAttach.getState())); - } - return pool; - } - - protected VolumeInfo createVolumeOnPrimaryForAttachIfNeeded(final VolumeInfo volumeToAttach, final UserVmVO vm, VolumeVO existingVolumeOfVm) { - VolumeInfo newVolumeOnPrimaryStorage = volumeToAttach; - boolean volumeOnSecondary = volumeToAttach.getState() == Volume.State.Uploaded; - if (!Arrays.asList(Volume.State.Allocated, Volume.State.Uploaded).contains(volumeToAttach.getState())) { - return newVolumeOnPrimaryStorage; - } - //don't create volume on primary storage if its being attached to the vm which Root's volume hasn't been created yet - StoragePool destPrimaryStorage = null; - if (existingVolumeOfVm != null && !existingVolumeOfVm.getState().equals(Volume.State.Allocated)) { - destPrimaryStorage = _storagePoolDao.findById(existingVolumeOfVm.getPoolId()); - if (s_logger.isTraceEnabled() && destPrimaryStorage != null) { - s_logger.trace(String.format("decided on target storage: %s/%s", destPrimaryStorage.getName(), destPrimaryStorage.getUuid())); - } - } - if (destPrimaryStorage == null) { - destPrimaryStorage = getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); - } - try { - if (volumeOnSecondary && Storage.StoragePoolType.PowerFlex.equals(destPrimaryStorage.getPoolType())) { - throw new InvalidParameterValueException("Cannot attach uploaded volume, this operation is unsupported on storage pool type " + destPrimaryStorage.getPoolType()); - } - newVolumeOnPrimaryStorage = _volumeMgr.createVolumeOnPrimaryStorage(vm, volumeToAttach, - vm.getHypervisorType(), destPrimaryStorage); - } catch (NoTransitionException e) { - s_logger.debug("Failed to create volume on primary storage", e); - throw new CloudRuntimeException("Failed to create volume on primary storage", e); - } - return newVolumeOnPrimaryStorage; - } - private Volume orchestrateAttachVolumeToVM(Long vmId, Long volumeId, Long deviceId) { VolumeInfo volumeToAttach = volFactory.getVolume(volumeId); @@ -2487,8 +2388,63 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } UserVmVO vm = _userVmDao.findById(vmId); - VolumeVO existingVolumeOfVm = getVmExistingVolumeForVolumeAttach(vm, volumeToAttach); - VolumeInfo newVolumeOnPrimaryStorage = createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, existingVolumeOfVm); + VolumeVO existingVolumeOfVm = null; + VMTemplateVO template = _templateDao.findById(vm.getTemplateId()); + List rootVolumesOfVm = _volsDao.findByInstanceAndType(vmId, Volume.Type.ROOT); + if (rootVolumesOfVm.size() > 1 && template != null && !template.isDeployAsIs()) { + throw new CloudRuntimeException("The VM " + vm.getHostName() + " has more than one ROOT volume and is in an invalid state."); + } else { + if (!rootVolumesOfVm.isEmpty()) { + existingVolumeOfVm = rootVolumesOfVm.get(0); + } else { + // locate data volume of the vm + List diskVolumesOfVm = _volsDao.findByInstanceAndType(vmId, Volume.Type.DATADISK); + for (VolumeVO diskVolume : diskVolumesOfVm) { + if (diskVolume.getState() != Volume.State.Allocated) { + existingVolumeOfVm = diskVolume; + break; + } + } + } + } + if (s_logger.isTraceEnabled()) { + String msg = "attaching volume %s/%s to a VM (%s/%s) with an existing volume %s/%s on primary storage %s"; + if (existingVolumeOfVm != null) { + s_logger.trace(String.format(msg, + volumeToAttach.getName(), volumeToAttach.getUuid(), + vm.getName(), vm.getUuid(), + existingVolumeOfVm.getName(), existingVolumeOfVm.getUuid(), + existingVolumeOfVm.getPoolId())); + } + } + + HypervisorType rootDiskHyperType = vm.getHypervisorType(); + HypervisorType volumeToAttachHyperType = _volsDao.getHypervisorType(volumeToAttach.getId()); + + VolumeInfo newVolumeOnPrimaryStorage = volumeToAttach; + + //don't create volume on primary storage if its being attached to the vm which Root's volume hasn't been created yet + StoragePoolVO destPrimaryStorage = null; + if (existingVolumeOfVm != null && !existingVolumeOfVm.getState().equals(Volume.State.Allocated)) { + destPrimaryStorage = _storagePoolDao.findById(existingVolumeOfVm.getPoolId()); + if (s_logger.isTraceEnabled() && destPrimaryStorage != null) { + s_logger.trace(String.format("decided on target storage: %s/%s", destPrimaryStorage.getName(), destPrimaryStorage.getUuid())); + } + } + + boolean volumeOnSecondary = volumeToAttach.getState() == Volume.State.Uploaded; + + if (destPrimaryStorage != null && (volumeToAttach.getState() == Volume.State.Allocated || volumeOnSecondary)) { + try { + if (volumeOnSecondary && destPrimaryStorage.getPoolType() == Storage.StoragePoolType.PowerFlex) { + throw new InvalidParameterValueException("Cannot attach uploaded volume, this operation is unsupported on storage pool type " + destPrimaryStorage.getPoolType()); + } + newVolumeOnPrimaryStorage = _volumeMgr.createVolumeOnPrimaryStorage(vm, volumeToAttach, rootDiskHyperType, destPrimaryStorage); + } catch (NoTransitionException e) { + s_logger.debug("Failed to create volume on primary storage", e); + throw new CloudRuntimeException("Failed to create volume on primary storage", e); + } + } // reload the volume from db newVolumeOnPrimaryStorage = volFactory.getVolume(newVolumeOnPrimaryStorage.getId()); @@ -2507,17 +2463,19 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic StoragePoolVO vmRootVolumePool = _storagePoolDao.findById(existingVolumeOfVm.getPoolId()); try { - HypervisorType volumeToAttachHyperType = _volsDao.getHypervisorType(volumeToAttach.getId()); newVolumeOnPrimaryStorage = _volumeMgr.moveVolume(newVolumeOnPrimaryStorage, vmRootVolumePool.getDataCenterId(), vmRootVolumePool.getPodId(), vmRootVolumePool.getClusterId(), volumeToAttachHyperType); - } catch (ConcurrentOperationException | StorageUnavailableException e) { + } catch (ConcurrentOperationException e) { + s_logger.debug("move volume failed", e); + throw new CloudRuntimeException("move volume failed", e); + } catch (StorageUnavailableException e) { s_logger.debug("move volume failed", e); throw new CloudRuntimeException("move volume failed", e); } } VolumeVO newVol = _volsDao.findById(newVolumeOnPrimaryStorage.getId()); // Getting the fresh vm object in case of volume migration to check the current state of VM - if (moveVolumeNeeded) { + if (moveVolumeNeeded || volumeOnSecondary) { vm = _userVmDao.findById(vmId); if (vm == null) { throw new InvalidParameterValueException("VM not found."); @@ -2701,6 +2659,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (!_volsDao.findByInstanceAndDeviceId(vm.getId(), 0).isEmpty()) { throw new InvalidParameterValueException("Vm already has root volume attached to it"); } + if (volumeToAttach.getState() == Volume.State.Uploaded) { + throw new InvalidParameterValueException("No support for Root volume attach in state " + Volume.State.Uploaded); + } } } diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 3e6f9ec63f9..a0f89956df5 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -45,7 +45,6 @@ import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -87,12 +86,8 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.api.query.dao.ServiceOfferingJoinDao; import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceType; -import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; -import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.dc.dao.HostPodDao; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; @@ -127,12 +122,10 @@ import com.cloud.utils.Pair; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; -import com.cloud.vm.DiskProfile; import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; -import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshotVO; @@ -206,15 +199,6 @@ public class VolumeApiServiceImplTest { private DataStoreManager dataStoreMgr; @Mock private SnapshotHelper snapshotHelper; - @Mock - VirtualMachineManager virtualMachineManager; - @Mock - HostPodDao podDao; - @Mock - ClusterDao clusterDao; - @Mock - VolumeOrchestrationService volumeOrchestrationService; - private DetachVolumeCmd detachCmd = new DetachVolumeCmd(); private Class _detachCmdClass = detachCmd.getClass(); @@ -1836,237 +1820,4 @@ public class VolumeApiServiceImplTest { volumeApiServiceImpl.validationsForCheckVolumeOperation(volume); } - - private UserVmVO getMockedVm() { - UserVmVO vm = Mockito.mock(UserVmVO.class); - Mockito.when(vm.getId()).thenReturn(1L); - Mockito.when(vm.getTemplateId()).thenReturn(10L); - Mockito.when(vm.getHostName()).thenReturn("test-vm"); - return vm; - } - - private VMTemplateVO getMockedTemplate() { - VMTemplateVO template = Mockito.mock(VMTemplateVO.class); - Mockito.when(template.isDeployAsIs()).thenReturn(false); - return template; - } - - @Test(expected = CloudRuntimeException.class) - public void testGetVmExistingVolumeForVolumeAttach_MultipleRootVolumes_ThrowsException() { - UserVmVO vm = getMockedVm(); - VMTemplateVO template = getMockedTemplate(); - when(templateDao.findById(10L)).thenReturn(template); - when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.ROOT)) - .thenReturn(Arrays.asList(Mockito.mock(VolumeVO.class), Mockito.mock(VolumeVO.class))); - volumeApiServiceImpl.getVmExistingVolumeForVolumeAttach(vm, Mockito.mock(VolumeInfo.class)); - } - - @Test - public void testGetVmExistingVolumeForVolumeAttach_SingleRootVolume() { - UserVmVO vm = getMockedVm(); - VMTemplateVO template = getMockedTemplate(); - VolumeVO rootVolume = Mockito.mock(VolumeVO.class); - Mockito.when(rootVolume.getId()).thenReturn(20L); - Mockito.when(templateDao.findById(10L)).thenReturn(template); - Mockito.when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.ROOT)) - .thenReturn(Collections.singletonList(rootVolume)); - VolumeVO result = volumeApiServiceImpl.getVmExistingVolumeForVolumeAttach(vm, Mockito.mock(VolumeInfo.class)); - Assert.assertNotNull(result); - Assert.assertEquals(20L, result.getId()); - } - - private VolumeVO getMockedDataVolume() { - VolumeVO volume = Mockito.mock(VolumeVO.class); - Mockito.when(volume.getId()).thenReturn(30L); - Mockito.when(volume.getState()).thenReturn(Volume.State.Ready); - return volume; - } - - @Test - public void testGetVmExistingVolumeForVolumeAttach_NoRootVolume_DataDiskAvailable() { - UserVmVO vm = getMockedVm(); - VMTemplateVO template = getMockedTemplate(); - VolumeVO dataDisk = getMockedDataVolume(); - List rootVolumes = Collections.emptyList(); - List dataVolumes = Collections.singletonList(dataDisk); - Mockito.when(templateDao.findById(10L)).thenReturn(template); - Mockito.when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.ROOT)).thenReturn(rootVolumes); - Mockito.when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.DATADISK)).thenReturn(dataVolumes); - VolumeVO result = volumeApiServiceImpl.getVmExistingVolumeForVolumeAttach(vm, Mockito.mock(VolumeInfo.class)); - Assert.assertNotNull(result); - Assert.assertEquals(30L, result.getId()); - } - - @Test - public void testGetVmExistingVolumeForVolumeAttach_NoVolumesAtAll() { - UserVmVO vm = getMockedVm(); - VMTemplateVO template = getMockedTemplate(); - Mockito.when(templateDao.findById(10L)).thenReturn(template); - Mockito.when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.ROOT)).thenReturn(Collections.emptyList()); - Mockito.when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.DATADISK)).thenReturn(Collections.emptyList()); - VolumeVO result = volumeApiServiceImpl.getVmExistingVolumeForVolumeAttach(vm, Mockito.mock(VolumeInfo.class)); - Assert.assertNull(result); - } - - private void mockDiskOffering() { - DiskOfferingVO offering = Mockito.mock(DiskOfferingVO.class); - Mockito.when(_diskOfferingDao.findById(1L)).thenReturn(offering); - Mockito.when(offering.isUseLocalStorage()).thenReturn(true); - Mockito.when(offering.isRecreatable()).thenReturn(false); - } - - private DataCenterVO mockZone() { - DataCenterVO zone = Mockito.mock(DataCenterVO.class); - Mockito.when(_dcDao.findById(1L)).thenReturn(zone); - return zone; - } - - @Test - public void testGetPoolForAllocatedOrUploadedVolumeForAttach_Success() { - VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); - UserVmVO vm = Mockito.mock(UserVmVO.class); - ClusterVO cluster = Mockito.mock(ClusterVO.class); - HostPodVO pod = Mockito.mock(HostPodVO.class); - DataCenterVO zone = mockZone(); - mockDiskOffering(); - StoragePool pool = Mockito.mock(StoragePool.class); - when(vm.getDataCenterId()).thenReturn(1L); - when(virtualMachineManager.findClusterAndHostIdForVm(vm, false)).thenReturn(new Pair<>(1L, 2L)); - when(clusterDao.findById(1L)).thenReturn(cluster); - when(cluster.getPodId()).thenReturn(1L); - when(podDao.findById(1L)).thenReturn(pod); - when(volumeToAttach.getDiskOfferingId()).thenReturn(1L); - when(volumeOrchestrationService.findStoragePool(any(DiskProfile.class), eq(zone), eq(pod), eq(1L), eq(2L), eq(vm), eq(Collections.emptySet()))) - .thenReturn(pool); - StoragePool result = volumeApiServiceImpl.getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); - Assert.assertNotNull(result); - Assert.assertEquals(pool, result); - } - - @Test(expected = CloudRuntimeException.class) - public void testGetPoolForAllocatedOrUploadedVolumeForAttach_NoPoolFound_ThrowsException() { - VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); - UserVmVO vm = Mockito.mock(UserVmVO.class); - DataCenterVO zone = mockZone(); - Pair clusterHostId = new Pair<>(1L, 2L); - ClusterVO cluster = Mockito.mock(ClusterVO.class); - HostPodVO pod = Mockito.mock(HostPodVO.class); - mockDiskOffering(); - when(vm.getDataCenterId()).thenReturn(1L); - when(clusterDao.findById(1L)).thenReturn(cluster); - when(virtualMachineManager.findClusterAndHostIdForVm(vm, false)).thenReturn(clusterHostId); - when(podDao.findById(anyLong())).thenReturn(pod); - when(volumeToAttach.getDiskOfferingId()).thenReturn(1L); - when(volumeOrchestrationService.findStoragePool(any(DiskProfile.class), eq(zone), eq(pod), eq(1L), eq(2L), eq(vm), eq(Collections.emptySet()))) - .thenReturn(null); - volumeApiServiceImpl.getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); - } - - @Test - public void testGetPoolForAllocatedOrUploadedVolumeForAttach_NoCluster() { - VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); - UserVmVO vm = Mockito.mock(UserVmVO.class); - DataCenterVO zone = mockZone(); - HostPodVO pod = Mockito.mock(HostPodVO.class); - mockDiskOffering(); - StoragePool pool = Mockito.mock(StoragePool.class); - when(vm.getDataCenterId()).thenReturn(1L); - when(vm.getPodIdToDeployIn()).thenReturn(2L); - when(virtualMachineManager.findClusterAndHostIdForVm(vm, false)).thenReturn(new Pair<>(null, 2L)); - when(podDao.findById(2L)).thenReturn(pod); - when(volumeToAttach.getDiskOfferingId()).thenReturn(1L); - when(volumeOrchestrationService.findStoragePool(any(DiskProfile.class), eq(zone), eq(pod), eq(null), eq(2L), eq(vm), eq(Collections.emptySet()))) - .thenReturn(pool); - StoragePool result = volumeApiServiceImpl.getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); - Assert.assertNotNull(result); - Assert.assertEquals(pool, result); - } - - - @Test - public void testCreateVolumeOnSecondaryForAttachIfNeeded_VolumeNotAllocatedOrUploaded() { - VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); - Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Ready); - VolumeInfo result = volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded( - volumeToAttach, Mockito.mock(UserVmVO.class), null); - Assert.assertSame(volumeToAttach, result); - Mockito.verifyNoInteractions(primaryDataStoreDaoMock, volumeOrchestrationService); - } - - @Test - public void testCreateVolumeOnSecondaryForAttachIfNeeded_ExistingVolumeDeterminesStoragePool() { - VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); - Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Uploaded); - UserVmVO vm = Mockito.mock(UserVmVO.class); - VolumeVO existingVolume = Mockito.mock(VolumeVO.class); - Mockito.when(existingVolume.getState()).thenReturn(Volume.State.Ready); - when(existingVolume.getPoolId()).thenReturn(1L); - StoragePoolVO destPrimaryStorage = Mockito.mock(StoragePoolVO.class); - Mockito.when(destPrimaryStorage.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); - Mockito.when(primaryDataStoreDaoMock.findById(1L)).thenReturn(destPrimaryStorage); - VolumeInfo newVolumeOnPrimaryStorage = Mockito.mock(VolumeInfo.class); - try { - Mockito.when(volumeOrchestrationService.createVolumeOnPrimaryStorage(vm, volumeToAttach, vm.getHypervisorType(), destPrimaryStorage)) - .thenReturn(newVolumeOnPrimaryStorage); - } catch (NoTransitionException nte) { - Assert.fail(nte.getMessage()); - } - VolumeInfo result = volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, existingVolume); - Assert.assertSame(newVolumeOnPrimaryStorage, result); - Mockito.verify(primaryDataStoreDaoMock).findById(1L); - } - - @Test - public void testCreateVolumeOnPrimaryForAttachIfNeeded_UsesGetPoolForAttach() { - VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); - Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Allocated); - UserVmVO vm = Mockito.mock(UserVmVO.class); - StoragePool destPrimaryStorage = Mockito.mock(StoragePool.class); - Mockito.doReturn(destPrimaryStorage).when(volumeApiServiceImpl) - .getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); - VolumeInfo newVolumeOnPrimaryStorage = Mockito.mock(VolumeInfo.class); - try { - Mockito.when(volumeOrchestrationService.createVolumeOnPrimaryStorage( - vm, volumeToAttach, vm.getHypervisorType(), destPrimaryStorage)) - .thenReturn(newVolumeOnPrimaryStorage); - } catch (NoTransitionException nte) { - Assert.fail(nte.getMessage()); - } - VolumeInfo result = volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null); - Assert.assertSame(newVolumeOnPrimaryStorage, result); - verify(volumeApiServiceImpl).getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); - } - - @Test(expected = InvalidParameterValueException.class) - public void testCreateVolumeOnPrimaryForAttachIfNeeded_UnsupportedPoolType_ThrowsException() { - VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); - when(volumeToAttach.getState()).thenReturn(Volume.State.Uploaded); - UserVmVO vm = Mockito.mock(UserVmVO.class); - StoragePool destPrimaryStorage = Mockito.mock(StoragePool.class); - when(destPrimaryStorage.getPoolType()).thenReturn(Storage.StoragePoolType.PowerFlex); - Mockito.doReturn(destPrimaryStorage).when(volumeApiServiceImpl) - .getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); - volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null); - } - - @Test - public void testCreateVolumeOnSecondaryForAttachIfNeeded_CreateVolumeFails_ThrowsException() { - VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); - Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Uploaded); - UserVmVO vm = Mockito.mock(UserVmVO.class); - StoragePool destPrimaryStorage = Mockito.mock(StoragePool.class); - Mockito.when(destPrimaryStorage.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); - Mockito.doReturn(destPrimaryStorage).when(volumeApiServiceImpl) - .getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); - try { - Mockito.when(volumeOrchestrationService.createVolumeOnPrimaryStorage(vm, volumeToAttach, vm.getHypervisorType(), destPrimaryStorage)) - .thenThrow(new NoTransitionException("Mocked exception")); - } catch (NoTransitionException nte) { - Assert.fail(nte.getMessage()); - } - CloudRuntimeException exception = Assert.assertThrows(CloudRuntimeException.class, () -> - volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null) - ); - Assert.assertTrue(exception.getMessage().contains("Failed to create volume on primary storage")); - } } From a627ab67c298345e92da013ebc0b786918a85f96 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 7 Feb 2025 17:29:23 +0530 Subject: [PATCH 11/21] server: fix pod retrieval during volume attach (#10324) Signed-off-by: Abhishek Kumar --- .../cloud/storage/VolumeApiServiceImpl.java | 121 +++++--- .../storage/VolumeApiServiceImplTest.java | 280 ++++++++++++++++++ 2 files changed, 361 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 7f867eb01a9..3ea8116764a 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -133,7 +133,9 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.Pod; +import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.HostPodDao; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEvent; @@ -153,6 +155,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorCapabilitiesVO; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.offering.DiskOffering; +import com.cloud.org.Cluster; import com.cloud.org.Grouping; import com.cloud.projects.Project; import com.cloud.projects.ProjectManager; @@ -323,6 +326,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject private VmWorkJobDao _workJobDao; @Inject + ClusterDao clusterDao; + @Inject private ClusterDetailsDao _clusterDetailsDao; @Inject private StorageManager storageMgr; @@ -346,6 +351,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic protected ProjectManager projectManager; @Inject protected StoragePoolDetailsDao storagePoolDetailsDao; + @Inject + HostPodDao podDao; protected Gson _gson; @@ -2380,17 +2387,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return attachVolumeToVM(command.getVirtualMachineId(), command.getId(), command.getDeviceId()); } - private Volume orchestrateAttachVolumeToVM(Long vmId, Long volumeId, Long deviceId) { - VolumeInfo volumeToAttach = volFactory.getVolume(volumeId); - - if (volumeToAttach.isAttachedVM()) { - throw new CloudRuntimeException("This volume is already attached to a VM."); - } - - UserVmVO vm = _userVmDao.findById(vmId); + protected VolumeVO getVmExistingVolumeForVolumeAttach(UserVmVO vm, VolumeInfo volumeToAttach) { VolumeVO existingVolumeOfVm = null; VMTemplateVO template = _templateDao.findById(vm.getTemplateId()); - List rootVolumesOfVm = _volsDao.findByInstanceAndType(vmId, Volume.Type.ROOT); + List rootVolumesOfVm = _volsDao.findByInstanceAndType(vm.getId(), Volume.Type.ROOT); if (rootVolumesOfVm.size() > 1 && template != null && !template.isDeployAsIs()) { throw new CloudRuntimeException("The VM " + vm.getHostName() + " has more than one ROOT volume and is in an invalid state."); } else { @@ -2398,7 +2398,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic existingVolumeOfVm = rootVolumesOfVm.get(0); } else { // locate data volume of the vm - List diskVolumesOfVm = _volsDao.findByInstanceAndType(vmId, Volume.Type.DATADISK); + List diskVolumesOfVm = _volsDao.findByInstanceAndType(vm.getId(), Volume.Type.DATADISK); for (VolumeVO diskVolume : diskVolumesOfVm) { if (diskVolume.getState() != Volume.State.Allocated) { existingVolumeOfVm = diskVolume; @@ -2407,44 +2407,90 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } } } + if (existingVolumeOfVm == null) { + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("No existing volume found for VM (%s/%s) to attach volume %s/%s", + vm.getName(), vm.getUuid(), + volumeToAttach.getName(), volumeToAttach.getUuid())); + } + return null; + } if (s_logger.isTraceEnabled()) { String msg = "attaching volume %s/%s to a VM (%s/%s) with an existing volume %s/%s on primary storage %s"; - if (existingVolumeOfVm != null) { - s_logger.trace(String.format(msg, - volumeToAttach.getName(), volumeToAttach.getUuid(), - vm.getName(), vm.getUuid(), - existingVolumeOfVm.getName(), existingVolumeOfVm.getUuid(), - existingVolumeOfVm.getPoolId())); - } + s_logger.trace(String.format(msg, + volumeToAttach.getName(), volumeToAttach.getUuid(), + vm.getName(), vm.getUuid(), + existingVolumeOfVm.getName(), existingVolumeOfVm.getUuid(), + existingVolumeOfVm.getPoolId())); } + return existingVolumeOfVm; + } - HypervisorType rootDiskHyperType = vm.getHypervisorType(); - HypervisorType volumeToAttachHyperType = _volsDao.getHypervisorType(volumeToAttach.getId()); + protected StoragePool getSuitablePoolForAllocatedOrUploadedVolumeForAttach(final VolumeInfo volumeToAttach, final UserVmVO vm) { + DataCenter zone = _dcDao.findById(vm.getDataCenterId()); + Pair clusterHostId = virtualMachineManager.findClusterAndHostIdForVm(vm, false); + Long podId = vm.getPodIdToDeployIn(); + if (clusterHostId.first() != null) { + Cluster cluster = clusterDao.findById(clusterHostId.first()); + podId = cluster.getPodId(); + } + Pod pod = podDao.findById(podId); + DiskOfferingVO offering = _diskOfferingDao.findById(volumeToAttach.getDiskOfferingId()); + DiskProfile diskProfile = new DiskProfile(volumeToAttach.getId(), volumeToAttach.getVolumeType(), + volumeToAttach.getName(), volumeToAttach.getId(), volumeToAttach.getSize(), offering.getTagsArray(), + offering.isUseLocalStorage(), offering.isRecreatable(), + volumeToAttach.getTemplateId()); + diskProfile.setHyperType(vm.getHypervisorType()); + return _volumeMgr.findStoragePool(diskProfile, zone, pod, clusterHostId.first(), + clusterHostId.second(), vm, Collections.emptySet()); + } + protected VolumeInfo createVolumeOnPrimaryForAttachIfNeeded(final VolumeInfo volumeToAttach, final UserVmVO vm, VolumeVO existingVolumeOfVm) { VolumeInfo newVolumeOnPrimaryStorage = volumeToAttach; - + boolean volumeOnSecondary = volumeToAttach.getState() == Volume.State.Uploaded; + if (!Arrays.asList(Volume.State.Allocated, Volume.State.Uploaded).contains(volumeToAttach.getState())) { + return newVolumeOnPrimaryStorage; + } //don't create volume on primary storage if its being attached to the vm which Root's volume hasn't been created yet - StoragePoolVO destPrimaryStorage = null; + StoragePool destPrimaryStorage = null; if (existingVolumeOfVm != null && !existingVolumeOfVm.getState().equals(Volume.State.Allocated)) { destPrimaryStorage = _storagePoolDao.findById(existingVolumeOfVm.getPoolId()); if (s_logger.isTraceEnabled() && destPrimaryStorage != null) { s_logger.trace(String.format("decided on target storage: %s/%s", destPrimaryStorage.getName(), destPrimaryStorage.getUuid())); } } - - boolean volumeOnSecondary = volumeToAttach.getState() == Volume.State.Uploaded; - - if (destPrimaryStorage != null && (volumeToAttach.getState() == Volume.State.Allocated || volumeOnSecondary)) { - try { - if (volumeOnSecondary && destPrimaryStorage.getPoolType() == Storage.StoragePoolType.PowerFlex) { - throw new InvalidParameterValueException("Cannot attach uploaded volume, this operation is unsupported on storage pool type " + destPrimaryStorage.getPoolType()); + if (destPrimaryStorage == null) { + destPrimaryStorage = getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + if (destPrimaryStorage == null) { + if (Volume.State.Allocated.equals(volumeToAttach.getState()) && State.Stopped.equals(vm.getState())) { + return newVolumeOnPrimaryStorage; } - newVolumeOnPrimaryStorage = _volumeMgr.createVolumeOnPrimaryStorage(vm, volumeToAttach, rootDiskHyperType, destPrimaryStorage); - } catch (NoTransitionException e) { - s_logger.debug("Failed to create volume on primary storage", e); - throw new CloudRuntimeException("Failed to create volume on primary storage", e); + throw new CloudRuntimeException(String.format("Failed to find a primary storage for volume in state: %s", volumeToAttach.getState())); } } + try { + if (volumeOnSecondary && Storage.StoragePoolType.PowerFlex.equals(destPrimaryStorage.getPoolType())) { + throw new InvalidParameterValueException("Cannot attach uploaded volume, this operation is unsupported on storage pool type " + destPrimaryStorage.getPoolType()); + } + newVolumeOnPrimaryStorage = _volumeMgr.createVolumeOnPrimaryStorage(vm, volumeToAttach, + vm.getHypervisorType(), destPrimaryStorage); + } catch (NoTransitionException e) { + s_logger.debug("Failed to create volume on primary storage", e); + throw new CloudRuntimeException("Failed to create volume on primary storage", e); + } + return newVolumeOnPrimaryStorage; + } + + private Volume orchestrateAttachVolumeToVM(Long vmId, Long volumeId, Long deviceId) { + VolumeInfo volumeToAttach = volFactory.getVolume(volumeId); + + if (volumeToAttach.isAttachedVM()) { + throw new CloudRuntimeException("This volume is already attached to a VM."); + } + + UserVmVO vm = _userVmDao.findById(vmId); + VolumeVO existingVolumeOfVm = getVmExistingVolumeForVolumeAttach(vm, volumeToAttach); + VolumeInfo newVolumeOnPrimaryStorage = createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, existingVolumeOfVm); // reload the volume from db newVolumeOnPrimaryStorage = volFactory.getVolume(newVolumeOnPrimaryStorage.getId()); @@ -2463,19 +2509,17 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic StoragePoolVO vmRootVolumePool = _storagePoolDao.findById(existingVolumeOfVm.getPoolId()); try { + HypervisorType volumeToAttachHyperType = _volsDao.getHypervisorType(volumeToAttach.getId()); newVolumeOnPrimaryStorage = _volumeMgr.moveVolume(newVolumeOnPrimaryStorage, vmRootVolumePool.getDataCenterId(), vmRootVolumePool.getPodId(), vmRootVolumePool.getClusterId(), volumeToAttachHyperType); - } catch (ConcurrentOperationException e) { - s_logger.debug("move volume failed", e); - throw new CloudRuntimeException("move volume failed", e); - } catch (StorageUnavailableException e) { + } catch (ConcurrentOperationException | StorageUnavailableException e) { s_logger.debug("move volume failed", e); throw new CloudRuntimeException("move volume failed", e); } } VolumeVO newVol = _volsDao.findById(newVolumeOnPrimaryStorage.getId()); // Getting the fresh vm object in case of volume migration to check the current state of VM - if (moveVolumeNeeded || volumeOnSecondary) { + if (moveVolumeNeeded) { vm = _userVmDao.findById(vmId); if (vm == null) { throw new InvalidParameterValueException("VM not found."); @@ -2659,9 +2703,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (!_volsDao.findByInstanceAndDeviceId(vm.getId(), 0).isEmpty()) { throw new InvalidParameterValueException("Vm already has root volume attached to it"); } - if (volumeToAttach.getState() == Volume.State.Uploaded) { - throw new InvalidParameterValueException("No support for Root volume attach in state " + Volume.State.Uploaded); - } } } diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index a0f89956df5..9b087bd384b 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -45,6 +45,7 @@ import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -86,8 +87,12 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.api.query.dao.ServiceOfferingJoinDao; import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceType; +import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.HostPodVO; +import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.HostPodDao; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; @@ -122,10 +127,12 @@ import com.cloud.utils.Pair; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.vm.DiskProfile; import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshotVO; @@ -199,6 +206,15 @@ public class VolumeApiServiceImplTest { private DataStoreManager dataStoreMgr; @Mock private SnapshotHelper snapshotHelper; + @Mock + VirtualMachineManager virtualMachineManager; + @Mock + HostPodDao podDao; + @Mock + ClusterDao clusterDao; + @Mock + VolumeOrchestrationService volumeOrchestrationService; + private DetachVolumeCmd detachCmd = new DetachVolumeCmd(); private Class _detachCmdClass = detachCmd.getClass(); @@ -1820,4 +1836,268 @@ public class VolumeApiServiceImplTest { volumeApiServiceImpl.validationsForCheckVolumeOperation(volume); } + + private UserVmVO getMockedVm() { + UserVmVO vm = Mockito.mock(UserVmVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getTemplateId()).thenReturn(10L); + Mockito.when(vm.getHostName()).thenReturn("test-vm"); + return vm; + } + + private VMTemplateVO getMockedTemplate() { + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Mockito.when(template.isDeployAsIs()).thenReturn(false); + return template; + } + + @Test(expected = CloudRuntimeException.class) + public void testGetVmExistingVolumeForVolumeAttach_MultipleRootVolumes_ThrowsException() { + UserVmVO vm = getMockedVm(); + VMTemplateVO template = getMockedTemplate(); + when(templateDao.findById(10L)).thenReturn(template); + when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.ROOT)) + .thenReturn(Arrays.asList(Mockito.mock(VolumeVO.class), Mockito.mock(VolumeVO.class))); + volumeApiServiceImpl.getVmExistingVolumeForVolumeAttach(vm, Mockito.mock(VolumeInfo.class)); + } + + @Test + public void testGetVmExistingVolumeForVolumeAttach_SingleRootVolume() { + UserVmVO vm = getMockedVm(); + VMTemplateVO template = getMockedTemplate(); + VolumeVO rootVolume = Mockito.mock(VolumeVO.class); + Mockito.when(rootVolume.getId()).thenReturn(20L); + Mockito.when(templateDao.findById(10L)).thenReturn(template); + Mockito.when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.ROOT)) + .thenReturn(Collections.singletonList(rootVolume)); + VolumeVO result = volumeApiServiceImpl.getVmExistingVolumeForVolumeAttach(vm, Mockito.mock(VolumeInfo.class)); + Assert.assertNotNull(result); + Assert.assertEquals(20L, result.getId()); + } + + private VolumeVO getMockedDataVolume() { + VolumeVO volume = Mockito.mock(VolumeVO.class); + Mockito.when(volume.getId()).thenReturn(30L); + Mockito.when(volume.getState()).thenReturn(Volume.State.Ready); + return volume; + } + + @Test + public void testGetVmExistingVolumeForVolumeAttach_NoRootVolume_DataDiskAvailable() { + UserVmVO vm = getMockedVm(); + VMTemplateVO template = getMockedTemplate(); + VolumeVO dataDisk = getMockedDataVolume(); + List rootVolumes = Collections.emptyList(); + List dataVolumes = Collections.singletonList(dataDisk); + Mockito.when(templateDao.findById(10L)).thenReturn(template); + Mockito.when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.ROOT)).thenReturn(rootVolumes); + Mockito.when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.DATADISK)).thenReturn(dataVolumes); + VolumeVO result = volumeApiServiceImpl.getVmExistingVolumeForVolumeAttach(vm, Mockito.mock(VolumeInfo.class)); + Assert.assertNotNull(result); + Assert.assertEquals(30L, result.getId()); + } + + @Test + public void testGetVmExistingVolumeForVolumeAttach_NoVolumesAtAll() { + UserVmVO vm = getMockedVm(); + VMTemplateVO template = getMockedTemplate(); + Mockito.when(templateDao.findById(10L)).thenReturn(template); + Mockito.when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.ROOT)).thenReturn(Collections.emptyList()); + Mockito.when(volumeDaoMock.findByInstanceAndType(1L, Volume.Type.DATADISK)).thenReturn(Collections.emptyList()); + VolumeVO result = volumeApiServiceImpl.getVmExistingVolumeForVolumeAttach(vm, Mockito.mock(VolumeInfo.class)); + Assert.assertNull(result); + } + + private void mockDiskOffering() { + DiskOfferingVO offering = Mockito.mock(DiskOfferingVO.class); + Mockito.when(_diskOfferingDao.findById(1L)).thenReturn(offering); + Mockito.when(offering.isUseLocalStorage()).thenReturn(true); + Mockito.when(offering.isRecreatable()).thenReturn(false); + } + + private DataCenterVO mockZone() { + DataCenterVO zone = Mockito.mock(DataCenterVO.class); + Mockito.when(_dcDao.findById(1L)).thenReturn(zone); + return zone; + } + + @Test + public void testGetPoolForAllocatedOrUploadedVolumeForAttach_Success() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + UserVmVO vm = Mockito.mock(UserVmVO.class); + ClusterVO cluster = Mockito.mock(ClusterVO.class); + HostPodVO pod = Mockito.mock(HostPodVO.class); + DataCenterVO zone = mockZone(); + mockDiskOffering(); + StoragePool pool = Mockito.mock(StoragePool.class); + when(vm.getDataCenterId()).thenReturn(1L); + when(virtualMachineManager.findClusterAndHostIdForVm(vm, false)).thenReturn(new Pair<>(1L, 2L)); + when(clusterDao.findById(1L)).thenReturn(cluster); + when(cluster.getPodId()).thenReturn(1L); + when(podDao.findById(1L)).thenReturn(pod); + when(volumeToAttach.getDiskOfferingId()).thenReturn(1L); + when(volumeOrchestrationService.findStoragePool(any(DiskProfile.class), eq(zone), eq(pod), eq(1L), eq(2L), eq(vm), eq(Collections.emptySet()))) + .thenReturn(pool); + StoragePool result = volumeApiServiceImpl.getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + Assert.assertNotNull(result); + Assert.assertEquals(pool, result); + } + + @Test + public void testGetPoolForAllocatedOrUploadedVolumeForAttach_NoSuitablePoolFound_ReturnsNull() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + UserVmVO vm = Mockito.mock(UserVmVO.class); + DataCenterVO zone = mockZone(); + Pair clusterHostId = new Pair<>(1L, 2L); + ClusterVO cluster = Mockito.mock(ClusterVO.class); + HostPodVO pod = Mockito.mock(HostPodVO.class); + mockDiskOffering(); + when(vm.getDataCenterId()).thenReturn(1L); + when(clusterDao.findById(1L)).thenReturn(cluster); + when(virtualMachineManager.findClusterAndHostIdForVm(vm, false)).thenReturn(clusterHostId); + when(podDao.findById(anyLong())).thenReturn(pod); + when(volumeToAttach.getDiskOfferingId()).thenReturn(1L); + when(volumeOrchestrationService.findStoragePool(any(DiskProfile.class), eq(zone), eq(pod), eq(1L), eq(2L), eq(vm), eq(Collections.emptySet()))) + .thenReturn(null); + Assert.assertNull(volumeApiServiceImpl.getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm)); + } + + @Test + public void testGetSuitablePoolForAllocatedOrUploadedVolumeForAttach_NoCluster() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + UserVmVO vm = Mockito.mock(UserVmVO.class); + DataCenterVO zone = mockZone(); + HostPodVO pod = Mockito.mock(HostPodVO.class); + mockDiskOffering(); + StoragePool pool = Mockito.mock(StoragePool.class); + when(vm.getDataCenterId()).thenReturn(1L); + when(vm.getPodIdToDeployIn()).thenReturn(2L); + when(virtualMachineManager.findClusterAndHostIdForVm(vm, false)).thenReturn(new Pair<>(null, 2L)); + when(podDao.findById(2L)).thenReturn(pod); + when(volumeToAttach.getDiskOfferingId()).thenReturn(1L); + when(volumeOrchestrationService.findStoragePool(any(DiskProfile.class), eq(zone), eq(pod), eq(null), eq(2L), eq(vm), eq(Collections.emptySet()))) + .thenReturn(pool); + StoragePool result = volumeApiServiceImpl.getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + Assert.assertNotNull(result); + Assert.assertEquals(pool, result); + } + + + @Test + public void testCreateVolumeOnSecondaryForAttachIfNeeded_VolumeNotAllocatedOrUploaded() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Ready); + VolumeInfo result = volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded( + volumeToAttach, Mockito.mock(UserVmVO.class), null); + Assert.assertSame(volumeToAttach, result); + Mockito.verifyNoInteractions(primaryDataStoreDaoMock, volumeOrchestrationService); + } + + @Test + public void testCreateVolumeOnSecondaryForAttachIfNeeded_ExistingVolumeDeterminesStoragePool() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Uploaded); + UserVmVO vm = Mockito.mock(UserVmVO.class); + VolumeVO existingVolume = Mockito.mock(VolumeVO.class); + Mockito.when(existingVolume.getState()).thenReturn(Volume.State.Ready); + when(existingVolume.getPoolId()).thenReturn(1L); + StoragePoolVO destPrimaryStorage = Mockito.mock(StoragePoolVO.class); + Mockito.when(destPrimaryStorage.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + Mockito.when(primaryDataStoreDaoMock.findById(1L)).thenReturn(destPrimaryStorage); + VolumeInfo newVolumeOnPrimaryStorage = Mockito.mock(VolumeInfo.class); + try { + Mockito.when(volumeOrchestrationService.createVolumeOnPrimaryStorage(vm, volumeToAttach, vm.getHypervisorType(), destPrimaryStorage)) + .thenReturn(newVolumeOnPrimaryStorage); + } catch (NoTransitionException nte) { + Assert.fail(nte.getMessage()); + } + VolumeInfo result = volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, existingVolume); + Assert.assertSame(newVolumeOnPrimaryStorage, result); + Mockito.verify(primaryDataStoreDaoMock).findById(1L); + } + + @Test + public void testCreateVolumeOnPrimaryForAttachIfNeeded_UsesGetPoolForAttach() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Allocated); + UserVmVO vm = Mockito.mock(UserVmVO.class); + StoragePool destPrimaryStorage = Mockito.mock(StoragePool.class); + Mockito.doReturn(destPrimaryStorage).when(volumeApiServiceImpl) + .getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + VolumeInfo newVolumeOnPrimaryStorage = Mockito.mock(VolumeInfo.class); + try { + Mockito.when(volumeOrchestrationService.createVolumeOnPrimaryStorage( + vm, volumeToAttach, vm.getHypervisorType(), destPrimaryStorage)) + .thenReturn(newVolumeOnPrimaryStorage); + } catch (NoTransitionException nte) { + Assert.fail(nte.getMessage()); + } + VolumeInfo result = volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null); + Assert.assertSame(newVolumeOnPrimaryStorage, result); + verify(volumeApiServiceImpl).getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCreateVolumeOnPrimaryForAttachIfNeeded_UnsupportedPoolType_ThrowsException() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + when(volumeToAttach.getState()).thenReturn(Volume.State.Uploaded); + UserVmVO vm = Mockito.mock(UserVmVO.class); + StoragePool destPrimaryStorage = Mockito.mock(StoragePool.class); + when(destPrimaryStorage.getPoolType()).thenReturn(Storage.StoragePoolType.PowerFlex); + Mockito.doReturn(destPrimaryStorage).when(volumeApiServiceImpl) + .getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null); + } + + @Test + public void testCreateVolumeOnSecondaryForAttachIfNeeded_CreateVolumeFails_ThrowsException() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Uploaded); + UserVmVO vm = Mockito.mock(UserVmVO.class); + StoragePool destPrimaryStorage = Mockito.mock(StoragePool.class); + Mockito.when(destPrimaryStorage.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + Mockito.doReturn(destPrimaryStorage).when(volumeApiServiceImpl) + .getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + try { + Mockito.when(volumeOrchestrationService.createVolumeOnPrimaryStorage(vm, volumeToAttach, vm.getHypervisorType(), destPrimaryStorage)) + .thenThrow(new NoTransitionException("Mocked exception")); + } catch (NoTransitionException nte) { + Assert.fail(nte.getMessage()); + } + CloudRuntimeException exception = Assert.assertThrows(CloudRuntimeException.class, () -> + volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null) + ); + Assert.assertTrue(exception.getMessage().contains("Failed to create volume on primary storage")); + } + + @Test + public void testCreateVolumeOnSecondaryForAttachIfNeeded_NoSuitablePool_ThrowsException() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Uploaded); + UserVmVO vm = Mockito.mock(UserVmVO.class); + Mockito.doReturn(null).when(volumeApiServiceImpl) + .getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + CloudRuntimeException exception = Assert.assertThrows(CloudRuntimeException.class, () -> + volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null) + ); + Assert.assertTrue(exception.getMessage().startsWith("Failed to find a primary storage for volume")); + } + + @Test + public void testCreateVolumeOnSecondaryForAttachIfNeeded_NoSuitablePool_ReturnSameVolumeInfo() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Allocated); + UserVmVO vm = Mockito.mock(UserVmVO.class); + Mockito.when(vm.getState()).thenReturn(State.Stopped); + Mockito.doReturn(null).when(volumeApiServiceImpl) + .getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + VolumeInfo result = volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null); + Assert.assertSame(volumeToAttach, result); + try { + Mockito.verify(volumeOrchestrationService, Mockito.never()).createVolumeOnPrimaryStorage(Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.any()); + } catch (NoTransitionException e) { + Assert.fail(); + } + } } From c09720a19a4608cbc480f03b97a56d4184b21229 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Fri, 7 Feb 2025 13:19:05 +0100 Subject: [PATCH 12/21] systemvm-registration: update seeded template_store_ref sizes (#10317) --- .../upgrade/SystemVmTemplateRegistration.java | 33 ++++++++++++++++--- .../com/cloud/storage/StorageManagerImpl.java | 4 +-- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 40a8cb4b11f..ec6ebf1680d 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -332,7 +332,7 @@ public class SystemVmTemplateRegistration { } }; - public static boolean validateIfSeeded(String url, String path, String nfsVersion) { + public boolean validateIfSeeded(TemplateDataStoreVO templDataStoreVO, String url, String path, String nfsVersion) { String filePath = null; try { filePath = Files.createTempDirectory(TEMPORARY_SECONDARY_STORE).toString(); @@ -345,6 +345,9 @@ public class SystemVmTemplateRegistration { String templatePath = filePath + File.separator + partialDirPath; File templateProps = new File(templatePath + "/template.properties"); if (templateProps.exists()) { + Pair templateSizes = readTemplatePropertiesSizes(templatePath + "/template.properties"); + updateSeededTemplateDetails(templDataStoreVO.getTemplateId(), templDataStoreVO.getDataStoreId(), + templateSizes.first(), templateSizes.second()); LOGGER.info("SystemVM template already seeded, skipping registration"); return true; } @@ -540,6 +543,21 @@ public class SystemVmTemplateRegistration { } } + public void updateSeededTemplateDetails(long templateId, long storeId, long size, long physicalSize) { + VMTemplateVO template = vmTemplateDao.findById(templateId); + template.setSize(size); + vmTemplateDao.update(template.getId(), template); + + TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeId, template.getId()); + templateDataStoreVO.setSize(size); + templateDataStoreVO.setPhysicalSize(physicalSize); + templateDataStoreVO.setLastUpdated(new Date(DateUtil.currentGMTTime().getTime())); + boolean updated = templateDataStoreDao.update(templateDataStoreVO.getId(), templateDataStoreVO); + if (!updated) { + throw new CloudRuntimeException("Failed to update template_store_ref entry for seeded systemVM template"); + } + } + public void updateSystemVMEntries(Long templateId, Hypervisor.HypervisorType hypervisorType) { vmInstanceDao.updateSystemVmTemplateId(templateId, hypervisorType); } @@ -553,7 +571,7 @@ public class SystemVmTemplateRegistration { } } - private static void readTemplateProperties(String path, SystemVMTemplateDetails details) { + private static Pair readTemplatePropertiesSizes(String path) { File tmpFile = new File(path); Long size = null; Long physicalSize = 0L; @@ -572,8 +590,13 @@ public class SystemVmTemplateRegistration { } catch (IOException ex) { LOGGER.warn("Failed to read from template.properties", ex); } - details.setSize(size); - details.setPhysicalSize(physicalSize); + return new Pair<>(size, physicalSize); + } + + public static void readTemplateProperties(String path, SystemVMTemplateDetails details) { + Pair templateSizes = readTemplatePropertiesSizes(path); + details.setSize(templateSizes.first()); + details.setPhysicalSize(templateSizes.second()); } private void updateTemplateTablesOnFailure(long templateId) { @@ -797,7 +820,7 @@ public class SystemVmTemplateRegistration { TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateId); if (templateDataStoreVO != null) { String installPath = templateDataStoreVO.getInstallPath(); - if (validateIfSeeded(storeUrlAndId.first(), installPath, nfsVersion)) { + if (validateIfSeeded(templateDataStoreVO, storeUrlAndId.first(), installPath, nfsVersion)) { continue; } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 3a6d804a762..fbe332c6fdf 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -3452,8 +3452,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C templateVO = _templateStoreDao.findByStoreTemplate(store.getId(), templateId); if (templateVO != null) { try { - if (SystemVmTemplateRegistration.validateIfSeeded( - url, templateVO.getInstallPath(), nfsVersion)) { + if (systemVmTemplateRegistration.validateIfSeeded( + templateVO, url, templateVO.getInstallPath(), nfsVersion)) { continue; } } catch (Exception e) { From 3337f425ffd52c5b278aceabbf0272015927acf6 Mon Sep 17 00:00:00 2001 From: Rene Glover Date: Fri, 7 Feb 2025 06:19:34 -0600 Subject: [PATCH 13/21] Primera pure patches & various small fixes (#10132) Co-authored-by: GLOVER RENE Co-authored-by: Suresh Kumar Anaparti --- .../exception/StorageAccessException.java | 4 +- .../orchestration/VolumeOrchestrator.java | 6 +-- .../StorageSystemDataMotionStrategy.java | 15 +++++- .../storage/volume/VolumeServiceImpl.java | 6 +-- .../CloudStackExtendedLifeCycle.java | 9 +++- .../lifecycle/registry/RegistryLifecycle.java | 13 +++-- .../hypervisor/kvm/resource/LibvirtVMDef.java | 5 +- ...rtGetUnmanagedInstancesCommandWrapper.java | 5 +- .../kvm/storage/KVMStorageProcessor.java | 28 +++++++--- .../kvm/storage/MultipathSCSIAdapterBase.java | 23 +++++++++ .../KubernetesClusterActionWorker.java | 5 ++ .../cluster/utils/KubernetesClusterUtil.java | 2 +- .../adapter/primera/PrimeraAdapter.java | 51 +++++++++++++++---- scripts/storage/multipath/cleanStaleMaps.sh | 10 +++- scripts/storage/multipath/disconnectVolume.sh | 3 ++ .../com/cloud/user/AccountManagerImpl.java | 6 ++- .../java/com/cloud/vm/UserVmManagerImpl.java | 8 ++- .../LocalNfsSecondaryStorageResource.java | 4 +- 18 files changed, 163 insertions(+), 40 deletions(-) diff --git a/api/src/main/java/com/cloud/exception/StorageAccessException.java b/api/src/main/java/com/cloud/exception/StorageAccessException.java index eefbcf5518a..d54d77d66f1 100644 --- a/api/src/main/java/com/cloud/exception/StorageAccessException.java +++ b/api/src/main/java/com/cloud/exception/StorageAccessException.java @@ -26,7 +26,7 @@ import com.cloud.utils.SerialVersionUID; public class StorageAccessException extends RuntimeException { private static final long serialVersionUID = SerialVersionUID.StorageAccessException; - public StorageAccessException(String message) { - super(message); + public StorageAccessException(String message, Exception causer) { + super(message, causer); } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index b461e50bf12..a08e74fc13c 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1827,7 +1827,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati try { volService.grantAccess(volFactory.getVolume(newVol.getId()), host, destPool); } catch (Exception e) { - throw new StorageAccessException(String.format("Unable to grant access to the volume [%s] on host [%s].", newVolToString, host)); + throw new StorageAccessException(String.format("Unable to grant access to the volume [%s] on host [%s].", newVolToString, host), e); } } @@ -1867,7 +1867,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati try { volService.grantAccess(volFactory.getVolume(volumeId), host, volumeStore); } catch (Exception e) { - throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host)); + throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host), e); } } @@ -1915,7 +1915,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati try { volService.grantAccess(volFactory.getVolume(vol.getId()), host, store); } catch (Exception e) { - throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host)); + throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host), e); } } else { grantVolumeAccessToHostIfNeeded(store, vol.getId(), host, volToString); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 398ce41db4a..f2d54823a0c 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -40,6 +40,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; @@ -1533,6 +1534,16 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { verifyFormat(templateInfo.getFormat()); } + // this blurb handles the case where the storage system can clone a volume from a template + String canCloneVolumeFromTemplate = templateInfo.getDataStore().getDriver().getCapabilities().get("CAN_CLONE_VOLUME_FROM_TEMPLATE"); + if (canCloneVolumeFromTemplate != null && canCloneVolumeFromTemplate.toLowerCase().equals("true")) { + DataStoreDriver driver = templateInfo.getDataStore().getDriver(); + driver.createAsync(volumeInfo.getDataStore(), volumeInfo, null); + volumeInfo = _volumeDataFactory.getVolume(volumeInfo.getId(), volumeInfo.getDataStore()); + driver.copyAsync(templateInfo, volumeInfo, null); + return; + } + HostVO hostVO = null; final boolean computeClusterSupportsVolumeClone; @@ -1640,7 +1651,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { errMsg = "Create volume from template failed: " + ex.getMessage(); } - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(errMsg, ex); } finally { if (copyCmdAnswer == null) { @@ -2633,7 +2644,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { catch (Exception ex) { errMsg = ex.getMessage(); - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(errMsg, ex); } finally { if (copyCmdAnswer == null) { diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 65fe25fe3cf..aba24b6956b 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -1035,7 +1035,7 @@ public class VolumeServiceImpl implements VolumeService { try { grantAccess(templateOnPrimary, destHost, destPrimaryDataStore); } catch (Exception e) { - throw new StorageAccessException("Unable to grant access to template: " + templateOnPrimary.getId() + " on host: " + destHost.getId()); + throw new StorageAccessException("Unable to grant access to template: " + templateOnPrimary.getId() + " on host: " + destHost.getId(), e); } templateOnPrimary.processEvent(Event.CopyingRequested); @@ -1161,7 +1161,7 @@ public class VolumeServiceImpl implements VolumeService { try { grantAccess(srcTemplateOnPrimary, destHost, destPrimaryDataStore); } catch (Exception e) { - throw new StorageAccessException("Unable to grant access to src template: " + srcTemplateOnPrimary.getId() + " on host: " + destHost.getId()); + throw new StorageAccessException("Unable to grant access to src template: " + srcTemplateOnPrimary.getId() + " on host: " + destHost.getId(), e); } _volumeDetailsDao.addDetail(volumeInfo.getId(), volumeDetailKey, String.valueOf(templatePoolRef.getId()), false); @@ -1406,7 +1406,7 @@ public class VolumeServiceImpl implements VolumeService { try { grantAccess(templateOnPrimary, destHost, destPrimaryDataStore); } catch (Exception e) { - throw new StorageAccessException("Unable to grant access to template: " + templateOnPrimary.getId() + " on host: " + destHost.getId()); + throw new StorageAccessException("Unable to grant access to template: " + templateOnPrimary.getId() + " on host: " + destHost.getId(), e); } templateOnPrimary.processEvent(Event.CopyingRequested); diff --git a/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycle.java b/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycle.java index b0c1dcc0760..c35360e3378 100644 --- a/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycle.java +++ b/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycle.java @@ -71,7 +71,11 @@ public class CloudStackExtendedLifeCycle extends AbstractBeanCollector { with(new WithComponentLifeCycle() { @Override public void with(ComponentLifecycle lifecycle) { - lifecycle.start(); + try { + lifecycle.start(); + } catch (Throwable e) { + log.warn("Unable to start component: " + lifecycle.getName(), e); + } if (lifecycle instanceof ManagementBean) { ManagementBean mbean = (ManagementBean)lifecycle; @@ -115,6 +119,9 @@ public class CloudStackExtendedLifeCycle extends AbstractBeanCollector { } catch (ConfigurationException e) { log.error("Failed to configure " + lifecycle.getName(), e); throw new CloudRuntimeException(e); + } catch (Throwable e) { + log.error("Failed to configure " + lifecycle.getName(), e); + throw new CloudRuntimeException(e); } } }); diff --git a/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/registry/RegistryLifecycle.java b/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/registry/RegistryLifecycle.java index 43efd846184..c82ef556a18 100644 --- a/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/registry/RegistryLifecycle.java +++ b/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/registry/RegistryLifecycle.java @@ -108,10 +108,15 @@ public class RegistryLifecycle implements BeanPostProcessor, SmartLifecycle, App while (iter.hasNext()) { Object next = iter.next(); - if (registry.register(next)) { - log.debug("Registered " + next); - } else { - iter.remove(); + try { + if (registry.register(next)) { + log.debug("Registered " + next); + } else { + log.warn("Bean registration failed for " + next.toString()); + iter.remove(); + } + } catch (Throwable e) { + log.warn("Bean registration attempt resulted in an exception for " + next.toString(), e); } } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index ec940942082..d220d133f37 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -184,6 +184,7 @@ public class LibvirtVMDef { guestDef.append("Apache Software Foundation\n"); guestDef.append("CloudStack " + _type.toString() + " Hypervisor\n"); guestDef.append("" + _uuid + "\n"); + guestDef.append("" + _uuid + "\n"); guestDef.append("\n"); guestDef.append("\n"); @@ -222,7 +223,9 @@ public class LibvirtVMDef { guestDef.append("\n"); } } - guestDef.append("\n"); + if (_arch == null || !_arch.equals("aarch64")) { + guestDef.append("\n"); + } guestDef.append("\n"); if (iothreads) { guestDef.append(String.format("%s", NUMBER_OF_IOTHREADS)); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java index d7818f7f5f3..3ea4bfcdf1d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java @@ -124,7 +124,10 @@ public final class LibvirtGetUnmanagedInstancesCommandWrapper extends CommandWra instance.setName(domain.getName()); instance.setCpuCores((int) LibvirtComputingResource.countDomainRunningVcpus(domain)); - instance.setCpuSpeed(parser.getCpuTuneDef().getShares()/instance.getCpuCores()); + + if (parser.getCpuTuneDef() != null && instance.getCpuCores() != null) { + instance.setCpuSpeed(parser.getCpuTuneDef().getShares()/instance.getCpuCores()); + } if (parser.getCpuModeDef() != null) { instance.setCpuCoresPerSocket(parser.getCpuModeDef().getCoresPerSocket()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index d58cef8c79d..d72ff47bb88 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -273,8 +273,13 @@ public class KVMStorageProcessor implements StorageProcessor { String path = derivePath(primaryStore, destData, details); - if (!storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path, details)) { + if (path == null) { + path = destTempl.getUuid(); + } + + if (path != null && !storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path, details)) { s_logger.warn("Failed to connect physical disk at path: " + path + ", in storage pool id: " + primaryStore.getUuid()); + return new PrimaryStorageDownloadAnswer("Failed to spool template disk at path: " + path + ", in storage pool id: " + primaryStore.getUuid()); } primaryVol = storagePoolMgr.copyPhysicalDisk(tmplVol, path != null ? path : destTempl.getUuid(), primaryPool, cmd.getWaitInMillSeconds()); @@ -338,6 +343,7 @@ public class KVMStorageProcessor implements StorageProcessor { } else { path = details != null ? details.get("managedStoreTarget") : null; } + return path; } @@ -418,7 +424,7 @@ public class KVMStorageProcessor implements StorageProcessor { if (primaryPool.getType() == StoragePoolType.CLVM) { templatePath = ((NfsTO)imageStore).getUrl() + File.separator + templatePath; vol = templateToPrimaryDownload(templatePath, primaryPool, volume.getUuid(), volume.getSize(), cmd.getWaitInMillSeconds()); - } if (primaryPool.getType() == StoragePoolType.PowerFlex) { + } if (primaryPool.getType() == StoragePoolType.PowerFlex || primaryPool.getType() == StoragePoolType.FiberChannel) { Map details = primaryStore.getDetails(); String path = derivePath(primaryStore, destData, details); @@ -772,15 +778,19 @@ public class KVMStorageProcessor implements StorageProcessor { KVMStoragePool secondaryStorage = null; + String path = null; try { // look for options indicating an overridden path or IQN. Used when snapshots have to be // temporarily copied on the manaaged storage device before the actual copy to target object Map details = cmd.getOptions(); - String path = details != null ? details.get(DiskTO.PATH) : null; + path = details != null ? details.get(DiskTO.PATH) : null; if (path == null) { path = details != null ? details.get(DiskTO.IQN) : null; if (path == null) { - new CloudRuntimeException("The 'path' or 'iqn' field must be specified."); + path = srcData.getPath(); + if (path == null) { + new CloudRuntimeException("The 'path' or 'iqn' field must be specified."); + } } } @@ -843,8 +853,6 @@ public class KVMStorageProcessor implements StorageProcessor { loc.addFormat(info); loc.save(); - storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path); - TemplateObjectTO newTemplate = new TemplateObjectTO(); newTemplate.setPath(templateFolder + File.separator + templateName + ".qcow2"); @@ -864,6 +872,10 @@ public class KVMStorageProcessor implements StorageProcessor { return new CopyCmdAnswer(ex.toString()); } finally { + if (path != null) { + storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path); + } + if (secondaryStorage != null) { secondaryStorage.delete(); } @@ -1039,7 +1051,9 @@ public class KVMStorageProcessor implements StorageProcessor { command.add(NAME_OPTION, snapshotName); command.add("-p", snapshotDestPath); - descName = UUID.randomUUID().toString(); + if (isCreatedFromVmSnapshot) { + descName = UUID.randomUUID().toString(); + } command.add("-t", descName); final String result = command.execute(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java index 558a5269ef4..2d702799109 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java @@ -160,6 +160,13 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { KVMPhysicalDisk disk = new KVMPhysicalDisk(address.getPath(), address.toString(), pool); disk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + // validate we have a connection, if not we need to connect first. + if (!isConnected(address.getPath())) { + if (!connectPhysicalDisk(address, pool, null)) { + throw new CloudRuntimeException("Unable to connect to volume " + address.getPath()); + } + } + long diskSize = getPhysicalDiskSize(address.getPath()); disk.setSize(diskSize); disk.setVirtualSize(diskSize); @@ -197,6 +204,10 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { // we expect WWN values in the volumePath so need to convert it to an actual physical path AddressInfo address = this.parseAndValidatePath(volumePath); + return connectPhysicalDisk(address, pool, details); + } + + private boolean connectPhysicalDisk(AddressInfo address, KVMStoragePool pool, Map details) { // validate we have a connection id - we can't proceed without that if (address.getConnectionId() == null) { LOGGER.error("Unable to connect volume with address [" + address.getPath() + "] of the storage pool: " + pool.getUuid() + " - connection id is not set in provided path"); @@ -508,6 +519,18 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { return false; } + boolean isConnected(String path) { + // run a command to test if this is a binary device at this path + Script blockTest = new Script("/bin/test", LOGGER); + blockTest.add("-b", path); + blockTest.execute(); + int rc = blockTest.getExitValue(); + if (rc == 0) { + return true; + } + return false; + } + long getPhysicalDiskSize(String diskPath) { if (StringUtils.isEmpty(diskPath)) { return 0; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 199f6da90d2..dda5c3f79fc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -362,6 +362,11 @@ public class KubernetesClusterActionWorker { IpAddress address = ipAddressDao.findByUuid(detailsVO.getValue()); if (address == null || network.getVpcId() != address.getVpcId()) { LOGGER.warn(String.format("Public IP with ID: %s linked to the Kubernetes cluster: %s is not usable", detailsVO.getValue(), kubernetesCluster.getName())); + if (address == null) { + LOGGER.warn(String.format("Public IP with ID: %s was not found by uuid", detailsVO.getValue())); + } else { + LOGGER.warn(String.format("Public IP with ID: %s was associated with vpc %d instead of %d", detailsVO.getValue(), address.getVpcId().longValue(), network.getVpcId().longValue())); + } return null; } return address; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index e1210a607e6..9ede7c0f830 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -192,7 +192,7 @@ public class KubernetesClusterUtil { while (System.currentTimeMillis() < timeoutTime) { try { Pair result = SshHelper.sshExecute(ipAddress, port, user, - sshKeyFile, null, "sudo cat /etc/kubernetes/admin.conf", + sshKeyFile, null, "sudo cat /etc/kubernetes/user.conf 2>/dev/null || sudo cat /etc/kubernetes/admin.conf", 10000, 10000, 10000); if (result.first() && StringUtils.isNotEmpty(result.second())) { diff --git a/plugins/storage/volume/primera/src/main/java/org/apache/cloudstack/storage/datastore/adapter/primera/PrimeraAdapter.java b/plugins/storage/volume/primera/src/main/java/org/apache/cloudstack/storage/datastore/adapter/primera/PrimeraAdapter.java index 1fdc92feedc..3e37b2efd9a 100644 --- a/plugins/storage/volume/primera/src/main/java/org/apache/cloudstack/storage/datastore/adapter/primera/PrimeraAdapter.java +++ b/plugins/storage/volume/primera/src/main/java/org/apache/cloudstack/storage/datastore/adapter/primera/PrimeraAdapter.java @@ -145,16 +145,18 @@ public class PrimeraAdapter implements ProviderAdapter { } // determine volume type based on offering - // THIN: tpvv=true, reduce=false - // SPARSE: tpvv=true, reduce=true - // THICK: tpvv=false, tpZeroFill=true (not supported) + // tpvv -- thin provisioned virtual volume (no deduplication) + // reduce -- thin provisioned virtual volume (with duplication and compression, also known as DECO) + // these are the only choices with newer Primera devices + // we will use THIN for the deduplicated/compressed type and SPARSE for thin-only without dedup/compress + // note: DECO/reduce type must be at least 16GB in size if (diskOffering != null) { if (diskOffering.getType() == ProvisioningType.THIN) { - request.setTpvv(true); - request.setReduce(false); - } else if (diskOffering.getType() == ProvisioningType.SPARSE) { request.setTpvv(false); request.setReduce(true); + } else if (diskOffering.getType() == ProvisioningType.SPARSE) { + request.setTpvv(true); + request.setReduce(false); } else if (diskOffering.getType() == ProvisioningType.FAT) { throw new RuntimeException("This storage provider does not support FAT provisioned volumes"); } @@ -165,8 +167,16 @@ public class PrimeraAdapter implements ProviderAdapter { } } else { // default to deduplicated volume - request.setReduce(true); request.setTpvv(false); + request.setReduce(true); + } + + if (request.getReduce() == true) { + // check if sizeMiB is less than 16GB adjust up to 16GB. The AdaptiveDatastoreDriver will automatically + // update this on the cloudstack side to match + if (request.getSizeMiB() < 16 * 1024) { + request.setSizeMiB(16 * 1024); + } } request.setComment(ProviderVolumeNamer.generateObjectComment(context, dataIn)); @@ -184,8 +194,11 @@ public class PrimeraAdapter implements ProviderAdapter { if (host == null) { throw new RuntimeException("Unable to find host " + hostname + " on storage provider"); } - request.setHostname(host.getName()); + // check if we already have a vlun for requested host + Integer vlun = hasVlun(hostname, hostname); + if (vlun == null) { + request.setHostname(host.getName()); request.setVolumeName(dataIn.getExternalName()); request.setAutoLun(true); // auto-lun returned here: Location: /api/v1/vluns/test_vv02,252,mysystem,2:2:4 @@ -197,7 +210,13 @@ public class PrimeraAdapter implements ProviderAdapter { if (toks.length <2) { throw new RuntimeException("Attach volume failed with invalid location response to vlun add command on storage provider. Provided location: " + location); } - return toks[1]; + try { + vlun = Integer.parseInt(toks[1]); + } catch (NumberFormatException e) { + throw new RuntimeException("VLUN attach request succeeded but the VLUN value is not a valid number: " + toks[1]); + } + } + return vlun.toString(); } /** @@ -232,6 +251,20 @@ public class PrimeraAdapter implements ProviderAdapter { } } + private Integer hasVlun(String externalName, String hostname) { + PrimeraVlunList list = getVluns(externalName); + if (list != null && list.getMembers().size() > 0) { + for (PrimeraVlun vlun: list.getMembers()) { + if (hostname != null) { + if (vlun.getHostname().equals(hostname) || vlun.getHostname().equals(hostname.split("\\.")[0])) { + return vlun.getLun(); + } + } + } + } + return null; + } + public void removeVlun(String name, Integer lunid, String hostString) { // hostString can be a hostname OR "set:". It is stored this way // in the appliance and returned as the vlun's name/string. diff --git a/scripts/storage/multipath/cleanStaleMaps.sh b/scripts/storage/multipath/cleanStaleMaps.sh index 90b9bef5a8d..c1ded42943c 100755 --- a/scripts/storage/multipath/cleanStaleMaps.sh +++ b/scripts/storage/multipath/cleanStaleMaps.sh @@ -22,10 +22,18 @@ # ############################################################################################# +SCRIPT_NAME=$(basename "$0") + +if [[ $(pgrep -f ${SCRIPT_NAME}) != "$$" ]]; then + echo "Another instance of ${SCRIPT_NAME} is already running! Exiting" + exit +fi + + cd $(dirname $0) for WWID in $(multipathd list maps status | awk '{ if ($4 == 0) { print substr($1,2); }}'); do - ./removeVolume.sh ${WWID} + ./disconnectVolume.sh ${WWID} done exit 0 diff --git a/scripts/storage/multipath/disconnectVolume.sh b/scripts/storage/multipath/disconnectVolume.sh index 067e561f8a3..f894076927f 100755 --- a/scripts/storage/multipath/disconnectVolume.sh +++ b/scripts/storage/multipath/disconnectVolume.sh @@ -66,6 +66,9 @@ fi logger -t CS_SCSI_VOL_REMOVE "${WWID} successfully purged from multipath along with slave devices" +# Added to give time for the event to be fired to the server +sleep 10 + echo "$(date): ${WWID} removed" exit 0 diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index 2358830e9a3..8e06c576881 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -694,7 +694,11 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M for (SecurityChecker checker : _securityCheckers) { if (checker.checkAccess(caller, entity, accessType, apiName)) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Access to " + entity + " granted to " + caller + " by " + checker.getName()); + User user = CallContext.current().getCallingUser(); + String userName = ""; + if (user != null) + userName = user.getUsername(); + s_logger.debug("Access to " + entity + " granted to " + caller + " by " + checker.getName() + " on behalf of user " + userName); } granted = true; break; diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 94034da4c8f..74b17459421 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -129,8 +129,8 @@ import org.apache.cloudstack.storage.template.VnfTemplateManager; import org.apache.cloudstack.userdata.UserDataManager; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.security.ParserUtils; -import org.apache.cloudstack.vm.UnmanagedVMsManager; import org.apache.cloudstack.vm.schedule.VMScheduleManager; +import org.apache.cloudstack.vm.UnmanagedVMsManager; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.math.NumberUtils; @@ -4406,7 +4406,11 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { - Long rootDiskSize = rootDiskSizeCustomParam * GiB_TO_BYTES; + Long rootDiskSize = NumbersUtil.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE), -1); + if (rootDiskSize <= 0) { + throw new InvalidParameterValueException("Root disk size should be a positive number."); + } + rootDiskSize = rootDiskSizeCustomParam * GiB_TO_BYTES; _volumeService.validateVolumeSizeInBytes(rootDiskSize); return rootDiskSize; } else { diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java index 6f189ef5f3c..08270086e8e 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java @@ -77,14 +77,14 @@ public class LocalNfsSecondaryStorageResource extends NfsSecondaryStorageResourc // Change permissions for the mountpoint - seems to bypass authentication Script script = new Script(true, "chmod", _timeout, s_logger); - script.add("777", localRootPath); + script.add("1777", localRootPath); String result = script.execute(); if (result != null) { String errMsg = "Unable to set permissions for " + localRootPath + " due to " + result; s_logger.error(errMsg); throw new CloudRuntimeException(errMsg); } - s_logger.debug("Successfully set 777 permission for " + localRootPath); + s_logger.debug("Successfully set 1777 permission for " + localRootPath); // XXX: Adding the check for creation of snapshots dir here. Might have // to move it somewhere more logical later. From a1117acbdfb8568ba2324f25b03762f405a22057 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 7 Feb 2025 08:18:11 -0500 Subject: [PATCH 14/21] List only untagged offerings for Shared networks when tag isn't passed (#10320) --- ui/src/views/network/CreateSharedNetworkForm.vue | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/src/views/network/CreateSharedNetworkForm.vue b/ui/src/views/network/CreateSharedNetworkForm.vue index 93bf693008f..fc648c293d0 100644 --- a/ui/src/views/network/CreateSharedNetworkForm.vue +++ b/ui/src/views/network/CreateSharedNetworkForm.vue @@ -791,6 +791,10 @@ export default { if (this.scopeType === 'domain') { params.domainid = this.selectedDomain.id } + console.log(params?.tags?.length === 0) + if (!params?.tags || params.tags.length === 0) { + params.istagged = false + } this.handleNetworkOfferingChange(null) this.networkOfferings = [] api('listNetworkOfferings', params).then(json => { From 58a63f64fd6c42d8e359baad4992ada7bf61b330 Mon Sep 17 00:00:00 2001 From: Gabriel Pordeus Santos Date: Sun, 9 Feb 2025 05:31:39 -0300 Subject: [PATCH 15/21] Fix VMWare leftovers when deleting VM without root disk (#9735) --- .../com/cloud/agent/api/CleanupVMCommand.java | 46 +++++++++++++++++++ .../com/cloud/hypervisor/guru/VMwareGuru.java | 8 ++++ .../vmware/resource/VmwareResource.java | 23 ++++++++++ 3 files changed, 77 insertions(+) create mode 100644 core/src/main/java/com/cloud/agent/api/CleanupVMCommand.java diff --git a/core/src/main/java/com/cloud/agent/api/CleanupVMCommand.java b/core/src/main/java/com/cloud/agent/api/CleanupVMCommand.java new file mode 100644 index 00000000000..a4d73a8b164 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/CleanupVMCommand.java @@ -0,0 +1,46 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +/** + * This command will destroy a leftover VM during the expunge process if it wasn't destroyed before. + * + */ +public class CleanupVMCommand extends Command { + String vmName; + boolean executeInSequence; + + public CleanupVMCommand(String vmName) { + this(vmName, false); + } + public CleanupVMCommand(String vmName, boolean executeInSequence) { + this.vmName = vmName; + this.executeInSequence = executeInSequence; + } + + @Override + public boolean executeInSequence() { + return executeInSequence; + } + + public String getVmName() { + return vmName; + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index 822fc870c67..9fc73e37082 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import com.cloud.agent.api.CleanupVMCommand; import javax.inject.Inject; import com.cloud.agent.api.to.NfsTO; @@ -371,6 +372,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co return tokens[0] + "@" + vCenterIp; } + @Override public List finalizeExpunge(VirtualMachine vm) { + List commands = new ArrayList(); + final CleanupVMCommand cleanupVMCommand = new CleanupVMCommand(vm.getInstanceName(), true); + commands.add(cleanupVMCommand); + return commands; + } + @Override public List finalizeExpungeNics(VirtualMachine vm, List nics) { List commands = new ArrayList(); List nicVOs = nicDao.listByVmId(vm.getId()); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 92821c7e26d..20495530909 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -45,6 +45,7 @@ import java.util.TimeZone; import java.util.UUID; import java.util.stream.Collectors; +import com.cloud.agent.api.CleanupVMCommand; import javax.naming.ConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; @@ -585,6 +586,8 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes return execute((ResizeVolumeCommand) cmd); } else if (clz == UnregisterVMCommand.class) { return execute((UnregisterVMCommand) cmd); + } else if (clz == CleanupVMCommand.class) { + return execute((CleanupVMCommand) cmd); } else if (cmd instanceof StorageSubSystemCommand) { checkStorageProcessorAndHandlerNfsVersionAttribute((StorageSubSystemCommand) cmd); return storageHandler.handleStorageCommands((StorageSubSystemCommand) cmd); @@ -5796,6 +5799,26 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes return new Answer(cmd, true, "success"); } + protected Answer execute(CleanupVMCommand cmd) { + VmwareContext context = getServiceContext(); + VmwareHypervisorHost hyperHost = getHyperHost(context); + + try { + VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName()); + if (vmMo == null) { + String msg = String.format("VM [%s] not found on vCenter, cleanup not needed.", cmd.getVmName()); + s_logger.debug(msg); + return new Answer(cmd, true, msg); + } + vmMo.destroy(); + String msg = String.format("VM [%s] remnants on vCenter cleaned up.", cmd.getVmName()); + s_logger.debug(msg); + return new Answer(cmd, true, msg); + } catch (Exception e) { + return new Answer(cmd, false, createLogMessageException(e, cmd)); + } + } + protected Answer execute(UnregisterVMCommand cmd) { VmwareContext context = getServiceContext(); VmwareHypervisorHost hyperHost = getHyperHost(context); From d453c63848f9367d50560bed103d0e8c4a31d283 Mon Sep 17 00:00:00 2001 From: dahn Date: Sun, 9 Feb 2025 09:34:20 +0100 Subject: [PATCH 16/21] cleanup VM IP after expunge in redundant VPC (#10183) Co-authored-by: Pearl Dsilva --- systemvm/debian/opt/cloud/bin/cs/CsDhcp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py index 8f1562b0b82..385cc49a90f 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py @@ -139,8 +139,7 @@ class CsDhcp(CsDataBag): # Listen Address if self.cl.is_redundant(): listen_address.append(gateway) - else: - listen_address.append(ip) + listen_address.append(ip) # Add localized "data-server" records in /etc/hosts for VPC routers if self.config.is_vpc() or self.config.is_router(): self.add_host(gateway, "%s data-server" % CsHelper.get_hostname()) From d920aba176c857ff1315b0d9be08626e14ee9eb2 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Sun, 9 Feb 2025 09:35:45 +0100 Subject: [PATCH 17/21] server: fix scale vm with same disk offering id (#10235) --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 74b17459421..173cec83d71 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2108,7 +2108,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir throw new InvalidParameterValueException("Unable to Scale VM, since disk offering strictness flag is not same for new service offering and old service offering"); } - if (currentServiceOffering.getDiskOfferingStrictness() && currentServiceOffering.getDiskOfferingId() != newServiceOffering.getDiskOfferingId()) { + if (currentServiceOffering.getDiskOfferingStrictness() && !currentServiceOffering.getDiskOfferingId().equals(newServiceOffering.getDiskOfferingId())) { throw new InvalidParameterValueException("Unable to Scale VM, since disk offering id associated with the old service offering is not same for new service offering"); } From d3170bfa165b2d4f8b31ab0863a0df79bdeef954 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Mon, 10 Feb 2025 16:45:39 +0530 Subject: [PATCH 18/21] UI: Fixup missing buttons (#10357) --- ui/src/config/section/compute.js | 8 ++++---- ui/src/config/section/image.js | 6 +++--- ui/src/config/section/network.js | 12 ++++++------ ui/src/config/section/storage.js | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 5ebcde5602d..ba7b3d75eb8 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -100,7 +100,7 @@ export default { label: 'label.vm.add', docHelp: 'adminguide/virtual_machines.html#creating-vms', listView: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, component: () => import('@/views/compute/DeployVM.vue') }, { @@ -569,7 +569,7 @@ export default { docHelp: 'plugins/cloudstack-kubernetes-service.html#creating-a-new-kubernetes-cluster', listView: true, popup: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, component: shallowRef(defineAsyncComponent(() => import('@/views/compute/CreateKubernetesCluster.vue'))) }, { @@ -698,7 +698,7 @@ export default { icon: 'plus-outlined', label: 'label.new.autoscale.vmgroup', listView: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, component: () => import('@/views/compute/CreateAutoScaleVmGroup.vue') }, { @@ -789,7 +789,7 @@ export default { icon: 'plus-outlined', label: 'label.new.instance.group', listView: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, args: ['name'] }, { diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index 2b5153c5865..49967bb80ee 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -111,7 +111,7 @@ export default { docHelp: 'adminguide/templates.html#uploading-templates-from-a-remote-http-server', listView: true, popup: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, component: shallowRef(defineAsyncComponent(() => import('@/views/image/RegisterOrUploadTemplate.vue'))) }, { @@ -272,7 +272,7 @@ export default { docHelp: 'adminguide/templates.html#id10', listView: true, popup: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, component: shallowRef(defineAsyncComponent(() => import('@/views/image/RegisterOrUploadIso.vue'))) }, { @@ -392,7 +392,7 @@ export default { label: 'label.kubernetes.version.add', listView: true, popup: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, component: shallowRef(defineAsyncComponent(() => import('@/views/image/AddKubernetesSupportedVersion.vue'))) }, { diff --git a/ui/src/config/section/network.js b/ui/src/config/section/network.js index f0415a783a5..6691afa247d 100644 --- a/ui/src/config/section/network.js +++ b/ui/src/config/section/network.js @@ -246,7 +246,7 @@ export default { icon: 'plus-outlined', label: 'label.add.vpc', docHelp: 'adminguide/networking_and_traffic.html#adding-a-virtual-private-cloud', - show: () => { isZoneCreated() }, + show: isZoneCreated, listView: true, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/network/CreateVpc.vue'))) @@ -396,7 +396,7 @@ export default { label: 'label.vnf.appliance.add', docHelp: 'adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances', listView: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, component: () => import('@/views/compute/DeployVnfAppliance.vue') }, { @@ -944,7 +944,7 @@ export default { label: 'label.add.vpn.gateway', docHelp: 'adminguide/networking_and_traffic.html#creating-a-vpn-gateway-for-the-vpc', listView: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, args: ['vpcid'] }, { @@ -1120,7 +1120,7 @@ export default { icon: 'plus-outlined', label: 'label.add.vpn.user', listView: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, args: (record, store) => { if (store.userInfo.roletype === 'User') { return ['username', 'password'] @@ -1200,7 +1200,7 @@ export default { docHelp: 'adminguide/networking_and_traffic.html#creating-and-updating-a-vpn-customer-gateway', listView: true, popup: true, - show: () => { isZoneCreated() }, + show: isZoneCreated, component: shallowRef(defineAsyncComponent(() => import('@/views/network/CreateVpnCustomerGateway.vue'))) }, { @@ -1390,7 +1390,7 @@ export default { component: shallowRef(defineAsyncComponent(() => import('@/views/network/GuestVlanNetworksTab.vue'))), show: (record) => { return (record.allocationstate === 'Allocated') } }], - show: () => { isZoneCreated() } + show: isZoneCreated } ] } diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 95894b33429..7e56d5c6eee 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -104,7 +104,7 @@ export default { icon: 'plus-outlined', docHelp: 'adminguide/storage.html#creating-a-new-volume', label: 'label.action.create.volume', - show: () => { isZoneCreated() }, + show: isZoneCreated, listView: true, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/storage/CreateVolume.vue'))) @@ -124,7 +124,7 @@ export default { icon: 'link-outlined', docHelp: 'adminguide/storage.html#uploading-an-existing-volume-to-a-virtual-machine', label: 'label.upload.volume.from.url', - show: () => { isZoneCreated() }, + show: isZoneCreated, listView: true, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/storage/UploadVolume.vue'))) From 4f604c00b65fc9f6cde4f460b72e6b12ba9f8723 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Mon, 10 Feb 2025 11:03:07 -0300 Subject: [PATCH 19/21] Support virtio-blk root disk controller (#10229) --- .../kvm/resource/LibvirtComputingResource.java | 2 +- .../cloud/hypervisor/kvm/resource/LibvirtVMDef.java | 2 +- .../hypervisor/kvm/storage/KVMStorageProcessor.java | 4 +++- .../kvm/resource/LibvirtComputingResourceTest.java | 10 ++++++++++ .../java/com/cloud/api/query/QueryManagerImpl.java | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 71f33d9be57..973f053fee6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3109,7 +3109,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv disk.setBusType(DiskDef.DiskBus.SCSI); } } else { - if (diskBusType == DiskDef.DiskBus.SCSI ) { + if (diskBusType == DiskDef.DiskBus.SCSI || diskBusType == DiskDef.DiskBus.VIRTIOBLK) { disk.setQemuDriver(true); disk.setDiscard(DiscardType.UNMAP); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index d220d133f37..1b007ed0696 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -639,7 +639,7 @@ public class LibvirtVMDef { } public enum DiskBus { - IDE("ide"), SCSI("scsi"), VIRTIO("virtio"), XEN("xen"), USB("usb"), UML("uml"), FDC("fdc"), SATA("sata"); + IDE("ide"), SCSI("scsi"), VIRTIO("virtio"), XEN("xen"), USB("usb"), UML("uml"), FDC("fdc"), SATA("sata"), VIRTIOBLK("virtio-blk"); String _bus; DiskBus(String bus) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index d72ff47bb88..193a3287361 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -1423,12 +1423,14 @@ public class KVMStorageProcessor implements StorageProcessor { if (disk.getDeviceType() == DeviceType.DISK) { if (disk.getBusType() == DiskDef.DiskBus.SCSI) { busT = DiskDef.DiskBus.SCSI; + } else if (disk.getBusType() == DiskDef.DiskBus.VIRTIOBLK) { + busT = DiskDef.DiskBus.VIRTIOBLK; } break; } } diskdef = new DiskDef(); - if (busT == DiskDef.DiskBus.SCSI) { + if (busT == DiskDef.DiskBus.SCSI || busT == DiskDef.DiskBus.VIRTIOBLK) { diskdef.setQemuDriver(true); diskdef.setDiscard(DiscardType.UNMAP); } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 9b1da988c29..9ef96deb691 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -6411,4 +6411,14 @@ public class LibvirtComputingResourceTest { assertEquals(DiskDef.DiscardType.UNMAP, rootDisk.getDiscard()); } } + + @Test + public void testGetDiskModelFromVMDetailVirtioBlk() { + VirtualMachineTO virtualMachineTO = Mockito.mock(VirtualMachineTO.class); + Map details = new HashMap<>(); + details.put(VmDetailConstants.ROOT_DISK_CONTROLLER, "virtio-blk"); + Mockito.when(virtualMachineTO.getDetails()).thenReturn(details); + DiskDef.DiskBus diskBus = libvirtComputingResourceSpy.getDiskModelFromVMDetail(virtualMachineTO); + assertEquals(DiskDef.DiskBus.VIRTIOBLK, diskBus); + } } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 3063b0d49a2..16cbd5ebe1b 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -4978,7 +4978,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (HypervisorType.KVM.equals(hypervisorType)) { options.put(VmDetailConstants.NIC_ADAPTER, Arrays.asList("e1000", "virtio", "rtl8139", "vmxnet3", "ne2k_pci")); - options.put(VmDetailConstants.ROOT_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "virtio")); + options.put(VmDetailConstants.ROOT_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "virtio", "virtio-blk")); options.put(VmDetailConstants.VIDEO_HARDWARE, Arrays.asList("cirrus", "vga", "qxl", "virtio")); options.put(VmDetailConstants.VIDEO_RAM, Collections.emptyList()); options.put(VmDetailConstants.IO_POLICY, Arrays.asList("threads", "native", "io_uring", "storage_specific")); From aa6c581e405d7652916a9290e042f0bd26c3913a Mon Sep 17 00:00:00 2001 From: dahn Date: Mon, 10 Feb 2025 17:06:39 +0100 Subject: [PATCH 20/21] Add the option to filter by host when retrieving of unregistered VMs (#9925) Co-authored-by: Nicolas Vazquez --- .../apache/cloudstack/api/ApiConstants.java | 9 +- .../cloudstack/api/response/HostResponse.java | 8 +- .../acl/DynamicRoleBasedAPIAccessChecker.java | 4 +- .../vmware/VmwareDatacenterService.java | 7 +- .../vmware/manager/VmwareManagerImpl.java | 292 ++++---- .../command/admin/zone/AddVmwareDcCmd.java | 12 +- .../zone/ImportVsphereStoragePoliciesCmd.java | 15 +- .../admin/zone/ListVmwareDcHostsCmd.java | 144 ++++ .../command/admin/zone/ListVmwareDcItems.java | 29 + .../admin/zone/ListVmwareDcVmsCmd.java | 52 +- .../command/admin/zone/ListVmwareDcsCmd.java | 26 +- .../zone/ListVsphereStoragePoliciesCmd.java | 17 +- ...sphereStoragePolicyCompatiblePoolsCmd.java | 2 +- .../command/admin/zone/RemoveVmwareDcCmd.java | 8 +- .../command/admin/zone/UpdateVmwareDcCmd.java | 11 +- .../admin/zone/VmwareRequestResponse.java | 38 ++ test/integration/smoke/test_dynamicroles.py | 1 + ui/public/locales/en.json | 1 + ui/src/views/tools/ManageInstances.vue | 1 + ui/src/views/tools/SelectVmwareVcenter.vue | 64 +- .../cloud/hypervisor/vmware/mo/BaseMO.java | 52 +- .../vmware/mo/CustomFieldsManagerMO.java | 7 +- .../hypervisor/vmware/mo/DatacenterMO.java | 161 +++-- .../cloud/hypervisor/vmware/mo/HostMO.java | 91 ++- .../vmware/mo/VirtualMachineMO.java | 625 ++++-------------- .../hypervisor/vmware/util/VmwareClient.java | 30 +- .../vmware/util/VmwareClientException.java | 33 + 27 files changed, 921 insertions(+), 819 deletions(-) create mode 100644 plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcHostsCmd.java create mode 100644 plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcItems.java create mode 100644 plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/VmwareRequestResponse.java create mode 100644 vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClientException.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index b2042c116a7..efceafd1a3a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -46,6 +46,7 @@ public class ApiConstants { public static final String BACKUP_OFFERING_NAME = "backupofferingname"; public static final String BACKUP_OFFERING_ID = "backupofferingid"; public static final String BASE64_IMAGE = "base64image"; + public static final String BATCH_SIZE = "batchsize"; public static final String BITS = "bits"; public static final String BOOTABLE = "bootable"; public static final String BIND_DN = "binddn"; @@ -437,11 +438,12 @@ public class ApiConstants { public static final String STATE = "state"; public static final String STATS = "stats"; public static final String STATUS = "status"; - public static final String STORAGE_TYPE = "storagetype"; - public static final String STORAGE_POLICY = "storagepolicy"; - public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled"; public static final String STORAGE_CAPABILITIES = "storagecapabilities"; public static final String STORAGE_CUSTOM_STATS = "storagecustomstats"; + public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled"; + public static final String STORAGE_POLICY = "storagepolicy"; + public static final String STORAGE_POOL = "storagepool"; + public static final String STORAGE_TYPE = "storagetype"; public static final String SUBNET = "subnet"; public static final String OWNER = "owner"; public static final String SWAP_OWNER = "swapowner"; @@ -1106,6 +1108,7 @@ public class ApiConstants { public static final String PARAMETER_DESCRIPTION_IS_TAG_A_RULE = "Whether the informed tag is a JS interpretable rule or not."; public static final String NFS_MOUNT_OPTIONS = "nfsmountopts"; + public static final String VMWARE_DC = "vmwaredc"; /** * This enum specifies IO Drivers, each option controls specific policies on I/O. diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index 99aabd88913..cb759d9ab6d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -152,7 +152,7 @@ public class HostResponse extends BaseResponseWithAnnotations { @Deprecated @SerializedName("memoryallocated") @Param(description = "the amount of the host's memory currently allocated") - private long memoryAllocated; + private Long memoryAllocated; @SerializedName("memoryallocatedpercentage") @Param(description = "the amount of the host's memory currently allocated in percentage") @@ -395,7 +395,7 @@ public class HostResponse extends BaseResponseWithAnnotations { this.memWithOverprovisioning=memWithOverprovisioning; } - public void setMemoryAllocated(long memoryAllocated) { + public void setMemoryAllocated(Long memoryAllocated) { this.memoryAllocated = memoryAllocated; } @@ -659,8 +659,8 @@ public class HostResponse extends BaseResponseWithAnnotations { return memoryTotal; } - public long getMemoryAllocated() { - return memoryAllocated; + public Long getMemoryAllocated() { + return memoryAllocated == null ? 0 : memoryAllocated; } public void setMemoryAllocatedPercentage(String memoryAllocatedPercentage) { diff --git a/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java b/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java index 1dfe20a10be..005eb2a8fd4 100644 --- a/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java +++ b/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java @@ -47,14 +47,14 @@ public class DynamicRoleBasedAPIAccessChecker extends AdapterBase implements API private RoleService roleService; private List services; - private Map> annotationRoleBasedApisMap = new HashMap>(); + private Map> annotationRoleBasedApisMap = new HashMap<>(); private static final Logger LOGGER = Logger.getLogger(DynamicRoleBasedAPIAccessChecker.class.getName()); protected DynamicRoleBasedAPIAccessChecker() { super(); for (RoleType roleType : RoleType.values()) { - annotationRoleBasedApisMap.put(roleType, new HashSet()); + annotationRoleBasedApisMap.put(roleType, new HashSet<>()); } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java index bbac78b879a..6bb473536b8 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java @@ -22,12 +22,15 @@ import com.cloud.dc.VmwareDatacenterVO; import com.cloud.dc.VsphereStoragePolicy; import com.cloud.exception.DiscoveryException; import com.cloud.exception.ResourceInUseException; +import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.storage.StoragePool; +import com.cloud.utils.Pair; import com.cloud.utils.component.PluggableService; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcVmsCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcHostsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePolicyCompatiblePoolsCmd; @@ -53,5 +56,7 @@ public interface VmwareDatacenterService extends PluggableService { List listVsphereStoragePolicyCompatibleStoragePools(ListVsphereStoragePolicyCompatiblePoolsCmd cmd); - List listVMsInDatacenter(ListVmwareDcVmsCmd cmd); + List listHostsInDatacenter(ListVmwareDcHostsCmd cmd); + + Pair> listVMsInDatacenter(ListVmwareDcVmsCmd cmd); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 61a949f42d3..2d733e8a16c 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -19,10 +19,12 @@ package com.cloud.hypervisor.vmware.manager; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.rmi.RemoteException; import java.time.Duration; import java.time.Instant; @@ -43,10 +45,11 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; -import com.cloud.hypervisor.vmware.util.VmwareClient; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcVmsCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcHostsCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcItems; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePolicyCompatiblePoolsCmd; @@ -87,6 +90,7 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterVO; import com.cloud.dc.ClusterVSMMapVO; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.VmwareDatacenter; import com.cloud.dc.VsphereStoragePolicy; import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.dc.dao.ClusterDao; @@ -112,7 +116,8 @@ import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.hypervisor.vmware.LegacyZoneVO; import com.cloud.hypervisor.vmware.VmwareCleanupMaid; -import com.cloud.dc.VmwareDatacenter; +import com.cloud.hypervisor.vmware.util.VmwareClient; +import com.cloud.hypervisor.vmware.util.VmwareClientException; import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.dc.VmwareDatacenterVO; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; @@ -165,13 +170,18 @@ import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; -import com.cloud.utils.ssh.SshHelper; -import com.cloud.vm.DomainRouterVO; import com.cloud.vm.dao.UserVmCloneSettingDao; import com.cloud.vm.dao.VMInstanceDao; + +// TODO move these items upstream? import com.vmware.pbm.PbmProfile; import com.vmware.vim25.AboutInfo; import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.InvalidLocaleFaultMsg; +import com.vmware.vim25.InvalidLoginFaultMsg; +import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.InvalidPropertyFaultMsg; +import org.jetbrains.annotations.NotNull; public class VmwareManagerImpl extends ManagerBase implements VmwareManager, VmwareStorageMount, Listener, VmwareDatacenterService, Configurable { private static final Logger s_logger = Logger.getLogger(VmwareManagerImpl.class); @@ -247,11 +257,11 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private StorageLayer _storage; private final String _privateNetworkVSwitchName = "vSwitch0"; - private int _portsPerDvPortGroup = DEFAULT_PORTS_PER_DV_PORT_GROUP; + private final int _portsPerDvPortGroup = DEFAULT_PORTS_PER_DV_PORT_GROUP; private boolean _fullCloneFlag; private boolean _instanceNameFlag; private String _serviceConsoleName; - private String _managemetPortGroupName; + private String _managementPortGroupName; private String _defaultSystemVmNicAdapterType = VirtualEthernetCardType.E1000.toString(); private String _recycleHungWorker = "false"; private int _additionalPortRangeStart; @@ -265,7 +275,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private final Random _rand = new Random(System.currentTimeMillis()); - private static ScheduledExecutorService templateCleanupScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Vmware-FullyClonedTemplateCheck"));; + private static final ScheduledExecutorService templateCleanupScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Vmware-FullyClonedTemplateCheck")); private final VmwareStorageManager _storageMgr; private final GlobalLock _exclusiveOpLock = GlobalLock.getInternLock("vmware.exclusive.op"); @@ -349,9 +359,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw _serviceConsoleName = "Service Console"; } - _managemetPortGroupName = _configDao.getValue(Config.VmwareManagementPortGroup.key()); - if (_managemetPortGroupName == null) { - _managemetPortGroupName = "Management Network"; + _managementPortGroupName = _configDao.getValue(Config.VmwareManagementPortGroup.key()); + if (_managementPortGroupName == null) { + _managementPortGroupName = "Management Network"; } _defaultSystemVmNicAdapterType = _configDao.getValue(Config.VmwareSystemVmNicDeviceType.key()); @@ -450,7 +460,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw s_logger.info("Preparing network on host " + hostMo.getContext().toString() + " for " + privateTrafficLabel); VirtualSwitchType vsType = VirtualSwitchType.getType(vSwitchType); - //The management network is probably always going to be a physical network with islation type of vlans, so assume BroadcastDomainType VLAN + //The management network is probably always going to be a physical network with isolation type of vlans, so assume BroadcastDomainType VLAN if (VirtualSwitchType.StandardVirtualSwitch == vsType) { HypervisorHostHelper.prepareNetwork(vSwitchName, "cloud.private", hostMo, vlanId, null, null, 180000, false, BroadcastDomainType.Vlan, null, null); } @@ -459,7 +469,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw AboutInfo about = hostMo.getHostAboutInfo(); if (about != null) { String version = about.getApiVersion(); - if (version != null && (version.equals("4.0") || version.equals("4.1")) && _portsPerDvPortGroup < DEFAULT_PORTS_PER_DV_PORT_GROUP_VSPHERE4_x) { + if (version != null && (version.equals("4.0") || version.equals("4.1")) ) { // && _portsPerDvPortGroup < DEFAULT_PORTS_PER_DV_PORT_GROUP_VSPHERE4_x) portsPerDvPortGroup = DEFAULT_PORTS_PER_DV_PORT_GROUP_VSPHERE4_x; } } @@ -482,7 +492,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } URI uriForHost = new URI(UriUtils.encodeURIComponent(clusterDetails.get("url") + "/" + host.getName())); - morSrcHost = serviceContext.getHostMorByPath(URLDecoder.decode(uriForHost.getPath(), "UTF-8")); + morSrcHost = serviceContext.getHostMorByPath(URLDecoder.decode(uriForHost.getPath(), StandardCharsets.UTF_8)); if (morSrcHost == null) { return null; } @@ -498,19 +508,18 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw throw new CloudRuntimeException("Invalid serviceContext"); } ManagedObjectReference mor = serviceContext.getHostMorByPath(hostInventoryPath); - String privateTrafficLabel = null; + String privateTrafficLabel; privateTrafficLabel = serviceContext.getStockObject("privateTrafficLabel"); if (privateTrafficLabel == null) { privateTrafficLabel = _privateNetworkVSwitchName; } if (mor != null) { - List returnedHostList = new ArrayList(); + List returnedHostList = new ArrayList<>(); if (mor.getType().equals("ComputeResource")) { List hosts = serviceContext.getVimClient().getDynamicProperty(mor, "host"); - assert (hosts != null && hosts.size() > 0); - + assert (CollectionUtils.isNullOrEmpty(hosts)); // For ESX host, we need to enable host firewall to allow VNC access HostMO hostMo = new HostMO(serviceContext, hosts.get(0)); @@ -521,8 +530,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw List hosts = serviceContext.getVimClient().getDynamicProperty(mor, "host"); assert (hosts != null); - if (hosts.size() > 0) { - AboutInfo about = (AboutInfo)(serviceContext.getVimClient().getDynamicProperty(hosts.get(0), "config.product")); + if (!hosts.isEmpty()) { + AboutInfo about = serviceContext.getVimClient().getDynamicProperty(hosts.get(0), "config.product"); String version = about.getApiVersion(); int maxHostsPerCluster = _hvCapabilitiesDao.getMaxHostsPerCluster(HypervisorType.VMware, version); if (hosts.size() > maxHostsPerCluster) { @@ -551,7 +560,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw returnedHostList.add(mor); return returnedHostList; } else { - s_logger.error("Unsupport host type " + mor.getType() + ":" + mor.getValue() + " from inventory path: " + hostInventoryPath); + s_logger.error("Unsupported host type " + mor.getType() + ":" + mor.getValue() + " from inventory path: " + hostInventoryPath); return null; } } @@ -616,13 +625,13 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override public String getManagementPortGroupName() { - return _managemetPortGroupName; + return _managementPortGroupName; } @Override public String getManagementPortGroupByHost(HostMO hostMo) throws Exception { if (hostMo.getHostType() == VmwareHostType.ESXi) { - return _managemetPortGroupName; + return _managementPortGroupName; } return _serviceConsoleName; } @@ -632,7 +641,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw params.put("vmware.create.full.clone", _fullCloneFlag); params.put("vm.instancename.flag", _instanceNameFlag); params.put("service.console.name", _serviceConsoleName); - params.put("management.portgroup.name", _managemetPortGroupName); + params.put("management.portgroup.name", _managementPortGroupName); params.put("vmware.root.disk.controller", _rootDiskController); params.put("vmware.data.disk.controller", _dataDiskController); params.put("vmware.recycle.hung.wokervm", _recycleHungWorker); @@ -659,25 +668,25 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return false; } - String tokens[] = workerTag.split("-"); + String[] tokens = workerTag.split("-"); if (tokens.length != 3) { s_logger.error("Invalid worker VM tag " + workerTag); return false; } long startTick = Long.parseLong(tokens[0]); - long msid = Long.parseLong(tokens[1]); - long runid = Long.parseLong(tokens[2]); + long msId = Long.parseLong(tokens[1]); + long runId = Long.parseLong(tokens[2]); - if (msHostPeerDao.countStateSeenInPeers(msid, runid, ManagementServerHost.State.Down) > 0) { + if (msHostPeerDao.countStateSeenInPeers(msId, runId, ManagementServerHost.State.Down) > 0) { if (s_logger.isInfoEnabled()) s_logger.info("Worker VM's owner management server node has been detected down from peer nodes, recycle it"); return true; } - if (runid != clusterManager.getManagementRunId(msid)) { + if (runId != clusterManager.getManagementRunId(msId)) { if (s_logger.isInfoEnabled()) - s_logger.info("Worker VM's owner management server has changed runid, recycle it"); + s_logger.info("Worker VM's owner management server has changed runId, recycle it"); return true; } @@ -712,7 +721,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw File patchFolder = new File(mountPoint + "/systemvm"); if (!patchFolder.exists()) { if (!patchFolder.mkdirs()) { - String msg = "Unable to create systemvm folder on secondary storage. location: " + patchFolder.toString(); + String msg = "Unable to create systemvm folder on secondary storage. location: " + patchFolder; s_logger.error(msg); throw new CloudRuntimeException(msg); } @@ -731,7 +740,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } catch (IOException e) { s_logger.error("Unexpected exception ", e); - String msg = "Unable to copy systemvm ISO on secondary storage. src location: " + srcIso.toString() + ", dest location: " + destIso; + String msg = "Unable to copy systemvm ISO on secondary storage. src location: " + srcIso + ", dest location: " + destIso; s_logger.error(msg); throw new CloudRuntimeException(msg); } @@ -773,9 +782,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw isoFile = new File("/usr/share/cloudstack-common/vms/systemvm.iso"); } - assert (isoFile != null); if (!isoFile.exists()) { - s_logger.error("Unable to locate systemvm.iso in your setup at " + isoFile.toString()); + s_logger.error("Unable to locate systemvm.iso in your setup at " + isoFile); } return isoFile; } @@ -790,16 +798,16 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw if (keyFile == null || !keyFile.exists()) { keyFile = new File("/usr/share/cloudstack-common/scripts/vm/systemvm/id_rsa.cloud"); } - assert (keyFile != null); + if (!keyFile.exists()) { - s_logger.error("Unable to locate id_rsa.cloud in your setup at " + keyFile.toString()); + s_logger.error("Unable to locate id_rsa.cloud in your setup at " + keyFile); } return keyFile; } @Override public String getMountPoint(String storageUrl, String nfsVersion) { - String mountPoint = null; + String mountPoint; synchronized (_storageMounts) { mountPoint = _storageMounts.get(storageUrl); if (mountPoint != null) { @@ -829,7 +837,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw String mountPoint = null; long mshostId = ManagementServerNode.getManagementServerId(); for (int i = 0; i < 10; i++) { - String mntPt = parent + File.separator + String.valueOf(mshostId) + "." + Integer.toHexString(_rand.nextInt(Integer.MAX_VALUE)); + String mntPt = parent + File.separator + mshostId + "." + Integer.toHexString(_rand.nextInt(Integer.MAX_VALUE)); File file = new File(mntPt); if (!file.exists()) { if (_storage.mkdir(mntPt)) { @@ -854,10 +862,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw for (String mountPoint : mounts) { s_logger.info("umount NFS mount from previous session: " + mountPoint); - String result = null; Script command = new Script(true, "umount", _timeout, s_logger); command.add(mountPoint); - result = command.execute(); + String result = command.execute(); if (result != null) { s_logger.warn("Unable to umount " + mountPoint + " due to " + result); } @@ -875,7 +882,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw for (String mountPoint : _storageMounts.values()) { s_logger.info("umount NFS mount: " + mountPoint); - String result = null; + String result; Script command = new Script(true, "umount", _timeout, s_logger); command.add(mountPoint); result = command.execute(); @@ -896,8 +903,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return null; } - Script script = null; - String result = null; + Script script; + String result; Script command = new Script(true, "mount", _timeout, s_logger); command.add("-t", "nfs"); if (nfsVersion != null){ @@ -984,40 +991,15 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override public void processConnect(Host host, StartupCommand cmd, boolean forRebalance) { - if (cmd instanceof StartupCommand) { + if (cmd != null) { if (host.getHypervisorType() == HypervisorType.VMware) { updateClusterNativeHAState(host, cmd); - } else { - return; } } } protected final static int DEFAULT_DOMR_SSHPORT = 3922; - protected boolean shutdownRouterVM(DomainRouterVO router) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Try to shutdown router VM " + router.getInstanceName() + " directly."); - } - - Pair result; - try { - result = SshHelper.sshExecute(router.getPrivateIpAddress(), DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "poweroff -f"); - - if (!result.first()) { - s_logger.debug("Unable to shutdown " + router.getInstanceName() + " directly"); - return false; - } - } catch (Throwable e) { - s_logger.warn("Unable to shutdown router " + router.getInstanceName() + " directly."); - return false; - } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Shutdown router " + router.getInstanceName() + " successful."); - } - return true; - } - @Override public boolean processDisconnect(long agentId, Status state) { return false; @@ -1058,16 +1040,16 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override public Pair getAddiionalVncPortRange() { - return new Pair(_additionalPortRangeStart, _additionalPortRangeSize); + return new Pair<>(_additionalPortRangeStart, _additionalPortRangeSize); } @Override public Map getNexusVSMCredentialsByClusterId(Long clusterId) { - CiscoNexusVSMDeviceVO nexusVSM = null; - ClusterVSMMapVO vsmMapVO = null; + CiscoNexusVSMDeviceVO nexusVSM; + ClusterVSMMapVO vsmMapVO; vsmMapVO = _vsmMapDao.findByClusterId(clusterId); - long vsmId = 0; + long vsmId; if (vsmMapVO != null) { vsmId = vsmMapVO.getVsmId(); s_logger.info("vsmId is " + vsmId); @@ -1078,7 +1060,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return null; } - Map nexusVSMCredentials = new HashMap(); + Map nexusVSMCredentials = new HashMap<>(); if (nexusVSM != null) { nexusVSMCredentials.put("vsmip", nexusVSM.getipaddr()); nexusVSMCredentials.put("vsmusername", nexusVSM.getUserName()); @@ -1105,7 +1087,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override public List> getCommands() { - List> cmdList = new ArrayList>(); + List> cmdList = new ArrayList<>(); cmdList.add(AddVmwareDcCmd.class); cmdList.add(UpdateVmwareDcCmd.class); cmdList.add(RemoveVmwareDcCmd.class); @@ -1114,13 +1096,14 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw cmdList.add(ListVsphereStoragePoliciesCmd.class); cmdList.add(ListVsphereStoragePolicyCompatiblePoolsCmd.class); cmdList.add(ListVmwareDcVmsCmd.class); + cmdList.add(ListVmwareDcHostsCmd.class); return cmdList; } @Override @DB public VmwareDatacenterVO addVmwareDatacenter(AddVmwareDcCmd cmd) throws ResourceInUseException { - VmwareDatacenterVO vmwareDc = null; + VmwareDatacenterVO vmwareDc; Long zoneId = cmd.getZoneId(); String userName = cmd.getUsername(); String password = cmd.getPassword(); @@ -1176,10 +1159,10 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw checkIfDcIsUsed(vCenterHost, vmwareDcName, zoneId); VmwareContext context = null; - DatacenterMO dcMo = null; + DatacenterMO dcMo; String dcCustomFieldValue; boolean addDcCustomFieldDef = false; - boolean dcInUse = false; + boolean dcInUse; String guid; ManagedObjectReference dcMor; try { @@ -1212,7 +1195,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw // Map zone with vmware datacenter vmwareDcZoneMap = new VmwareDatacenterZoneMapVO(zoneId, vmwareDc.getId()); - vmwareDcZoneMap = vmwareDatacenterZoneMapDao.persist(vmwareDcZoneMap); + vmwareDatacenterZoneMapDao.persist(vmwareDcZoneMap); // Set custom field for this DC if (addDcCustomFieldDef) { @@ -1232,7 +1215,6 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw if (context != null) { context.close(); } - context = null; } importVsphereStoragePoliciesInternal(zoneId, vmwareDc.getId()); return vmwareDc; @@ -1257,9 +1239,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw * Check if DC is already part of zone * In that case vmware_data_center table should have the DC and a dc zone mapping should exist * - * @param vCenterHost - * @param vmwareDcName - * @param zoneId + * @param vCenterHost the vcenter appliance hostname + * @param vmwareDcName the name of the vmware DC + * @param zoneId zone that the DC should be connected to * @throws ResourceInUseException if the DC can not be used. */ private void checkIfDcIsUsed(String vCenterHost, String vmwareDcName, Long zoneId) throws ResourceInUseException { @@ -1267,7 +1249,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw vmwareDc = vmwareDcDao.getVmwareDatacenterByGuid(vmwareDcName + "@" + vCenterHost); if (vmwareDc != null) { VmwareDatacenterZoneMapVO mapping = vmwareDatacenterZoneMapDao.findByVmwareDcId(vmwareDc.getId()); - if (mapping != null && Long.compare(zoneId, mapping.getZoneId()) == 0) { + if (mapping != null && zoneId == mapping.getZoneId()) { throw new ResourceInUseException(String.format("This DC (%s) is already part of other CloudStack zone (%d). Cannot add this DC to more zones.", vmwareDc.getUuid(), zoneId)); } } @@ -1276,7 +1258,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override @ActionEvent(eventType = EventTypes.EVENT_ZONE_EDIT, eventDescription = "updating VMware datacenter") public VmwareDatacenter updateVmwareDatacenter(UpdateVmwareDcCmd cmd) { - final Long zoneId = cmd.getZoneId(); + final long zoneId = cmd.getZoneId(); final String userName = cmd.getUsername(); final String password = cmd.getPassword(); final String vCenterHost = cmd.getVcenter(); @@ -1304,7 +1286,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } vmwareDc.setGuid(String.format("%s@%s", vmwareDc.getVmwareDatacenterName(), vmwareDc.getVcenterHost())); - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public VmwareDatacenter doInTransaction(TransactionStatus status) { if (vmwareDcDao.update(vmwareDc.getId(), vmwareDc)) { @@ -1353,7 +1335,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw String vCenterHost; String userName; String password; - DatacenterMO dcMo = null; + DatacenterMO dcMo; final VmwareDatacenterZoneMapVO vmwareDcZoneMap = vmwareDatacenterZoneMapDao.findByZoneId(zoneId); // Check if zone is associated with VMware DC if (vmwareDcZoneMap == null) { @@ -1390,11 +1372,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw throw new DiscoveryException(msg); } - assert (dcMo != null); - // Reset custom field property cloud.zone over this DC dcMo.setCustomFieldValue(CustomFieldConstants.CLOUD_ZONE, "false"); - s_logger.info("Sucessfully reset custom field property cloud.zone over DC " + vmwareDcName); + s_logger.info("Successfully reset custom field property cloud.zone over DC " + vmwareDcName); } catch (Exception e) { String msg = "Unable to reset custom field property cloud.zone over DC " + vmwareDcName + " due to : " + VmwareHelper.getExceptionMessage(e); s_logger.error(msg); @@ -1403,7 +1383,6 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw if (context != null) { context.close(); } - context = null; } return true; } @@ -1424,7 +1403,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private void validateZoneWithResources(Long zoneId, String errStr) throws ResourceInUseException { // Check if zone has resources? - For now look for clusters List clusters = clusterDao.listByZoneId(zoneId); - if (clusters != null && clusters.size() > 0) { + if (!CollectionUtils.isNullOrEmpty(clusters)) { // Look for VMware hypervisor. for (ClusterVO cluster : clusters) { if (cluster.getHypervisorType().equals(HypervisorType.VMware)) { @@ -1445,9 +1424,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } @Override - public List listVmwareDatacenters(ListVmwareDcsCmd cmd) throws CloudRuntimeException, InvalidParameterValueException { + public List listVmwareDatacenters(ListVmwareDcsCmd cmd) throws CloudRuntimeException { Long zoneId = cmd.getZoneId(); - List vmwareDcList = new ArrayList(); + List vmwareDcList = new ArrayList<>(); VmwareDatacenterZoneMapVO vmwareDcZoneMap; VmwareDatacenterVO vmwareDatacenter; long vmwareDcId; @@ -1505,7 +1484,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw String vCenterHost = vmwareDatacenter.getVcenterHost(); String userName = vmwareDatacenter.getUser(); String password = vmwareDatacenter.getPassword(); - List storageProfiles = null; + List storageProfiles; try { s_logger.debug(String.format("Importing vSphere Storage Policies for the vmware DC %d in zone %d", vmwareDcId, zoneId)); VmwareContext context = VmwareContextFactory.getContext(vCenterHost, userName, password); @@ -1533,16 +1512,15 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw List allStoragePolicies = vsphereStoragePolicyDao.listAll(); List finalStorageProfiles = storageProfiles; List needToMarkRemoved = allStoragePolicies.stream() - .filter(existingPolicy -> !finalStorageProfiles.stream() - .anyMatch(storageProfile -> storageProfile.getProfileId().getUniqueId().equals(existingPolicy.getPolicyId()))) + .filter(existingPolicy -> finalStorageProfiles.stream() + .noneMatch(storageProfile -> storageProfile.getProfileId().getUniqueId().equals(existingPolicy.getPolicyId()))) .collect(Collectors.toList()); for (VsphereStoragePolicyVO storagePolicy : needToMarkRemoved) { vsphereStoragePolicyDao.remove(storagePolicy.getId()); } - List storagePolicies = vsphereStoragePolicyDao.listAll(); - return storagePolicies; + return vsphereStoragePolicyDao.listAll(); } @Override @@ -1588,13 +1566,87 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } @Override - public List listVMsInDatacenter(ListVmwareDcVmsCmd cmd) { + public List listHostsInDatacenter(ListVmwareDcHostsCmd cmd) { + VcenterData vmwaredc = getVcenterData(cmd); + + try { + VmwareContext context = getVmwareContext(vmwaredc); + DatacenterMO dcMo = getDatacenterMO(context, vmwaredc); + return dcMo.getAllHostsOnDatacenter(); + } catch (RuntimeFaultFaultMsg | URISyntaxException | VmwareClientException | InvalidLocaleFaultMsg | + InvalidLoginFaultMsg | InvalidPropertyFaultMsg e) { + String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s", + vmwaredc.vcenter, vmwaredc.datacenterName, e.getMessage()); + s_logger.error(errorMsg, e); + throw new CloudRuntimeException(errorMsg); + } + + } + + @Override + public Pair> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) { + Integer maxObjects = cmd.getBatchSize(); + String token = cmd.getToken(); + String host = cmd.getHost(); + + VcenterData vmwaredc = getVcenterData(cmd); + + try { + VmwareContext context = getVmwareContext(vmwaredc); + + DatacenterMO dcMo = getDatacenterMO(context, vmwaredc); + + if (com.cloud.utils.StringUtils.isNotBlank(host)) { + ManagedObjectReference hostMor = dcMo.findHost(host); + if (hostMor == null) { + throw new VmwareClientException(String.format("No host '%s' found on DC: %s.", host, dcMo.getName())); + } + HostMO hostMo = new HostMO(context, hostMor); + return hostMo.getVms(maxObjects, token); + } else { + return dcMo.getVms(maxObjects, token); + } + } catch (InvalidParameterValueException | VmwareClientException | InvalidLocaleFaultMsg | InvalidLoginFaultMsg | + RuntimeFaultFaultMsg | URISyntaxException | InvalidPropertyFaultMsg | InvocationTargetException | + NoSuchMethodException | IllegalAccessException e) { + String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s", + vmwaredc.vcenter, vmwaredc.datacenterName, e.getMessage()); + s_logger.error(errorMsg, e); + throw new CloudRuntimeException(errorMsg); + } + } + + @NotNull + private static DatacenterMO getDatacenterMO(VmwareContext context, VcenterData vmwaredc) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { + DatacenterMO dcMo = new DatacenterMO(context, vmwaredc.datacenterName); + ManagedObjectReference dcMor = dcMo.getMor(); + if (dcMor == null) { + String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", + vmwaredc.datacenterName, vmwaredc.vcenter); + s_logger.error(msg); + throw new InvalidParameterValueException(msg); + } + return dcMo; + } + + @NotNull + private static VmwareContext getVmwareContext(VcenterData vmwaredc) throws RuntimeFaultFaultMsg, URISyntaxException, VmwareClientException, InvalidLocaleFaultMsg, InvalidLoginFaultMsg { + s_logger.debug(String.format("Connecting to the VMware datacenter %s at vCenter %s to retrieve VMs", + vmwaredc.datacenterName, vmwaredc.vcenter)); + String serviceUrl = String.format("https://%s/sdk/vimService", vmwaredc.vcenter); + VmwareClient vimClient = new VmwareClient(vmwaredc.vcenter); + vimClient.connect(serviceUrl, vmwaredc.username, vmwaredc.password); + VmwareContext context = new VmwareContext(vimClient, vmwaredc.vcenter); + return context; + } + + @NotNull + private VcenterData getVcenterData(ListVmwareDcItems cmd) { String vcenter = cmd.getVcenter(); String datacenterName = cmd.getDatacenterName(); String username = cmd.getUsername(); String password = cmd.getPassword(); Long existingVcenterId = cmd.getExistingVcenterId(); - String keyword = cmd.getKeyword(); if ((existingVcenterId == null && StringUtils.isBlank(vcenter)) || (existingVcenterId != null && StringUtils.isNotBlank(vcenter))) { @@ -1615,37 +1667,27 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw username = vmwareDc.getUser(); password = vmwareDc.getPassword(); } + VcenterData vmwaredc = new VcenterData(vcenter, datacenterName, username, password); + return vmwaredc; + } - try { - s_logger.debug(String.format("Connecting to the VMware datacenter %s at vCenter %s to retrieve VMs", - datacenterName, vcenter)); - String serviceUrl = String.format("https://%s/sdk/vimService", vcenter); - VmwareClient vimClient = new VmwareClient(vcenter); - vimClient.connect(serviceUrl, username, password); - VmwareContext context = new VmwareContext(vimClient, vcenter); + private static class VcenterData { + public final String vcenter; + public final String datacenterName; + public final String username; + public final String password; - DatacenterMO dcMo = new DatacenterMO(context, datacenterName); - ManagedObjectReference dcMor = dcMo.getMor(); - if (dcMor == null) { - String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", - datacenterName, vcenter); - s_logger.error(msg); - throw new InvalidParameterValueException(msg); - } - List instances = dcMo.getAllVmsOnDatacenter(); - return StringUtils.isBlank(keyword) ? instances : - instances.stream().filter(x -> x.getName().toLowerCase().contains(keyword.toLowerCase())).collect(Collectors.toList()); - } catch (Exception e) { - String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s", - vcenter, datacenterName, e.getMessage()); - s_logger.error(errorMsg, e); - throw new CloudRuntimeException(errorMsg); + public VcenterData(String vcenter, String datacenterName, String username, String password) { + this.vcenter = vcenter; + this.datacenterName = datacenterName; + this.username = username; + this.password = password; } } @Override public boolean hasNexusVSM(Long clusterId) { - ClusterVSMMapVO vsmMapVo = null; + ClusterVSMMapVO vsmMapVo; vsmMapVo = _vsmMapDao.findByClusterId(clusterId); if (vsmMapVo == null) { @@ -1695,7 +1737,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } /** - * This task is to cleanup templates from primary storage that are otherwise not cleaned by the {@link com.cloud.storage.StorageManagerImpl.StorageGarbageCollector}. + * This task is to clean-up templates from primary storage that are otherwise not cleaned by the {@see com.cloud.storage.StorageManagerImpl.StorageGarbageCollector}. * it is called at regular intervals when storage.template.cleanup.enabled == true * It collect all templates that * - are deleted from cloudstack diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java index 9f4985a1363..5da2c7b7003 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java @@ -37,8 +37,8 @@ import com.cloud.dc.VmwareDatacenterVO; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; -@APICommand(name = "addVmwareDc", description = "Adds a VMware datacenter to specified zone", responseObject = VmwareDatacenterResponse.class, - requestHasSensitiveInfo = true, responseHasSensitiveInfo = false) +@APICommand(name = "addVmwareDc", description = "Adds a Vmware datacenter to specified zone", + responseObject = VmwareDatacenterResponse.class, responseHasSensitiveInfo = false) public class AddVmwareDcCmd extends BaseCmd { @Inject @@ -47,7 +47,7 @@ public class AddVmwareDcCmd extends BaseCmd { public static final Logger s_logger = Logger.getLogger(AddVmwareDcCmd.class.getName()); - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Name of VMware datacenter to be added to specified zone.") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Name of Vmware datacenter to be added to specified zone.") private String name; @Parameter(name = ApiConstants.VCENTER, @@ -56,10 +56,10 @@ public class AddVmwareDcCmd extends BaseCmd { description = "The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") private String vCenter; - @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, required = false, description = "The Username required to connect to resource.") + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "The Username required to connect to resource.") private String username; - @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, required = false, description = "The password for specified username.") + @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.") private String password; @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "The Zone ID.") @@ -101,7 +101,7 @@ public class AddVmwareDcCmd extends BaseCmd { response.setResponseName(getCommandName()); response.setObjectName("vmwaredc"); } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add VMware Datacenter to zone."); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add Vmware Datacenter to zone."); } this.setResponseObject(response); } catch (DiscoveryException ex) { diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java index c7ba63c02a7..40b479f809c 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java @@ -37,7 +37,6 @@ import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; -import org.apache.log4j.Logger; import javax.inject.Inject; import java.util.ArrayList; @@ -49,9 +48,6 @@ import java.util.List; authorized = {RoleType.Admin}) public class ImportVsphereStoragePoliciesCmd extends BaseCmd { - public static final Logger LOGGER = Logger.getLogger(ImportVsphereStoragePoliciesCmd.class.getName()); - - @Inject public VmwareDatacenterService _vmwareDatacenterService; @@ -76,6 +72,13 @@ public class ImportVsphereStoragePoliciesCmd extends BaseCmd { List storagePolicies = _vmwareDatacenterService.importVsphereStoragePolicies(this); final ListResponse responseList = new ListResponse<>(); + final List storagePoliciesResponseList = getVsphereStoragePoliciesResponses(storagePolicies, dataCenter); + responseList.setResponses(storagePoliciesResponseList); + responseList.setResponseName(getCommandName()); + setResponseObject(responseList); + } + + private static List getVsphereStoragePoliciesResponses(List storagePolicies, DataCenter dataCenter) { final List storagePoliciesResponseList = new ArrayList<>(); for (VsphereStoragePolicy storagePolicy : storagePolicies) { final VsphereStoragePoliciesResponse storagePoliciesResponse = new VsphereStoragePoliciesResponse(); @@ -88,9 +91,7 @@ public class ImportVsphereStoragePoliciesCmd extends BaseCmd { storagePoliciesResponseList.add(storagePoliciesResponse); } - responseList.setResponses(storagePoliciesResponseList); - responseList.setResponseName(getCommandName()); - setResponseObject(responseList); + return storagePoliciesResponseList; } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcHostsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcHostsCmd.java new file mode 100644 index 00000000000..6f193c9c1b2 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcHostsCmd.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.zone; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.vmware.VmwareDatacenterService; +import com.cloud.hypervisor.vmware.mo.HostMO; +import com.cloud.user.Account; +import com.cloud.utils.exception.CloudRuntimeException; + +import com.vmware.vim25.InvalidPropertyFaultMsg; +import com.vmware.vim25.RuntimeFaultFaultMsg; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.api.response.VmwareDatacenterResponse; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import javax.inject.Inject; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = "listVmwareDcHosts", responseObject = VmwareRequestResponse.class, + description = "Lists the VMs in a Vmware Datacenter", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class ListVmwareDcHostsCmd extends BaseCmd implements ListVmwareDcItems { + + @Inject + public VmwareDatacenterService _vmwareDatacenterService; + + @Parameter(name = ApiConstants.EXISTING_VCENTER_ID, + type = CommandType.UUID, + entityType = VmwareDatacenterResponse.class, + description = "UUID of a linked existing vCenter") + private Long existingVcenterId; + + @Parameter(name = ApiConstants.VCENTER, + type = CommandType.STRING, + description = "The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") + private String vcenter; + + @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, description = "Name of Vmware datacenter.") + private String datacenterName; + + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "The Username required to connect to resource.") + private String username; + + @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.") + private String password; + + public String getVcenter() { + return vcenter; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getDatacenterName() { + return datacenterName; + } + + public Long getExistingVcenterId() { + return existingVcenterId; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + checkParameters(); + try { + List hosts = _vmwareDatacenterService.listHostsInDatacenter(this); + List baseResponseList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(hosts)) { + for (HostMO vmwareHost : hosts) { + HostResponse resp = createHostResponse(vmwareHost); + baseResponseList.add(resp); + } + } + VmwareRequestResponse response = new VmwareRequestResponse<>(); + response.setResponses(baseResponseList, baseResponseList.size()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException | InvalidPropertyFaultMsg | RuntimeFaultFaultMsg | InvocationTargetException | + NoSuchMethodException | IllegalAccessException e) { + String errorMsg = String.format("Error retrieving VMs from Vmware VC: %s", e.getMessage()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, errorMsg); + } + } + + private HostResponse createHostResponse(HostMO hostInstance) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { + HostResponse response = new HostResponse(); + response.setHypervisor(Hypervisor.HypervisorType.VMware.toString()); + response.setName(hostInstance.getHostName()); + response.setObjectName("host"); + return response; + } + + private void checkParameters() { + if ((existingVcenterId == null && vcenter == null) || (existingVcenterId != null && vcenter != null)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Please provide an existing vCenter ID or a vCenter IP/Name, parameters are mutually exclusive"); + } + if (existingVcenterId == null && StringUtils.isAnyBlank(vcenter, datacenterName, username, password)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Please set all the information for a vCenter IP/Name, datacenter, username and password"); + } + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcItems.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcItems.java new file mode 100644 index 00000000000..580fb3bad9b --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcItems.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.zone; + +public interface ListVmwareDcItems { + String getVcenter(); + + String getDatacenterName(); + + String getUsername(); + + String getPassword(); + + Long getExistingVcenterId(); +} diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java index 4dd1b4beb09..544e756fe80 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java @@ -23,15 +23,15 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.user.Account; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UnmanagedInstanceResponse; import org.apache.cloudstack.api.response.VmwareDatacenterResponse; import org.apache.cloudstack.vm.UnmanagedInstanceTO; @@ -42,10 +42,10 @@ import javax.inject.Inject; import java.util.ArrayList; import java.util.List; -@APICommand(name = "listVmwareDcVms", responseObject = UnmanagedInstanceResponse.class, - description = "Lists the VMs in a VMware Datacenter", +@APICommand(name = "listVmwareDcVms", responseObject = VmwareRequestResponse.class, + description = "Lists the VMs in a Vmware Datacenter", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) -public class ListVmwareDcVmsCmd extends BaseListCmd { +public class ListVmwareDcVmsCmd extends BaseCmd implements ListVmwareDcItems { @Inject public VmwareDatacenterService _vmwareDatacenterService; @@ -61,7 +61,7 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { description = "The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") private String vcenter; - @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, description = "Name of VMware datacenter.") + @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, description = "Name of Vmware datacenter.") private String datacenterName; @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "The Username required to connect to resource.") @@ -70,6 +70,18 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.") private String password; + @Parameter(name = ApiConstants.HOST, type = CommandType.STRING, description = "get only the VMs from the specified host.") + private String host; + + @Parameter(name = ApiConstants.BATCH_SIZE, type = CommandType.INTEGER, description = "The maximum number of results to return.") + private Integer batchSize; + + @Parameter(name = ApiConstants.TOKEN, type = CommandType.STRING, + description = "For listVmwareDcVms, if the maximum number of results (the `batchsize`) is exceeded, " + + " a token is returned. This token can be used in subsequent calls to retrieve more results." + + " As long as a token is returned, more results can be retrieved.") + private String token; + public String getVcenter() { return vcenter; } @@ -82,6 +94,18 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { return password; } + public Integer getBatchSize() { + return batchSize; + } + + public String getHost() { + return host; + } + + public String getToken() { + return token; + } + public String getDatacenterName() { return datacenterName; } @@ -94,7 +118,8 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { checkParameters(); try { - List vms = _vmwareDatacenterService.listVMsInDatacenter(this); + Pair> results = _vmwareDatacenterService.listVMsInDatacenter(this); + List vms = results.second(); List baseResponseList = new ArrayList<>(); if (CollectionUtils.isNotEmpty(vms)) { for (UnmanagedInstanceTO vmwareVm : vms) { @@ -102,16 +127,13 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { baseResponseList.add(resp); } } - List pagingList = com.cloud.utils.StringUtils.applyPagination(baseResponseList, this.getStartIndex(), this.getPageSizeVal()); - if (CollectionUtils.isEmpty(pagingList)) { - pagingList = baseResponseList; - } - ListResponse response = new ListResponse<>(); - response.setResponses(pagingList, baseResponseList.size()); + VmwareRequestResponse response = new VmwareRequestResponse<>(); + response.setResponses(baseResponseList, baseResponseList.size()); response.setResponseName(getCommandName()); + response.setToken(results.first()); setResponseObject(response); } catch (CloudRuntimeException e) { - String errorMsg = String.format("Error retrieving VMs from VMware VC: %s", e.getMessage()); + String errorMsg = String.format("Error retrieving VMs from Vmware VC: %s", e.getMessage()); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, errorMsg); } } @@ -134,6 +156,6 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { @Override public String getCommandName() { - return "listvmwaredcvmsresponse"; + return "listVmwareDcVmsResponse".toLowerCase(); } } diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java index 61b5210bb3a..9f7c4cf2944 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java @@ -43,7 +43,7 @@ import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.user.Account; -@APICommand(name = "listVmwareDcs", responseObject = VmwareDatacenterResponse.class, description = "Retrieves VMware DC(s) associated with a zone.", +@APICommand(name = "listVmwareDcs", responseObject = VmwareDatacenterResponse.class, description = "Retrieves Vmware DC(s) associated with a zone.", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListVmwareDcsCmd extends BaseListCmd { @@ -52,7 +52,6 @@ public class ListVmwareDcsCmd extends BaseListCmd { public static final Logger s_logger = Logger.getLogger(ListVmwareDcsCmd.class.getName()); - ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// @@ -75,20 +74,27 @@ public class ListVmwareDcsCmd extends BaseListCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException { - List vmwareDcList = null; + List vmwareDcList; try { vmwareDcList = _vmwareDatacenterService.listVmwareDatacenters(this); } catch (InvalidParameterValueException ie) { throw new InvalidParameterValueException("Invalid zone id " + getZoneId()); } catch (Exception e) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to find associated VMware DCs associated with zone " + getZoneId()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to find associated Vmware DCs associated with zone " + getZoneId()); } - ListResponse response = new ListResponse(); - List vmwareDcResponses = new ArrayList(); + ListResponse response = new ListResponse<>(); + List vmwareDcResponses = getVmwareDatacenterResponses(vmwareDcList); + response.setResponses(vmwareDcResponses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } - if (vmwareDcList != null && vmwareDcList.size() > 0) { + private List getVmwareDatacenterResponses(List vmwareDcList) { + List vmwareDcResponses = new ArrayList<>(); + + if (vmwareDcList != null && !vmwareDcList.isEmpty()) { for (VmwareDatacenter vmwareDc : vmwareDcList) { VmwareDatacenterResponse vmwareDcResponse = new VmwareDatacenterResponse(); @@ -96,14 +102,12 @@ public class ListVmwareDcsCmd extends BaseListCmd { vmwareDcResponse.setVcenter(vmwareDc.getVcenterHost()); vmwareDcResponse.setName(vmwareDc.getVmwareDatacenterName()); vmwareDcResponse.setZoneId(getZoneId()); - vmwareDcResponse.setObjectName("VMwareDC"); + vmwareDcResponse.setObjectName(ApiConstants.VMWARE_DC); vmwareDcResponses.add(vmwareDcResponse); } } - response.setResponses(vmwareDcResponses); - response.setResponseName(getCommandName()); - setResponseObject(response); + return vmwareDcResponses; } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java index ac909a0dc64..35631ba1315 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java @@ -36,7 +36,6 @@ import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; -import org.apache.log4j.Logger; import javax.inject.Inject; import java.util.ArrayList; @@ -48,9 +47,6 @@ import java.util.List; authorized = {RoleType.Admin}) public class ListVsphereStoragePoliciesCmd extends BaseCmd { - public static final Logger LOGGER = Logger.getLogger(ListVsphereStoragePoliciesCmd.class.getName()); - - @Inject public VmwareDatacenterService _vmwareDatacenterService; @@ -75,6 +71,13 @@ public class ListVsphereStoragePoliciesCmd extends BaseCmd { List storagePolicies = _vmwareDatacenterService.listVsphereStoragePolicies(this); final ListResponse responseList = new ListResponse<>(); + final List storagePoliciesResponseList = getVsphereStoragePoliciesResponses(storagePolicies, dataCenter); + responseList.setResponses(storagePoliciesResponseList); + responseList.setResponseName(getCommandName()); + setResponseObject(responseList); + } + + private static List getVsphereStoragePoliciesResponses(List storagePolicies, DataCenter dataCenter) { final List storagePoliciesResponseList = new ArrayList<>(); for (VsphereStoragePolicy storagePolicy : storagePolicies) { final VsphereStoragePoliciesResponse storagePoliciesResponse = new VsphereStoragePoliciesResponse(); @@ -83,13 +86,11 @@ public class ListVsphereStoragePoliciesCmd extends BaseCmd { storagePoliciesResponse.setName(storagePolicy.getName()); storagePoliciesResponse.setPolicyId(storagePolicy.getPolicyId()); storagePoliciesResponse.setDescription(storagePolicy.getDescription()); - storagePoliciesResponse.setObjectName("StoragePolicy"); + storagePoliciesResponse.setObjectName(ApiConstants.STORAGE_POLICY); storagePoliciesResponseList.add(storagePoliciesResponse); } - responseList.setResponses(storagePoliciesResponseList); - responseList.setResponseName(getCommandName()); - setResponseObject(responseList); + return storagePoliciesResponseList; } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java index d66972ded2e..ab697fbad67 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java @@ -68,7 +68,7 @@ public class ListVsphereStoragePolicyCompatiblePoolsCmd extends BaseListCmd { List poolResponses = new ArrayList<>(); for (StoragePool pool : pools) { StoragePoolResponse poolResponse = _responseGenerator.createStoragePoolForMigrationResponse(pool); - poolResponse.setObjectName("storagepool"); + poolResponse.setObjectName(ApiConstants.STORAGE_POOL); poolResponses.add(poolResponse); } response.setResponses(poolResponses); diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/RemoveVmwareDcCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/RemoveVmwareDcCmd.java index 735d00d6ca9..d15369d2983 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/RemoveVmwareDcCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/RemoveVmwareDcCmd.java @@ -35,7 +35,7 @@ import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; -@APICommand(name = "removeVmwareDc", responseObject = SuccessResponse.class, description = "Remove a VMware datacenter from a zone.", +@APICommand(name = "removeVmwareDc", responseObject = SuccessResponse.class, description = "Remove a Vmware datacenter from a zone.", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class RemoveVmwareDcCmd extends BaseCmd { @@ -49,7 +49,7 @@ public class RemoveVmwareDcCmd extends BaseCmd { type = CommandType.UUID, entityType = ZoneResponse.class, required = true, - description = "The id of Zone from which VMware datacenter has to be removed.") + description = "The id of Zone from which Vmware datacenter has to be removed.") private Long zoneId; public Long getZoneId() { @@ -65,10 +65,10 @@ public class RemoveVmwareDcCmd extends BaseCmd { response.setResponseName(getCommandName()); setResponseObject(response); } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to remove VMware datacenter from zone"); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to remove Vmware datacenter from zone"); } } catch (ResourceInUseException ex) { - s_logger.warn("The zone has one or more resources (like cluster), hence not able to remove VMware datacenter from zone." + s_logger.warn("The zone has one or more resources (like cluster), hence not able to remove Vmware datacenter from zone." + " Please remove all resource from zone, and retry. Exception: ", ex); ServerApiException e = new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); for (String proxyObj : ex.getIdProxyList()) { diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java index 2b6cf5984c5..5e02d5a96c2 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java @@ -28,18 +28,15 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.VmwareDatacenterResponse; import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.log4j.Logger; import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.user.Account; -@APICommand(name = "updateVmwareDc", description = "Updates a VMware datacenter details for a zone", +@APICommand(name = "updateVmwareDc", description = "Updates a Vmware datacenter details for a zone", responseObject = VmwareDatacenterResponse.class, responseHasSensitiveInfo = false, since = "4.12.0", authorized = {RoleType.Admin}) public class UpdateVmwareDcCmd extends BaseCmd { - public static final Logger LOG = Logger.getLogger(UpdateVmwareDcCmd.class); - @Inject public VmwareDatacenterService vmwareDatacenterService; @@ -53,7 +50,7 @@ public class UpdateVmwareDcCmd extends BaseCmd { private Long zoneId; @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, - description = "VMware datacenter name.") + description = "Vmware datacenter name.") private String name; @Parameter(name = ApiConstants.VCENTER, type = CommandType.STRING, @@ -108,13 +105,13 @@ public class UpdateVmwareDcCmd extends BaseCmd { public void execute() { final VmwareDatacenter vmwareDatacenter = vmwareDatacenterService.updateVmwareDatacenter(this); if (vmwareDatacenter == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update VMware datacenter"); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update Vmware datacenter"); } final VmwareDatacenterResponse response = new VmwareDatacenterResponse(); response.setId(vmwareDatacenter.getUuid()); response.setName(vmwareDatacenter.getVmwareDatacenterName()); response.setResponseName(getCommandName()); - response.setObjectName("vmwaredc"); + response.setObjectName(ApiConstants.VMWARE_DC); setResponseObject(response); } diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/VmwareRequestResponse.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/VmwareRequestResponse.java new file mode 100644 index 00000000000..81c58ef27ba --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/VmwareRequestResponse.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.zone; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.response.ListResponse; + +public class VmwareRequestResponse extends ListResponse { + @SerializedName(ApiConstants.TOKEN) + @Param(description = "The Vmware API token to use for retrieving further responses with") + private String token; + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } +} diff --git a/test/integration/smoke/test_dynamicroles.py b/test/integration/smoke/test_dynamicroles.py index b91ba9c2eba..39f995a14a3 100644 --- a/test/integration/smoke/test_dynamicroles.py +++ b/test/integration/smoke/test_dynamicroles.py @@ -73,6 +73,7 @@ class TestData(object): "listApis": "allow", "listAccounts": "allow", "listClusters": "deny", + "*VmwareDc*": "allow", "*VM*": "allow", "*Host*": "deny" } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index cb113d41b2f..78179cb8108 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -3006,6 +3006,7 @@ "message.license.agreements.not.accepted": "License agreements not accepted.", "message.linstor.resourcegroup.description": "Linstor resource group to use for primary storage.", "message.list.zone.vmware.datacenter.empty": "No VMware Datacenter exists in the selected Zone", +"message.list.zone.vmware.hosts.empty": "No VMware hosts were found in the selected Datacenter", "message.listnsp.not.return.providerid": "error: listNetworkServiceProviders API doesn't return VirtualRouter provider ID.", "message.load.host.failed": "Failed to load hosts.", "message.loadbalancer.stickypolicy.configuration": "Customize the load balancer stickiness policy:", diff --git a/ui/src/views/tools/ManageInstances.vue b/ui/src/views/tools/ManageInstances.vue index 2ff9052d8b9..27ad4cc9696 100644 --- a/ui/src/views/tools/ManageInstances.vue +++ b/ui/src/views/tools/ManageInstances.vue @@ -1184,6 +1184,7 @@ export default { } else { params.existingvcenterid = this.selectedVmwareVcenter.existingvcenterid } + params.host = this.selectedVmwareVcenter.host } api(apiName, params).then(json => { diff --git a/ui/src/views/tools/SelectVmwareVcenter.vue b/ui/src/views/tools/SelectVmwareVcenter.vue index 78e38f107f3..1d25657b11b 100644 --- a/ui/src/views/tools/SelectVmwareVcenter.vue +++ b/ui/src/views/tools/SelectVmwareVcenter.vue @@ -89,6 +89,7 @@ @@ -98,6 +99,7 @@ @@ -107,6 +109,7 @@ @@ -116,14 +119,36 @@ +
+ + + + {{ 'ESXi: ' + opt.name }} + + + +
+ {{ $t('message.list.zone.vmware.hosts.empty') }} +
+
{{ $t('label.list.vmware.vcenter.vms') }} @@ -154,6 +179,8 @@ export default { zones: {}, vcenterSelectedOption: '', existingvcenter: [], + hosts: [], + selectedHost: '', selectedExistingVcenterId: '', selectedPoweredOnVm: false, vmwareDcVms: [], @@ -217,6 +244,7 @@ export default { } else { params.existingvcenterid = this.selectedExistingVcenterId } + params.host = this.selectedHost api('listVmwareDcVms', params).then(json => { const obj = { params: params, @@ -246,8 +274,29 @@ export default { listZoneVmwareDcs () { this.loading = true api('listVmwareDcs', { zoneid: this.sourcezoneid }).then(response => { - if (response.listvmwaredcsresponse.VMwareDC && response.listvmwaredcsresponse.VMwareDC.length > 0) { - this.existingvcenter = response.listvmwaredcsresponse.VMwareDC + if (response.listvmwaredcsresponse.vmwaredc && response.listvmwaredcsresponse.vmwaredc.length > 0) { + this.existingvcenter = response.listvmwaredcsresponse.vmwaredc + } + }).catch(error => { + this.$notifyError(error) + }).finally(() => { + this.loading = false + }) + }, + listZoneVmwareDcHosts () { + this.loading = true + const params = {} + if (this.vcenterSelectedOption === 'new') { + params.datacentername = this.datacenter + params.vcenter = this.vcenter + params.username = this.username + params.password = this.password + } else { + params.existingvcenterid = this.selectedExistingVcenterId + } + api('listVmwareDcHosts', params).then(response => { + if (response.listvmwaredchostsresponse.host && response.listvmwaredchostsresponse.host.length > 0) { + this.hosts = response.listvmwaredchostsresponse.host } }).catch(error => { this.$notifyError(error) @@ -257,6 +306,15 @@ export default { }, onSelectExistingVmwareDatacenter (value) { this.selectedExistingVcenterId = value + this.listZoneVmwareDcHosts() + }, + onSelectExternalVmwareDatacenter (value) { + if (this.vcenterSelectedOption === 'new' && !(this.vcenter === '' || this.datacentername === '' || this.username === '' || this.password === '')) { + this.listZoneVmwareDcHosts() + } + }, + onSelectExistingVmwareHost (value) { + this.selectedHost = value }, onVcenterTypeChange () { this.$emit('onVcenterTypeChanged', this.vcenterSelectedOption) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/BaseMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/BaseMO.java index 88f6c871fbd..15fc4ab8b2e 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/BaseMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/BaseMO.java @@ -16,13 +16,24 @@ // under the License. package com.cloud.hypervisor.vmware.mo; +import com.cloud.hypervisor.vmware.util.VmwareHelper; +import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.utils.Pair; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; + import org.apache.log4j.Logger; import com.vmware.vim25.CustomFieldDef; import com.vmware.vim25.CustomFieldStringValue; import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.InvalidPropertyFaultMsg; +import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.ObjectContent; +import com.vmware.vim25.RetrieveResult; -import com.cloud.hypervisor.vmware.util.VmwareContext; +import java.lang.reflect.InvocationTargetException; +import java.util.List; public class BaseMO { private static final Logger s_logger = Logger.getLogger(BaseMO.class); @@ -50,6 +61,15 @@ public class BaseMO { _mor.setValue(morValue); } + protected static Pair> createReturnObjectPair(RetrieveResult result) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("vmware result : " + ReflectionToStringBuilderUtils.reflectCollection(result)); + } + String tokenForRetrievingNewResults = result.getToken(); + List listOfObjects = result.getObjects(); + return new Pair<>(tokenForRetrievingNewResults, listOfObjects); + } + public VmwareContext getContext() { return _context; } @@ -137,11 +157,11 @@ public class BaseMO { return null; } - public int getCustomFieldKey(String fieldName) throws Exception { + public int getCustomFieldKey(String fieldName) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { return getCustomFieldKey(getMor().getType(), fieldName); } - public int getCustomFieldKey(String morType, String fieldName) throws Exception { + public int getCustomFieldKey(String morType, String fieldName) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { assert (morType != null); ManagedObjectReference cfmMor = _context.getServiceContent().getCustomFieldsManager(); @@ -153,4 +173,30 @@ public class BaseMO { return cfmMo.getCustomFieldKey(morType, fieldName); } + + protected Pair> retrieveNextSetOfProperties(String tokenForPriorQuery) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { + RetrieveResult result = _context.getService().continueRetrievePropertiesEx(_context.getPropertyCollector(), tokenForPriorQuery); + return BaseMO.createReturnObjectPair(result); + } + + protected void objectContentToUnmanagedInstanceTO(Pair> objectContents, List vms) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { + List ocs = objectContents.second(); + if (ocs != null) { + for (ObjectContent oc : ocs) { + ManagedObjectReference vmMor = oc.getObj(); + if (vmMor != null) { + VirtualMachineMO vmMo = new VirtualMachineMO(_context, vmMor); + try { + if (!vmMo.isTemplate()) { + HostMO hostMO = vmMo.getRunningHost(); + UnmanagedInstanceTO unmanagedInstance = VmwareHelper.getUnmanagedInstance(hostMO, vmMo); + vms.add(unmanagedInstance); + } + } catch (Exception e) { + s_logger.debug(String.format("Unexpected error checking unmanaged instance %s, excluding it: %s", vmMo.getVmName(), e.getMessage()), e); + } + } + } + } + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/CustomFieldsManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/CustomFieldsManagerMO.java index 78ce9a83c68..61958d1b952 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/CustomFieldsManagerMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/CustomFieldsManagerMO.java @@ -16,11 +16,14 @@ // under the License. package com.cloud.hypervisor.vmware.mo; +import java.lang.reflect.InvocationTargetException; import java.util.List; import com.vmware.vim25.CustomFieldDef; import com.vmware.vim25.ManagedObjectReference; import com.vmware.vim25.PrivilegePolicyDef; +import com.vmware.vim25.InvalidPropertyFaultMsg; +import com.vmware.vim25.RuntimeFaultFaultMsg; import com.cloud.hypervisor.vmware.util.VmwareContext; @@ -50,12 +53,12 @@ public class CustomFieldsManagerMO extends BaseMO { _context.getService().setField(getMor(), morEntity, key, value); } - public List getFields() throws Exception { + public List getFields() throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { return _context.getVimClient().getDynamicProperty(getMor(), "field"); } @Override - public int getCustomFieldKey(String morType, String fieldName) throws Exception { + public int getCustomFieldKey(String morType, String fieldName) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { List fields = getFields(); if (fields != null) { for (CustomFieldDef field : fields) { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java index d8c7e8a61ea..211972f2efc 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java @@ -17,11 +17,12 @@ package com.cloud.hypervisor.vmware.mo; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import com.cloud.hypervisor.vmware.util.VmwareHelper; +import com.cloud.utils.StringUtils; import org.apache.cloudstack.vm.UnmanagedInstanceTO; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -39,6 +40,10 @@ import com.vmware.vim25.PropertySpec; import com.vmware.vim25.SelectionSpec; import com.vmware.vim25.TraversalSpec; import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo; +import com.vmware.vim25.RetrieveOptions; +import com.vmware.vim25.RetrieveResult; +import com.vmware.vim25.InvalidPropertyFaultMsg; +import com.vmware.vim25.RuntimeFaultFaultMsg; import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.utils.Pair; @@ -54,7 +59,7 @@ public class DatacenterMO extends BaseMO { super(context, morType, morValue); } - public DatacenterMO(VmwareContext context, String dcName) throws Exception { + public DatacenterMO(VmwareContext context, String dcName) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { super(context, null); _mor = _context.getVimClient().getDecendentMoRef(_context.getRootFolder(), "Datacenter", dcName); @@ -64,25 +69,8 @@ public class DatacenterMO extends BaseMO { } @Override - public String getName() throws Exception { - return (String)_context.getVimClient().getDynamicProperty(_mor, "name"); - } - - public void registerTemplate(ManagedObjectReference morHost, String datastoreName, String templateName, String templateFileName) throws Exception { - - ManagedObjectReference morFolder = (ManagedObjectReference)_context.getVimClient().getDynamicProperty(_mor, "vmFolder"); - assert (morFolder != null); - - ManagedObjectReference morTask = - _context.getService() - .registerVMTask(morFolder, String.format("[%s] %s/%s", datastoreName, templateName, templateFileName), templateName, true, null, morHost); - - boolean result = _context.getVimClient().waitForTask(morTask); - if (!result) { - throw new Exception("Unable to register template due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } else { - _context.waitForTaskProgressDone(morTask); - } + public String getName() throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { + return _context.getVimClient().getDynamicProperty(_mor, "name"); } public VirtualMachineMO findVm(String vmName) throws Exception { @@ -91,7 +79,7 @@ public class DatacenterMO extends BaseMO { s_logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!"); } String instanceNameCustomField = "value[" + key + "]"; - List ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name", instanceNameCustomField}); + List ocs = getVmProperties(new String[] {"name", instanceNameCustomField}); return HypervisorHostHelper.findVmFromObjectContent(_context, ocs.toArray(new ObjectContent[0]), vmName, instanceNameCustomField); } @@ -100,10 +88,10 @@ public class DatacenterMO extends BaseMO { int key = cfmMo.getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_UUID); assert (key != 0); - List list = new ArrayList(); + List list = new ArrayList<>(); - List ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name", String.format("value[%d]", key)}); - if (ocs != null && ocs.size() > 0) { + List ocs = getVmProperties(new String[] {"name", String.format("value[%d]", key)}); + if (CollectionUtils.isNotEmpty(ocs)) { for (ObjectContent oc : ocs) { List props = oc.getPropSet(); if (props != null) { @@ -135,8 +123,8 @@ public class DatacenterMO extends BaseMO { s_logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!"); } - List ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name", String.format("value[%d]", key)}); - if (ocs != null && ocs.size() > 0) { + List ocs = getVmProperties(new String[] {"name", String.format("value[%d]", key)}); + if (CollectionUtils.isNotEmpty(ocs)) { for (ObjectContent oc : ocs) { List props = oc.getPropSet(); if (props != null) { @@ -161,31 +149,20 @@ public class DatacenterMO extends BaseMO { return null; } - public List getAllVmsOnDatacenter() throws Exception { + public Pair> getVms(Integer maxObjects, String token) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { List vms = new ArrayList<>(); - List ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name"}); - if (ocs != null) { - for (ObjectContent oc : ocs) { - ManagedObjectReference vmMor = oc.getObj(); - if (vmMor != null) { - VirtualMachineMO vmMo = new VirtualMachineMO(_context, vmMor); - try { - if (!vmMo.isTemplate()) { - HostMO hostMO = vmMo.getRunningHost(); - UnmanagedInstanceTO unmanagedInstance = VmwareHelper.getUnmanagedInstance(hostMO, vmMo); - vms.add(unmanagedInstance); - } - } catch (Exception e) { - s_logger.debug(String.format("Unexpected error checking unmanaged instance %s, excluding it: %s", vmMo.getVmName(), e.getMessage()), e); - } - } - } + Pair> objectContents = getVmProperties(new String[] {"name"}, maxObjects, token); + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("returning token %s for future retrievals, currently %d objects retrieved.", objectContents.first(), objectContents.second().size())); } + Pair> retval = new Pair<>(objectContents.first(), vms); - return vms; + objectContentToUnmanagedInstanceTO(objectContents, vms); + + return retval; } - public List getAllHostsOnDatacenter() throws Exception { + public List getAllHostsOnDatacenter() throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { List hosts = new ArrayList<>(); List ocs = getHostPropertiesOnDatacenterHostFolder(new String[] {"name"}); @@ -212,21 +189,7 @@ public class DatacenterMO extends BaseMO { return null; } - public ManagedObjectReference listDatastore(String name) throws Exception { - assert (name != null); - - List ocs = getDatastorePropertiesOnDatacenter(new String[] {"name"}); - if (ocs != null) { - for (ObjectContent oc : ocs) { - if (oc.getPropSet().get(0).getVal().toString().equals(name)) { - return oc.getObj(); - } - } - } - return null; - } - - public ManagedObjectReference findHost(String name) throws Exception { + public ManagedObjectReference findHost(String name) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { List ocs = getHostPropertiesOnDatacenterHostFolder(new String[] {"name"}); if (ocs != null) { @@ -240,10 +203,10 @@ public class DatacenterMO extends BaseMO { } public ManagedObjectReference getVmFolder() throws Exception { - return (ManagedObjectReference)_context.getVimClient().getDynamicProperty(_mor, "vmFolder"); + return _context.getVimClient().getDynamicProperty(_mor, "vmFolder"); } - public List getHostPropertiesOnDatacenterHostFolder(String[] propertyPaths) throws Exception { + public List getHostPropertiesOnDatacenterHostFolder(String[] propertyPaths) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { PropertySpec pSpec = new PropertySpec(); pSpec.setType("HostSystem"); pSpec.getPathSet().addAll(Arrays.asList(propertyPaths)); @@ -277,7 +240,7 @@ public class DatacenterMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); @@ -303,14 +266,41 @@ public class DatacenterMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); } - public List getVmPropertiesOnDatacenterVmFolder(String[] propertyPaths) throws Exception { + public List getVmProperties(String[] propertyPaths) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { + return getVmProperties(propertyPaths, null, null).second(); + } + + /** + * + * @param propertyPaths Vmware side property names to query, for instance {"name"} + * @param maxObjects the number of objects to retrieve + * @param tokenForPriorQuery restart the query or continue a previous query + * @return The propertyPaths requested for the objects of type "VirtualMachine" in a list are found/returned by the DC + * @throws InvalidPropertyFaultMsg property does not exist as thrown by Vmware. + * @throws RuntimeFaultFaultMsg generic vmware runtime exception + */ + public Pair> getVmProperties(String[] propertyPaths, Integer maxObjects, String tokenForPriorQuery) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { + if(StringUtils.isNotBlank(tokenForPriorQuery)) { + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("running repeat query with token \'%s\'", tokenForPriorQuery)); + } + return retrieveNextSetOfProperties(tokenForPriorQuery); + } else { + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("running query for %d propertypaths and max %d objects", propertyPaths.length, maxObjects)); + } + return retrieveNextSetOfProperties(propertyPaths, maxObjects); + } + } + + private Pair> retrieveNextSetOfProperties(String[] propertyPaths, Integer maxObjects) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { PropertySpec pSpec = new PropertySpec(); pSpec.setType("VirtualMachine"); pSpec.getPathSet().addAll(Arrays.asList(propertyPaths)); @@ -338,10 +328,16 @@ public class DatacenterMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); - return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); + RetrieveOptions ro = new RetrieveOptions(); + if (maxObjects != null && maxObjects > 0) { + ro.setMaxObjects(maxObjects); + } + + RetrieveResult result = _context.getService().retrievePropertiesEx(_context.getPropertyCollector(), pfSpecArr, ro); + return createReturnObjectPair(result); } public static Pair getOwnerDatacenter(VmwareContext context, ManagedObjectReference morEntity) throws Exception { @@ -366,18 +362,18 @@ public class DatacenterMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); List ocs = context.getService().retrieveProperties(context.getPropertyCollector(), pfSpecArr); - assert (ocs != null && ocs.size() > 0); + assert (CollectionUtils.isNotEmpty(ocs)); assert (ocs.get(0).getObj() != null); assert (ocs.get(0).getPropSet().get(0) != null); assert (ocs.get(0).getPropSet().get(0).getVal() != null); String dcName = ocs.get(0).getPropSet().get(0).getVal().toString(); - return new Pair(new DatacenterMO(context, ocs.get(0).getObj()), dcName); + return new Pair<>(new DatacenterMO(context, ocs.get(0).getObj()), dcName); } public ManagedObjectReference getDvPortGroupMor(String dvPortGroupName) throws Exception { @@ -398,7 +394,7 @@ public class DatacenterMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); List ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); @@ -419,9 +415,7 @@ public class DatacenterMO extends BaseMO { public boolean hasDvPortGroup(String dvPortGroupName) throws Exception { ManagedObjectReference morNetwork = getDvPortGroupMor(dvPortGroupName); - if (morNetwork != null) - return true; - return false; + return morNetwork != null; } public DVPortgroupConfigInfo getDvPortGroupSpec(String dvPortGroupName) throws Exception { @@ -445,7 +439,7 @@ public class DatacenterMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); List ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); @@ -462,7 +456,7 @@ public class DatacenterMO extends BaseMO { nameProperty = prop.getVal().toString(); } } - if (nameProperty.equalsIgnoreCase(dvPortGroupName)) { + if (nameProperty != null && nameProperty.equalsIgnoreCase(dvPortGroupName)) { return configSpec; } } @@ -492,7 +486,7 @@ public class DatacenterMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); List ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); @@ -520,7 +514,7 @@ public class DatacenterMO extends BaseMO { public String getDvSwitchUuid(ManagedObjectReference dvSwitchMor) throws Exception { assert (dvSwitchMor != null); - return (String)_context.getVimClient().getDynamicProperty(dvSwitchMor, "uuid"); + return _context.getVimClient().getDynamicProperty(dvSwitchMor, "uuid"); } public VirtualEthernetCardDistributedVirtualPortBackingInfo getDvPortBackingInfo(Pair networkInfo) throws Exception { @@ -538,8 +532,8 @@ public class DatacenterMO extends BaseMO { } public ManagedObjectReference getDvSwitchMor(String dvSwitchName) throws Exception { - ManagedObjectReference dvSwitchMor = null; - ManagedObjectReference networkFolderMor = null; + ManagedObjectReference dvSwitchMor; + ManagedObjectReference networkFolderMor; networkFolderMor = _context.getVimClient().getMoRefProp(_mor, "networkFolder"); dvSwitchMor = _context.getVimClient().getDecendentMoRef(networkFolderMor, "VmwareDistributedVirtualSwitch", dvSwitchName); return dvSwitchMor; @@ -551,7 +545,6 @@ public class DatacenterMO extends BaseMO { } public DatacenterConfigInfo getDatacenterConfigInfo() throws Exception { - DatacenterConfigInfo configInfo = (DatacenterConfigInfo)_context.getVimClient().getDynamicProperty(_mor, "configuration"); - return configInfo; + return _context.getVimClient().getDynamicProperty(_mor, "configuration"); } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java index 3b96e7e1999..edffd8b0906 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.hypervisor.vmware.mo; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -24,11 +25,20 @@ import java.util.List; import java.util.Map; import java.util.regex.Pattern; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; + +import com.cloud.hypervisor.vmware.util.VmwareClientException; +import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.hypervisor.vmware.util.VmwareHelper; +import com.cloud.utils.LogUtils; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; + import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.google.gson.Gson; + import com.vmware.vim25.AboutInfo; import com.vmware.vim25.AlreadyExistsFaultMsg; import com.vmware.vim25.ClusterDasConfigInfo; @@ -58,6 +68,7 @@ import com.vmware.vim25.HostRuntimeInfo; import com.vmware.vim25.HostSystemConnectionState; import com.vmware.vim25.HostVirtualNic; import com.vmware.vim25.HostVirtualSwitch; +import com.vmware.vim25.InvalidPropertyFaultMsg; import com.vmware.vim25.ManagedObjectReference; import com.vmware.vim25.NasDatastoreInfo; import com.vmware.vim25.ObjectContent; @@ -65,13 +76,12 @@ import com.vmware.vim25.ObjectSpec; import com.vmware.vim25.OptionValue; import com.vmware.vim25.PropertyFilterSpec; import com.vmware.vim25.PropertySpec; +import com.vmware.vim25.RetrieveOptions; +import com.vmware.vim25.RetrieveResult; +import com.vmware.vim25.RuntimeFaultFaultMsg; import com.vmware.vim25.TraversalSpec; import com.vmware.vim25.VirtualMachineConfigSpec; import com.vmware.vim25.VirtualNicManagerNetConfig; -import com.cloud.hypervisor.vmware.util.VmwareContext; -import com.cloud.hypervisor.vmware.util.VmwareHelper; -import com.cloud.utils.LogUtils; -import com.cloud.utils.Pair; public class HostMO extends BaseMO implements VmwareHypervisorHost { private static final Logger s_logger = Logger.getLogger(HostMO.class); @@ -535,18 +545,22 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { return null; } - public String getHostName() throws Exception { + public String getHostName() throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { return (String)_context.getVimClient().getDynamicProperty(_mor, "name"); } @Override - public synchronized List listVmsOnHyperHostWithHypervisorName(String vmName) throws Exception { + public synchronized List listVmsOnHyperHostWithHypervisorName(String vmName) throws VmwareClientException { List vms = new ArrayList<>(); - if (StringUtils.isNotEmpty(vmName)) { - vms.add(findVmOnHyperHostWithHypervisorName(vmName)); - } else { - loadVmCache(); - vms.addAll(_vmCache.values()); + try { + if (StringUtils.isNotEmpty(vmName)) { + vms.add(findVmOnHyperHostWithHypervisorName(vmName)); + } else { + loadVmCache(); + vms.addAll(_vmCache.values()); + } + } catch (InvalidPropertyFaultMsg | RuntimeFaultFaultMsg | InvocationTargetException | NoSuchMethodException | IllegalAccessException e) { + throw new VmwareClientException("problem loading vm cache.", e); } return vms; } @@ -582,7 +596,7 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { } } - private void loadVmCache() throws Exception { + private void loadVmCache() throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { if (s_logger.isDebugEnabled()) s_logger.debug("load VM cache on host"); @@ -704,11 +718,16 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { } @Override - public ObjectContent[] getVmPropertiesOnHyperHost(String[] propertyPaths) throws Exception { - if (s_logger.isTraceEnabled()) + public ObjectContent[] getVmPropertiesOnHyperHost(String[] propertyPaths) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { + List properties = getVmProperties(propertyPaths, null, null).second(); + return properties.toArray(new ObjectContent[properties.size()]); + } + + public Pair> getVmProperties(String[] propertyPaths, Integer maxObjects) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { + if (s_logger.isTraceEnabled()) { s_logger.trace("vCenter API trace - retrieveProperties() for VM properties. target MOR: " + _mor.getValue() + ", properties: " + new Gson().toJson(propertyPaths)); - + } PropertySpec pSpec = new PropertySpec(); pSpec.setType("VirtualMachine"); pSpec.getPathSet().addAll(Arrays.asList(propertyPaths)); @@ -726,14 +745,19 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); - List properties = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); + RetrieveOptions ro = new RetrieveOptions(); + if (maxObjects != null && maxObjects > 0) { + ro.setMaxObjects(maxObjects); + } + + RetrieveResult result = _context.getService().retrievePropertiesEx(_context.getPropertyCollector(), pfSpecArr, ro); if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - retrieveProperties() done"); - return properties.toArray(new ObjectContent[properties.size()]); + return createReturnObjectPair(result); } @Override @@ -759,7 +783,7 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); List properties = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); @@ -1318,7 +1342,7 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { return false; } - private synchronized VirtualMachineMO findVmOnHyperHostWithHypervisorName(String vmName) throws Exception { + private synchronized VirtualMachineMO findVmOnHyperHostWithHypervisorName(String vmName) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { if (s_logger.isDebugEnabled()) s_logger.debug("find VM hypervisor name: " + vmName + " on host"); @@ -1350,4 +1374,29 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { return null; } + public Pair> getVms(Integer maxObjects, String token) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { + List vms = new ArrayList<>(); + Pair> objectContents = getVmProperties(new String[] {"name"}, maxObjects, token); + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("returning token %s for future retrievals, currently %d objects retrieved.", objectContents.first(), objectContents.second().size())); + } + Pair> retval = new Pair<>(objectContents.first(), vms); + + objectContentToUnmanagedInstanceTO(objectContents, vms); + + return retval; + } + public Pair> getVmProperties(String[] propertyPaths, Integer maxObjects, String tokenForPriorQuery) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { + if(com.cloud.utils.StringUtils.isNotBlank(tokenForPriorQuery)) { + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("running repeat query with token \'%s\'", tokenForPriorQuery)); + } + return retrieveNextSetOfProperties(tokenForPriorQuery); + } else { + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("running query for %d propertypaths and max %d objects", propertyPaths.length, maxObjects)); + } + return getVmProperties(propertyPaths, maxObjects); + } + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index e53d14fc43b..20c62c30c0b 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -26,7 +26,8 @@ import java.io.File; import java.io.FileOutputStream; import java.io.InputStreamReader; import java.io.OutputStreamWriter; -import java.net.URLEncoder; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -41,11 +42,6 @@ import java.util.stream.Collectors; import com.cloud.storage.Storage; import com.cloud.utils.exception.CloudRuntimeException; -import com.vmware.vim25.InvalidStateFaultMsg; -import com.vmware.vim25.RuntimeFaultFaultMsg; -import com.vmware.vim25.TaskInfo; -import com.vmware.vim25.TaskInfoState; -import com.vmware.vim25.VirtualMachineTicket; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -55,12 +51,13 @@ import com.cloud.hypervisor.vmware.mo.SnapshotDescriptor.SnapshotInfo; import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.hypervisor.vmware.util.VmwareHelper; import com.cloud.utils.ActionDelegate; -import com.cloud.utils.LogUtils; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.script.Script; + import com.google.gson.Gson; + import com.vmware.vim25.ArrayOfManagedObjectReference; import com.vmware.vim25.ChoiceOption; import com.vmware.vim25.CustomFieldStringValue; @@ -72,6 +69,8 @@ import com.vmware.vim25.GuestOsDescriptor; import com.vmware.vim25.HttpNfcLeaseDeviceUrl; import com.vmware.vim25.HttpNfcLeaseInfo; import com.vmware.vim25.HttpNfcLeaseState; +import com.vmware.vim25.InvalidPropertyFaultMsg; +import com.vmware.vim25.InvalidStateFaultMsg; import com.vmware.vim25.ManagedObjectReference; import com.vmware.vim25.ObjectContent; import com.vmware.vim25.ObjectSpec; @@ -82,12 +81,14 @@ import com.vmware.vim25.OvfFile; import com.vmware.vim25.ParaVirtualSCSIController; import com.vmware.vim25.PropertyFilterSpec; import com.vmware.vim25.PropertySpec; +import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.TaskInfo; +import com.vmware.vim25.TaskInfoState; import com.vmware.vim25.TraversalSpec; import com.vmware.vim25.VirtualBusLogicController; import com.vmware.vim25.VirtualCdrom; import com.vmware.vim25.VirtualCdromIsoBackingInfo; import com.vmware.vim25.VirtualCdromRemotePassthroughBackingInfo; -import com.vmware.vim25.VirtualController; import com.vmware.vim25.VirtualDevice; import com.vmware.vim25.VirtualDeviceBackingInfo; import com.vmware.vim25.VirtualDeviceConfigSpec; @@ -126,6 +127,7 @@ import com.vmware.vim25.VirtualMachineRelocateSpecDiskLocator; import com.vmware.vim25.VirtualMachineRuntimeInfo; import com.vmware.vim25.VirtualMachineSnapshotInfo; import com.vmware.vim25.VirtualMachineSnapshotTree; +import com.vmware.vim25.VirtualMachineTicket; import com.vmware.vim25.VirtualSCSIController; import com.vmware.vim25.VirtualSCSISharing; @@ -160,54 +162,17 @@ public class VirtualMachineMO extends BaseMO { return DatacenterMO.getOwnerDatacenter(getContext(), getMor()); } - public Pair getOwnerDatastore(String dsFullPath) throws Exception { - String dsName = DatastoreFile.getDatastoreNameFromPath(dsFullPath); - - PropertySpec pSpec = new PropertySpec(); - pSpec.setType("Datastore"); - pSpec.getPathSet().add("name"); - - TraversalSpec vmDatastoreTraversal = new TraversalSpec(); - vmDatastoreTraversal.setType("VirtualMachine"); - vmDatastoreTraversal.setPath("datastore"); - vmDatastoreTraversal.setName("vmDatastoreTraversal"); - - ObjectSpec oSpec = new ObjectSpec(); - oSpec.setObj(_mor); - oSpec.setSkip(Boolean.TRUE); - oSpec.getSelectSet().add(vmDatastoreTraversal); - - PropertyFilterSpec pfSpec = new PropertyFilterSpec(); - pfSpec.getPropSet().add(pSpec); - pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); - pfSpecArr.add(pfSpec); - - List ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); - - if (ocs != null) { - for (ObjectContent oc : ocs) { - DynamicProperty prop = oc.getPropSet().get(0); - if (prop.getVal().toString().equals(dsName)) { - return new Pair(new DatastoreMO(_context, oc.getObj()), dsName); - } - } - } - - return null; - } - public HostMO getRunningHost() throws Exception { VirtualMachineRuntimeInfo runtimeInfo = getRuntimeInfo(); return new HostMO(_context, runtimeInfo.getHost()); } - public String getVmName() throws Exception { - return (String)getContext().getVimClient().getDynamicProperty(_mor, "name"); + public String getVmName() throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { + return getContext().getVimClient().getDynamicProperty(_mor, "name"); } public GuestInfo getVmGuestInfo() throws Exception { - return (GuestInfo)getContext().getVimClient().getDynamicProperty(_mor, "guest"); + return getContext().getVimClient().getDynamicProperty(_mor, "guest"); } public void answerVM(String questionId, String choice) throws Exception { @@ -216,11 +181,7 @@ public class VirtualMachineMO extends BaseMO { public boolean isVMwareToolsRunning() throws Exception { GuestInfo guestInfo = getVmGuestInfo(); - if (guestInfo != null) { - if ("guestToolsRunning".equalsIgnoreCase(guestInfo.getToolsRunningStatus())) - return true; - } - return false; + return guestInfo != null && "guestToolsRunning".equalsIgnoreCase(guestInfo.getToolsRunningStatus()); } public boolean powerOn() throws Exception { @@ -389,7 +350,7 @@ public class VirtualMachineMO extends BaseMO { // In the future, VMsync should not kick off CloudStack action (this is not a HA case) based on VM // state report, until then we can remove this hacking fix for (int i = 0; i < 3; i++) { - powerState = (VirtualMachinePowerState)getContext().getVimClient().getDynamicProperty(_mor, "runtime.powerState"); + powerState = getContext().getVimClient().getDynamicProperty(_mor, "runtime.powerState"); if (powerState == VirtualMachinePowerState.POWERED_OFF) { try { Thread.sleep(1000); @@ -405,7 +366,7 @@ public class VirtualMachineMO extends BaseMO { } public VirtualMachinePowerState getPowerState() throws Exception { - return (VirtualMachinePowerState)getContext().getVimClient().getDynamicProperty(_mor, "runtime.powerState"); + return getContext().getVimClient().getDynamicProperty(_mor, "runtime.powerState"); } public boolean reset() throws Exception { @@ -468,39 +429,6 @@ public class VirtualMachineMO extends BaseMO { return false; } - public boolean changeHost(VirtualMachineRelocateSpec relocateSpec) throws Exception { - ManagedObjectReference morTask = _context.getService().relocateVMTask(_mor, relocateSpec, VirtualMachineMovePriority.DEFAULT_PRIORITY); - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - s_logger.error("VMware RelocateVM_Task to change host failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - return false; - } - - public boolean changeDatastore(ManagedObjectReference morDataStore, VmwareHypervisorHost targetHost) throws Exception { - VirtualMachineRelocateSpec relocateSpec = new VirtualMachineRelocateSpec(); - relocateSpec.setDatastore(morDataStore); - if (targetHost != null) { - relocateSpec.setHost(targetHost.getMor()); - relocateSpec.setPool(targetHost.getHyperHostOwnerResourcePool()); - } - - ManagedObjectReference morTask = _context.getService().relocateVMTask(_mor, relocateSpec, null); - - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - s_logger.error("VMware change datastore relocateVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - - return false; - } - public boolean relocate(ManagedObjectReference morTargetHost) throws Exception { VirtualMachineRelocateSpec relocateSpec = new VirtualMachineRelocateSpec(); relocateSpec.setHost(morTargetHost); @@ -519,7 +447,7 @@ public class VirtualMachineMO extends BaseMO { } public VirtualMachineSnapshotInfo getSnapshotInfo() throws Exception { - return (VirtualMachineSnapshotInfo)_context.getVimClient().getDynamicProperty(_mor, "snapshot"); + return _context.getVimClient().getDynamicProperty(_mor, "snapshot"); } public boolean createSnapshot(String snapshotName, String snapshotDescription, boolean dumpMemory, boolean quiesce) throws Exception { @@ -589,39 +517,6 @@ public class VirtualMachineMO extends BaseMO { return false; } - public boolean revertToSnapshot(String snapshotName) throws Exception { - ManagedObjectReference morSnapshot = getSnapshotMor(snapshotName); - if (morSnapshot == null) { - s_logger.warn("Unable to find snapshot: " + snapshotName); - return false; - } - ManagedObjectReference morTask = _context.getService().revertToSnapshotTask(morSnapshot, _mor, null); - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - s_logger.error("VMware revert to snapshot failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - - return false; - } - - /** - * Deletes all of the snapshots of a VM. - */ - public void consolidateAllSnapshots() throws Exception { - ManagedObjectReference task = _context.getService().removeAllSnapshotsTask(_mor, true); - - boolean result = _context.getVimClient().waitForTask(task); - - if (result) { - _context.waitForTaskProgressDone(task); - } else { - throw new Exception("Unable to register VM due to the following issue: " + TaskMO.getTaskFailureInfo(_context, task)); - } - } - public boolean removeAllSnapshots() throws Exception { VirtualMachineSnapshotInfo snapshotInfo = getSnapshotInfo(); @@ -701,7 +596,7 @@ public class VirtualMachineMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); List ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); @@ -750,9 +645,7 @@ public class VirtualMachineMO extends BaseMO { return true; } List rootSnapshotList = info.getRootSnapshotList(); - if (rootSnapshotList != null && rootSnapshotList.size() > 0) { - return true; - } + return CollectionUtils.isNotEmpty(rootSnapshotList); } return false; } @@ -905,11 +798,11 @@ public class VirtualMachineMO extends BaseMO { VirtualDisk[] independentDisks = getAllIndependentDiskDevice(); VirtualMachineRelocateSpec rSpec = new VirtualMachineRelocateSpec(); if (independentDisks.length > 0) { - List diskLocator = new ArrayList(independentDisks.length); - for (int i = 0; i < independentDisks.length; i++) { + List diskLocator = new ArrayList<>(independentDisks.length); + for (VirtualDisk independentDisk : independentDisks) { VirtualMachineRelocateSpecDiskLocator loc = new VirtualMachineRelocateSpecDiskLocator(); loc.setDatastore(morDs); - loc.setDiskId(independentDisks[i].getKey()); + loc.setDiskId(independentDisk.getKey()); loc.setDiskMoveType(VirtualMachineRelocateDiskMoveOptions.MOVE_ALL_DISK_BACKINGS_AND_DISALLOW_SHARING.value()); diskLocator.add(loc); } @@ -941,26 +834,26 @@ public class VirtualMachineMO extends BaseMO { } public VirtualMachineRuntimeInfo getRuntimeInfo() throws Exception { - return (VirtualMachineRuntimeInfo)_context.getVimClient().getDynamicProperty(_mor, "runtime"); + return _context.getVimClient().getDynamicProperty(_mor, "runtime"); } public VirtualMachineConfigInfo getConfigInfo() throws Exception { - return (VirtualMachineConfigInfo)_context.getVimClient().getDynamicProperty(_mor, "config"); + return _context.getVimClient().getDynamicProperty(_mor, "config"); } public boolean isToolsInstallerMounted() throws Exception { return _context.getVimClient().getDynamicProperty(_mor, "runtime.toolsInstallerMounted"); } public GuestInfo getGuestInfo() throws Exception { - return (GuestInfo)_context.getVimClient().getDynamicProperty(_mor, "guest"); + return _context.getVimClient().getDynamicProperty(_mor, "guest"); } public VirtualMachineConfigSummary getConfigSummary() throws Exception { - return (VirtualMachineConfigSummary)_context.getVimClient().getDynamicProperty(_mor, "summary.config"); + return _context.getVimClient().getDynamicProperty(_mor, "summary.config"); } public VirtualMachineFileInfo getFileInfo() throws Exception { - return (VirtualMachineFileInfo)_context.getVimClient().getDynamicProperty(_mor, "config.files"); + return _context.getVimClient().getDynamicProperty(_mor, "config.files"); } public VirtualMachineFileLayoutEx getFileLayout() throws Exception { @@ -976,7 +869,7 @@ public class VirtualMachineMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); List ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); @@ -1000,7 +893,7 @@ public class VirtualMachineMO extends BaseMO { @Override public ManagedObjectReference getParentMor() throws Exception { - return (ManagedObjectReference)_context.getVimClient().getDynamicProperty(_mor, "parent"); + return _context.getVimClient().getDynamicProperty(_mor, "parent"); } public String[] getNetworks() throws Exception { @@ -1021,13 +914,13 @@ public class VirtualMachineMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); List ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); - List networks = new ArrayList(); - if (ocs != null && ocs.size() > 0) { + List networks = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(ocs)) { for (ObjectContent oc : ocs) { networks.add(oc.getPropSet().get(0).getVal().toString()); } @@ -1036,7 +929,7 @@ public class VirtualMachineMO extends BaseMO { } public List getNetworksWithDetails() throws Exception { - List networks = new ArrayList(); + List networks = new ArrayList<>(); int gcTagKey = getCustomFieldKey("Network", CustomFieldConstants.CLOUD_GC); @@ -1064,12 +957,12 @@ public class VirtualMachineMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); List ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); - if (ocs != null && ocs.size() > 0) { + if (CollectionUtils.isNotEmpty(ocs)) { for (ObjectContent oc : ocs) { ArrayOfManagedObjectReference morVms = null; String gcTagValue = null; @@ -1117,12 +1010,12 @@ public class VirtualMachineMO extends BaseMO { PropertyFilterSpec pfSpec = new PropertyFilterSpec(); pfSpec.getPropSet().add(pSpec); pfSpec.getObjectSet().add(oSpec); - List pfSpecArr = new ArrayList(); + List pfSpecArr = new ArrayList<>(); pfSpecArr.add(pfSpec); List ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); - List datastores = new ArrayList(); + List datastores = new ArrayList<>(); if (CollectionUtils.isNotEmpty(ocs)) { for (ObjectContent oc : ocs) { datastores.add(new DatastoreMO(_context, oc.getObj())); @@ -1131,61 +1024,6 @@ public class VirtualMachineMO extends BaseMO { return datastores; } - /** - * Retrieve path info to access VM files via vSphere web interface - * @return [0] vm-name, [1] data-center-name, [2] datastore-name - * @throws Exception - */ - public String[] getHttpAccessPathInfo() throws Exception { - String[] pathInfo = new String[3]; - - Pair dcInfo = getOwnerDatacenter(); - - VirtualMachineFileInfo fileInfo = getFileInfo(); - String vmxFilePath = fileInfo.getVmPathName(); - String vmxPathTokens[] = vmxFilePath.split("\\[|\\]|/"); - assert (vmxPathTokens.length == 4); - pathInfo[1] = vmxPathTokens[1].trim(); // vSphere vm name - pathInfo[2] = dcInfo.second(); // vSphere datacenter name - pathInfo[3] = vmxPathTokens[0].trim(); // vSphere datastore name - return pathInfo; - } - - public String getVmxHttpAccessUrl() throws Exception { - Pair dcInfo = getOwnerDatacenter(); - - VirtualMachineFileInfo fileInfo = getFileInfo(); - String vmxFilePath = fileInfo.getVmPathName(); - String vmxPathTokens[] = vmxFilePath.split("\\[|\\]|/"); - - StringBuffer sb = new StringBuffer("https://" + _context.getServerAddress() + "/folder/"); - sb.append(URLEncoder.encode(vmxPathTokens[2].trim(), "UTF-8")); - sb.append("/"); - sb.append(URLEncoder.encode(vmxPathTokens[3].trim(), "UTF-8")); - sb.append("?dcPath="); - sb.append(URLEncoder.encode(dcInfo.second(), "UTF-8")); - sb.append("&dsName="); - sb.append(URLEncoder.encode(vmxPathTokens[1].trim(), "UTF-8")); - - return sb.toString(); - } - - public boolean setVncConfigInfo(boolean enableVnc, String vncPassword, int vncPort, String keyboard) throws Exception { - VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec(); - OptionValue[] vncOptions = VmwareHelper.composeVncOptions(null, enableVnc, vncPassword, vncPort, keyboard); - vmConfigSpec.getExtraConfig().addAll(Arrays.asList(vncOptions)); - ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, vmConfigSpec); - - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - s_logger.error("VMware reconfigVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - return false; - } - public boolean configureVm(VirtualMachineConfigSpec vmConfigSpec) throws Exception { ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, vmConfigSpec); @@ -1239,12 +1077,12 @@ public class VirtualMachineMO extends BaseMO { if (option.getKey().equals("RemoteDisplay.vnc.port")) { String value = (String)option.getValue(); if (value != null) { - return new Pair(summary.getHostIp(), Integer.parseInt(value)); + return new Pair<>(summary.getHostIp(), Integer.parseInt(value)); } } } } - return new Pair(summary.getHostIp(), 0); + return new Pair<>(summary.getHostIp(), 0); } // vmdkDatastorePath: [datastore name] vmdkFilePath @@ -1255,11 +1093,11 @@ public class VirtualMachineMO extends BaseMO { // vmdkDatastorePath: [datastore name] vmdkFilePath public void createDisk(String vmdkDatastorePath, VirtualDiskType diskType, VirtualDiskMode diskMode, String rdmDeviceName, long sizeInMb, ManagedObjectReference morDs, int controllerKey, String vSphereStoragePolicyId) throws Exception { + assert (morDs != null); s_logger.trace(String.format("Creating disk in target MOR [%s] with values: vmdkDatastorePath [%s], sizeInMb [%s], diskType [%s], diskMode [%s], rdmDeviceName [%s]" + ", datastore [%s], controllerKey [%s].", _mor.getValue(), vmdkDatastorePath, sizeInMb, diskType, diskMode, rdmDeviceName, morDs.getValue(), controllerKey)); assert (vmdkDatastorePath != null); - assert (morDs != null); int ideControllerKey = getIDEDeviceControllerKey(); if (controllerKey < 0) { @@ -1271,17 +1109,9 @@ public class VirtualMachineMO extends BaseMO { VirtualDiskFlatVer2BackingInfo backingInfo = new VirtualDiskFlatVer2BackingInfo(); backingInfo.setDiskMode(VirtualDiskMode.PERSISTENT.value()); - if (diskType == VirtualDiskType.THIN) { - backingInfo.setThinProvisioned(true); - } else { - backingInfo.setThinProvisioned(false); - } + backingInfo.setThinProvisioned(diskType == VirtualDiskType.THIN); - if (diskType == VirtualDiskType.EAGER_ZEROED_THICK) { - backingInfo.setEagerlyScrub(true); - } else { - backingInfo.setEagerlyScrub(false); - } + backingInfo.setEagerlyScrub(diskType == VirtualDiskType.EAGER_ZEROED_THICK); backingInfo.setDatastore(morDs); backingInfo.setFileName(vmdkDatastorePath); @@ -1358,7 +1188,7 @@ public class VirtualMachineMO extends BaseMO { if (!currentAdapterType.equalsIgnoreCase(newAdapterType)) { s_logger.info("Updating adapter type to " + newAdapterType + " for VMDK file " + vmdkFileName); Pair dcInfo = getOwnerDatacenter(); - byte[] newVmdkContent = vmdkFileDescriptor.changeVmdkAdapterType(vmdkInfo.second(), newAdapterType); + byte[] newVmdkContent = VmdkFileDescriptor.changeVmdkAdapterType(vmdkInfo.second(), newAdapterType); String vmdkUploadUrl = getContext().composeDatastoreBrowseUrl(dcInfo.first().getName(), vmdkFileName); getContext().uploadResourceContent(vmdkUploadUrl, newVmdkContent); s_logger.info("Updated VMDK file " + vmdkFileName); @@ -1385,7 +1215,7 @@ public class VirtualMachineMO extends BaseMO { VmdkAdapterType newAdapterType = VmdkAdapterType.lsilogic; s_logger.debug("Updating adapter type to " + newAdapterType + " from " + currentAdapterTypeStr + " for VMDK file " + vmdkFileName); Pair dcInfo = getOwnerDatacenter(); - byte[] newVmdkContent = vmdkFileDescriptor.changeVmdkAdapterType(vmdkInfo.second(), newAdapterType.toString()); + byte[] newVmdkContent = VmdkFileDescriptor.changeVmdkAdapterType(vmdkInfo.second(), newAdapterType.toString()); String vmdkUploadUrl = getContext().composeDatastoreBrowseUrl(dcInfo.first().getName(), vmdkFileName); getContext().uploadResourceContent(vmdkUploadUrl, newVmdkContent); @@ -1406,8 +1236,8 @@ public class VirtualMachineMO extends BaseMO { if(s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - attachDisk(). target MOR: " + _mor.getValue() + ", vmdkDatastorePath: " + GSON.toJson(vmdkDatastorePathChain) + ", datastore: " + morDs.getValue()); - int controllerKey = 0; - int unitNumber = 0; + int controllerKey; + int unitNumber; if (DiskControllerType.getType(diskController) == DiskControllerType.ide) { // IDE virtual disk cannot be added if VM is running @@ -1468,21 +1298,6 @@ public class VirtualMachineMO extends BaseMO { s_logger.trace("vCenter API trace - attachDisk() done(successfully)"); } - private int getControllerBusNumber(int controllerKey) throws Exception { - List devices = (List)_context.getVimClient(). - getDynamicProperty(_mor, "config.hardware.device"); - - if (devices != null && devices.size() > 0) { - for (VirtualDevice device : devices) { - if (device instanceof VirtualController && device.getKey() == controllerKey) { - return ((VirtualController)device).getBusNumber(); - } - } - } - throw new Exception("SCSI Controller with key " + controllerKey + " is Not Found"); - - } - // vmdkDatastorePath: [datastore name] vmdkFilePath public List> detachDisk(String vmdkDatastorePath, boolean deleteBackingFile) throws Exception { @@ -1792,7 +1607,7 @@ public class VirtualMachineMO extends BaseMO { VmdkFileDescriptor descriptor = new VmdkFileDescriptor(); descriptor.parse(content); - Pair result = new Pair(descriptor, content); + Pair result = new Pair<>(descriptor, content); if (s_logger.isTraceEnabled()) { s_logger.trace("vCenter API trace - getVmdkFileInfo() done"); s_logger.trace("VMDK file descriptor: " + GSON.toJson(result.first())); @@ -1871,7 +1686,7 @@ public class VirtualMachineMO extends BaseMO { if (s_logger.isInfoEnabled()) { s_logger.info("Download VMDK file for export url: " + deviceUrlStr + ", size: " + diskFileSize); } - long lengthOfDiskFile = _context.downloadVmdkFile(diskUrlStr, diskLocalPath, totalBytesDownloaded, new ActionDelegate() { + long lengthOfDiskFile = _context.downloadVmdkFile(diskUrlStr, diskLocalPath, totalBytesDownloaded, new ActionDelegate<>() { @Override public void action(Long param) { if (s_logger.isTraceEnabled()) { @@ -1896,7 +1711,7 @@ public class VirtualMachineMO extends BaseMO { CompletableFuture future = CompletableFuture.supplyAsync(() -> { long lengthOfDiskFile = 0; try { - lengthOfDiskFile = _context.downloadVmdkFile(diskUrl, diskLocalPath, totalBytesDownloaded, new ActionDelegate() { + lengthOfDiskFile = _context.downloadVmdkFile(diskUrl, diskLocalPath, totalBytesDownloaded, new ActionDelegate<>() { @Override public void action(Long param) { if (s_logger.isTraceEnabled()) { @@ -1947,7 +1762,7 @@ public class VirtualMachineMO extends BaseMO { String ovfPath = exportDir + File.separator + exportName + ".ovf"; fileNames.add(ovfPath); - OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(ovfPath),"UTF-8"); + OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(ovfPath), StandardCharsets.UTF_8); out.write(ovfCreateDescriptorResult.getOvfDescriptor()); out.close(); @@ -1969,7 +1784,7 @@ public class VirtualMachineMO extends BaseMO { } } - s_logger.info("Package OVA with command: " + command.toString()); + s_logger.info("Package OVA with command: " + command); command.execute(); // to be safe, physically test existence of the target OVA file @@ -2018,8 +1833,8 @@ public class VirtualMachineMO extends BaseMO { boolean replaced = false; try { - in = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(vmxContent),"UTF-8")); - out = new BufferedWriter(new OutputStreamWriter(bos,"UTF-8")); + in = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(vmxContent), StandardCharsets.UTF_8)); + out = new BufferedWriter(new OutputStreamWriter(bos, StandardCharsets.UTF_8)); String line; while ((line = in.readLine()) != null) { if (line.startsWith("workingDir")) { @@ -2052,70 +1867,6 @@ public class VirtualMachineMO extends BaseMO { // redoRegistration(); } - // destName does not contain extension name - public void backupCurrentSnapshot(String deviceName, ManagedObjectReference morDestDs, String destDsDirectory, String destName, boolean includeBase) throws Exception { - - SnapshotDescriptor descriptor = getSnapshotDescriptor(); - SnapshotInfo[] snapshotInfo = descriptor.getCurrentDiskChain(); - if (snapshotInfo.length == 0) { - String msg = "No snapshot found in this VM"; - throw new Exception(msg); - } - - HostMO hostMo = getRunningHost(); - DatacenterMO dcMo = getOwnerDatacenter().first(); - List> mounts = hostMo.getDatastoreMountsOnHost(); - VirtualMachineFileInfo vmFileInfo = getFileInfo(); - - List> backupInfo = new ArrayList>(); - - for (int i = 0; i < snapshotInfo.length; i++) { - if (!includeBase && i == snapshotInfo.length - 1) { - break; - } - - SnapshotDescriptor.DiskInfo[] disks = snapshotInfo[i].getDisks(); - if (disks != null) { - String destBaseFileName; - String destFileName; - String destParentFileName; - for (SnapshotDescriptor.DiskInfo disk : disks) { - if (deviceName == null || deviceName.equals(disk.getDeviceName())) { - String srcVmdkFullDsPath = getSnapshotDiskFileDatastorePath(vmFileInfo, mounts, disk.getDiskFileName()); - Pair srcDsInfo = getOwnerDatastore(srcVmdkFullDsPath); - - Pair vmdkInfo = getVmdkFileInfo(srcVmdkFullDsPath); - String srcVmdkBaseFilePath = DatastoreFile.getCompanionDatastorePath(srcVmdkFullDsPath, vmdkInfo.first().getBaseFileName()); - - destFileName = destName + (snapshotInfo.length - i - 1) + ".vmdk"; - if (vmdkInfo.first().getParentFileName() != null) { - destBaseFileName = destName + (snapshotInfo.length - i - 1) + "-delta.vmdk"; - destParentFileName = destName + (snapshotInfo.length - i - 2) + ".vmdk"; - } else { - destBaseFileName = destName + (snapshotInfo.length - i - 1) + "-flat.vmdk"; - destParentFileName = null; - } - - s_logger.info("Copy VMDK base file " + srcVmdkBaseFilePath + " to " + destDsDirectory + "/" + destBaseFileName); - srcDsInfo.first().copyDatastoreFile(srcVmdkBaseFilePath, dcMo.getMor(), morDestDs, destDsDirectory + "/" + destBaseFileName, dcMo.getMor(), true); - - byte[] newVmdkContent = VmdkFileDescriptor.changeVmdkContentBaseInfo(vmdkInfo.second(), destBaseFileName, destParentFileName); - String vmdkUploadUrl = getContext().composeDatastoreBrowseUrl(dcMo.getName(), destDsDirectory + "/" + destFileName); - - s_logger.info("Upload VMDK content file to " + destDsDirectory + "/" + destFileName); - getContext().uploadResourceContent(vmdkUploadUrl, newVmdkContent); - - backupInfo.add(new Ternary(destFileName, destBaseFileName, destParentFileName)); - } - } - } - } - - byte[] vdiskInfo = VmwareHelper.composeDiskInfo(backupInfo, snapshotInfo.length, includeBase); - String vdiskUploadUrl = getContext().composeDatastoreBrowseUrl(dcMo.getName(), destDsDirectory + "/" + destName + ".vdisk"); - getContext().uploadResourceContent(vdiskUploadUrl, vdiskInfo); - } - public String[] getCurrentSnapshotDiskChainDatastorePaths(String diskDevice) throws Exception { HostMO hostMo = getRunningHost(); List> mounts = hostMo.getDatastoreMountsOnHost(); @@ -2124,9 +1875,9 @@ public class VirtualMachineMO extends BaseMO { SnapshotDescriptor descriptor = getSnapshotDescriptor(); SnapshotInfo[] snapshotInfo = descriptor.getCurrentDiskChain(); - List diskDsFullPaths = new ArrayList(); - for (int i = 0; i < snapshotInfo.length; i++) { - SnapshotDescriptor.DiskInfo[] disks = snapshotInfo[i].getDisks(); + List diskDsFullPaths = new ArrayList<>(); + for (SnapshotInfo info : snapshotInfo) { + SnapshotDescriptor.DiskInfo[] disks = info.getDisks(); if (disks != null) { for (SnapshotDescriptor.DiskInfo disk : disks) { String deviceNameInDisk = disk.getDeviceName(); @@ -2146,7 +1897,7 @@ public class VirtualMachineMO extends BaseMO { assert (morDs != null); String[] disks = getCurrentSnapshotDiskChainDatastorePaths(diskDevice); VirtualMachineMO clonedVm = cloneFromDiskChain(clonedVmName, cpuSpeedMHz, memoryMb, disks, morDs, virtualHardwareVersion); - return new Pair(clonedVm, disks); + return new Pair<>(clonedVm, disks); } public VirtualMachineMO cloneFromDiskChain(String clonedVmName, int cpuSpeedMHz, int memoryMb, String[] disks, ManagedObjectReference morDs, String cloneHardwareVersion) throws Exception { @@ -2197,19 +1948,6 @@ public class VirtualMachineMO extends BaseMO { return guestOsDescriptor; } - public void plugDevice(VirtualDevice device) throws Exception { - s_logger.debug(LogUtils.logGsonWithoutException("Pluging device [%s] to VM [%s].", device, getVmName())); - VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec(); - VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec(); - deviceConfigSpec.setDevice(device); - deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - - vmConfigSpec.getDeviceChange().add(deviceConfigSpec); - if (!configureVm(vmConfigSpec)) { - throw new Exception("Failed to add devices"); - } - } - public void tearDownDevice(VirtualDevice device) throws Exception { VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec(); VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec(); @@ -2242,40 +1980,8 @@ public class VirtualMachineMO extends BaseMO { } } - public void copyAllVmDiskFiles(DatastoreMO destDsMo, String destDsDir, boolean followDiskChain) throws Exception { - VirtualDevice[] disks = getAllDiskDevice(); - DatacenterMO dcMo = getOwnerDatacenter().first(); - if (disks != null) { - for (VirtualDevice disk : disks) { - List> vmdkFiles = getDiskDatastorePathChain((VirtualDisk)disk, followDiskChain); - for (Pair fileItem : vmdkFiles) { - DatastoreMO srcDsMo = new DatastoreMO(_context, fileItem.second()); - - DatastoreFile srcFile = new DatastoreFile(fileItem.first()); - DatastoreFile destFile = new DatastoreFile(destDsMo.getName(), destDsDir, srcFile.getFileName()); - - Pair vmdkDescriptor = null; - - vmdkDescriptor = getVmdkFileInfo(fileItem.first()); - - s_logger.info("Copy VM disk file " + srcFile.getPath() + " to " + destFile.getPath()); - srcDsMo.copyDatastoreFile(fileItem.first(), dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true); - - if (vmdkDescriptor != null) { - String vmdkBaseFileName = vmdkDescriptor.first().getBaseFileName(); - String baseFilePath = srcFile.getCompanionPath(vmdkBaseFileName); - destFile = new DatastoreFile(destDsMo.getName(), destDsDir, vmdkBaseFileName); - - s_logger.info("Copy VM disk file " + baseFilePath + " to " + destFile.getPath()); - srcDsMo.copyDatastoreFile(baseFilePath, dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true); - } - } - } - } - } - public List getVmdkFileBaseNames() throws Exception { - List vmdkFileBaseNames = new ArrayList(); + List vmdkFileBaseNames = new ArrayList<>(); VirtualDevice[] devices = getAllDiskDevice(); for(VirtualDevice device : devices) { if(device instanceof VirtualDisk) { @@ -2310,7 +2016,7 @@ public class VirtualMachineMO extends BaseMO { DatastoreFile srcFile = new DatastoreFile(fileItem.first()); DatastoreFile destFile = new DatastoreFile(destDsMo.getName(), destDsDir, srcFile.getFileName()); - Pair vmdkDescriptor = null; + Pair vmdkDescriptor; vmdkDescriptor = getVmdkFileInfo(fileItem.first()); s_logger.info("Move VM disk file " + srcFile.getPath() + " to " + destFile.getPath()); @@ -2329,37 +2035,6 @@ public class VirtualMachineMO extends BaseMO { } } - public int getPvScsiDeviceControllerKeyNoException() throws Exception { - List devices = (List)_context.getVimClient(). - getDynamicProperty(_mor, "config.hardware.device"); - - if (devices != null && devices.size() > 0) { - for (VirtualDevice device : devices) { - if (device instanceof ParaVirtualSCSIController) { - return device.getKey(); - } - } - } - - return -1; - } - - public int getPvScsiDeviceControllerKey() throws Exception { - List devices = (List)_context.getVimClient(). - getDynamicProperty(_mor, "config.hardware.device"); - - if (devices != null && devices.size() > 0) { - for (VirtualDevice device : devices) { - if (device instanceof ParaVirtualSCSIController) { - return device.getKey(); - } - } - } - - assert (false); - throw new Exception("VMware Paravirtual SCSI Controller Not Found"); - } - protected VirtualSCSIController getScsiController(DiskControllerType type) { switch (type) { case pvscsi: @@ -2443,23 +2118,23 @@ public class VirtualMachineMO extends BaseMO { // Would be useful if there exists multiple sub types of SCSI controllers per VM are supported in CloudStack f public int getScsiDiskControllerKey(String diskController) throws Exception { - List devices = (List)_context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); + List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); if (CollectionUtils.isNotEmpty(devices)) { DiskControllerType diskControllerType = DiskControllerType.getType(diskController); for (VirtualDevice device : devices) { if ((diskControllerType == DiskControllerType.lsilogic || diskControllerType == DiskControllerType.scsi) && device instanceof VirtualLsiLogicController && isValidScsiDiskController((VirtualLsiLogicController)device)) { - return ((VirtualLsiLogicController)device).getKey(); + return device.getKey(); } else if ((diskControllerType == DiskControllerType.lsisas1068 || diskControllerType == DiskControllerType.scsi) && device instanceof VirtualLsiLogicSASController && isValidScsiDiskController((VirtualLsiLogicSASController)device)) { - return ((VirtualLsiLogicSASController)device).getKey(); + return device.getKey(); } else if ((diskControllerType == DiskControllerType.pvscsi || diskControllerType == DiskControllerType.scsi) && device instanceof ParaVirtualSCSIController && isValidScsiDiskController((ParaVirtualSCSIController)device)) { - return ((ParaVirtualSCSIController)device).getKey(); + return device.getKey(); } else if ((diskControllerType == DiskControllerType.buslogic || diskControllerType == DiskControllerType.scsi) && device instanceof VirtualBusLogicController && isValidScsiDiskController((VirtualBusLogicController)device)) { - return ((VirtualBusLogicController)device).getKey(); + return device.getKey(); } } } @@ -2469,7 +2144,7 @@ public class VirtualMachineMO extends BaseMO { } public int getScsiDiskControllerKeyNoException(String diskController, int scsiUnitNumber) throws Exception { - List devices = (List)_context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); + List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); if (CollectionUtils.isNotEmpty(devices) && scsiUnitNumber >= 0) { int requiredScsiController = scsiUnitNumber / VmwareHelper.MAX_ALLOWED_DEVICES_SCSI_CONTROLLER; @@ -2479,7 +2154,7 @@ public class VirtualMachineMO extends BaseMO { if ((diskControllerType == DiskControllerType.lsilogic || diskControllerType == DiskControllerType.scsi) && device instanceof VirtualLsiLogicController) { if (scsiControllerDeviceCount == requiredScsiController) { if (isValidScsiDiskController((VirtualLsiLogicController)device)) { - return ((VirtualLsiLogicController)device).getKey(); + return device.getKey(); } break; } @@ -2487,7 +2162,7 @@ public class VirtualMachineMO extends BaseMO { } else if ((diskControllerType == DiskControllerType.lsisas1068 || diskControllerType == DiskControllerType.scsi) && device instanceof VirtualLsiLogicSASController) { if (scsiControllerDeviceCount == requiredScsiController) { if (isValidScsiDiskController((VirtualLsiLogicSASController)device)) { - return ((VirtualLsiLogicSASController)device).getKey(); + return device.getKey(); } break; } @@ -2495,7 +2170,7 @@ public class VirtualMachineMO extends BaseMO { } else if ((diskControllerType == DiskControllerType.pvscsi || diskControllerType == DiskControllerType.scsi) && device instanceof ParaVirtualSCSIController) { if (scsiControllerDeviceCount == requiredScsiController) { if (isValidScsiDiskController((ParaVirtualSCSIController)device)) { - return ((ParaVirtualSCSIController)device).getKey(); + return device.getKey(); } break; } @@ -2503,7 +2178,7 @@ public class VirtualMachineMO extends BaseMO { } else if ((diskControllerType == DiskControllerType.buslogic || diskControllerType == DiskControllerType.scsi) && device instanceof VirtualBusLogicController) { if (scsiControllerDeviceCount == requiredScsiController) { if (isValidScsiDiskController((VirtualBusLogicController)device)) { - return ((VirtualBusLogicController)device).getKey(); + return device.getKey(); } break; } @@ -2516,15 +2191,13 @@ public class VirtualMachineMO extends BaseMO { public int getNextScsiDiskDeviceNumber() throws Exception { int scsiControllerKey = getScsiDeviceControllerKey(); - int deviceNumber = getNextDeviceNumber(scsiControllerKey); - - return deviceNumber; + return getNextDeviceNumber(scsiControllerKey); } public int getScsiDeviceControllerKey() throws Exception { List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); - if (devices != null && devices.size() > 0) { + if (CollectionUtils.isNotEmpty(devices)) { for (VirtualDevice device : devices) { if (device instanceof VirtualSCSIController && isValidScsiDiskController((VirtualSCSIController)device)) { return device.getKey(); @@ -2539,7 +2212,7 @@ public class VirtualMachineMO extends BaseMO { public int getScsiDeviceControllerKeyNoException() throws Exception { List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); - if (devices != null && devices.size() > 0) { + if (CollectionUtils.isNotEmpty(devices)) { for (VirtualDevice device : devices) { if (device instanceof VirtualSCSIController && isValidScsiDiskController((VirtualSCSIController)device)) { return device.getKey(); @@ -2577,10 +2250,10 @@ public class VirtualMachineMO extends BaseMO { } private int getLsiLogicDeviceControllerKeyNoException() throws Exception { - List devices = (List)_context.getVimClient(). + List devices = _context.getVimClient(). getDynamicProperty(_mor, "config.hardware.device"); - if (devices != null && devices.size() > 0) { + if (CollectionUtils.isNotEmpty(devices)) { for (VirtualDevice device : devices) { if (device instanceof VirtualLsiLogicController) { return device.getKey(); @@ -2612,32 +2285,6 @@ public class VirtualMachineMO extends BaseMO { } } - public void ensureScsiDeviceControllers(int count, int availableBusNum) throws Exception { - int scsiControllerKey = getScsiDeviceControllerKeyNoException(); - if (scsiControllerKey < 0) { - VirtualMachineConfigSpec vmConfig = new VirtualMachineConfigSpec(); - - int busNum = availableBusNum; - while (busNum < count) { - VirtualLsiLogicController scsiController = new VirtualLsiLogicController(); - scsiController.setSharedBus(VirtualSCSISharing.NO_SHARING); - scsiController.setBusNumber(busNum); - scsiController.setKey(busNum - VmwareHelper.MAX_SCSI_CONTROLLER_COUNT); - VirtualDeviceConfigSpec scsiControllerSpec = new VirtualDeviceConfigSpec(); - scsiControllerSpec.setDevice(scsiController); - scsiControllerSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - - vmConfig.getDeviceChange().add(scsiControllerSpec); - busNum++; - } - if (configureVm(vmConfig)) { - throw new Exception("Unable to add Scsi controllers to the VM " + getName()); - } else { - s_logger.info("Successfully added " + count + " SCSI controllers."); - } - } - } - private boolean isValidScsiDiskController(VirtualSCSIController scsiDiskController) { if (scsiDiskController == null) { return false; @@ -2648,11 +2295,7 @@ public class VirtualMachineMO extends BaseMO { return false; } - if (scsiDiskController.getBusNumber() >= VmwareHelper.MAX_SCSI_CONTROLLER_COUNT) { - return false; - } - - return true; + return scsiDiskController.getBusNumber() < VmwareHelper.MAX_SCSI_CONTROLLER_COUNT; } // return pair of VirtualDisk and disk device bus name(ide0:0, etc) @@ -2670,7 +2313,7 @@ public class VirtualMachineMO extends BaseMO { s_logger.info(String.format("Looking for disk device info for volume [%s] with base name [%s].", vmdkDatastorePath, srcBaseName)); - if (devices != null && devices.size() > 0) { + if (CollectionUtils.isNotEmpty(devices)) { for (VirtualDevice device : devices) { if (device instanceof VirtualDisk) { s_logger.info(String.format("Testing if disk device with controller key [%s] and unit number [%s] has backing of type VirtualDiskFlatVer2BackingInfo.", @@ -2754,12 +2397,12 @@ public class VirtualMachineMO extends BaseMO { s_logger.info("Look for disk device info from volume : " + vmdkDatastorePath + " with trimmed base name: " + trimmedSrcBaseName); } - if (devices != null && devices.size() > 0) { + if (CollectionUtils.isNotEmpty(devices)) { for (VirtualDevice device : devices) { if (device instanceof VirtualDisk) { s_logger.info("Test against disk device, controller key: " + device.getControllerKey() + ", unit number: " + device.getUnitNumber()); - VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); + VirtualDeviceBackingInfo backingInfo = device.getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; do { @@ -2772,14 +2415,14 @@ public class VirtualMachineMO extends BaseMO { String deviceNumbering = getDeviceBusName(devices, device); s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); - return new Pair((VirtualDisk)device, deviceNumbering); + return new Pair<>((VirtualDisk)device, deviceNumbering); } } else { if (backingBaseName.contains(trimmedSrcBaseName)) { String deviceNumbering = getDeviceBusName(devices, device); s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); - return new Pair((VirtualDisk)device, deviceNumbering); + return new Pair<>((VirtualDisk)device, deviceNumbering); } } @@ -2795,12 +2438,12 @@ public class VirtualMachineMO extends BaseMO { public String getDiskCurrentTopBackingFileInChain(String deviceBusName) throws Exception { List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); - if (devices != null && devices.size() > 0) { + if (CollectionUtils.isNotEmpty(devices)) { for (VirtualDevice device : devices) { if (device instanceof VirtualDisk) { s_logger.info("Test against disk device, controller key: " + device.getControllerKey() + ", unit number: " + device.getUnitNumber()); - VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); + VirtualDeviceBackingInfo backingInfo = device.getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; @@ -2815,31 +2458,15 @@ public class VirtualMachineMO extends BaseMO { return null; } - public VirtualDisk getDiskDeviceByDeviceBusName(String deviceBusName) throws Exception { - List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); - - if (devices != null && devices.size() > 0) { - for (VirtualDevice device : devices) { - if (device instanceof VirtualDisk) { - String deviceNumbering = getDeviceBusName(devices, device); - if (deviceNumbering.equals(deviceBusName)) - return (VirtualDisk)device; - } - } - } - - return null; - } - public VirtualMachineDiskInfoBuilder getDiskInfoBuilder() throws Exception { VirtualMachineDiskInfoBuilder builder = new VirtualMachineDiskInfoBuilder(); List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); - if (devices != null && devices.size() > 0) { + if (CollectionUtils.isNotEmpty(devices)) { for (VirtualDevice device : devices) { if (device instanceof VirtualDisk) { - VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); + VirtualDeviceBackingInfo backingInfo = device.getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; while (diskBackingInfo != null) { @@ -2856,16 +2483,16 @@ public class VirtualMachineMO extends BaseMO { } public List> getAllDiskDatastores() throws Exception { - List> disks = new ArrayList>(); + List> disks = new ArrayList<>(); List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); if (devices != null && devices.size() > 0) { for (VirtualDevice device : devices) { if (device instanceof VirtualDisk) { - VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); + VirtualDeviceBackingInfo backingInfo = device.getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; - disks.add(new Pair(new Integer(device.getKey()), diskBackingInfo.getDatastore())); + disks.add(new Pair(Integer.valueOf(device.getKey()), diskBackingInfo.getDatastore())); } } } @@ -3072,24 +2699,22 @@ public class VirtualMachineMO extends BaseMO { public VirtualDisk[] getAllIndependentDiskDevice() throws Exception { List independentDisks = new ArrayList(); VirtualDisk[] allDisks = getAllDiskDevice(); - if (allDisks.length > 0) { - for (VirtualDisk disk : allDisks) { - String diskMode = ""; - if (disk.getBacking() instanceof VirtualDiskFlatVer1BackingInfo) { - diskMode = ((VirtualDiskFlatVer1BackingInfo)disk.getBacking()).getDiskMode(); - } else if (disk.getBacking() instanceof VirtualDiskFlatVer2BackingInfo) { - diskMode = ((VirtualDiskFlatVer2BackingInfo)disk.getBacking()).getDiskMode(); - } else if (disk.getBacking() instanceof VirtualDiskRawDiskMappingVer1BackingInfo) { - diskMode = ((VirtualDiskRawDiskMappingVer1BackingInfo)disk.getBacking()).getDiskMode(); - } else if (disk.getBacking() instanceof VirtualDiskSparseVer1BackingInfo) { - diskMode = ((VirtualDiskSparseVer1BackingInfo)disk.getBacking()).getDiskMode(); - } else if (disk.getBacking() instanceof VirtualDiskSparseVer2BackingInfo) { - diskMode = ((VirtualDiskSparseVer2BackingInfo)disk.getBacking()).getDiskMode(); - } + for (VirtualDisk disk : allDisks) { + String diskMode = ""; + if (disk.getBacking() instanceof VirtualDiskFlatVer1BackingInfo) { + diskMode = ((VirtualDiskFlatVer1BackingInfo) disk.getBacking()).getDiskMode(); + } else if (disk.getBacking() instanceof VirtualDiskFlatVer2BackingInfo) { + diskMode = ((VirtualDiskFlatVer2BackingInfo) disk.getBacking()).getDiskMode(); + } else if (disk.getBacking() instanceof VirtualDiskRawDiskMappingVer1BackingInfo) { + diskMode = ((VirtualDiskRawDiskMappingVer1BackingInfo) disk.getBacking()).getDiskMode(); + } else if (disk.getBacking() instanceof VirtualDiskSparseVer1BackingInfo) { + diskMode = ((VirtualDiskSparseVer1BackingInfo) disk.getBacking()).getDiskMode(); + } else if (disk.getBacking() instanceof VirtualDiskSparseVer2BackingInfo) { + diskMode = ((VirtualDiskSparseVer2BackingInfo) disk.getBacking()).getDiskMode(); + } - if (diskMode.indexOf("independent") != -1) { - independentDisks.add(disk); - } + if (diskMode.indexOf("independent") != -1) { + independentDisks.add(disk); } } @@ -3102,7 +2727,7 @@ public class VirtualMachineMO extends BaseMO { if (devices != null && devices.size() > 0) { for (VirtualDevice device : devices) { if (device instanceof VirtualIDEController) { - return ((VirtualIDEController)device).getKey(); + return device.getKey(); } } } @@ -3116,7 +2741,7 @@ public class VirtualMachineMO extends BaseMO { if (devices != null && devices.size() > 0) { for (VirtualDevice device : devices) { if (device instanceof VirtualIDEController) { - return ((VirtualIDEController)device).getKey(); + return device.getKey(); } } } @@ -3126,7 +2751,7 @@ public class VirtualMachineMO extends BaseMO { } public int getIDEControllerKey(int ideUnitNumber) throws Exception { - List devices = (List)_context.getVimClient(). + List devices = _context.getVimClient(). getDynamicProperty(_mor, "config.hardware.device"); int requiredIdeController = ideUnitNumber / VmwareHelper.MAX_IDE_CONTROLLER_COUNT; @@ -3136,7 +2761,7 @@ public class VirtualMachineMO extends BaseMO { for(VirtualDevice device : devices) { if(device instanceof VirtualIDEController) { if (ideControllerCount == requiredIdeController) { - return ((VirtualIDEController)device).getKey(); + return device.getKey(); } ideControllerCount++; } @@ -3149,7 +2774,7 @@ public class VirtualMachineMO extends BaseMO { public int getNumberOfIDEDevices() throws Exception { int ideDeviceCount = 0; - List devices = (List)_context.getVimClient(). + List devices = _context.getVimClient(). getDynamicProperty(_mor, "config.hardware.device"); if (devices != null && devices.size() > 0) { @@ -3164,7 +2789,7 @@ public class VirtualMachineMO extends BaseMO { public int getFreeUnitNumberOnIDEController(int controllerKey) throws Exception { int freeUnitNumber = 0; - List devices = (List)_context.getVimClient(). + List devices = _context.getVimClient(). getDynamicProperty(_mor, "config.hardware.device"); int deviceCount = 0; @@ -3227,7 +2852,7 @@ public class VirtualMachineMO extends BaseMO { } public VirtualDevice getIsoDevice(String filename) throws Exception { - List devices = (List)_context.getVimClient(). + List devices = _context.getVimClient(). getDynamicProperty(_mor, "config.hardware.device"); if(devices != null && devices.size() > 0) { long isoDevices = devices.stream() @@ -3327,19 +2952,19 @@ public class VirtualMachineMO extends BaseMO { String attachedNetworkSummary; String dvPortGroupName; for (VirtualDevice nic : nics) { - attachedNetworkSummary = ((VirtualEthernetCard)nic).getDeviceInfo().getSummary(); + attachedNetworkSummary = nic.getDeviceInfo().getSummary(); if (attachedNetworkSummary.startsWith(networkNamePrefix)) { - return new Pair(new Integer(index), nic); + return new Pair(Integer.valueOf(index), nic); } else if (attachedNetworkSummary.endsWith("DistributedVirtualPortBackingInfo.summary") || attachedNetworkSummary.startsWith("DVSwitch")) { dvPortGroupName = getDvPortGroupName((VirtualEthernetCard)nic); if (dvPortGroupName != null && dvPortGroupName.startsWith(networkNamePrefix)) { s_logger.debug("Found a dvPortGroup already associated with public NIC."); - return new Pair(new Integer(index), nic); + return new Pair(Integer.valueOf(index), nic); } } index++; } - return new Pair(new Integer(-1), null); + return new Pair(Integer.valueOf(-1), null); } public String getDvPortGroupName(VirtualEthernetCard nic) throws Exception { @@ -3349,7 +2974,7 @@ public class VirtualMachineMO extends BaseMO { ManagedObjectReference dvPortGroupMor = new ManagedObjectReference(); dvPortGroupMor.setValue(dvPortGroupKey); dvPortGroupMor.setType("DistributedVirtualPortgroup"); - return (String)_context.getVimClient().getDynamicProperty(dvPortGroupMor, "name"); + return _context.getVimClient().getDynamicProperty(dvPortGroupMor, "name"); } public VirtualDevice[] getMatchedDevices(Class[] deviceClasses) throws Exception { @@ -3501,14 +3126,14 @@ public class VirtualMachineMO extends BaseMO { } public long getHotAddMemoryIncrementSizeInMb() throws Exception { - return (Long)_context.getVimClient().getDynamicProperty(_mor, "config.hotPlugMemoryIncrementSize"); + return _context.getVimClient().getDynamicProperty(_mor, "config.hotPlugMemoryIncrementSize"); } public long getHotAddMemoryLimitInMb() throws Exception { - return (Long)_context.getVimClient().getDynamicProperty(_mor, "config.hotPlugMemoryLimit"); + return _context.getVimClient().getDynamicProperty(_mor, "config.hotPlugMemoryLimit"); } public String getGuestId() throws Exception { - return (String)_context.getVimClient().getDynamicProperty(_mor, "config.guestId"); + return _context.getVimClient().getDynamicProperty(_mor, "config.guestId"); } public int getCoresPerSocket() throws Exception { @@ -3517,7 +3142,7 @@ public class VirtualMachineMO extends BaseMO { if (apiVersion.compareTo("5.0") < 0) { return 1; } - Integer coresPerSocket = (Integer)_context.getVimClient().getDynamicProperty(_mor, "config.hardware.numCoresPerSocket"); + Integer coresPerSocket = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.numCoresPerSocket"); return coresPerSocket != null ? coresPerSocket : 1; } @@ -3624,7 +3249,7 @@ public class VirtualMachineMO extends BaseMO { } private int getLsiLogicSasDeviceControllerKeyNoException() throws Exception { - List devices = (List)_context.getVimClient(). + List devices = _context.getVimClient(). getDynamicProperty(_mor, "config.hardware.device"); if (devices != null && devices.size() > 0) { @@ -3667,7 +3292,7 @@ public class VirtualMachineMO extends BaseMO { } private int getBusLogicDeviceControllerKeyNoException() throws Exception { - List devices = (List)_context.getVimClient(). + List devices = _context.getVimClient(). getDynamicProperty(_mor, "config.hardware.device"); if (devices != null && devices.size() > 0) { @@ -3682,7 +3307,7 @@ public class VirtualMachineMO extends BaseMO { } public Ternary getScsiControllerInfo() throws Exception { - List devices = (List)_context.getVimClient(). + List devices = _context.getVimClient(). getDynamicProperty(_mor, "config.hardware.device"); int scsiControllerCount = 0; @@ -3713,7 +3338,7 @@ public class VirtualMachineMO extends BaseMO { } public int getNumberOfVirtualDisks() throws Exception { - List devices = (List)_context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); + List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); s_logger.info("Counting disk devices attached to VM " + getVmName()); int count = 0; @@ -3729,7 +3354,7 @@ public class VirtualMachineMO extends BaseMO { } public String getExternalDiskUUID(String datastoreVolumePath) throws Exception{ - List devices = (List)_context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); + List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); if (CollectionUtils.isEmpty(devices) || datastoreVolumePath == null) { return null; } @@ -3801,7 +3426,7 @@ public class VirtualMachineMO extends BaseMO { int vmTasks = 0, vmPendingTasks = 0; for (ManagedObjectReference task : tasks) { - TaskInfo info = (TaskInfo) (_context.getVimClient().getDynamicProperty(task, "info")); + TaskInfo info = _context.getVimClient().getDynamicProperty(task, "info"); if (info.getEntityName().equals(vmName)) { vmTasks++; if (!(info.getState().equals(TaskInfoState.SUCCESS) || info.getState().equals(TaskInfoState.ERROR))) { @@ -3838,7 +3463,7 @@ public class VirtualMachineMO extends BaseMO { if (content == null || content.length == 0) { break; } - byte[] newVmdkContent = vmdkFileDescriptor.removeChangeTrackPath(content); + byte[] newVmdkContent = VmdkFileDescriptor.removeChangeTrackPath(content); Pair dcPair = getOwnerDatacenter(); String vmdkUrl = getContext().composeDatastoreBrowseUrl(dcPair.second(), diskBackingInfo.getFileName()); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index ccc68283d06..6ef0840f89a 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java @@ -16,8 +16,10 @@ // under the License. package com.cloud.hypervisor.vmware.util; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -36,19 +38,23 @@ import javax.xml.ws.handler.Handler; import javax.xml.ws.handler.HandlerResolver; import javax.xml.ws.handler.PortInfo; - import org.apache.cloudstack.utils.security.SSLUtils; import org.apache.cloudstack.utils.security.SecureSSLSocketFactory; + +import com.cloud.utils.StringUtils; + +import org.apache.log4j.Logger; + +import org.w3c.dom.Element; + import com.vmware.pbm.PbmPortType; import com.vmware.pbm.PbmService; import com.vmware.pbm.PbmServiceInstanceContent; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; -import org.w3c.dom.Element; - import com.vmware.vim25.DynamicProperty; import com.vmware.vim25.InvalidCollectorVersionFaultMsg; +import com.vmware.vim25.InvalidLocaleFaultMsg; +import com.vmware.vim25.InvalidLoginFaultMsg; import com.vmware.vim25.InvalidPropertyFaultMsg; import com.vmware.vim25.LocalizedMethodFault; import com.vmware.vim25.ManagedObjectReference; @@ -158,7 +164,7 @@ public class VmwareClient { * @throws Exception * the exception */ - public void connect(String url, String userName, String password) throws Exception { + public void connect(String url, String userName, String password) throws RuntimeFaultFaultMsg, URISyntaxException, VmwareClientException, InvalidLocaleFaultMsg, InvalidLoginFaultMsg { svcInstRef.setType(SVC_INST_NAME); svcInstRef.setValue(SVC_INST_NAME); @@ -188,7 +194,7 @@ public class VmwareClient { if (cookies == null) { String msg = "Login successful, but failed to get server cookies from url :[" + url + "]"; s_logger.error(msg); - throw new Exception(msg); + throw new VmwareClientException(msg); } } @@ -204,7 +210,7 @@ public class VmwareClient { isConnected = true; } - private void pbmConnect(String url, String cookieValue) throws Exception { + private void pbmConnect(String url, String cookieValue) throws URISyntaxException { URI uri = new URI(url); String pbmurl = "https://" + uri.getHost() + "/pbm"; String[] tokens = cookieValue.split("="); @@ -348,8 +354,8 @@ public class VmwareClient { * in case of error. */ @SuppressWarnings("unchecked") - public T getDynamicProperty(ManagedObjectReference mor, String propertyName) throws Exception { - List props = new ArrayList(); + public T getDynamicProperty(ManagedObjectReference mor, String propertyName) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, NoSuchMethodException, InvocationTargetException, IllegalAccessException { + List props = new ArrayList<>(); props.add(propertyName); List objContent = retrieveMoRefProperties(mor, props); @@ -379,7 +385,7 @@ public class VmwareClient { return (T)propertyValue; } - private List retrieveMoRefProperties(ManagedObjectReference mObj, List props) throws Exception { + private List retrieveMoRefProperties(ManagedObjectReference mObj, List props) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { PropertySpec pSpec = new PropertySpec(); pSpec.setAll(false); pSpec.setType(mObj.getType()); @@ -704,7 +710,7 @@ public class VmwareClient { * * @return First ManagedObjectReference of the type / name pair found */ - public ManagedObjectReference getDecendentMoRef(ManagedObjectReference root, String type, String name) throws Exception { + public ManagedObjectReference getDecendentMoRef(ManagedObjectReference root, String type, String name) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { if (name == null || name.length() == 0) { return null; } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClientException.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClientException.java new file mode 100644 index 00000000000..828eed4622d --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClientException.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.util; + +import com.cloud.exception.CloudException; + +public class VmwareClientException extends CloudException { + public VmwareClientException(String message, Throwable cause) { + super(message, cause); + } + + public VmwareClientException(String msg) { + super(msg); + } + // TODO embed vmware classes in this one for use downstream + public VmwareClientException(String msg, Exception embedded) { + super(msg, embedded); + } +} From ae1d7cc86091ce1612dba9981c01d63544c5c426 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Mon, 10 Feb 2025 23:20:30 +0530 Subject: [PATCH 21/21] Fix private gateway acl on static routes (#10262) --- systemvm/debian/opt/cloud/bin/cs/CsAddress.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index 3d6d1f6f722..7e7060a6cde 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -24,6 +24,7 @@ from CsDatabag import CsDataBag from CsApp import CsApache, CsDnsmasq, CsPasswdSvc from CsRoute import CsRoute from CsRule import CsRule +from CsStaticRoutes import CsStaticRoutes VRRP_TYPES = ['guest'] @@ -553,6 +554,23 @@ class CsIP: self.fw.append(["mangle", "front", "-A PREROUTING -s %s -d %s -m state --state NEW -j MARK --set-xmark %s/0xffffffff" % (self.cl.get_vpccidr(), self.address['network'], hex(100 + int(self.dev[3:])))]) + + static_routes = CsStaticRoutes("staticroutes", self.config) + if static_routes: + for item in static_routes.get_bag(): + if item == "id": + continue + static_route = static_routes.get_bag()[item] + if static_route['ip_address'] == self.address['public_ip'] and not static_route['revoke']: + self.fw.append(["mangle", "", + "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" % + (self.dev, static_route['network'], static_route['ip_address'], self.dev)]) + self.fw.append(["filter", "front", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % + (static_route['network'], self.dev, self.dev)]) + self.fw.append(["filter", "front", + "-A FORWARD -d %s -o %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % + (static_route['network'], self.dev)]) + if self.address["source_nat"]: self.fw.append(["nat", "front", "-A POSTROUTING -o %s -j SNAT --to-source %s" %