From cbd2b5a022476e8b85f8ce745d01db5370fb185f Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Thu, 19 Jun 2025 04:03:58 -0400 Subject: [PATCH 01/23] Add check for ldap truststore password (#11055) --- .../cloudstack/ldap/LdapContextFactory.java | 34 +++++++++++++++++-- .../cloudstack/ldap/LdapManagerImpl.java | 5 +++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java index 0161adf9fda..ee48e8cc027 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.ldap; +import java.io.FileInputStream; import java.io.IOException; import java.util.Hashtable; @@ -24,6 +25,7 @@ import javax.naming.Context; import javax.naming.NamingException; import javax.naming.ldap.InitialLdapContext; import javax.naming.ldap.LdapContext; +import java.security.KeyStore; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -72,8 +74,36 @@ public class LdapContextFactory { if (sslStatus) { s_logger.info("LDAP SSL enabled."); environment.put(Context.SECURITY_PROTOCOL, "ssl"); - System.setProperty("javax.net.ssl.trustStore", _ldapConfiguration.getTrustStore(domainId)); - System.setProperty("javax.net.ssl.trustStorePassword", _ldapConfiguration.getTrustStorePassword(domainId)); + String trustStore = _ldapConfiguration.getTrustStore(domainId); + String trustStorePassword = _ldapConfiguration.getTrustStorePassword(domainId); + + if (!validateTrustStore(trustStore, trustStorePassword)) { + throw new RuntimeException("Invalid truststore or truststore password"); + } + + System.setProperty("javax.net.ssl.trustStore", trustStore); + System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword); + } + } + + private boolean validateTrustStore(String trustStore, String trustStorePassword) { + if (trustStore == null) { + return true; + } + + if (trustStorePassword == null) { + return false; + } + + try { + KeyStore.getInstance("JKS").load( + new FileInputStream(trustStore), + trustStorePassword.toCharArray() + ); + return true; + } catch (Exception e) { + s_logger.warn("Failed to validate truststore: " + e.getMessage()); + return false; } } diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java index 6ed79a0c69f..352e439b50c 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java @@ -186,6 +186,11 @@ public class LdapManagerImpl extends ComponentLifecycleBase implements LdapManag } catch (NamingException | IOException e) { LOGGER.debug("NamingException while doing an LDAP bind", e); throw new InvalidParameterValueException("Unable to bind to the given LDAP server"); + } catch (RuntimeException e) { + if (e.getMessage().contains("Invalid truststore")) { + throw new InvalidParameterValueException("Invalid truststore or truststore password"); + } + throw e; } finally { closeContext(context); } From 544028ca8e6b80f7cdd1c9b9e0a2156f8fd69789 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 2 Jul 2025 15:45:32 +0530 Subject: [PATCH 02/23] Do not rely on Memory engine even transiently in DB setup scripts (#11106) It is not safe for use with replication, and is straight up incompatible with highly-available active-active type MySQL distributions such as Galera Co-authored-by: Tristan Deloche --- setup/db/create-schema.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index 3f14fccd010..10141185eb4 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -398,7 +398,7 @@ CREATE TABLE `cloud`.`op_lock` ( `waiters` int NOT NULL DEFAULT 0 COMMENT 'How many have the thread acquired this lock (reentrant)', PRIMARY KEY (`key`), INDEX `i_op_lock__mac_ip_thread`(`mac`, `ip`, `thread`) -) ENGINE=Memory DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`configuration` ( `category` varchar(255) NOT NULL DEFAULT 'Advanced', @@ -1793,7 +1793,7 @@ CREATE TABLE `cloud`.`op_nwgrp_work` ( INDEX `i_op_nwgrp_work__taken`(`taken`), INDEX `i_op_nwgrp_work__step`(`step`), INDEX `i_op_nwgrp_work__seq_no`(`seq_no`) -) ENGINE=MEMORY DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`op_vm_ruleset_log` ( `id` bigint unsigned UNIQUE NOT NULL AUTO_INCREMENT COMMENT 'id', From 3b54194aef18c44a37489614ce3391eceab55b2f Mon Sep 17 00:00:00 2001 From: Harikrishna Date: Thu, 3 Jul 2025 13:15:35 +0530 Subject: [PATCH 03/23] Correct quota type indexes (#11085) --- ui/src/utils/quota.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/utils/quota.js b/ui/src/utils/quota.js index 485c99131d2..b8adbb93518 100644 --- a/ui/src/utils/quota.js +++ b/ui/src/utils/quota.js @@ -107,7 +107,7 @@ export const QUOTA_TYPES = [ }, { id: 29, - type: 'VPC' + type: 'BUCKET' }, { id: 30, @@ -115,7 +115,7 @@ export const QUOTA_TYPES = [ }, { id: 31, - type: 'BACKUP_OBJECT' + type: 'VPC' } ] From c24e4eea855bd21e72ce7aa182fada2671dbe857 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 3 Jul 2025 13:20:36 +0530 Subject: [PATCH 04/23] server: fix orphan db transaction issue (#11095) Signed-off-by: Abhishek Kumar --- .../com/cloud/alert/AlertManagerImpl.java | 32 ++++++++++++----- .../com/cloud/storage/StorageManagerImpl.java | 36 +++++++++++-------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java index f35a0664a85..377b2134d78 100644 --- a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java +++ b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java @@ -89,6 +89,10 @@ import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; + import org.jetbrains.annotations.Nullable; public class AlertManagerImpl extends ManagerBase implements AlertManager, Configurable { @@ -290,8 +294,13 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi Math.min(CapacityManager.CapacityCalculateWorkers.value(), hostIds.size()))); for (Long hostId : hostIds) { futures.put(hostId, executorService.submit(() -> { - final HostVO host = hostDao.findById(hostId); - _capacityMgr.updateCapacityForHost(host); + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final HostVO host = hostDao.findById(hostId); + _capacityMgr.updateCapacityForHost(host); + } + }); return null; })); } @@ -316,13 +325,18 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi Math.min(CapacityManager.CapacityCalculateWorkers.value(), storagePoolIds.size()))); for (Long poolId: storagePoolIds) { futures.put(poolId, executorService.submit(() -> { - final StoragePoolVO pool = _storagePoolDao.findById(poolId); - long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null); - if (pool.isShared()) { - _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk); - } else { - _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk); - } + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final StoragePoolVO pool = _storagePoolDao.findById(poolId); + long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null); + if (pool.isShared()) { + _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk); + } else { + _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk); + } + } + }); return null; })); } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index f9199b94288..90113f66aaf 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -257,6 +257,7 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; @@ -1591,22 +1592,27 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if (exceptionOccurred.get()) { return null; } - HostVO host = _hostDao.findById(hostId); - try { - connectHostToSharedPool(host, primaryStore.getId()); - poolHostIds.add(hostId); - } catch (Exception e) { - if (handleExceptionsPartially && e.getCause() instanceof StorageConflictException) { - exceptionOccurred.set(true); - throw e; + Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) throws Exception { + HostVO host = _hostDao.findById(hostId); + try { + connectHostToSharedPool(host, primaryStore.getId()); + poolHostIds.add(hostId); + } catch (Exception e) { + if (handleExceptionsPartially && e.getCause() instanceof StorageConflictException) { + exceptionOccurred.set(true); + throw e; + } + logger.warn("Unable to establish a connection between {} and {}", host, primaryStore, e); + String reason = getStoragePoolMountFailureReason(e.getMessage()); + if (handleExceptionsPartially && reason != null) { + exceptionOccurred.set(true); + throw new CloudRuntimeException(reason); + } + } } - logger.warn("Unable to establish a connection between {} and {}", host, primaryStore, e); - String reason = getStoragePoolMountFailureReason(e.getMessage()); - if (handleExceptionsPartially && reason != null) { - exceptionOccurred.set(true); - throw new CloudRuntimeException(reason); - } - } + }); return null; })); } From 1a251c8b7899f5cee9d586047aaa469d9bec4ea6 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Thu, 3 Jul 2025 12:55:21 +0200 Subject: [PATCH 05/23] merge forward fix --- .../org/apache/cloudstack/ldap/LdapContextFactory.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java index 4199b0757d8..1c759c6a45e 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java @@ -54,14 +54,14 @@ public class LdapContextFactory { return createInitialDirContext(bindPrincipal, bindPassword, providerUrl, true, domainId); } - private LdapContext createInitialDirContext(final String principal, final String password, final boolean isSystemContext, Long domainId) throws NamingException, IOException { + private LdapContext createInitialDirContext(final String principal, final String password, final boolean isSystemContext, Long domainId) throws NamingException { return createInitialDirContext(principal, password, null, isSystemContext, domainId); } private LdapContext createInitialDirContext(final String principal, final String password, final String providerUrl, final boolean isSystemContext, Long domainId) - throws NamingException, IOException { + throws NamingException { Hashtable environment = getEnvironment(principal, password, providerUrl, isSystemContext, domainId); - logger.debug("initializing ldap with provider url: " + environment.get(Context.PROVIDER_URL)); + logger.debug("initializing ldap with provider url: {}", environment.get(Context.PROVIDER_URL)); return new InitialLdapContext(environment, null); } @@ -103,7 +103,7 @@ public class LdapContextFactory { ); return true; } catch (Exception e) { - s_logger.warn("Failed to validate truststore: " + e.getMessage()); + logger.warn("Failed to validate truststore: {}", e.getMessage()); return false; } } From ed7bd5e5804c2d834b061f64bfd2d711e1ae5715 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 3 Jul 2025 19:00:42 +0530 Subject: [PATCH 06/23] ui: fix handler for deploy button menu (#11116) Signed-off-by: Abhishek Kumar --- ui/src/views/compute/DeployVM.vue | 4 +- ui/src/views/compute/DeployVnfAppliance.vue | 4 +- ui/src/views/compute/wizard/DeployButtons.vue | 47 +++++++++---------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/ui/src/views/compute/DeployVM.vue b/ui/src/views/compute/DeployVM.vue index 73ea3fcc59d..d61c9e08ea9 100644 --- a/ui/src/views/compute/DeployVM.vue +++ b/ui/src/views/compute/DeployVM.vue @@ -845,7 +845,7 @@ :deployButtonMenuOptions="deployMenuOptions" @handle-cancel="() => $router.back()" @handle-deploy="handleSubmit" - @handle-deploy-menu="handleSubmitAndStay" /> + @handle-deploy-menu="(index, e) => handleSubmitAndStay(e)" /> @@ -860,7 +860,7 @@ :deployButtonMenuOptions="deployMenuOptions" @handle-cancel="() => $router.back()" @handle-deploy="handleSubmit" - @handle-deploy-menu="handleSubmitAndStay" /> + @handle-deploy-menu="(index, e) => handleSubmitAndStay(e)" /> diff --git a/ui/src/views/compute/DeployVnfAppliance.vue b/ui/src/views/compute/DeployVnfAppliance.vue index 49a2a9ef8f3..b7e797ee57c 100644 --- a/ui/src/views/compute/DeployVnfAppliance.vue +++ b/ui/src/views/compute/DeployVnfAppliance.vue @@ -825,7 +825,7 @@ :deployButtonMenuOptions="deployMenuOptions" @handle-cancel="() => $router.back()" @handle-deploy="handleSubmit" - @handle-deploy-menu="handleSubmitAndStay" /> + @handle-deploy-menu="(index, e) => handleSubmitAndStay(e)" /> @@ -840,7 +840,7 @@ :deployButtonMenuOptions="deployMenuOptions" @handle-cancel="() => $router.back()" @handle-deploy="handleSubmit" - @handle-deploy-menu="handleSubmitAndStay" /> + @handle-deploy-menu="(index, e) => handleSubmitAndStay(e)" /> diff --git a/ui/src/views/compute/wizard/DeployButtons.vue b/ui/src/views/compute/wizard/DeployButtons.vue index 2bd2408e4ed..6fc9aee096d 100644 --- a/ui/src/views/compute/wizard/DeployButtons.vue +++ b/ui/src/views/compute/wizard/DeployButtons.vue @@ -86,7 +86,7 @@ export default { this.$emit('handle-deploy', e) }, handleMenu (e) { - this.$emit('handle-deploy-menu', e.key - 1) + this.$emit('handle-deploy-menu', e.key - 1, e) } } } @@ -94,37 +94,36 @@ export default { From 80f46ad55d63bea88da0d1d38b00b045499aa105 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 4 Jul 2025 13:54:54 +0530 Subject: [PATCH 07/23] [VMware to KVM Migration] Fix for converted instance npe issue when source vmware instance ovf is exported from management server (#11003) --- .../com/cloud/storage/VolumeApiService.java | 7 + .../agent/api/ConvertInstanceAnswer.java | 16 -- .../agent/api/ConvertInstanceCommand.java | 11 +- .../LibvirtConvertInstanceCommandWrapper.java | 205 +----------------- ...virtConvertInstanceCommandWrapperTest.java | 70 ------ .../vm/UnmanagedVMsManagerImpl.java | 59 +---- .../vm/UnmanagedVMsManagerImplTest.java | 2 - 7 files changed, 20 insertions(+), 350 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index bb69b5b6650..4182728c204 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -171,6 +171,13 @@ public interface VolumeApiService { * */ boolean doesStoragePoolSupportDiskOffering(StoragePool destPool, DiskOffering diskOffering); + + /** + * Checks if the storage pool supports the required disk offering tags + * destPool the storage pool to check the disk offering tags + * diskOfferingTags the tags that should be supported + * return whether the tags are supported in the storage pool + */ boolean doesStoragePoolSupportDiskOfferingTags(StoragePool destPool, String diskOfferingTags); Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge); diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java index 174348f4f18..8092ab9b43f 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java @@ -16,8 +16,6 @@ // under the License. package com.cloud.agent.api; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; - public class ConvertInstanceAnswer extends Answer { private String temporaryConvertUuid; @@ -25,16 +23,6 @@ public class ConvertInstanceAnswer extends Answer { public ConvertInstanceAnswer() { super(); } - private UnmanagedInstanceTO convertedInstance; - - public ConvertInstanceAnswer(Command command, boolean success, String details) { - super(command, success, details); - } - - public ConvertInstanceAnswer(Command command, UnmanagedInstanceTO convertedInstance) { - super(command, true, ""); - this.convertedInstance = convertedInstance; - } public ConvertInstanceAnswer(Command command, String temporaryConvertUuid) { super(command, true, ""); @@ -44,8 +32,4 @@ public class ConvertInstanceAnswer extends Answer { public String getTemporaryConvertUuid() { return temporaryConvertUuid; } - - public UnmanagedInstanceTO getConvertedInstance() { - return convertedInstance; - } } diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java index b8250903f85..f938d0ac55d 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -20,13 +20,10 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.RemoteInstanceTO; import com.cloud.hypervisor.Hypervisor; -import java.util.List; - public class ConvertInstanceCommand extends Command { private RemoteInstanceTO sourceInstance; private Hypervisor.HypervisorType destinationHypervisorType; - private List destinationStoragePools; private DataStoreTO conversionTemporaryLocation; private String templateDirOnConversionLocation; private boolean checkConversionSupport; @@ -36,12 +33,10 @@ public class ConvertInstanceCommand extends Command { public ConvertInstanceCommand() { } - public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, - List destinationStoragePools, DataStoreTO conversionTemporaryLocation, + public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, DataStoreTO conversionTemporaryLocation, String templateDirOnConversionLocation, boolean checkConversionSupport, boolean exportOvfToConversionLocation) { this.sourceInstance = sourceInstance; this.destinationHypervisorType = destinationHypervisorType; - this.destinationStoragePools = destinationStoragePools; this.conversionTemporaryLocation = conversionTemporaryLocation; this.templateDirOnConversionLocation = templateDirOnConversionLocation; this.checkConversionSupport = checkConversionSupport; @@ -56,10 +51,6 @@ public class ConvertInstanceCommand extends Command { return destinationHypervisorType; } - public List getDestinationStoragePools() { - return destinationStoragePools; - } - public DataStoreTO getConversionTemporaryLocation() { return conversionTemporaryLocation; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java index a11730a1240..e79a8da9dda 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -18,22 +18,12 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.net.URLEncoder; import java.nio.charset.Charset; -import java.util.ArrayList; import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import com.cloud.agent.api.Answer; @@ -44,17 +34,12 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.RemoteInstanceTO; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; -import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; -import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; -import com.cloud.storage.Storage; import com.cloud.utils.FileUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; @@ -77,7 +62,7 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) { - List disksDefs = xmlParser.getDisks(); - disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE && - x.getDeviceType() == LibvirtVMDef.DiskDef.DeviceType.DISK).collect(Collectors.toList()); - if (CollectionUtils.isEmpty(disksDefs)) { - String err = String.format("Cannot find any disk defined on the converted XML domain %s.xml", convertedBasePath); - logger.error(err); - throw new CloudRuntimeException(err); - } - sanitizeDisksPath(disksDefs); - return getPhysicalDisksFromDefPaths(disksDefs, pool); - } - - private List getPhysicalDisksFromDefPaths(List disksDefs, KVMStoragePool pool) { - List disks = new ArrayList<>(); - for (LibvirtVMDef.DiskDef diskDef : disksDefs) { - KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(diskDef.getDiskPath()); - disks.add(physicalDisk); - } - return disks; - } - - protected List getTemporaryDisksWithPrefixFromTemporaryPool(KVMStoragePool pool, String path, String prefix) { - String msg = String.format("Could not parse correctly the converted XML domain, checking for disks on %s with prefix %s", path, prefix); - logger.info(msg); - pool.refresh(); - List disksWithPrefix = pool.listPhysicalDisks() - .stream() - .filter(x -> x.getName().startsWith(prefix) && !x.getName().endsWith(".xml")) - .collect(Collectors.toList()); - if (CollectionUtils.isEmpty(disksWithPrefix)) { - msg = String.format("Could not find any converted disk with prefix %s on temporary location %s", prefix, path); - logger.error(msg); - throw new CloudRuntimeException(msg); - } - return disksWithPrefix; - } - - private void cleanupDisksAndDomainFromTemporaryLocation(List disks, - KVMStoragePool temporaryStoragePool, - String temporaryConvertUuid) { - for (KVMPhysicalDisk disk : disks) { - logger.info(String.format("Cleaning up temporary disk %s after conversion from temporary location", disk.getName())); - temporaryStoragePool.deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2); - } - logger.info(String.format("Cleaning up temporary domain %s after conversion from temporary location", temporaryConvertUuid)); - FileUtil.deleteFiles(temporaryStoragePool.getLocalPath(), temporaryConvertUuid, ".xml"); - } - protected void sanitizeDisksPath(List disks) { for (LibvirtVMDef.DiskDef disk : disks) { String[] diskPathParts = disk.getDiskPath().split("/"); @@ -262,114 +198,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper moveTemporaryDisksToDestination(List temporaryDisks, - List destinationStoragePools, - KVMStoragePoolManager storagePoolMgr) { - List targetDisks = new ArrayList<>(); - if (temporaryDisks.size() != destinationStoragePools.size()) { - String warn = String.format("Discrepancy between the converted instance disks (%s) " + - "and the expected number of disks (%s)", temporaryDisks.size(), destinationStoragePools.size()); - logger.warn(warn); - } - for (int i = 0; i < temporaryDisks.size(); i++) { - String poolPath = destinationStoragePools.get(i); - KVMStoragePool destinationPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, poolPath); - if (destinationPool == null) { - String err = String.format("Could not find a storage pool by URI: %s", poolPath); - logger.error(err); - continue; - } - if (destinationPool.getType() != Storage.StoragePoolType.NetworkFilesystem) { - String err = String.format("Storage pool by URI: %s is not an NFS storage", poolPath); - logger.error(err); - continue; - } - KVMPhysicalDisk sourceDisk = temporaryDisks.get(i); - if (logger.isDebugEnabled()) { - String msg = String.format("Trying to copy converted instance disk number %s from the temporary location %s" + - " to destination storage pool %s", i, sourceDisk.getPool().getLocalPath(), destinationPool.getUuid()); - logger.debug(msg); - } - - String destinationName = UUID.randomUUID().toString(); - - KVMPhysicalDisk destinationDisk = storagePoolMgr.copyPhysicalDisk(sourceDisk, destinationName, destinationPool, 7200 * 1000); - targetDisks.add(destinationDisk); - } - return targetDisks; - } - - private UnmanagedInstanceTO getConvertedUnmanagedInstance(String baseName, - List vmDisks, - LibvirtDomainXMLParser xmlParser) { - UnmanagedInstanceTO instanceTO = new UnmanagedInstanceTO(); - instanceTO.setName(baseName); - instanceTO.setDisks(getUnmanagedInstanceDisks(vmDisks, xmlParser)); - instanceTO.setNics(getUnmanagedInstanceNics(xmlParser)); - return instanceTO; - } - - private List getUnmanagedInstanceNics(LibvirtDomainXMLParser xmlParser) { - List nics = new ArrayList<>(); - if (xmlParser != null) { - List interfaces = xmlParser.getInterfaces(); - for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) { - UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic(); - nic.setMacAddress(interfaceDef.getMacAddress()); - nic.setNicId(interfaceDef.getBrName()); - nic.setAdapterType(interfaceDef.getModel().toString()); - nics.add(nic); - } - } - return nics; - } - - protected List getUnmanagedInstanceDisks(List vmDisks, LibvirtDomainXMLParser xmlParser) { - List instanceDisks = new ArrayList<>(); - List diskDefs = xmlParser != null ? xmlParser.getDisks() : null; - for (int i = 0; i< vmDisks.size(); i++) { - KVMPhysicalDisk physicalDisk = vmDisks.get(i); - KVMStoragePool storagePool = physicalDisk.getPool(); - UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk(); - disk.setPosition(i); - Pair storagePoolHostAndPath = getNfsStoragePoolHostAndPath(storagePool); - disk.setDatastoreHost(storagePoolHostAndPath.first()); - disk.setDatastorePath(storagePoolHostAndPath.second()); - disk.setDatastoreName(storagePool.getUuid()); - disk.setDatastoreType(storagePool.getType().name()); - disk.setCapacity(physicalDisk.getVirtualSize()); - disk.setFileBaseName(physicalDisk.getName()); - if (CollectionUtils.isNotEmpty(diskDefs)) { - LibvirtVMDef.DiskDef diskDef = diskDefs.get(i); - disk.setController(diskDef.getBusType() != null ? diskDef.getBusType().toString() : LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); - } else { - // If the job is finished but we cannot parse the XML, the guest VM can use the virtio driver - disk.setController(LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); - } - instanceDisks.add(disk); - } - return instanceDisks; - } - - protected Pair getNfsStoragePoolHostAndPath(KVMStoragePool storagePool) { - String sourceHostIp = null; - String sourcePath = null; - List commands = new ArrayList<>(); - commands.add(new String[]{Script.getExecutableAbsolutePath("mount")}); - commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), storagePool.getLocalPath()}); - String storagePoolMountPoint = Script.executePipedCommands(commands, 0).second(); - logger.debug(String.format("NFS Storage pool: %s - local path: %s, mount point: %s", storagePool.getUuid(), storagePool.getLocalPath(), storagePoolMountPoint)); - if (StringUtils.isNotEmpty(storagePoolMountPoint)) { - String[] res = storagePoolMountPoint.strip().split(" "); - res = res[0].split(":"); - if (res.length > 1) { - sourceHostIp = res[0].strip(); - sourcePath = res[1].strip(); - } - } - return new Pair<>(sourceHostIp, sourcePath); - } - private boolean exportOVAFromVMOnVcenter(String vmExportUrl, String targetOvfDir, int noOfThreads, @@ -412,27 +240,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper disks = List.of(sourceDisk); - String destinationPoolUuid = UUID.randomUUID().toString(); - List destinationPools = List.of(destinationPoolUuid); - - KVMPhysicalDisk destDisk = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(destDisk.getPath()).thenReturn("xyz"); - Mockito.when(storagePoolManager.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, destinationPoolUuid)) - .thenReturn(destinationPool); - Mockito.when(destinationPool.getType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); - Mockito.when(storagePoolManager.copyPhysicalDisk(Mockito.eq(sourceDisk), Mockito.anyString(), Mockito.eq(destinationPool), Mockito.anyInt())) - .thenReturn(destDisk); - - List movedDisks = convertInstanceCommandWrapper.moveTemporaryDisksToDestination(disks, destinationPools, storagePoolManager); - Assert.assertEquals(1, movedDisks.size()); - Assert.assertEquals("xyz", movedDisks.get(0).getPath()); - } - - @Test - public void testGetUnmanagedInstanceDisks() { - try (MockedStatic