diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java index 033a338fa7d..10c50dc380b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java @@ -86,7 +86,8 @@ public class StartVMCmd extends BaseAsyncCmd implements UserCmd { type = CommandType.BOOLEAN, description = "True by default, CloudStack will firstly try to start the VM on the last host where it run on before stopping, if destination host is not specified. " + "If false, CloudStack will not consider the last host and start the VM by normal process.", - since = "4.18.0") + since = "4.18.0", + authorized = {RoleType.Admin}) private Boolean considerLastHost; @Parameter(name = ApiConstants.DEPLOYMENT_PLANNER, type = CommandType.STRING, description = "Deployment planner to use for vm allocation. Available to ROOT admin only", since = "4.4", authorized = { RoleType.Admin }) diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index ada94cd057c..82396cf4635 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -264,6 +264,8 @@ public interface VirtualMachineManager extends Manager { Pair findClusterAndHostIdForVm(long vmId); + Pair findClusterAndHostIdForVm(VirtualMachine vm, boolean skipCurrentHostForStartingVm); + /** * Obtains statistics for a list of VMs; CPU and network utilization * @param hostId ID of the host diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 1b3d914b27a..9b37132d07c 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -212,6 +212,7 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateVO; @@ -1054,6 +1055,26 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } + protected void checkAndAttemptMigrateVmAcrossCluster(final VMInstanceVO vm, final Long destinationClusterId, final Map volumePoolMap) { + if (!HypervisorType.VMware.equals(vm.getHypervisorType()) || vm.getLastHostId() == null) { + return; + } + Host lastHost = _hostDao.findById(vm.getLastHostId()); + if (destinationClusterId.equals(lastHost.getClusterId())) { + return; + } + if (volumePoolMap.values().stream().noneMatch(s -> destinationClusterId.equals(s.getClusterId()))) { + return; + } + Answer[] answer = attemptHypervisorMigration(vm, volumePoolMap, lastHost.getId()); + if (answer == null) { + s_logger.warn("Hypervisor inter-cluster migration during VM start failed"); + return; + } + // Other network related updates will be done using caller + markVolumesInPool(vm, answer); + } + @Override public void orchestrateStart(final String vmUuid, final Map params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { @@ -1227,6 +1248,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac resetVmNicsDeviceId(vm.getId()); _networkMgr.prepare(vmProfile, dest, ctx); if (vm.getHypervisorType() != HypervisorType.BareMetal) { + checkAndAttemptMigrateVmAcrossCluster(vm, cluster_id, dest.getStorageForDisks()); volumeMgr.prepare(vmProfile, dest); } @@ -2355,7 +2377,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { return _agentMgr.send(hostId, commandsContainer); } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException(String.format("Failed to migrate VM: %s", vm.getUuid()),e); + s_logger.warn(String.format("Hypervisor migration failed for the VM: %s", vm), e); } } return null; @@ -2904,7 +2926,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac * */ protected void executeManagedStorageChecksWhenTargetStoragePoolProvided(StoragePoolVO currentPool, VolumeVO volume, StoragePoolVO targetPool) { - if (!currentPool.isManaged()) { + if (!currentPool.isManaged() || currentPool.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) { return; } if (currentPool.getId() == targetPool.getId()) { @@ -5712,8 +5734,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac return new Pair<>(clusterId, hostId); } - private Pair findClusterAndHostIdForVm(VirtualMachine vm) { - Long hostId = vm.getHostId(); + @Override + public Pair findClusterAndHostIdForVm(VirtualMachine vm, boolean skipCurrentHostForStartingVm) { + Long hostId = null; + if (!skipCurrentHostForStartingVm || !State.Starting.equals(vm.getState())) { + hostId = vm.getHostId(); + } Long clusterId = null; if(hostId == null) { hostId = vm.getLastHostId(); @@ -5731,6 +5757,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac return new Pair<>(clusterId, hostId); } + private Pair findClusterAndHostIdForVm(VirtualMachine vm) { + return findClusterAndHostIdForVm(vm, false); + } + @Override public Pair findClusterAndHostIdForVm(long vmId) { VMInstanceVO vm = _vmDao.findById(vmId); diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index 742bb3dda89..15a2f2c0ac1 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -32,6 +32,8 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.framework.config.ConfigKey; @@ -46,6 +48,7 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.Spy; +import org.mockito.stubbing.Answer; import org.mockito.runners.MockitoJUnitRunner; import com.cloud.agent.AgentManager; @@ -68,6 +71,7 @@ import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; @@ -373,9 +377,26 @@ public class VirtualMachineManagerImplTest { Mockito.verify(storagePoolVoMock, Mockito.times(0)).getId(); } + @Test + public void allowVolumeMigrationsForPowerFlexStorage() { + Mockito.doReturn(true).when(storagePoolVoMock).isManaged(); + Mockito.doReturn(Storage.StoragePoolType.PowerFlex).when(storagePoolVoMock).getPoolType(); + + virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, Mockito.mock(StoragePoolVO.class)); + + Mockito.verify(storagePoolVoMock).isManaged(); + Mockito.verify(storagePoolVoMock, Mockito.times(0)).getId(); + } + @Test public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentStoragePoolEqualsTargetPool() { Mockito.doReturn(true).when(storagePoolVoMock).isManaged(); + // return any storage type except powerflex/scaleio + List values = Arrays.asList(Storage.StoragePoolType.values()); + when(storagePoolVoMock.getPoolType()).thenAnswer((Answer) invocation -> { + List filteredValues = values.stream().filter(v -> v != Storage.StoragePoolType.PowerFlex).collect(Collectors.toList()); + int randomIndex = new Random().nextInt(filteredValues.size()); + return filteredValues.get(randomIndex); }); virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, storagePoolVoMock); @@ -386,6 +407,12 @@ public class VirtualMachineManagerImplTest { @Test(expected = CloudRuntimeException.class) public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentStoragePoolNotEqualsTargetPool() { Mockito.doReturn(true).when(storagePoolVoMock).isManaged(); + // return any storage type except powerflex/scaleio + List values = Arrays.asList(Storage.StoragePoolType.values()); + when(storagePoolVoMock.getPoolType()).thenAnswer((Answer) invocation -> { + List filteredValues = values.stream().filter(v -> v != Storage.StoragePoolType.PowerFlex).collect(Collectors.toList()); + int randomIndex = new Random().nextInt(filteredValues.size()); + return filteredValues.get(randomIndex); }); virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, Mockito.mock(StoragePoolVO.class)); } @@ -838,4 +865,34 @@ public class VirtualMachineManagerImplTest { Mockito.when(templateZoneDao.findByZoneTemplate(dcId, templateId)).thenReturn(Mockito.mock(VMTemplateZoneVO.class)); virtualMachineManagerImpl.checkIfTemplateNeededForCreatingVmVolumes(vm); } + + @Test + public void checkAndAttemptMigrateVmAcrossClusterNonValid() { + // Below scenarios shouldn't result in VM migration + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.KVM); + virtualMachineManagerImpl.checkAndAttemptMigrateVmAcrossCluster(vm, 1L, new HashMap<>()); + + Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.VMware); + Mockito.when(vm.getLastHostId()).thenReturn(null); + virtualMachineManagerImpl.checkAndAttemptMigrateVmAcrossCluster(vm, 1L, new HashMap<>()); + + Long destinationClusterId = 10L; + Mockito.when(vm.getLastHostId()).thenReturn(1L); + HostVO hostVO = Mockito.mock(HostVO.class); + Mockito.when(hostVO.getClusterId()).thenReturn(destinationClusterId); + Mockito.when(hostDaoMock.findById(1L)).thenReturn(hostVO); + virtualMachineManagerImpl.checkAndAttemptMigrateVmAcrossCluster(vm, destinationClusterId, new HashMap<>()); + + destinationClusterId = 20L; + Map map = new HashMap<>(); + StoragePool pool1 = Mockito.mock(StoragePool.class); + Mockito.when(pool1.getClusterId()).thenReturn(10L); + map.put(Mockito.mock(Volume.class), pool1); + StoragePool pool2 = Mockito.mock(StoragePool.class); + Mockito.when(pool2.getClusterId()).thenReturn(null); + map.put(Mockito.mock(Volume.class), pool2); + virtualMachineManagerImpl.checkAndAttemptMigrateVmAcrossCluster(vm, destinationClusterId, map); + } } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 48de0eb016b..ffc12b98c84 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -1648,6 +1648,10 @@ public class VolumeServiceImpl implements VolumeService { newVol.setPoolType(pool.getPoolType()); newVol.setLastPoolId(lastPoolId); newVol.setPodId(pool.getPodId()); + if (volume.getPassphraseId() != null) { + newVol.setPassphraseId(volume.getPassphraseId()); + newVol.setEncryptFormat(volume.getEncryptFormat()); + } return volDao.persist(newVol); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java index c70a72f399c..0f8031e3aaa 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java @@ -21,6 +21,7 @@ import java.util.Map; import org.apache.log4j.Logger; import org.libvirt.Connect; +import org.libvirt.Library; import org.libvirt.LibvirtException; import com.cloud.hypervisor.Hypervisor; @@ -44,6 +45,7 @@ public class LibvirtConnection { if (conn == null) { s_logger.info("No existing libvirtd connection found. Opening a new one"); conn = new Connect(hypervisorURI, false); + Library.initEventLoop(); s_logger.debug("Successfully connected to libvirt at: " + hypervisorURI); s_connections.put(hypervisorURI, conn); } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 311eb670e99..5c893e5d12f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -29,27 +29,255 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; import java.util.Map; import java.util.UUID; +import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; +import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainBlockJobInfo; +import org.libvirt.DomainInfo; +import org.libvirt.TypedParameter; +import org.libvirt.TypedUlongParameter; +import org.libvirt.LibvirtException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; @ResourceWrapper(handles = MigrateVolumeCommand.class) -public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { +public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { private static final Logger LOGGER = Logger.getLogger(LibvirtMigrateVolumeCommandWrapper.class); @Override public Answer execute(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); + PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); + + MigrateVolumeAnswer answer; + if (srcPrimaryDataStore.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) { + answer = migratePowerFlexVolume(command, libvirtComputingResource); + } else { + answer = migrateRegularVolume(command, libvirtComputingResource); + } + + return answer; + } + + protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + + // Source Details + VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); + String srcPath = srcVolumeObjectTO.getPath(); + final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumeObjectTO.getPath()); + final String vmName = srcVolumeObjectTO.getVmName(); + + // Destination Details + VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); + String destPath = destVolumeObjectTO.getPath(); + final String destVolumeId = ScaleIOUtil.getVolumePath(destVolumeObjectTO.getPath()); + Map destDetails = command.getDestDetails(); + final String destSystemId = destDetails.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + String destDiskLabel = null; + + final String destDiskFileName = ScaleIOUtil.DISK_NAME_PREFIX + destSystemId + "-" + destVolumeId; + final String diskFilePath = ScaleIOUtil.DISK_PATH + File.separator + destDiskFileName; + + Domain dm = null; + try { + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + Connect conn = libvirtUtilitiesHelper.getConnection(); + dm = libvirtComputingResource.getDomain(conn, vmName); + if (dm == null) { + return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to can not find vm: " + vmName, null); + } + + DomainInfo.DomainState domainState = dm.getInfo().state ; + if (domainState != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) { + return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null); + } + + final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + PrimaryDataStoreTO spool = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore(); + KVMStoragePool pool = storagePoolMgr.getStoragePool(spool.getPoolType(), spool.getUuid()); + pool.connectPhysicalDisk(destVolumeObjectTO.getPath(), null); + + String srcSecretUUID = null; + String destSecretUUID = null; + if (ArrayUtils.isNotEmpty(destVolumeObjectTO.getPassphrase())) { + srcSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, srcVolumeObjectTO.getPath(), srcVolumeObjectTO.getPassphrase()); + destSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, destVolumeObjectTO.getPath(), destVolumeObjectTO.getPassphrase()); + } + + String diskdef = generateDestinationDiskXML(dm, srcVolumeId, diskFilePath, destSecretUUID); + destDiskLabel = generateDestinationDiskLabel(diskdef); + + TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); + TypedParameter[] parameters = new TypedParameter[1]; + parameters[0] = parameter; + + dm.blockCopy(destDiskLabel, diskdef, parameters, Domain.BlockCopyFlags.REUSE_EXT); + LOGGER.info(String.format("Block copy has started for the volume %s : %s ", destDiskLabel, srcPath)); + + return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath, libvirtComputingResource, conn, srcSecretUUID); + + } catch (Exception e) { + String msg = "Migrate volume failed due to " + e.toString(); + LOGGER.warn(msg, e); + if (destDiskLabel != null) { + try { + dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.ASYNC); + } catch (LibvirtException ex) { + LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); + } + } + return new MigrateVolumeAnswer(command, false, msg, null); + } finally { + if (dm != null) { + try { + dm.free(); + } catch (LibvirtException l) { + LOGGER.trace("Ignoring libvirt error.", l); + }; + } + } + } + + protected MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException { + int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found + int waitTimeInSec = command.getWait(); + while (waitTimeInSec > 0) { + DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(diskLabel, 0); + if (blockJobInfo != null) { + LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% current value:%s end value:%s", diskLabel, srcPath, (blockJobInfo.end == 0)? 0 : 100*(blockJobInfo.cur / (double) blockJobInfo.end), blockJobInfo.cur, blockJobInfo.end)); + if (blockJobInfo.cur == blockJobInfo.end) { + LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath)); + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT); + if (StringUtils.isNotEmpty(srcSecretUUID)) { + libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID); + } + break; + } + } else { + LOGGER.info("Failed to get the block copy status, trying to abort the job"); + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC); + } + waitTimeInSec--; + + try { + Thread.sleep(timeBetweenTries); + } catch (Exception ex) { + // don't do anything + } + } + + if (waitTimeInSec <= 0) { + String msg = "Block copy is taking long time, failing the job"; + LOGGER.error(msg); + try { + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC); + } catch (LibvirtException ex) { + LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); + } + return new MigrateVolumeAnswer(command, false, msg, null); + } + + return new MigrateVolumeAnswer(command, true, null, destPath); + } + + private String generateDestinationDiskLabel(String diskXml) throws ParserConfigurationException, IOException, SAXException { + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(new ByteArrayInputStream(diskXml.getBytes("UTF-8"))); + doc.getDocumentElement().normalize(); + + Element disk = doc.getDocumentElement(); + String diskLabel = getAttrValue("target", "dev", disk); + + return diskLabel; + } + + protected String generateDestinationDiskXML(Domain dm, String srcVolumeId, String diskFilePath, String destSecretUUID) throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException { + final String domXml = dm.getXMLDesc(0); + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(new ByteArrayInputStream(domXml.getBytes("UTF-8"))); + doc.getDocumentElement().normalize(); + + NodeList disks = doc.getElementsByTagName("disk"); + + for (int i = 0; i < disks.getLength(); i++) { + Element disk = (Element)disks.item(i); + String type = disk.getAttribute("type"); + if (!type.equalsIgnoreCase("network")) { + String diskDev = getAttrValue("source", "dev", disk); + if (StringUtils.isNotEmpty(diskDev) && diskDev.contains(srcVolumeId)) { + setAttrValue("source", "dev", diskFilePath, disk); + if (StringUtils.isNotEmpty(destSecretUUID)) { + setAttrValue("secret", "uuid", destSecretUUID, disk); + } + StringWriter diskSection = new StringWriter(); + Transformer xformer = TransformerFactory.newInstance().newTransformer(); + xformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + xformer.transform(new DOMSource(disk), new StreamResult(diskSection)); + + return diskSection.toString(); + } + } + } + + return null; + } + + private static String getAttrValue(String tag, String attr, Element eElement) { + NodeList tagNode = eElement.getElementsByTagName(tag); + if (tagNode.getLength() == 0) { + return null; + } + Element node = (Element)tagNode.item(0); + return node.getAttribute(attr); + } + + private static void setAttrValue(String tag, String attr, String newValue, Element eElement) { + NodeList tagNode = eElement.getElementsByTagName(tag); + if (tagNode.getLength() == 0) { + return; + } + Element node = (Element)tagNode.item(0); + node.setAttribute(attr, newValue); + } + + protected MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { KVMStoragePoolManager storagePoolManager = libvirtComputingResource.getStoragePoolMgr(); VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); Map srcDetails = command.getSrcDetails(); - String srcPath = srcDetails != null ? srcDetails.get(DiskTO.IQN) : srcVolumeObjectTO.getPath(); VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index cae872e287f..a8a7d6f5694 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -37,6 +37,7 @@ import java.util.UUID; import javax.naming.ConfigurationException; import com.cloud.storage.ScopeType; +import com.cloud.storage.Volume; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand; @@ -2448,7 +2449,12 @@ public class KVMStorageProcessor implements StorageProcessor { destPool = storagePoolMgr.getStoragePool(destPrimaryStore.getPoolType(), destPrimaryStore.getUuid()); try { - storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds()); + if (srcVol.getPassphrase() != null && srcVol.getVolumeType().equals(Volume.Type.ROOT)) { + volume.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); + storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds(), srcVol.getPassphrase(), destVol.getPassphrase(), srcVol.getProvisioningType()); + } else { + storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds()); + } } catch (Exception e) { // Any exceptions while copying the disk, should send failed answer with the error message String errMsg = String.format("Failed to copy volume: %s to dest storage: %s, due to %s", srcVol.getName(), destPrimaryStore.getName(), e.toString()); s_logger.debug(errMsg, e); @@ -2467,6 +2473,7 @@ public class KVMStorageProcessor implements StorageProcessor { String path = destPrimaryStore.isManaged() ? destVolumeName : destVolumePath + File.separator + destVolumeName; newVol.setPath(path); newVol.setFormat(destFormat); + newVol.setEncryptFormat(destVol.getEncryptFormat()); return new CopyCmdAnswer(newVol); } catch (final CloudRuntimeException e) { s_logger.debug("Failed to copyVolumeFromPrimaryToPrimary: ", e); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index 09c7e146e49..607dd620f61 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -387,6 +387,7 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { boolean forceSourceFormat = srcQemuFile.getFormat() == QemuImg.PhysicalDiskFormat.RAW; LOGGER.debug(String.format("Starting copy from source disk %s(%s) to PowerFlex volume %s(%s), forcing source format is %b", srcQemuFile.getFileName(), srcQemuFile.getFormat(), destQemuFile.getFileName(), destQemuFile.getFormat(), forceSourceFormat)); + qemuImageOpts.setImageOptsFlag(true); qemu.convert(srcQemuFile, destQemuFile, options, qemuObjects, qemuImageOpts,null, forceSourceFormat); LOGGER.debug("Successfully converted source disk image " + srcQemuFile.getFileName() + " to PowerFlex volume: " + destDisk.getPath()); diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java index f9a2e4ba652..4a577ef3400 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java @@ -30,6 +30,7 @@ public class QemuImageOptions { private static final String LUKS_KEY_SECRET_PARAM_KEY = "key-secret"; private static final String QCOW2_KEY_SECRET_PARAM_KEY = "encrypt.key-secret"; private static final String DRIVER = "driver"; + private boolean addImageOpts = false; private QemuImg.PhysicalDiskFormat format; private static final List DISK_FORMATS_THAT_SUPPORT_OPTION_IMAGE_OPTS = Arrays.asList(QemuImg.PhysicalDiskFormat.QCOW2, QemuImg.PhysicalDiskFormat.LUKS); @@ -71,13 +72,19 @@ public class QemuImageOptions { } } + public void setImageOptsFlag(boolean addImageOpts) { + this.addImageOpts = addImageOpts; + } + /** * Converts QemuImageOptions into the command strings required by qemu-img flags * @return array of strings representing command flag and value (--image-opts) */ public String[] toCommandFlag() { - if (format == null || !DISK_FORMATS_THAT_SUPPORT_OPTION_IMAGE_OPTS.contains(format)) { - return new String[] { params.get(FILENAME_PARAM_KEY) }; + if (!addImageOpts) { + if (format == null || !DISK_FORMATS_THAT_SUPPORT_OPTION_IMAGE_OPTS.contains(format)) { + return new String[] { params.get(FILENAME_PARAM_KEY) }; + } } Map sorted = new TreeMap<>(params); String paramString = Joiner.on(",").withKeyValueSeparator("=").join(sorted); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java new file mode 100644 index 00000000000..c278144b4e1 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java @@ -0,0 +1,388 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.MigrateVolumeAnswer; +import com.cloud.agent.api.storage.MigrateVolumeCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage; +import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainBlockJobInfo; +import org.libvirt.DomainInfo; +import org.libvirt.LibvirtException; +import org.libvirt.TypedParameter; +import org.mockito.InjectMocks; +import org.mockito.Matchers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.runners.MockitoJUnitRunner; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtMigrateVolumeCommandWrapperTest { + + @Spy + @InjectMocks + private LibvirtMigrateVolumeCommandWrapper libvirtMigrateVolumeCommandWrapper; + + @Mock + MigrateVolumeCommand command; + + @Mock + LibvirtComputingResource libvirtComputingResource; + + @Mock + LibvirtUtilitiesHelper libvirtUtilitiesHelper; + + private String domxml = "\n" + + " i-2-27-VM\n" + + " 2d37fe1a-621a-4903-9ab5-5c9544c733f8\n" + + " Ubuntu 18.04 LTS\n" + + " 524288\n" + + " 524288\n" + + " 1\n" + + " \n" + + " 256\n" + + " \n" + + " \n" + + " /machine\n" + + " \n" + + " \n" + + " \n" + + " Apache Software Foundation\n" + + " CloudStack KVM Hypervisor\n" + + " 2d37fe1a-621a-4903-9ab5-5c9544c733f8\n" + + " \n" + + " \n" + + " \n" + + " hvm\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " qemu64\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " destroy\n" + + " restart\n" + + " destroy\n" + + " \n" + + " /usr/libexec/qemu-kvm\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 38a54bf719f24af6b070\n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 0ceeb7c643b447aba5ce\n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "