From 0d3ac9f8d9604575065d7841cd021f222a63551e Mon Sep 17 00:00:00 2001 From: dahn Date: Fri, 9 Jun 2023 10:29:42 +0200 Subject: [PATCH 1/7] interpret /etc/redhet-release better (#7570) --- scripts/vm/hypervisor/versions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/vm/hypervisor/versions.sh b/scripts/vm/hypervisor/versions.sh index ea27dc18e01..bf2029e5959 100755 --- a/scripts/vm/hypervisor/versions.sh +++ b/scripts/vm/hypervisor/versions.sh @@ -30,9 +30,9 @@ REV="X.Y" CODENAME="" function get_from_redhat_release { - DIST=`cat /etc/redhat-release | awk '{print $1}'` + DIST=`cat /etc/redhat-release | awk -F 'release' '{print $1}'` CODENAME=`cat /etc/redhat-release | sed s/.*\(// | sed s/\)//` - REV=`cat /etc/redhat-release | awk '{print $3,$4}' | grep -o "[0-9.]*"` + REV=`cat /etc/redhat-release | awk -F 'release' '{print $2}' | awk '{print $1}'` } function get_from_lsb_release { From 4ef7ebbded87bcc5db04678d0b63e590aca7e128 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 12 Jun 2023 12:32:42 +0530 Subject: [PATCH 2/7] ssvm: pass all accessible secondary storage to ssvm (#7410) * ssvm: pass all accessible secondary storage to ssvm Fixes #7162 Signed-off-by: Abhishek Kumar * changes Signed-off-by: Abhishek Kumar * test Signed-off-by: Abhishek Kumar * license Signed-off-by: Abhishek Kumar --------- Signed-off-by: Abhishek Kumar --- .../SecondaryStorageManagerImpl.java | 49 ++++++---- .../SecondaryStorageManagerImplTest.java | 89 +++++++++++++++++++ systemvm/agent/scripts/ssvm-check.sh | 27 +++--- 3 files changed, 139 insertions(+), 26 deletions(-) create mode 100644 services/secondary-storage/controller/src/test/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImplTest.java diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index aa03211d9c0..f93d3e28a38 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -1065,11 +1066,12 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar Map details = _vmDetailsDao.listDetailsKeyPairs(vm.getId()); vm.setDetails(details); - DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dest.getDataCenter().getId()); - if (secStore == null) { + List secStores= _dataStoreMgr.listImageStoresWithFreeCapacity(dest.getDataCenter().getId()); + if (CollectionUtils.isEmpty(secStores)) { s_logger.warn(String.format("Unable to finalize virtual machine profile [%s] as it has no secondary storage available to satisfy storage needs for zone [%s].", profile.toString(), dest.getDataCenter().getUuid())); return false; } + Collections.shuffle(secStores); final Map sshAccessDetails = _networkMgr.getSystemVMAccessDetails(profile.getVirtualMachine()); final Map ipAddressDetails = new HashMap<>(sshAccessDetails); @@ -1163,7 +1165,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar if (dc.getDns2() != null) { buf.append(" dns2=").append(dc.getDns2()); } - String nfsVersion = imageStoreDetailsUtil != null ? imageStoreDetailsUtil.getNfsVersion(secStore.getId()) : null; + String nfsVersion = imageStoreDetailsUtil != null ? imageStoreDetailsUtil.getNfsVersion(secStores.get(0).getId()) : null; buf.append(" nfsVersion=").append(nfsVersion); buf.append(" keystore_password=").append(VirtualMachineGuru.getEncodedString(PasswordGenerator.generateRandomPassword(16))); String bootArgs = buf.toString(); @@ -1175,27 +1177,44 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar s_logger.debug(String.format("Setting UseHttpsToUpload config on cmdline with [%s] value.", useHttpsToUpload)); buf.append(" useHttpsToUpload=").append(useHttpsToUpload); - addSecondaryStorageServerAddressToBuffer(buf, secStore, vmName); + addSecondaryStorageServerAddressToBuffer(buf, secStores, vmName); return true; } /** - * Adds the secondary storage address to the buffer if it is in the following pattern: //
/... + * Adds the secondary storages address to the buffer if it is in the following pattern: //
/... */ - protected void addSecondaryStorageServerAddressToBuffer(StringBuilder buffer, DataStore dataStore, String vmName) { - String url = dataStore.getTO().getUrl(); - String[] urlArray = url.split("/"); + protected void addSecondaryStorageServerAddressToBuffer(StringBuilder buffer, List dataStores, String vmName) { + List addresses = new ArrayList<>(); + for (DataStore dataStore: dataStores) { + String url = dataStore.getTO().getUrl(); + String[] urlArray = url.split("/"); - s_logger.debug(String.format("Found [%s] as secondary storage's URL for SSVM [%s].", url, vmName)); - if (ArrayUtils.getLength(urlArray) < 3) { - s_logger.debug(String.format("Could not retrieve secondary storage address from URL [%s] of SSVM [%s].", url, vmName)); + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Found [%s] as secondary storage [%s] URL for SSVM [%s].", dataStore.getName(), url, vmName)); + } + if (ArrayUtils.getLength(urlArray) < 3) { + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Could not retrieve secondary storage [%s] address from URL [%s] of SSVM [%s].", dataStore.getName(), url, vmName)); + } + continue; + } + + String address = urlArray[2]; + s_logger.info(String.format("Using [%s] as address of secondary storage [%s] of SSVM [%s].", address, dataStore.getName(), vmName)); + if (!addresses.contains(address)) { + addresses.add(address); + } + + } + if (addresses.isEmpty()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("No address found for the secondary storages: [%s] of SSVM: [%s]", StringUtils.join(dataStores.stream().map(DataStore::getName).collect(Collectors.toList()), ","), vmName)); + } return; } - - String address = urlArray[2]; - s_logger.info(String.format("Using [%s] as address of secondary storage of SSVM [%s].", address, vmName)); - buffer.append(" secondaryStorageServerAddress=").append(address); + buffer.append(" secondaryStorageServerAddress=").append(StringUtils.join(addresses, ",")); } @Override diff --git a/services/secondary-storage/controller/src/test/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImplTest.java b/services/secondary-storage/controller/src/test/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImplTest.java new file mode 100644 index 00000000000..6df205e1aac --- /dev/null +++ b/services/secondary-storage/controller/src/test/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImplTest.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secondarystorage; + +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.List; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.utils.net.NetUtils; +import com.google.common.net.InetAddresses; + +@RunWith(MockitoJUnitRunner.class) +public class SecondaryStorageManagerImplTest { + private final SecureRandom secureRandom = new SecureRandom(); + + @Spy + @InjectMocks + private SecondaryStorageManagerImpl secondaryStorageManager; + + private List mockDataStoresForTestAddSecondaryStorageServerAddressToBuffer(List addresses) { + List dataStores = new ArrayList<>(); + for (String address: addresses) { + DataStore dataStore = Mockito.mock(DataStore.class); + DataStoreTO dataStoreTO = Mockito.mock(DataStoreTO.class); + Mockito.when(dataStoreTO.getUrl()).thenReturn(NetUtils.isValidIp4(address) ? String.format("http://%s", address) : address); + Mockito.when(dataStore.getTO()).thenReturn(dataStoreTO); + dataStores.add(dataStore); + } + return dataStores; + } + + private void runAddSecondaryStorageServerAddressToBufferTest(List addresses, String expected) { + List dataStores = mockDataStoresForTestAddSecondaryStorageServerAddressToBuffer(addresses); + StringBuilder builder = new StringBuilder(); + secondaryStorageManager.addSecondaryStorageServerAddressToBuffer(builder, dataStores, "VM"); + String result = builder.toString(); + result = result.contains("=") ? result.split("=")[1] : null; + Assert.assertEquals(expected, result); + } + + @Test + public void testAddSecondaryStorageServerAddressToBufferDifferentAddress() { + String randomIp1 = InetAddresses.fromInteger(secureRandom.nextInt()).getHostAddress(); + String randomIp2 = InetAddresses.fromInteger(secureRandom.nextInt()).getHostAddress(); + List addresses = List.of(randomIp1, randomIp2); + String expected = StringUtils.join(addresses, ","); + runAddSecondaryStorageServerAddressToBufferTest(addresses, expected); + } + + @Test + public void testAddSecondaryStorageServerAddressToBufferSameAddress() { + String randomIp1 = InetAddresses.fromInteger(secureRandom.nextInt()).getHostAddress(); + List addresses = List.of(randomIp1, randomIp1); + runAddSecondaryStorageServerAddressToBufferTest(addresses, randomIp1); + } + + @Test + public void testAddSecondaryStorageServerAddressToBufferInvalidAddress() { + String randomIp1 = InetAddresses.fromInteger(secureRandom.nextInt()).getHostAddress(); + String randomIp2 = InetAddresses.fromInteger(secureRandom.nextInt()).getHostAddress(); + List addresses = List.of(randomIp1, "garbage", randomIp2); + runAddSecondaryStorageServerAddressToBufferTest(addresses, StringUtils.join(List.of(randomIp1, randomIp2), ",")); + } +} diff --git a/systemvm/agent/scripts/ssvm-check.sh b/systemvm/agent/scripts/ssvm-check.sh index 5fdc244debd..b2721a93b3f 100644 --- a/systemvm/agent/scripts/ssvm-check.sh +++ b/systemvm/agent/scripts/ssvm-check.sh @@ -101,7 +101,7 @@ then else echo "ERROR: Storage $storage is not currently mounted" echo "Verifying if we can at least ping the storage" - STORAGE_ADDRESS=`grep "secondaryStorageServerAddress" $CMDLINE | sed -E 's/.*secondaryStorageServerAddress=([^ ]*).*/\1/g'` + STORAGE_ADDRESSES=`grep "secondaryStorageServerAddress" $CMDLINE | sed -E 's/.*secondaryStorageServerAddress=([^ ]*).*/\1/g'` if [[ -z "$STORAGE_ADDRESS" ]] then @@ -117,16 +117,21 @@ else route -n fi else - echo "Storage address is $STORAGE_ADDRESS, trying to ping it" - ping -c 2 $STORAGE_ADDRESS - if [ $? -eq 0 ] - then - echo "Good: Can ping $storage storage address" - else - echo "WARNING: Cannot ping $storage storage address" - echo routing table follows - route -n - fi + echo "Storage address(s): $STORAGE_ADDRESSES, trying to ping" + STORAGE_ADDRESS_LIST=$(echo $STORAGE_ADDRESSES | tr ",") + for STORAGE_ADDRESS in $STORAGE_ADDRESS_LIST + do + echo "Pinging storage address: $STORAGE_ADDRESS" + ping -c 2 $STORAGE_ADDRESS + if [ $? -eq 0 ] + then + echo "Good: Can ping $storage storage address" + else + echo "WARNING: Cannot ping $storage storage address" + echo routing table follows + route -n + fi + done fi fi From fb29608f61012f2529056e7ad9f15572ec944c2b Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 14 Jun 2023 09:35:55 +0200 Subject: [PATCH 3/7] consider last host only for root-admin (#7597) --- .../cloudstack/api/command/user/vm/StartVMCmd.java | 3 ++- ui/src/views/compute/StartVirtualMachine.vue | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java index 033a338fa7d..10c50dc380b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java @@ -86,7 +86,8 @@ public class StartVMCmd extends BaseAsyncCmd implements UserCmd { type = CommandType.BOOLEAN, description = "True by default, CloudStack will firstly try to start the VM on the last host where it run on before stopping, if destination host is not specified. " + "If false, CloudStack will not consider the last host and start the VM by normal process.", - since = "4.18.0") + since = "4.18.0", + authorized = {RoleType.Admin}) private Boolean considerLastHost; @Parameter(name = ApiConstants.DEPLOYMENT_PLANNER, type = CommandType.STRING, description = "Deployment planner to use for vm allocation. Available to ROOT admin only", since = "4.4", authorized = { RoleType.Admin }) diff --git a/ui/src/views/compute/StartVirtualMachine.vue b/ui/src/views/compute/StartVirtualMachine.vue index 828a4a31f59..4440153d150 100644 --- a/ui/src/views/compute/StartVirtualMachine.vue +++ b/ui/src/views/compute/StartVirtualMachine.vue @@ -88,6 +88,13 @@ + + + + + @@ -97,13 +104,6 @@ - - - - -
{{ $t('label.cancel') }} {{ $t('label.ok') }} From 1aa4f8074197ecb38fe9bf5a3431f5cc73b7d635 Mon Sep 17 00:00:00 2001 From: dahn Date: Thu, 15 Jun 2023 09:09:31 +0200 Subject: [PATCH 4/7] accept first word from host os string for backwards compatibility (#7620) --- .../kvm/discoverer/LibvirtServerDiscoverer.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index f4706be185d..440961dd7b0 100644 --- a/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -471,8 +471,12 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements String hostOsInCluster = oneHost.getDetail("Host.OS"); String hostOs = ssCmd.getHostDetails().get("Host.OS"); if (!hostOsInCluster.equalsIgnoreCase(hostOs)) { - throw new IllegalArgumentException("Can't add host: " + firstCmd.getPrivateIpAddress() + " with hostOS: " + hostOs + " into a cluster," + - "in which there are " + hostOsInCluster + " hosts added"); + String msg = String.format("host: %s with hostOS, \"%s\"into a cluster, in which there are \"%s\" hosts added", firstCmd.getPrivateIpAddress(), hostOs, hostOsInCluster); + if (hostOs != null && hostOs.startsWith(hostOsInCluster)) { + s_logger.warn(String.format("Adding %s. This may or may not be ok!", msg)); + } else { + throw new IllegalArgumentException(String.format("Can't add %s.", msg)); + } } } From 658daef7158f23f73fc446db6ddb4320eb6d3463 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 19 Jun 2023 12:23:22 +0530 Subject: [PATCH 5/7] utils: fix check for mrtalink url (#7636) Fixes incorrect condition while checking metalink URLs for accesibility --- utils/src/main/java/com/cloud/utils/UriUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index 4370ba28ec2..11e62255fae 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -416,7 +416,7 @@ public class UriUtils { List urls = metalinkUrls.get("url"); boolean validUrl = false; for (String u : urls) { - if (url.endsWith("torrent")) { + if (u.endsWith("torrent")) { continue; } try { From 3748f32bc7f44b8730c0ba77441c6b0ec8525d7b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 20 Jun 2023 20:43:08 +0530 Subject: [PATCH 6/7] engine-orchestration,vmware: hypervisor migration during start vm migration (#7444) Signed-off-by: Abhishek Kumar --- .../com/cloud/vm/VirtualMachineManager.java | 2 + .../cloud/vm/VirtualMachineManagerImpl.java | 35 +++++++++++++-- .../vm/VirtualMachineManagerImplTest.java | 30 +++++++++++++ .../com/cloud/hypervisor/guru/VMwareGuru.java | 14 +++--- .../cloud/hypervisor/guru/VMwareGuruTest.java | 43 ++++++++++--------- 5 files changed, 93 insertions(+), 31 deletions(-) diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index ada94cd057c..82396cf4635 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -264,6 +264,8 @@ public interface VirtualMachineManager extends Manager { Pair findClusterAndHostIdForVm(long vmId); + Pair findClusterAndHostIdForVm(VirtualMachine vm, boolean skipCurrentHostForStartingVm); + /** * Obtains statistics for a list of VMs; CPU and network utilization * @param hostId ID of the host diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 1b3d914b27a..db76fec9b86 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1054,6 +1054,26 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } + protected void checkAndAttemptMigrateVmAcrossCluster(final VMInstanceVO vm, final Long destinationClusterId, final Map volumePoolMap) { + if (!HypervisorType.VMware.equals(vm.getHypervisorType()) || vm.getLastHostId() == null) { + return; + } + Host lastHost = _hostDao.findById(vm.getLastHostId()); + if (destinationClusterId.equals(lastHost.getClusterId())) { + return; + } + if (volumePoolMap.values().stream().noneMatch(s -> destinationClusterId.equals(s.getClusterId()))) { + return; + } + Answer[] answer = attemptHypervisorMigration(vm, volumePoolMap, lastHost.getId()); + if (answer == null) { + s_logger.warn("Hypervisor inter-cluster migration during VM start failed"); + return; + } + // Other network related updates will be done using caller + markVolumesInPool(vm, answer); + } + @Override public void orchestrateStart(final String vmUuid, final Map params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { @@ -1227,6 +1247,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac resetVmNicsDeviceId(vm.getId()); _networkMgr.prepare(vmProfile, dest, ctx); if (vm.getHypervisorType() != HypervisorType.BareMetal) { + checkAndAttemptMigrateVmAcrossCluster(vm, cluster_id, dest.getStorageForDisks()); volumeMgr.prepare(vmProfile, dest); } @@ -2355,7 +2376,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { return _agentMgr.send(hostId, commandsContainer); } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException(String.format("Failed to migrate VM: %s", vm.getUuid()),e); + s_logger.warn(String.format("Hypervisor migration failed for the VM: %s", vm), e); } } return null; @@ -5712,8 +5733,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac return new Pair<>(clusterId, hostId); } - private Pair findClusterAndHostIdForVm(VirtualMachine vm) { - Long hostId = vm.getHostId(); + @Override + public Pair findClusterAndHostIdForVm(VirtualMachine vm, boolean skipCurrentHostForStartingVm) { + Long hostId = null; + if (!skipCurrentHostForStartingVm || !State.Starting.equals(vm.getState())) { + hostId = vm.getHostId(); + } Long clusterId = null; if(hostId == null) { hostId = vm.getLastHostId(); @@ -5731,6 +5756,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac return new Pair<>(clusterId, hostId); } + private Pair findClusterAndHostIdForVm(VirtualMachine vm) { + return findClusterAndHostIdForVm(vm, false); + } + @Override public Pair findClusterAndHostIdForVm(long vmId) { VMInstanceVO vm = _vmDao.findById(vmId); diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index 742bb3dda89..c4be16ea255 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -838,4 +838,34 @@ public class VirtualMachineManagerImplTest { Mockito.when(templateZoneDao.findByZoneTemplate(dcId, templateId)).thenReturn(Mockito.mock(VMTemplateZoneVO.class)); virtualMachineManagerImpl.checkIfTemplateNeededForCreatingVmVolumes(vm); } + + @Test + public void checkAndAttemptMigrateVmAcrossClusterNonValid() { + // Below scenarios shouldn't result in VM migration + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.KVM); + virtualMachineManagerImpl.checkAndAttemptMigrateVmAcrossCluster(vm, 1L, new HashMap<>()); + + Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.VMware); + Mockito.when(vm.getLastHostId()).thenReturn(null); + virtualMachineManagerImpl.checkAndAttemptMigrateVmAcrossCluster(vm, 1L, new HashMap<>()); + + Long destinationClusterId = 10L; + Mockito.when(vm.getLastHostId()).thenReturn(1L); + HostVO hostVO = Mockito.mock(HostVO.class); + Mockito.when(hostVO.getClusterId()).thenReturn(destinationClusterId); + Mockito.when(hostDaoMock.findById(1L)).thenReturn(hostVO); + virtualMachineManagerImpl.checkAndAttemptMigrateVmAcrossCluster(vm, destinationClusterId, new HashMap<>()); + + destinationClusterId = 20L; + Map map = new HashMap<>(); + StoragePool pool1 = Mockito.mock(StoragePool.class); + Mockito.when(pool1.getClusterId()).thenReturn(10L); + map.put(Mockito.mock(Volume.class), pool1); + StoragePool pool2 = Mockito.mock(StoragePool.class); + Mockito.when(pool2.getClusterId()).thenReturn(null); + map.put(Mockito.mock(Volume.class), pool2); + virtualMachineManagerImpl.checkAndAttemptMigrateVmAcrossCluster(vm, destinationClusterId, map); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index 0b67074e937..fe35d565088 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -27,9 +27,6 @@ import java.util.UUID; import javax.inject.Inject; -import com.cloud.storage.StorageManager; -import com.cloud.storage.StoragePoolHostVO; -import com.cloud.storage.dao.StoragePoolHostDao; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; @@ -112,7 +109,9 @@ import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.GuestOSVO; import com.cloud.storage.Storage; +import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; @@ -120,6 +119,7 @@ import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VolumeDao; @@ -1149,10 +1149,10 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co @Override public List finalizeMigrate(VirtualMachine vm, Map volumeToPool) { - List commands = new ArrayList(); + List commands = new ArrayList<>(); // OfflineVmwareMigration: specialised migration command - List> volumeToFilerTo = new ArrayList>(); + List> volumeToFilerTo = new ArrayList<>(); Long poolClusterId = null; StoragePool targetLocalPoolForVM = null; for (Map.Entry entry : volumeToPool.entrySet()) { @@ -1166,10 +1166,10 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co if (volume.getVolumeType().equals(Volume.Type.ROOT) && pool.isLocal()) { targetLocalPoolForVM = pool; } - volumeToFilerTo.add(new Pair(volumeTo, filerTo)); + volumeToFilerTo.add(new Pair<>(volumeTo, filerTo)); } final Long destClusterId = poolClusterId; - final Long srcClusterId = vmManager.findClusterAndHostIdForVm(vm.getId()).first(); + final Long srcClusterId = vmManager.findClusterAndHostIdForVm(vm, true).first(); final boolean isInterClusterMigration = isInterClusterMigration(destClusterId, srcClusterId); String targetHostGuid = getTargetHostGuid(targetLocalPoolForVM, destClusterId, isInterClusterMigration); diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java index 0db8271c8e5..7398ef99a8c 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java @@ -16,20 +16,13 @@ // under the License. package com.cloud.hypervisor.guru; -import com.cloud.agent.api.Command; -import com.cloud.agent.api.MigrateVmToPoolCommand; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.storage.Storage.ProvisioningType; -import com.cloud.storage.StoragePool; -import com.cloud.storage.StoragePoolHostVO; -import com.cloud.storage.Volume; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.StoragePoolHostDao; -import com.cloud.utils.Pair; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineManager; +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.junit.Assert; @@ -46,12 +39,20 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.support.AnnotationConfigContextLoader; -import static org.junit.Assert.assertEquals; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.MigrateVmToPoolCommand; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.storage.Storage.ProvisioningType; +import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.utils.Pair; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineManager; @RunWith(PowerMockRunner.class) @PrepareForTest({VMwareGuru.class}) @@ -105,7 +106,7 @@ public class VMwareGuruTest { Mockito.when(localStorage.isLocal()).thenReturn(true); Pair clusterAndHost = new Pair<>(1L, 1L); - Mockito.when(vmManager.findClusterAndHostIdForVm(1L)).thenReturn(clusterAndHost); + Mockito.when(vmManager.findClusterAndHostIdForVm(vm, true)).thenReturn(clusterAndHost); List storagePoolHostVOS = new ArrayList<>(); storagePoolHostVOS.add(storagePoolHostVO); From 40cc10a73deb3211d76caa70f7f3b66e450b3214 Mon Sep 17 00:00:00 2001 From: Harikrishna Date: Wed, 21 Jun 2023 11:57:05 +0530 Subject: [PATCH 7/7] Allow volume migrations in ScaleIO within and across ScaleIO storage clusters (#7408) * Live storage migration of volume in scaleIO within same storage scaleio cluster * Added migrate command * Recent changes of migration across clusters * Fixed uuid * recent changes * Pivot changes * working blockcopy api in libvirt * Checking block copy status * Formatting code * Fixed failures * code refactoring and some changes * Removed unused methods * removed unused imports * Unit tests to check if volume belongs to same or different storage scaleio cluster * Unit tests for volume livemigration in ScaleIOPrimaryDataStoreDriver * Fixed offline volume migration case and allowed encrypted volume migration * Added more integration tests * Support for migration of encrypted volumes across different scaleio clusters * Fix UI notifications for migrate volume * Data volume offline migration: save encryption details to destination volume entry * Offline storage migration for scaleio encrypted volumes * Allow multiple Volumes to be migrated with migrateVirtualMachineWithVolume API * Removed unused unittests * Removed duplicate keys in migrate volume vue file * Fix Unit tests * Add volume secrets if does not exists during volume migrations. secrets are getting cleared on package upgrades. * Fix secret UUID for encrypted volume migration * Added a null check for secret before removing * Added more unit tests * Fixed passphrase check * Add image options to the encypted volume conversion --- .../cloud/vm/VirtualMachineManagerImpl.java | 3 +- .../vm/VirtualMachineManagerImplTest.java | 27 + .../storage/volume/VolumeServiceImpl.java | 4 + .../kvm/resource/LibvirtConnection.java | 2 + .../LibvirtMigrateVolumeCommandWrapper.java | 232 +++++++- .../kvm/storage/KVMStorageProcessor.java | 9 +- .../kvm/storage/ScaleIOStorageAdaptor.java | 1 + .../utils/qemu/QemuImageOptions.java | 11 +- ...ibvirtMigrateVolumeCommandWrapperTest.java | 388 +++++++++++++ .../client/ScaleIOGatewayClientImpl.java | 2 +- .../driver/ScaleIOPrimaryDataStoreDriver.java | 281 +++++++++- .../ScaleIOPrimaryDataStoreDriverTest.java | 527 ++++++++++++++++++ .../cloud/storage/VolumeApiServiceImpl.java | 21 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 10 - .../plugins/scaleio/test_scaleio_volumes.py | 243 ++++++++ 15 files changed, 1722 insertions(+), 39 deletions(-) create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java create mode 100644 plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriverTest.java diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index db76fec9b86..9b37132d07c 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -212,6 +212,7 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateVO; @@ -2925,7 +2926,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac * */ protected void executeManagedStorageChecksWhenTargetStoragePoolProvided(StoragePoolVO currentPool, VolumeVO volume, StoragePoolVO targetPool) { - if (!currentPool.isManaged()) { + if (!currentPool.isManaged() || currentPool.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) { return; } if (currentPool.getId() == targetPool.getId()) { diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index c4be16ea255..15a2f2c0ac1 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -32,6 +32,8 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.framework.config.ConfigKey; @@ -46,6 +48,7 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.Spy; +import org.mockito.stubbing.Answer; import org.mockito.runners.MockitoJUnitRunner; import com.cloud.agent.AgentManager; @@ -68,6 +71,7 @@ import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; @@ -373,9 +377,26 @@ public class VirtualMachineManagerImplTest { Mockito.verify(storagePoolVoMock, Mockito.times(0)).getId(); } + @Test + public void allowVolumeMigrationsForPowerFlexStorage() { + Mockito.doReturn(true).when(storagePoolVoMock).isManaged(); + Mockito.doReturn(Storage.StoragePoolType.PowerFlex).when(storagePoolVoMock).getPoolType(); + + virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, Mockito.mock(StoragePoolVO.class)); + + Mockito.verify(storagePoolVoMock).isManaged(); + Mockito.verify(storagePoolVoMock, Mockito.times(0)).getId(); + } + @Test public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentStoragePoolEqualsTargetPool() { Mockito.doReturn(true).when(storagePoolVoMock).isManaged(); + // return any storage type except powerflex/scaleio + List values = Arrays.asList(Storage.StoragePoolType.values()); + when(storagePoolVoMock.getPoolType()).thenAnswer((Answer) invocation -> { + List filteredValues = values.stream().filter(v -> v != Storage.StoragePoolType.PowerFlex).collect(Collectors.toList()); + int randomIndex = new Random().nextInt(filteredValues.size()); + return filteredValues.get(randomIndex); }); virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, storagePoolVoMock); @@ -386,6 +407,12 @@ public class VirtualMachineManagerImplTest { @Test(expected = CloudRuntimeException.class) public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentStoragePoolNotEqualsTargetPool() { Mockito.doReturn(true).when(storagePoolVoMock).isManaged(); + // return any storage type except powerflex/scaleio + List values = Arrays.asList(Storage.StoragePoolType.values()); + when(storagePoolVoMock.getPoolType()).thenAnswer((Answer) invocation -> { + List filteredValues = values.stream().filter(v -> v != Storage.StoragePoolType.PowerFlex).collect(Collectors.toList()); + int randomIndex = new Random().nextInt(filteredValues.size()); + return filteredValues.get(randomIndex); }); virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, Mockito.mock(StoragePoolVO.class)); } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 48de0eb016b..ffc12b98c84 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -1648,6 +1648,10 @@ public class VolumeServiceImpl implements VolumeService { newVol.setPoolType(pool.getPoolType()); newVol.setLastPoolId(lastPoolId); newVol.setPodId(pool.getPodId()); + if (volume.getPassphraseId() != null) { + newVol.setPassphraseId(volume.getPassphraseId()); + newVol.setEncryptFormat(volume.getEncryptFormat()); + } return volDao.persist(newVol); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java index c70a72f399c..0f8031e3aaa 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java @@ -21,6 +21,7 @@ import java.util.Map; import org.apache.log4j.Logger; import org.libvirt.Connect; +import org.libvirt.Library; import org.libvirt.LibvirtException; import com.cloud.hypervisor.Hypervisor; @@ -44,6 +45,7 @@ public class LibvirtConnection { if (conn == null) { s_logger.info("No existing libvirtd connection found. Opening a new one"); conn = new Connect(hypervisorURI, false); + Library.initEventLoop(); s_logger.debug("Successfully connected to libvirt at: " + hypervisorURI); s_connections.put(hypervisorURI, conn); } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 311eb670e99..5c893e5d12f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -29,27 +29,255 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; import java.util.Map; import java.util.UUID; +import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; +import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainBlockJobInfo; +import org.libvirt.DomainInfo; +import org.libvirt.TypedParameter; +import org.libvirt.TypedUlongParameter; +import org.libvirt.LibvirtException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; @ResourceWrapper(handles = MigrateVolumeCommand.class) -public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { +public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { private static final Logger LOGGER = Logger.getLogger(LibvirtMigrateVolumeCommandWrapper.class); @Override public Answer execute(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); + PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); + + MigrateVolumeAnswer answer; + if (srcPrimaryDataStore.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) { + answer = migratePowerFlexVolume(command, libvirtComputingResource); + } else { + answer = migrateRegularVolume(command, libvirtComputingResource); + } + + return answer; + } + + protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + + // Source Details + VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); + String srcPath = srcVolumeObjectTO.getPath(); + final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumeObjectTO.getPath()); + final String vmName = srcVolumeObjectTO.getVmName(); + + // Destination Details + VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); + String destPath = destVolumeObjectTO.getPath(); + final String destVolumeId = ScaleIOUtil.getVolumePath(destVolumeObjectTO.getPath()); + Map destDetails = command.getDestDetails(); + final String destSystemId = destDetails.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + String destDiskLabel = null; + + final String destDiskFileName = ScaleIOUtil.DISK_NAME_PREFIX + destSystemId + "-" + destVolumeId; + final String diskFilePath = ScaleIOUtil.DISK_PATH + File.separator + destDiskFileName; + + Domain dm = null; + try { + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + Connect conn = libvirtUtilitiesHelper.getConnection(); + dm = libvirtComputingResource.getDomain(conn, vmName); + if (dm == null) { + return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to can not find vm: " + vmName, null); + } + + DomainInfo.DomainState domainState = dm.getInfo().state ; + if (domainState != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) { + return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null); + } + + final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + PrimaryDataStoreTO spool = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore(); + KVMStoragePool pool = storagePoolMgr.getStoragePool(spool.getPoolType(), spool.getUuid()); + pool.connectPhysicalDisk(destVolumeObjectTO.getPath(), null); + + String srcSecretUUID = null; + String destSecretUUID = null; + if (ArrayUtils.isNotEmpty(destVolumeObjectTO.getPassphrase())) { + srcSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, srcVolumeObjectTO.getPath(), srcVolumeObjectTO.getPassphrase()); + destSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, destVolumeObjectTO.getPath(), destVolumeObjectTO.getPassphrase()); + } + + String diskdef = generateDestinationDiskXML(dm, srcVolumeId, diskFilePath, destSecretUUID); + destDiskLabel = generateDestinationDiskLabel(diskdef); + + TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); + TypedParameter[] parameters = new TypedParameter[1]; + parameters[0] = parameter; + + dm.blockCopy(destDiskLabel, diskdef, parameters, Domain.BlockCopyFlags.REUSE_EXT); + LOGGER.info(String.format("Block copy has started for the volume %s : %s ", destDiskLabel, srcPath)); + + return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath, libvirtComputingResource, conn, srcSecretUUID); + + } catch (Exception e) { + String msg = "Migrate volume failed due to " + e.toString(); + LOGGER.warn(msg, e); + if (destDiskLabel != null) { + try { + dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.ASYNC); + } catch (LibvirtException ex) { + LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); + } + } + return new MigrateVolumeAnswer(command, false, msg, null); + } finally { + if (dm != null) { + try { + dm.free(); + } catch (LibvirtException l) { + LOGGER.trace("Ignoring libvirt error.", l); + }; + } + } + } + + protected MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException { + int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found + int waitTimeInSec = command.getWait(); + while (waitTimeInSec > 0) { + DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(diskLabel, 0); + if (blockJobInfo != null) { + LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% current value:%s end value:%s", diskLabel, srcPath, (blockJobInfo.end == 0)? 0 : 100*(blockJobInfo.cur / (double) blockJobInfo.end), blockJobInfo.cur, blockJobInfo.end)); + if (blockJobInfo.cur == blockJobInfo.end) { + LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath)); + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT); + if (StringUtils.isNotEmpty(srcSecretUUID)) { + libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID); + } + break; + } + } else { + LOGGER.info("Failed to get the block copy status, trying to abort the job"); + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC); + } + waitTimeInSec--; + + try { + Thread.sleep(timeBetweenTries); + } catch (Exception ex) { + // don't do anything + } + } + + if (waitTimeInSec <= 0) { + String msg = "Block copy is taking long time, failing the job"; + LOGGER.error(msg); + try { + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC); + } catch (LibvirtException ex) { + LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); + } + return new MigrateVolumeAnswer(command, false, msg, null); + } + + return new MigrateVolumeAnswer(command, true, null, destPath); + } + + private String generateDestinationDiskLabel(String diskXml) throws ParserConfigurationException, IOException, SAXException { + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(new ByteArrayInputStream(diskXml.getBytes("UTF-8"))); + doc.getDocumentElement().normalize(); + + Element disk = doc.getDocumentElement(); + String diskLabel = getAttrValue("target", "dev", disk); + + return diskLabel; + } + + protected String generateDestinationDiskXML(Domain dm, String srcVolumeId, String diskFilePath, String destSecretUUID) throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException { + final String domXml = dm.getXMLDesc(0); + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(new ByteArrayInputStream(domXml.getBytes("UTF-8"))); + doc.getDocumentElement().normalize(); + + NodeList disks = doc.getElementsByTagName("disk"); + + for (int i = 0; i < disks.getLength(); i++) { + Element disk = (Element)disks.item(i); + String type = disk.getAttribute("type"); + if (!type.equalsIgnoreCase("network")) { + String diskDev = getAttrValue("source", "dev", disk); + if (StringUtils.isNotEmpty(diskDev) && diskDev.contains(srcVolumeId)) { + setAttrValue("source", "dev", diskFilePath, disk); + if (StringUtils.isNotEmpty(destSecretUUID)) { + setAttrValue("secret", "uuid", destSecretUUID, disk); + } + StringWriter diskSection = new StringWriter(); + Transformer xformer = TransformerFactory.newInstance().newTransformer(); + xformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + xformer.transform(new DOMSource(disk), new StreamResult(diskSection)); + + return diskSection.toString(); + } + } + } + + return null; + } + + private static String getAttrValue(String tag, String attr, Element eElement) { + NodeList tagNode = eElement.getElementsByTagName(tag); + if (tagNode.getLength() == 0) { + return null; + } + Element node = (Element)tagNode.item(0); + return node.getAttribute(attr); + } + + private static void setAttrValue(String tag, String attr, String newValue, Element eElement) { + NodeList tagNode = eElement.getElementsByTagName(tag); + if (tagNode.getLength() == 0) { + return; + } + Element node = (Element)tagNode.item(0); + node.setAttribute(attr, newValue); + } + + protected MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { KVMStoragePoolManager storagePoolManager = libvirtComputingResource.getStoragePoolMgr(); VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); Map srcDetails = command.getSrcDetails(); - String srcPath = srcDetails != null ? srcDetails.get(DiskTO.IQN) : srcVolumeObjectTO.getPath(); VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index cae872e287f..a8a7d6f5694 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -37,6 +37,7 @@ import java.util.UUID; import javax.naming.ConfigurationException; import com.cloud.storage.ScopeType; +import com.cloud.storage.Volume; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand; @@ -2448,7 +2449,12 @@ public class KVMStorageProcessor implements StorageProcessor { destPool = storagePoolMgr.getStoragePool(destPrimaryStore.getPoolType(), destPrimaryStore.getUuid()); try { - storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds()); + if (srcVol.getPassphrase() != null && srcVol.getVolumeType().equals(Volume.Type.ROOT)) { + volume.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); + storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds(), srcVol.getPassphrase(), destVol.getPassphrase(), srcVol.getProvisioningType()); + } else { + storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds()); + } } catch (Exception e) { // Any exceptions while copying the disk, should send failed answer with the error message String errMsg = String.format("Failed to copy volume: %s to dest storage: %s, due to %s", srcVol.getName(), destPrimaryStore.getName(), e.toString()); s_logger.debug(errMsg, e); @@ -2467,6 +2473,7 @@ public class KVMStorageProcessor implements StorageProcessor { String path = destPrimaryStore.isManaged() ? destVolumeName : destVolumePath + File.separator + destVolumeName; newVol.setPath(path); newVol.setFormat(destFormat); + newVol.setEncryptFormat(destVol.getEncryptFormat()); return new CopyCmdAnswer(newVol); } catch (final CloudRuntimeException e) { s_logger.debug("Failed to copyVolumeFromPrimaryToPrimary: ", e); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index 09c7e146e49..607dd620f61 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -387,6 +387,7 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { boolean forceSourceFormat = srcQemuFile.getFormat() == QemuImg.PhysicalDiskFormat.RAW; LOGGER.debug(String.format("Starting copy from source disk %s(%s) to PowerFlex volume %s(%s), forcing source format is %b", srcQemuFile.getFileName(), srcQemuFile.getFormat(), destQemuFile.getFileName(), destQemuFile.getFormat(), forceSourceFormat)); + qemuImageOpts.setImageOptsFlag(true); qemu.convert(srcQemuFile, destQemuFile, options, qemuObjects, qemuImageOpts,null, forceSourceFormat); LOGGER.debug("Successfully converted source disk image " + srcQemuFile.getFileName() + " to PowerFlex volume: " + destDisk.getPath()); diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java index f9a2e4ba652..4a577ef3400 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java @@ -30,6 +30,7 @@ public class QemuImageOptions { private static final String LUKS_KEY_SECRET_PARAM_KEY = "key-secret"; private static final String QCOW2_KEY_SECRET_PARAM_KEY = "encrypt.key-secret"; private static final String DRIVER = "driver"; + private boolean addImageOpts = false; private QemuImg.PhysicalDiskFormat format; private static final List DISK_FORMATS_THAT_SUPPORT_OPTION_IMAGE_OPTS = Arrays.asList(QemuImg.PhysicalDiskFormat.QCOW2, QemuImg.PhysicalDiskFormat.LUKS); @@ -71,13 +72,19 @@ public class QemuImageOptions { } } + public void setImageOptsFlag(boolean addImageOpts) { + this.addImageOpts = addImageOpts; + } + /** * Converts QemuImageOptions into the command strings required by qemu-img flags * @return array of strings representing command flag and value (--image-opts) */ public String[] toCommandFlag() { - if (format == null || !DISK_FORMATS_THAT_SUPPORT_OPTION_IMAGE_OPTS.contains(format)) { - return new String[] { params.get(FILENAME_PARAM_KEY) }; + if (!addImageOpts) { + if (format == null || !DISK_FORMATS_THAT_SUPPORT_OPTION_IMAGE_OPTS.contains(format)) { + return new String[] { params.get(FILENAME_PARAM_KEY) }; + } } Map sorted = new TreeMap<>(params); String paramString = Joiner.on(",").withKeyValueSeparator("=").join(sorted); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java new file mode 100644 index 00000000000..c278144b4e1 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java @@ -0,0 +1,388 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.MigrateVolumeAnswer; +import com.cloud.agent.api.storage.MigrateVolumeCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage; +import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainBlockJobInfo; +import org.libvirt.DomainInfo; +import org.libvirt.LibvirtException; +import org.libvirt.TypedParameter; +import org.mockito.InjectMocks; +import org.mockito.Matchers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.runners.MockitoJUnitRunner; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtMigrateVolumeCommandWrapperTest { + + @Spy + @InjectMocks + private LibvirtMigrateVolumeCommandWrapper libvirtMigrateVolumeCommandWrapper; + + @Mock + MigrateVolumeCommand command; + + @Mock + LibvirtComputingResource libvirtComputingResource; + + @Mock + LibvirtUtilitiesHelper libvirtUtilitiesHelper; + + private String domxml = "\n" + + " i-2-27-VM\n" + + " 2d37fe1a-621a-4903-9ab5-5c9544c733f8\n" + + " Ubuntu 18.04 LTS\n" + + " 524288\n" + + " 524288\n" + + " 1\n" + + " \n" + + " 256\n" + + " \n" + + " \n" + + " /machine\n" + + " \n" + + " \n" + + " \n" + + " Apache Software Foundation\n" + + " CloudStack KVM Hypervisor\n" + + " 2d37fe1a-621a-4903-9ab5-5c9544c733f8\n" + + " \n" + + " \n" + + " \n" + + " hvm\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " qemu64\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " destroy\n" + + " restart\n" + + " destroy\n" + + " \n" + + " /usr/libexec/qemu-kvm\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 38a54bf719f24af6b070\n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 0ceeb7c643b447aba5ce\n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "