From b34f09313738c7352aed04d1e0b7de6ab5d6c722 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 29 Jan 2024 11:40:43 +0100 Subject: [PATCH] veeam: fix some issues with restoring volume from backup and attaching it to VM (#8570) * veeam: detach only the restored volume during backup restore Steps to reproduce the issue 1. create a VM (A) with ROOT and DATA disk 2. assign to a backup offering 3. create backup 4. create another VM (B) 5. restore the DATA disk of VM A, and attach to VM B 6. When operation is done, check the datastore Without this change, the ROOT image is not removed and left over on the datastore. ``` [root@ref-trl-5933-v-Mr8-wei-zhou-esxi2:/vmfs/volumes/5f60667d-18d828eb] ls -l /vmfs/volumes/5f60667d-18d828eb/CS-RSTR-dfb6f21c-a941-49db-9963-4f0286a17dac total 1784840 -rw------- 1 root root 5242880000 Jan 24 09:23 ROOT-722_2-flat.vmdk -rw------- 1 root root 499 Jan 24 09:23 ROOT-722_2.vmdk ``` With this change, the whole temporary vm has been destroyed. ``` [root@ref-trl-5933-v-Mr8-wei-zhou-esxi2:/vmfs/volumes/5f60667d-18d828eb] ls -l /vmfs/volumes/5f60667d-18d828eb/CS-RSTR-734bee3b-640c-4ff0-a34b-bc45358565b2 ls: /vmfs/volumes/5f60667d-18d828eb/CS-RSTR-734bee3b-640c-4ff0-a34b-bc45358565b2: No such file or directory ``` * veeam: fix wrong disk size in debug message * veeam: sync backup repository after operations are done got exception of some operations which succeeds due to the following error ``` 2024-01-19 10:59:52,846 DEBUG [o.a.c.b.v.VeeamClient] (API-Job-Executor-42:ctx-716501bb job-4373 ctx-2359b76d) (logid:b5e19a17) Veeam response for PowerShell commands [PowerShell Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue;$restorePoint = Get-VBRRestorePoint ^| Where-Object { $_.Id -eq '1d99106a-b5c8-4a1e-958d-066a987caa5f' };if ($restorePoint) { Remove-VBRRestorePoint -Oib $restorePoint -Confirm:$false;$repo = Get-VBRBackupRepository;Sync-VBRBackupRepository -Repository $repo;} else { ; Write-Output 'Failed to delete'; Exit 1;}] is: [^M Restore Type Job Name State Start Time End Time Description ^M ------------ -------- ----- ---------- -------- ----------- ^M ConfResynchronize Configuration Dat... Starting 19/01/2024 10:59:52 01/01/1900 00:00:00 ^M ^M ^M Remove-VBRRestorePoint : Win32 internal error "Access is denied" 0x5 occurred while reading the console output buffer. ^M Contact Microsoft Customer Support Services.^M At line:1 char:196^M + ... orePoint) { Remove-VBRRestorePoint -Oib $restorePoint -Confirm:$false ...^M + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^M + CategoryInfo : ReadError: (:) [Remove-VBRRestorePoint], HostException^M + FullyQualifiedErrorId : ReadConsoleOutput,Veeam.Backup.PowerShell.Cmdlets.RemoveVBRRestorePoint^M ^M ]. ``` * veeam: fix unable to detach volume when restore backup and attach to vm then detach the volume It also happened when destroy the original or backup VM ``` 2024-01-24 10:10:03,401 ERROR [c.c.s.r.VmwareStorageProcessor] (DirectAgent-74:ctx-95b24ac7 10.0.35.53, job-25995/job-25996, cmd: DettachCommand) (logid:7260ffb8) Failed to detach volume! java.lang.RuntimeException: Unable to access file [de52fdd3386b3d67b27b3960ecdb08f4] i-2-723-VM/7c2197c129464035bab062edec536a09-flat.vmdk at com.cloud.hypervisor.vmware.util.VmwareClient.waitForTask(VmwareClient.java:426) at com.cloud.hypervisor.vmware.mo.DatastoreMO.moveDatastoreFile(DatastoreMO.java:290) at com.cloud.storage.resource.VmwareStorageLayoutHelper.syncVolumeToRootFolder(VmwareStorageLayoutHelper.java:241) at com.cloud.storage.resource.VmwareStorageProcessor.attachVolume(VmwareStorageProcessor.java:2150) at com.cloud.storage.resource.VmwareStorageProcessor.dettachVolume(VmwareStorageProcessor.java:2408) at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:174) at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:71) at com.cloud.hypervisor.vmware.resource.VmwareResource.executeRequest(VmwareResource.java:589) at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:315) at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:48) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:102) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52) at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:45) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) 2024-01-24 10:10:03,402 INFO [c.c.h.v.u.VmwareHelper] (DirectAgent-74:ctx-95b24ac7 10.0.35.53, job-25995/job-25996, cmd: DettachCommand) (logid:7260ffb8) [ignored]failed to get message for exception: Unable to access file [de52fdd3386b3d67b27b3960ecdb08f4] i-2-723-VM/7c2197c129464035bab062edec536a09-flat.vmdk ``` * vmware: create restored volume with new UUID and attach to VM --- .../backup/VeeamBackupProvider.java | 3 +++ .../cloudstack/backup/veeam/VeeamClient.java | 18 +++++++++++---- .../com/cloud/hypervisor/guru/VMwareGuru.java | 23 +++++++++++-------- .../resource/VmwareStorageLayoutHelper.java | 21 ++++++++++++++++- .../resource/VmwareStorageProcessor.java | 23 ++----------------- .../smoke/test_backup_recovery_veeam.py | 14 +++++++---- 6 files changed, 61 insertions(+), 41 deletions(-) diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 1a445080b5c..5c96e4b7057 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -212,6 +212,7 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, LOG.warn("Failed to remove Veeam job and backup for job: " + clonedJobName); throw new CloudRuntimeException("Failed to delete Veeam B&R job and backup, an operation may be in progress. Please try again after some time."); } + client.syncBackupRepository(); return true; } @@ -245,6 +246,8 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, return false; } + client.syncBackupRepository(); + List allBackups = backupDao.listByVmId(backup.getZoneId(), backup.getVmId()); for (Backup b : allBackups) { if (b.getId() != backup.getId()) { diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index 7d3bfc50d18..bdc5723f79d 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -604,6 +604,7 @@ public class VeeamClient { joiner.add("PowerShell Add-PSSnapin VeeamPSSnapin"); } else { joiner.add("PowerShell Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue"); + joiner.add("$ProgressPreference='SilentlyContinue'"); } for (String cmd : cmds) { joiner.add(cmd); @@ -646,9 +647,7 @@ public class VeeamClient { String.format("$job = Get-VBRJob -Name '%s'", jobName), "if ($job) { Remove-VBRJob -Job $job -Confirm:$false }", String.format("$backup = Get-VBRBackup -Name '%s'", jobName), - "if ($backup) { Remove-VBRBackup -Backup $backup -FromDisk -Confirm:$false }", - "$repo = Get-VBRBackupRepository", - "Sync-VBRBackupRepository -Repository $repo" + "if ($backup) { Remove-VBRBackup -Backup $backup -FromDisk -Confirm:$false }" )); return result != null && result.first() && !result.second().contains(FAILED_TO_DELETE); } @@ -658,8 +657,6 @@ public class VeeamClient { Pair result = executePowerShellCommands(Arrays.asList( String.format("$restorePoint = Get-VBRRestorePoint ^| Where-Object { $_.Id -eq '%s' }", restorePointId), "if ($restorePoint) { Remove-VBRRestorePoint -Oib $restorePoint -Confirm:$false", - "$repo = Get-VBRBackupRepository", - "Sync-VBRBackupRepository -Repository $repo", "} else { ", " Write-Output 'Failed to delete'", " Exit 1", @@ -668,6 +665,17 @@ public class VeeamClient { return result != null && result.first() && !result.second().contains(FAILED_TO_DELETE); } + public boolean syncBackupRepository() { + LOG.debug("Trying to sync backup repository."); + Pair result = executePowerShellCommands(Arrays.asList( + "$repo = Get-VBRBackupRepository", + "$Syncs = Sync-VBRBackupRepository -Repository $repo", + "while ((Get-VBRSession -ID $Syncs.ID).Result -ne 'Success') { Start-Sleep -Seconds 10 }" + )); + LOG.debug("Done syncing backup repository."); + return result != null && result.first(); + } + public Map getBackupMetrics() { if (isLegacyServer()) { return getBackupMetricsLegacy(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index aa3b314fb3f..410fc7c2a7c 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -1022,12 +1022,12 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co /** * Get dest volume full path */ - private String getDestVolumeFullPath(VirtualDisk restoredDisk, VirtualMachineMO restoredVm, VirtualMachineMO vmMo) throws Exception { + private String getDestVolumeFullPath(VirtualMachineMO vmMo) throws Exception { VirtualDisk vmDisk = vmMo.getVirtualDisks().get(0); String vmDiskPath = vmMo.getVmdkFileBaseName(vmDisk); String vmDiskFullPath = getVolumeFullPath(vmMo.getVirtualDisks().get(0)); - String restoredVolumePath = restoredVm.getVmdkFileBaseName(restoredDisk); - return vmDiskFullPath.replace(vmDiskPath, restoredVolumePath); + String uuid = UUID.randomUUID().toString().replace("-", ""); + return vmDiskFullPath.replace(vmDiskPath, uuid); } /** @@ -1079,17 +1079,18 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co VirtualDisk restoredDisk = findRestoredVolume(volumeInfo, vmRestored); String diskPath = vmRestored.getVmdkFileBaseName(restoredDisk); - s_logger.debug("Restored disk size=" + toHumanReadableSize(restoredDisk.getCapacityInKB()) + " path=" + diskPath); + s_logger.debug("Restored disk size=" + toHumanReadableSize(restoredDisk.getCapacityInKB() * Resource.ResourceType.bytesToKiB) + " path=" + diskPath); // Detach restored VM disks - vmRestored.detachAllDisks(); + vmRestored.detachDisk(String.format("%s/%s.vmdk", location, diskPath), false); String srcPath = getVolumeFullPath(restoredDisk); - String destPath = getDestVolumeFullPath(restoredDisk, vmRestored, vmMo); + String destPath = getDestVolumeFullPath(vmMo); VirtualDiskManagerMO virtualDiskManagerMO = new VirtualDiskManagerMO(dcMo.getContext()); // Copy volume to the VM folder + s_logger.debug(String.format("Moving volume from %s to %s", srcPath, destPath)); virtualDiskManagerMO.moveVirtualDisk(srcPath, dcMo.getMor(), destPath, dcMo.getMor(), true); try { @@ -1103,11 +1104,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co vmRestored.destroy(); } - VirtualDisk attachedDisk = getAttachedDisk(vmMo, diskPath); + s_logger.debug(String.format("Attaching disk %s to vm %s", destPath, vm.getId())); + VirtualDisk attachedDisk = getAttachedDisk(vmMo, destPath); if (attachedDisk == null) { - s_logger.error("Failed to get the attached the (restored) volume " + diskPath); + s_logger.error("Failed to get the attached the (restored) volume " + destPath); return false; } + s_logger.debug(String.format("Creating volume entry for disk %s attached to vm %s", destPath, vm.getId())); createVolume(attachedDisk, vmMo, vm.getDomainId(), vm.getDataCenterId(), vm.getAccountId(), vm.getId(), poolId, vm.getTemplateId(), backup, false); if (vm.getBackupOfferingId() == null) { @@ -1119,9 +1122,9 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co return true; } - private VirtualDisk getAttachedDisk(VirtualMachineMO vmMo, String diskPath) throws Exception { + private VirtualDisk getAttachedDisk(VirtualMachineMO vmMo, String diskFullPath) throws Exception { for (VirtualDisk disk : vmMo.getVirtualDisks()) { - if (vmMo.getVmdkFileBaseName(disk).equals(diskPath)) { + if (getVolumeFullPath(disk).equals(diskFullPath)) { return disk; } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java index 1cb57c37813..b6b92f67ec5 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -218,7 +218,10 @@ public class VmwareStorageLayoutHelper implements Configurable { } public static void syncVolumeToRootFolder(DatacenterMO dcMo, DatastoreMO ds, String vmdkName, String vmName, String excludeFolders) throws Exception { - String fileDsFullPath = ds.searchFileInSubFolders(vmdkName + ".vmdk", false, excludeFolders); + String fileDsFullPath = ds.searchFileInSubFolders(String.format("%s/%s.vmdk", vmName, vmdkName), false, excludeFolders); + if (fileDsFullPath == null) { + fileDsFullPath = ds.searchFileInSubFolders(vmdkName + ".vmdk", false, excludeFolders); + } if (fileDsFullPath == null) return; @@ -409,6 +412,22 @@ public class VmwareStorageLayoutHelper implements Configurable { return String.format("[%s] %s/%s", dsMo.getName(), vmName, vmdkFileName); } + public static String getDatastoreVolumePath(DatastoreMO dsMo, String vmName, String volumePath) throws Exception { + String datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumePath + ".vmdk"); + if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) && dsMo.fileExists(datastoreVolumePath)) { + return datastoreVolumePath; + } + datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumePath, volumePath + ".vmdk"); + if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumePath) && dsMo.fileExists(datastoreVolumePath)) { + return datastoreVolumePath; + } + datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumePath + ".vmdk"); + if (dsMo.fileExists(datastoreVolumePath)) { + return datastoreVolumePath; + } + return dsMo.searchFileInSubFolders(volumePath + ".vmdk", false, null); + } + @Override public String getConfigComponentName() { return VmwareStorageLayoutHelper.class.getSimpleName(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index ba9d16f8186..7f4f517a293 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2062,17 +2062,7 @@ public class VmwareStorageProcessor implements StorageProcessor { datastoreVolumePath = dsMo.getDatastorePath((vmdkPath != null ? vmdkPath : dsMo.getName()) + ".vmdk"); } else { if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { - datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumePath + ".vmdk"); - if (!dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumePath + ".vmdk"); - } - if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) || !dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumePath, volumePath + ".vmdk"); - } - if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumePath) || !dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = dsMo.searchFileInSubFolders(volumePath + ".vmdk", true, null); - } - + datastoreVolumePath = VmwareStorageLayoutHelper.getDatastoreVolumePath(dsMo, vmName, volumePath); } else { datastoreVolumePath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dsMo.getOwnerDatacenter().first(), vmName, dsMo, volumePath, VmwareManager.s_vmwareSearchExcludeFolder.value()); } @@ -2101,16 +2091,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } dsMo = new DatastoreMO(context, morDs); - datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumePath + ".vmdk"); - if (!dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumePath + ".vmdk"); - } - if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) || !dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumePath, volumePath + ".vmdk"); - } - if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumePath) || !dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = dsMo.searchFileInSubFolders(volumePath + ".vmdk", true, null); - } + datastoreVolumePath = VmwareStorageLayoutHelper.getDatastoreVolumePath(dsMo, vmName, volumePath); } } diff --git a/test/integration/smoke/test_backup_recovery_veeam.py b/test/integration/smoke/test_backup_recovery_veeam.py index d0da66fa7c2..f99b3db17d4 100644 --- a/test/integration/smoke/test_backup_recovery_veeam.py +++ b/test/integration/smoke/test_backup_recovery_veeam.py @@ -235,13 +235,13 @@ class TestVeeamBackupAndRecovery(cloudstackTestCase): if self.offering: self.cleanup.insert(0, self.offering) - self.vm = VirtualMachine.create(self.user_apiclient, self.services["small"], accountid=self.account.name, - domainid=self.account.domainid, serviceofferingid=self.service_offering.id) - self.vm_with_datadisk = VirtualMachine.create(self.user_apiclient, self.services["small"], accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering.id, diskofferingid=self.disk_offering.id) + self.vm = VirtualMachine.create(self.user_apiclient, self.services["small"], accountid=self.account.name, + domainid=self.account.domainid, serviceofferingid=self.service_offering.id) + # Assign VM to offering and create ad-hoc backup self.offering.assignOffering(self.user_apiclient, self.vm_with_datadisk.id) @@ -296,7 +296,13 @@ class TestVeeamBackupAndRecovery(cloudstackTestCase): listall=True ) self.assertTrue(isinstance(vm_volumes, list), "List volumes should return a valid list") - self.assertEqual(3, len(vm_volumes), "The number of volumes should be 2") + self.assertEqual(3, len(vm_volumes), "The number of volumes should be 3") finally: # Delete backup Backup.delete(self.user_apiclient, backup.id, forced=True) + # Remove VM from offering + self.offering.removeOffering(self.user_apiclient, self.vm_with_datadisk.id) + # Delete vm + self.vm.delete(self.apiclient) + # Delete vm with datadisk + self.vm_with_datadisk.delete(self.apiclient)