mirror of
https://github.com/apache/cloudstack.git
synced 2025-10-26 08:42:29 +01:00
veeam: fix some issues with restoring volume from backup and attaching it to VM (#8570)
* veeam: detach only the restored volume during backup restore
Steps to reproduce the issue
1. create a VM (A) with ROOT and DATA disk
2. assign to a backup offering
3. create backup
4. create another VM (B)
5. restore the DATA disk of VM A, and attach to VM B
6. When operation is done, check the datastore
Without this change, the ROOT image is not removed and left over on the datastore.
```
[root@ref-trl-5933-v-Mr8-wei-zhou-esxi2:/vmfs/volumes/5f60667d-18d828eb] ls -l /vmfs/volumes/5f60667d-18d828eb/CS-RSTR-dfb6f21c-a941-49db-9963-4f0286a17dac
total 1784840
-rw------- 1 root root 5242880000 Jan 24 09:23 ROOT-722_2-flat.vmdk
-rw------- 1 root root 499 Jan 24 09:23 ROOT-722_2.vmdk
```
With this change, the whole temporary vm has been destroyed.
```
[root@ref-trl-5933-v-Mr8-wei-zhou-esxi2:/vmfs/volumes/5f60667d-18d828eb] ls -l /vmfs/volumes/5f60667d-18d828eb/CS-RSTR-734bee3b-640c-4ff0-a34b-bc45358565b2
ls: /vmfs/volumes/5f60667d-18d828eb/CS-RSTR-734bee3b-640c-4ff0-a34b-bc45358565b2: No such file or directory
```
* veeam: fix wrong disk size in debug message
* veeam: sync backup repository after operations are done
got exception of some operations which succeeds due to the following error
```
2024-01-19 10:59:52,846 DEBUG [o.a.c.b.v.VeeamClient] (API-Job-Executor-42:ctx-716501bb job-4373 ctx-2359b76d) (logid:b5e19a17) Veeam response for PowerShell commands [PowerShell Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue;$restorePoint = Get-VBRRestorePoint ^| Where-Object { $_.Id -eq '1d99106a-b5c8-4a1e-958d-066a987caa5f' };if ($restorePoint) { Remove-VBRRestorePoint -Oib $restorePoint -Confirm:$false;$repo = Get-VBRBackupRepository;Sync-VBRBackupRepository -Repository $repo;} else { ; Write-Output 'Failed to delete'; Exit 1;}] is: [^M
Restore Type Job Name State Start Time End Time Description ^M
------------ -------- ----- ---------- -------- ----------- ^M
ConfResynchronize Configuration Dat... Starting 19/01/2024 10:59:52 01/01/1900 00:00:00 ^M
^M
^M
Remove-VBRRestorePoint : Win32 internal error "Access is denied" 0x5 occurred while reading the console output buffer. ^M
Contact Microsoft Customer Support Services.^M
At line:1 char:196^M
+ ... orePoint) { Remove-VBRRestorePoint -Oib $restorePoint -Confirm:$false ...^M
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^M
+ CategoryInfo : ReadError: (:) [Remove-VBRRestorePoint], HostException^M
+ FullyQualifiedErrorId : ReadConsoleOutput,Veeam.Backup.PowerShell.Cmdlets.RemoveVBRRestorePoint^M
^M
].
```
* veeam: fix unable to detach volume when restore backup and attach to vm then detach the volume
It also happened when destroy the original or backup VM
```
2024-01-24 10:10:03,401 ERROR [c.c.s.r.VmwareStorageProcessor] (DirectAgent-74:ctx-95b24ac7 10.0.35.53, job-25995/job-25996, cmd: DettachCommand) (logid:7260ffb8) Failed to detach volume!
java.lang.RuntimeException: Unable to access file [de52fdd3386b3d67b27b3960ecdb08f4] i-2-723-VM/7c2197c129464035bab062edec536a09-flat.vmdk
at com.cloud.hypervisor.vmware.util.VmwareClient.waitForTask(VmwareClient.java:426)
at com.cloud.hypervisor.vmware.mo.DatastoreMO.moveDatastoreFile(DatastoreMO.java:290)
at com.cloud.storage.resource.VmwareStorageLayoutHelper.syncVolumeToRootFolder(VmwareStorageLayoutHelper.java:241)
at com.cloud.storage.resource.VmwareStorageProcessor.attachVolume(VmwareStorageProcessor.java:2150)
at com.cloud.storage.resource.VmwareStorageProcessor.dettachVolume(VmwareStorageProcessor.java:2408)
at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:174)
at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:71)
at com.cloud.hypervisor.vmware.resource.VmwareResource.executeRequest(VmwareResource.java:589)
at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:315)
at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:48)
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55)
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:102)
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52)
at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:45)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
2024-01-24 10:10:03,402 INFO [c.c.h.v.u.VmwareHelper] (DirectAgent-74:ctx-95b24ac7 10.0.35.53, job-25995/job-25996, cmd: DettachCommand) (logid:7260ffb8) [ignored]failed to get message for exception: Unable to access file [de52fdd3386b3d67b27b3960ecdb08f4] i-2-723-VM/7c2197c129464035bab062edec536a09-flat.vmdk
```
* vmware: create restored volume with new UUID and attach to VM
This commit is contained in:
Wei Zhou
GitHub
parent
33bb92acce
commit
b34f093137
@ -212,6 +212,7 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider,
|
||||
LOG.warn("Failed to remove Veeam job and backup for job: " + clonedJobName);
|
||||
throw new CloudRuntimeException("Failed to delete Veeam B&R job and backup, an operation may be in progress. Please try again after some time.");
|
||||
}
|
||||
client.syncBackupRepository();
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -245,6 +246,8 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider,
|
||||
return false;
|
||||
}
|
||||
|
||||
client.syncBackupRepository();
|
||||
|
||||
List<Backup> allBackups = backupDao.listByVmId(backup.getZoneId(), backup.getVmId());
|
||||
for (Backup b : allBackups) {
|
||||
if (b.getId() != backup.getId()) {
|
||||
|
||||
@ -604,6 +604,7 @@ public class VeeamClient {
|
||||
joiner.add("PowerShell Add-PSSnapin VeeamPSSnapin");
|
||||
} else {
|
||||
joiner.add("PowerShell Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue");
|
||||
joiner.add("$ProgressPreference='SilentlyContinue'");
|
||||
}
|
||||
for (String cmd : cmds) {
|
||||
joiner.add(cmd);
|
||||
@ -646,9 +647,7 @@ public class VeeamClient {
|
||||
String.format("$job = Get-VBRJob -Name '%s'", jobName),
|
||||
"if ($job) { Remove-VBRJob -Job $job -Confirm:$false }",
|
||||
String.format("$backup = Get-VBRBackup -Name '%s'", jobName),
|
||||
"if ($backup) { Remove-VBRBackup -Backup $backup -FromDisk -Confirm:$false }",
|
||||
"$repo = Get-VBRBackupRepository",
|
||||
"Sync-VBRBackupRepository -Repository $repo"
|
||||
"if ($backup) { Remove-VBRBackup -Backup $backup -FromDisk -Confirm:$false }"
|
||||
));
|
||||
return result != null && result.first() && !result.second().contains(FAILED_TO_DELETE);
|
||||
}
|
||||
@ -658,8 +657,6 @@ public class VeeamClient {
|
||||
Pair<Boolean, String> result = executePowerShellCommands(Arrays.asList(
|
||||
String.format("$restorePoint = Get-VBRRestorePoint ^| Where-Object { $_.Id -eq '%s' }", restorePointId),
|
||||
"if ($restorePoint) { Remove-VBRRestorePoint -Oib $restorePoint -Confirm:$false",
|
||||
"$repo = Get-VBRBackupRepository",
|
||||
"Sync-VBRBackupRepository -Repository $repo",
|
||||
"} else { ",
|
||||
" Write-Output 'Failed to delete'",
|
||||
" Exit 1",
|
||||
@ -668,6 +665,17 @@ public class VeeamClient {
|
||||
return result != null && result.first() && !result.second().contains(FAILED_TO_DELETE);
|
||||
}
|
||||
|
||||
public boolean syncBackupRepository() {
|
||||
LOG.debug("Trying to sync backup repository.");
|
||||
Pair<Boolean, String> result = executePowerShellCommands(Arrays.asList(
|
||||
"$repo = Get-VBRBackupRepository",
|
||||
"$Syncs = Sync-VBRBackupRepository -Repository $repo",
|
||||
"while ((Get-VBRSession -ID $Syncs.ID).Result -ne 'Success') { Start-Sleep -Seconds 10 }"
|
||||
));
|
||||
LOG.debug("Done syncing backup repository.");
|
||||
return result != null && result.first();
|
||||
}
|
||||
|
||||
public Map<String, Backup.Metric> getBackupMetrics() {
|
||||
if (isLegacyServer()) {
|
||||
return getBackupMetricsLegacy();
|
||||
|
||||
@ -1022,12 +1022,12 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
|
||||
/**
|
||||
* Get dest volume full path
|
||||
*/
|
||||
private String getDestVolumeFullPath(VirtualDisk restoredDisk, VirtualMachineMO restoredVm, VirtualMachineMO vmMo) throws Exception {
|
||||
private String getDestVolumeFullPath(VirtualMachineMO vmMo) throws Exception {
|
||||
VirtualDisk vmDisk = vmMo.getVirtualDisks().get(0);
|
||||
String vmDiskPath = vmMo.getVmdkFileBaseName(vmDisk);
|
||||
String vmDiskFullPath = getVolumeFullPath(vmMo.getVirtualDisks().get(0));
|
||||
String restoredVolumePath = restoredVm.getVmdkFileBaseName(restoredDisk);
|
||||
return vmDiskFullPath.replace(vmDiskPath, restoredVolumePath);
|
||||
String uuid = UUID.randomUUID().toString().replace("-", "");
|
||||
return vmDiskFullPath.replace(vmDiskPath, uuid);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -1079,17 +1079,18 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
|
||||
VirtualDisk restoredDisk = findRestoredVolume(volumeInfo, vmRestored);
|
||||
String diskPath = vmRestored.getVmdkFileBaseName(restoredDisk);
|
||||
|
||||
s_logger.debug("Restored disk size=" + toHumanReadableSize(restoredDisk.getCapacityInKB()) + " path=" + diskPath);
|
||||
s_logger.debug("Restored disk size=" + toHumanReadableSize(restoredDisk.getCapacityInKB() * Resource.ResourceType.bytesToKiB) + " path=" + diskPath);
|
||||
|
||||
// Detach restored VM disks
|
||||
vmRestored.detachAllDisks();
|
||||
vmRestored.detachDisk(String.format("%s/%s.vmdk", location, diskPath), false);
|
||||
|
||||
String srcPath = getVolumeFullPath(restoredDisk);
|
||||
String destPath = getDestVolumeFullPath(restoredDisk, vmRestored, vmMo);
|
||||
String destPath = getDestVolumeFullPath(vmMo);
|
||||
|
||||
VirtualDiskManagerMO virtualDiskManagerMO = new VirtualDiskManagerMO(dcMo.getContext());
|
||||
|
||||
// Copy volume to the VM folder
|
||||
s_logger.debug(String.format("Moving volume from %s to %s", srcPath, destPath));
|
||||
virtualDiskManagerMO.moveVirtualDisk(srcPath, dcMo.getMor(), destPath, dcMo.getMor(), true);
|
||||
|
||||
try {
|
||||
@ -1103,11 +1104,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
|
||||
vmRestored.destroy();
|
||||
}
|
||||
|
||||
VirtualDisk attachedDisk = getAttachedDisk(vmMo, diskPath);
|
||||
s_logger.debug(String.format("Attaching disk %s to vm %s", destPath, vm.getId()));
|
||||
VirtualDisk attachedDisk = getAttachedDisk(vmMo, destPath);
|
||||
if (attachedDisk == null) {
|
||||
s_logger.error("Failed to get the attached the (restored) volume " + diskPath);
|
||||
s_logger.error("Failed to get the attached the (restored) volume " + destPath);
|
||||
return false;
|
||||
}
|
||||
s_logger.debug(String.format("Creating volume entry for disk %s attached to vm %s", destPath, vm.getId()));
|
||||
createVolume(attachedDisk, vmMo, vm.getDomainId(), vm.getDataCenterId(), vm.getAccountId(), vm.getId(), poolId, vm.getTemplateId(), backup, false);
|
||||
|
||||
if (vm.getBackupOfferingId() == null) {
|
||||
@ -1119,9 +1122,9 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
|
||||
return true;
|
||||
}
|
||||
|
||||
private VirtualDisk getAttachedDisk(VirtualMachineMO vmMo, String diskPath) throws Exception {
|
||||
private VirtualDisk getAttachedDisk(VirtualMachineMO vmMo, String diskFullPath) throws Exception {
|
||||
for (VirtualDisk disk : vmMo.getVirtualDisks()) {
|
||||
if (vmMo.getVmdkFileBaseName(disk).equals(diskPath)) {
|
||||
if (getVolumeFullPath(disk).equals(diskFullPath)) {
|
||||
return disk;
|
||||
}
|
||||
}
|
||||
|
||||
@ -218,7 +218,10 @@ public class VmwareStorageLayoutHelper implements Configurable {
|
||||
}
|
||||
|
||||
public static void syncVolumeToRootFolder(DatacenterMO dcMo, DatastoreMO ds, String vmdkName, String vmName, String excludeFolders) throws Exception {
|
||||
String fileDsFullPath = ds.searchFileInSubFolders(vmdkName + ".vmdk", false, excludeFolders);
|
||||
String fileDsFullPath = ds.searchFileInSubFolders(String.format("%s/%s.vmdk", vmName, vmdkName), false, excludeFolders);
|
||||
if (fileDsFullPath == null) {
|
||||
fileDsFullPath = ds.searchFileInSubFolders(vmdkName + ".vmdk", false, excludeFolders);
|
||||
}
|
||||
if (fileDsFullPath == null)
|
||||
return;
|
||||
|
||||
@ -409,6 +412,22 @@ public class VmwareStorageLayoutHelper implements Configurable {
|
||||
return String.format("[%s] %s/%s", dsMo.getName(), vmName, vmdkFileName);
|
||||
}
|
||||
|
||||
public static String getDatastoreVolumePath(DatastoreMO dsMo, String vmName, String volumePath) throws Exception {
|
||||
String datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumePath + ".vmdk");
|
||||
if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) && dsMo.fileExists(datastoreVolumePath)) {
|
||||
return datastoreVolumePath;
|
||||
}
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumePath, volumePath + ".vmdk");
|
||||
if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumePath) && dsMo.fileExists(datastoreVolumePath)) {
|
||||
return datastoreVolumePath;
|
||||
}
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumePath + ".vmdk");
|
||||
if (dsMo.fileExists(datastoreVolumePath)) {
|
||||
return datastoreVolumePath;
|
||||
}
|
||||
return dsMo.searchFileInSubFolders(volumePath + ".vmdk", false, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getConfigComponentName() {
|
||||
return VmwareStorageLayoutHelper.class.getSimpleName();
|
||||
|
||||
@ -2062,17 +2062,7 @@ public class VmwareStorageProcessor implements StorageProcessor {
|
||||
datastoreVolumePath = dsMo.getDatastorePath((vmdkPath != null ? vmdkPath : dsMo.getName()) + ".vmdk");
|
||||
} else {
|
||||
if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) {
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumePath + ".vmdk");
|
||||
if (!dsMo.fileExists(datastoreVolumePath)) {
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumePath + ".vmdk");
|
||||
}
|
||||
if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) || !dsMo.fileExists(datastoreVolumePath)) {
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumePath, volumePath + ".vmdk");
|
||||
}
|
||||
if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumePath) || !dsMo.fileExists(datastoreVolumePath)) {
|
||||
datastoreVolumePath = dsMo.searchFileInSubFolders(volumePath + ".vmdk", true, null);
|
||||
}
|
||||
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.getDatastoreVolumePath(dsMo, vmName, volumePath);
|
||||
} else {
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dsMo.getOwnerDatacenter().first(), vmName, dsMo, volumePath, VmwareManager.s_vmwareSearchExcludeFolder.value());
|
||||
}
|
||||
@ -2101,16 +2091,7 @@ public class VmwareStorageProcessor implements StorageProcessor {
|
||||
}
|
||||
dsMo = new DatastoreMO(context, morDs);
|
||||
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumePath + ".vmdk");
|
||||
if (!dsMo.fileExists(datastoreVolumePath)) {
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumePath + ".vmdk");
|
||||
}
|
||||
if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) || !dsMo.fileExists(datastoreVolumePath)) {
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumePath, volumePath + ".vmdk");
|
||||
}
|
||||
if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumePath) || !dsMo.fileExists(datastoreVolumePath)) {
|
||||
datastoreVolumePath = dsMo.searchFileInSubFolders(volumePath + ".vmdk", true, null);
|
||||
}
|
||||
datastoreVolumePath = VmwareStorageLayoutHelper.getDatastoreVolumePath(dsMo, vmName, volumePath);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -235,13 +235,13 @@ class TestVeeamBackupAndRecovery(cloudstackTestCase):
|
||||
if self.offering:
|
||||
self.cleanup.insert(0, self.offering)
|
||||
|
||||
self.vm = VirtualMachine.create(self.user_apiclient, self.services["small"], accountid=self.account.name,
|
||||
domainid=self.account.domainid, serviceofferingid=self.service_offering.id)
|
||||
|
||||
self.vm_with_datadisk = VirtualMachine.create(self.user_apiclient, self.services["small"], accountid=self.account.name,
|
||||
domainid=self.account.domainid, serviceofferingid=self.service_offering.id,
|
||||
diskofferingid=self.disk_offering.id)
|
||||
|
||||
self.vm = VirtualMachine.create(self.user_apiclient, self.services["small"], accountid=self.account.name,
|
||||
domainid=self.account.domainid, serviceofferingid=self.service_offering.id)
|
||||
|
||||
# Assign VM to offering and create ad-hoc backup
|
||||
self.offering.assignOffering(self.user_apiclient, self.vm_with_datadisk.id)
|
||||
|
||||
@ -296,7 +296,13 @@ class TestVeeamBackupAndRecovery(cloudstackTestCase):
|
||||
listall=True
|
||||
)
|
||||
self.assertTrue(isinstance(vm_volumes, list), "List volumes should return a valid list")
|
||||
self.assertEqual(3, len(vm_volumes), "The number of volumes should be 2")
|
||||
self.assertEqual(3, len(vm_volumes), "The number of volumes should be 3")
|
||||
finally:
|
||||
# Delete backup
|
||||
Backup.delete(self.user_apiclient, backup.id, forced=True)
|
||||
# Remove VM from offering
|
||||
self.offering.removeOffering(self.user_apiclient, self.vm_with_datadisk.id)
|
||||
# Delete vm
|
||||
self.vm.delete(self.apiclient)
|
||||
# Delete vm with datadisk
|
||||
self.vm_with_datadisk.delete(self.apiclient)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user