From 23c9e830473ef9a45a28439ea7ec18b1f05205af Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 25 Sep 2025 13:28:29 +0530 Subject: [PATCH] Create Instance from backup on another Zone (DRaaS use case) (#11560) * draas initial changes * Added option to enable disaster recovery on a backup respository. Added UpdateBackupRepositoryCmd api. * Added timeout for mount operation in backup restore configurable via global setting * Addressed review comments * fix for simulator test failures * Added UT for coverage * Fix create instance from backup ui for other providers * Added events to add/update backup repository * Fix race in fetchZones * One more fix in fetchZones in DeployVMFromBackup.vue * Fix zone selection in createNetwork via Create Instance from backup form. * Allow template/iso selection in create instance from backup ui * rename draasenabled to crosszoneinstancecreation * Added Cross-zone instance creation in test_backup_recovery_nas.py * Added UT in BackupManagerTest and UserVmManagerImplTest * Integration test added for Cross-zone instance creation in test_backup_recovery_nas.py --- .../main/java/com/cloud/event/EventTypes.java | 9 + .../com/cloud/vm/VirtualMachineProfile.java | 1 + .../apache/cloudstack/api/ApiConstants.java | 1 + .../repository/AddBackupRepositoryCmd.java | 8 +- .../repository/UpdateBackupRepositoryCmd.java | 116 ++++ .../response/BackupRepositoryResponse.java | 12 + .../cloudstack/backup/BackupManager.java | 2 + .../cloudstack/backup/BackupProvider.java | 4 +- .../cloudstack/backup/BackupRepository.java | 3 + .../backup/BackupRepositoryService.java | 2 + .../backup/RestoreBackupCommand.java | 9 + .../cloud/vm/VirtualMachineManagerImpl.java | 15 + .../cloudstack/backup/BackupRepositoryVO.java | 21 +- .../META-INF/db/schema-42100to42200.sql | 3 + .../backup/DummyBackupProvider.java | 9 +- .../cloudstack/backup/NASBackupProvider.java | 73 ++- .../backup/NASBackupProviderTest.java | 138 ++++- .../backup/NetworkerBackupProvider.java | 9 +- .../backup/VeeamBackupProvider.java | 9 +- .../LibvirtRestoreBackupCommandWrapper.java | 103 ++-- ...ibvirtRestoreBackupCommandWrapperTest.java | 499 ++++++++++++++++++ .../java/com/cloud/api/ApiResponseHelper.java | 1 + .../java/com/cloud/vm/UserVmManagerImpl.java | 49 +- .../cloudstack/backup/BackupManagerImpl.java | 50 +- .../backup/BackupRepositoryServiceImpl.java | 53 +- .../com/cloud/vm/UserVmManagerImplTest.java | 126 ++++- .../cloudstack/backup/BackupManagerTest.java | 274 +++++++++- .../BackupRepositoryServiceImplTest.java | 243 +++++++++ .../smoke/test_backup_recovery_nas.py | 124 +++-- tools/marvin/marvin/lib/base.py | 14 +- ui/public/locales/en.json | 4 + ui/src/components/view/DeployVMFromBackup.vue | 111 +++- ui/src/config/section/config.js | 13 +- .../compute/wizard/TemplateIsoRadioGroup.vue | 10 +- .../network/CreateIsolatedNetworkForm.vue | 2 +- ui/src/views/network/CreateL2NetworkForm.vue | 2 +- ui/src/views/network/CreateNetwork.vue | 2 +- .../views/network/CreateSharedNetworkForm.vue | 2 +- ui/src/views/storage/CreateVMFromBackup.vue | 41 +- 39 files changed, 1961 insertions(+), 206 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java create mode 100644 server/src/test/java/org/apache/cloudstack/backup/BackupRepositoryServiceImplTest.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index be21f13267b..38e601c790a 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.backup.BackupRepositoryService; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.datacenter.DataCenterIpv4GuestSubnet; import org.apache.cloudstack.extension.Extension; @@ -852,6 +853,10 @@ public class EventTypes { // Custom Action public static final String EVENT_CUSTOM_ACTION = "CUSTOM.ACTION"; + // Backup Repository + public static final String EVENT_BACKUP_REPOSITORY_ADD = "BACKUP.REPOSITORY.ADD"; + public static final String EVENT_BACKUP_REPOSITORY_UPDATE = "BACKUP.REPOSITORY.UPDATE"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking @@ -1385,6 +1390,10 @@ public class EventTypes { entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_ADD, ExtensionCustomAction.class); entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_UPDATE, ExtensionCustomAction.class); entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_DELETE, ExtensionCustomAction.class); + + // Backup Repository + entityEventDetails.put(EVENT_BACKUP_REPOSITORY_ADD, BackupRepositoryService.class); + entityEventDetails.put(EVENT_BACKUP_REPOSITORY_UPDATE, BackupRepositoryService.class); } public static boolean isNetworkEvent(String eventType) { diff --git a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java index c67ee4eabc2..5c78d6bedd6 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java @@ -78,6 +78,7 @@ public interface VirtualMachineProfile { public static final Param BootIntoSetup = new Param("enterHardwareSetup"); public static final Param PreserveNics = new Param("PreserveNics"); public static final Param ConsiderLastHost = new Param("ConsiderLastHost"); + public static final Param ReturnAfterVolumePrepare = new Param("ReturnAfterVolumePrepare"); private String name; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 489d737b5bb..6c84e54b2d1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -139,6 +139,7 @@ public class ApiConstants { public static final String CPU_SPEED = "cpuspeed"; public static final String CPU_LOAD_AVERAGE = "cpuloadaverage"; public static final String CREATED = "created"; + public static final String CROSS_ZONE_INSTANCE_CREATION = "crosszoneinstancecreation"; public static final String CTX_ACCOUNT_ID = "ctxaccountid"; public static final String CTX_DETAILS = "ctxDetails"; public static final String CTX_USER_ID = "ctxuserid"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java index 5d0c838bc37..64998a74954 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java @@ -63,12 +63,14 @@ public class AddBackupRepositoryCmd extends BaseCmd { type = CommandType.UUID, entityType = ZoneResponse.class, required = true, - description = "ID of the zone where the backup repository is to be added") + description = "ID of the zone where the backup repository is to be added for taking backups") private Long zoneId; @Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, description = "capacity of this backup repository") private Long capacityBytes; + @Parameter(name = ApiConstants.CROSS_ZONE_INSTANCE_CREATION, type = CommandType.BOOLEAN, description = "backups on this repository can be used to create Instances on all Zones", since = "4.22.0") + private Boolean crossZoneInstanceCreation; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -109,6 +111,10 @@ public class AddBackupRepositoryCmd extends BaseCmd { return capacityBytes; } + public Boolean crossZoneInstanceCreationEnabled() { + return crossZoneInstanceCreation; + } + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java new file mode 100644 index 00000000000..5ffd79e497e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup.repository; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupRepositoryResponse; +import org.apache.cloudstack.backup.BackupRepository; +import org.apache.cloudstack.backup.BackupRepositoryService; +import org.apache.cloudstack.context.CallContext; + +import javax.inject.Inject; + +@APICommand(name = "updateBackupRepository", + description = "Update a backup repository", + responseObject = BackupRepositoryResponse.class, since = "4.22.0", + authorized = {RoleType.Admin}) +public class UpdateBackupRepositoryCmd extends BaseCmd { + + @Inject + private BackupRepositoryService backupRepositoryService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, entityType = BackupRepositoryResponse.class, description = "ID of the backup repository") + private Long id; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "name of the backup repository") + private String name; + + @Parameter(name = ApiConstants.ADDRESS, type = CommandType.STRING, description = "address of the backup repository") + private String address; + + @Parameter(name = ApiConstants.MOUNT_OPTIONS, type = CommandType.STRING, description = "shared storage mount options") + private String mountOptions; + + @Parameter(name = ApiConstants.CROSS_ZONE_INSTANCE_CREATION, type = CommandType.BOOLEAN, description = "backups in this repository can be used to create Instances on all Zones") + private Boolean crossZoneInstanceCreation; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public BackupRepositoryService getBackupRepositoryService() { + return backupRepositoryService; + } + + public Long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getAddress() { + return address; + } + + public String getMountOptions() { + return mountOptions == null ? "" : mountOptions; + } + + public Boolean crossZoneInstanceCreationEnabled() { + return crossZoneInstanceCreation; + } + + ///////////////////////////////////////////////////// + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + try { + BackupRepository result = backupRepositoryService.updateBackupRepository(this); + if (result != null) { + BackupRepositoryResponse response = _responseGenerator.createBackupRepositoryResponse(result); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update the backup repository"); + } + } catch (Exception ex4) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex4.getMessage()); + } + + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java index 0db51f04034..327bbae0051 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java @@ -61,6 +61,10 @@ public class BackupRepositoryResponse extends BaseResponse { @Param(description = "capacity of the backup repository") private Long capacityBytes; + @SerializedName(ApiConstants.CROSS_ZONE_INSTANCE_CREATION) + @Param(description = "the backups in this repository can be used to create Instances on all Zones") + private Boolean crossZoneInstanceCreation; + @SerializedName("created") @Param(description = "the date and time the backup repository was added") private Date created; @@ -132,6 +136,14 @@ public class BackupRepositoryResponse extends BaseResponse { this.capacityBytes = capacityBytes; } + public Boolean getCrossZoneInstanceCreation() { + return crossZoneInstanceCreation; + } + + public void setCrossZoneInstanceCreation(Boolean crossZoneInstanceCreation) { + this.crossZoneInstanceCreation = crossZoneInstanceCreation; + } + public Date getCreated() { return created; } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index c4b92fc9e05..37d21613c3d 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -205,6 +205,8 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer Boolean canCreateInstanceFromBackup(Long backupId); + Boolean canCreateInstanceFromBackupAcrossZones(Long backupId); + /** * Restore a backup to a new Instance */ diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 1eb36f89556..32a714370df 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -23,6 +23,8 @@ import com.cloud.vm.VirtualMachine; public interface BackupProvider { + Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering); + /** * Returns the unique name of the provider * @return returns provider name @@ -85,7 +87,7 @@ public interface BackupProvider { */ boolean deleteBackup(Backup backup, boolean forced); - boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid); + Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid); /** * Restore VM from backup diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java index be539a0eb04..886d13c13f9 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java @@ -28,9 +28,12 @@ public interface BackupRepository extends InternalIdentity, Identity { String getType(); String getAddress(); String getMountOptions(); + void setMountOptions(String mountOptions); void setUsedBytes(Long usedBytes); Long getCapacityBytes(); Long getUsedBytes(); void setCapacityBytes(Long capacityBytes); + Boolean crossZoneInstanceCreationEnabled(); + void setCrossZoneInstanceCreation(Boolean crossZoneInstanceCreation); Date getCreated(); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java index ae71053e400..875fc3b3d90 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java @@ -23,11 +23,13 @@ import com.cloud.utils.Pair; import org.apache.cloudstack.api.command.user.backup.repository.AddBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.backup.repository.DeleteBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.backup.repository.ListBackupRepositoriesCmd; +import org.apache.cloudstack.api.command.user.backup.repository.UpdateBackupRepositoryCmd; import java.util.List; public interface BackupRepositoryService { BackupRepository addBackupRepository(AddBackupRepositoryCmd cmd); + BackupRepository updateBackupRepository(UpdateBackupRepositoryCmd cmd); boolean deleteBackupRepository(DeleteBackupRepositoryCmd cmd); Pair, Integer> listBackupRepositories(ListBackupRepositoriesCmd cmd); diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java index f447fbe3d00..453b236df6b 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -36,6 +36,7 @@ public class RestoreBackupCommand extends Command { private Boolean vmExists; private String restoreVolumeUUID; private VirtualMachine.State vmState; + private Integer mountTimeout; protected RestoreBackupCommand() { super(); @@ -136,4 +137,12 @@ public class RestoreBackupCommand extends Command { public void setBackupVolumesUUIDs(List backupVolumesUUIDs) { this.backupVolumesUUIDs = backupVolumesUUIDs; } + + public Integer getMountTimeout() { + return this.mountTimeout; + } + + public void setMountTimeout(Integer mountTimeout) { + this.mountTimeout = mountTimeout; + } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 3a6e1b62277..b5597280364 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1482,6 +1482,21 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac volumeMgr.prepare(vmProfile, dest); } + if (params != null) { + Boolean returnAfterVolumePrepare = (Boolean) params.get(VirtualMachineProfile.Param.ReturnAfterVolumePrepare); + if (Boolean.TRUE.equals(returnAfterVolumePrepare)) { + logger.info("Returning from VM start command execution for VM {} as requested. Volumes are prepared and ready.", vm.getUuid()); + + if (!changeState(vm, Event.AgentReportStopped, destHostId, work, Step.Done)) { + logger.error("Unable to transition to a new state. VM uuid: {}, VM oldstate: {}, Event: {}", vm, vm.getState(), Event.AgentReportStopped); + throw new ConcurrentOperationException(String.format("Failed to deploy VM %s", vm)); + } + + logger.debug("Volume preparation completed for VM {} (VM state set to Stopped)", vm); + return; + } + } + if (!reuseVolume) { reuseVolume = true; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java index 98efa94ceca..1764496a6c0 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java @@ -67,6 +67,9 @@ public class BackupRepositoryVO implements BackupRepository { @Column(name = "capacity_bytes", nullable = true) private Long capacityBytes; + @Column(name = "cross_zone_instance_creation") + private Boolean crossZoneInstanceCreation; + @Column(name = "created") @Temporal(value = TemporalType.TIMESTAMP) private Date created; @@ -79,7 +82,7 @@ public class BackupRepositoryVO implements BackupRepository { this.uuid = UUID.randomUUID().toString(); } - public BackupRepositoryVO(final long zoneId, final String provider, final String name, final String type, final String address, final String mountOptions, final Long capacityBytes) { + public BackupRepositoryVO(final long zoneId, final String provider, final String name, final String type, final String address, final String mountOptions, final Long capacityBytes, final Boolean crossZoneInstanceCreation) { this(); this.zoneId = zoneId; this.provider = provider; @@ -88,6 +91,7 @@ public class BackupRepositoryVO implements BackupRepository { this.address = address; this.mountOptions = mountOptions; this.capacityBytes = capacityBytes; + this.crossZoneInstanceCreation = crossZoneInstanceCreation; this.created = new Date(); } @@ -139,6 +143,11 @@ public class BackupRepositoryVO implements BackupRepository { return mountOptions; } + @Override + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + @Override public Long getUsedBytes() { return usedBytes; @@ -154,6 +163,16 @@ public class BackupRepositoryVO implements BackupRepository { return capacityBytes; } + @Override + public Boolean crossZoneInstanceCreationEnabled() { + return crossZoneInstanceCreation; + } + + @Override + public void setCrossZoneInstanceCreation(Boolean crossZoneInstanceCreation) { + this.crossZoneInstanceCreation = crossZoneInstanceCreation; + } + @Override public void setCapacityBytes(Long capacityBytes) { this.capacityBytes = capacityBytes; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 0538e45d1dc..4fcb2b75de5 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -25,3 +25,6 @@ CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', ' -- Increase length of scripts_version column to 128 due to md5sum to sha512sum change CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.domain_router', 'scripts_version', 'scripts_version', 'VARCHAR(128)'); + +-- Add the column cross_zone_instance_creation to cloud.backup_repository. if enabled it means that new Instance can be created on all Zones from Backups on this Repository. +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_instance_creation', 'TINYINT(1) DEFAULT NULL COMMENT ''Backup Repository can be used for disaster recovery on another zone'''); diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index 48082c1c8a4..b228a9f8ce0 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -53,6 +53,11 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { @Inject private DiskOfferingDao diskOfferingDao; + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return true; + } + @Override public String getName() { return "dummy"; @@ -199,7 +204,7 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { - return true; + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + return new Pair<>(true, null); } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index e5f98ad291b..9cd2f20e386 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -70,9 +70,18 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; +import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; + public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); + ConfigKey NASBackupRestoreMountTimeout = new ConfigKey<>("Advanced", Integer.class, + "nas.backup.restore.mount.timeout", + "30", + "Timeout in seconds after which backup repository mount for restore fails.", + true, + BackupFrameworkEnabled.key()); + @Inject private BackupDao backupDao; @@ -115,30 +124,45 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private DiskOfferingDao diskOfferingDao; - protected Host getLastVMHypervisorHost(VirtualMachine vm) { - Long hostId = vm.getLastHostId(); - if (hostId == null) { - LOG.debug("Cannot find last host for vm. This should never happen, please check your database."); + private Long getClusterIdFromRootVolume(VirtualMachine vm) { + VolumeVO rootVolume = volumeDao.getInstanceRootVolume(vm.getId()); + StoragePoolVO rootDiskPool = primaryDataStoreDao.findById(rootVolume.getPoolId()); + if (rootDiskPool == null) { return null; } - Host host = hostDao.findById(hostId); + return rootDiskPool.getClusterId(); + } - if (host.getStatus() == Status.Up) { - return host; - } else { + protected Host getVMHypervisorHost(VirtualMachine vm) { + Long hostId = vm.getLastHostId(); + Long clusterId = null; + + if (hostId != null) { + Host host = hostDao.findById(hostId); + if (host.getStatus() == Status.Up) { + return host; + } // Try to find any Up host in the same cluster - for (final Host hostInCluster : hostDao.findHypervisorHostInCluster(host.getClusterId())) { + clusterId = host.getClusterId(); + } else { + // Try to find any Up host in the same cluster as the root volume + clusterId = getClusterIdFromRootVolume(vm); + } + + if (clusterId != null) { + for (final Host hostInCluster : hostDao.findHypervisorHostInCluster(clusterId)) { if (hostInCluster.getStatus() == Status.Up) { - LOG.debug("Found Host {} in cluster {}", hostInCluster, host.getClusterId()); + LOG.debug("Found Host {} in cluster {}", hostInCluster, clusterId); return hostInCluster; } } } + // Try to find any Host in the zone - return resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, host.getDataCenterId()); + return resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, vm.getDataCenterId()); } - protected Host getVMHypervisorHost(VirtualMachine vm) { + protected Host getVMHypervisorHostForBackup(VirtualMachine vm) { Long hostId = vm.getHostId(); if (hostId == null && VirtualMachine.State.Running.equals(vm.getState())) { throw new CloudRuntimeException(String.format("Unable to find the hypervisor host for %s. Make sure the virtual machine is running", vm.getName())); @@ -158,7 +182,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Override public Pair takeBackup(final VirtualMachine vm, Boolean quiesceVM) { - final Host host = getVMHypervisorHost(vm); + final Host host = getVMHypervisorHostForBackup(vm); final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); if (backupRepository == null) { @@ -249,16 +273,16 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { return restoreVMBackup(vm, backup); } @Override public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { - return restoreVMBackup(vm, backup); + return restoreVMBackup(vm, backup).first(); } - private boolean restoreVMBackup(VirtualMachine vm, Backup backup) { + private Pair restoreVMBackup(VirtualMachine vm, Backup backup) { List backedVolumesUUIDs = backup.getBackedUpVolumes().stream() .sorted(Comparator.comparingLong(Backup.VolumeInfo::getDeviceId)) .map(Backup.VolumeInfo::getUuid) @@ -271,7 +295,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co LOG.debug("Restoring vm {} from backup {} on the NAS Backup Provider", vm, backup); BackupRepository backupRepository = getBackupRepository(backup); - final Host host = getLastVMHypervisorHost(vm); + final Host host = getVMHypervisorHost(vm); RestoreBackupCommand restoreCommand = new RestoreBackupCommand(); restoreCommand.setBackupPath(backup.getExternalId()); restoreCommand.setBackupRepoType(backupRepository.getType()); @@ -282,6 +306,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co restoreCommand.setRestoreVolumePaths(getVolumePaths(restoreVolumes)); restoreCommand.setVmExists(vm.getRemoved() == null); restoreCommand.setVmState(vm.getState()); + restoreCommand.setMountTimeout(NASBackupRestoreMountTimeout.value()); BackupAnswer answer; try { @@ -291,7 +316,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to restore backup timed out, please try again"); } - return answer.getResult(); + return new Pair<>(answer.getResult(), answer.getDetails()); } private List getVolumePaths(List volumes) { @@ -398,7 +423,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co final Host host; final VirtualMachine vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); if (vm != null) { - host = getLastVMHypervisorHost(vm); + host = getVMHypervisorHost(vm); } else { host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, backup.getZoneId()); } @@ -513,9 +538,19 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co return true; } + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(backupOffering.getId()); + if (backupRepository == null) { + throw new CloudRuntimeException("Backup repository not found for the backup offering" + backupOffering.getName()); + } + return Boolean.TRUE.equals(backupRepository.crossZoneInstanceCreationEnabled()); + } + @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ + NASBackupRestoreMountTimeout }; } diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java index d6f29dc1aac..7540cabbbf5 100644 --- a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java @@ -21,11 +21,9 @@ import static org.mockito.Mockito.mock; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; -import org.apache.cloudstack.backup.dao.BackupDao; -import org.apache.cloudstack.backup.dao.BackupRepositoryDao; -import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -39,6 +37,7 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.agent.AgentManager; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; @@ -51,6 +50,12 @@ import com.cloud.utils.Pair; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupRepositoryDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; + @RunWith(MockitoJUnitRunner.class) public class NASBackupProviderTest { @Spy @@ -84,6 +89,9 @@ public class NASBackupProviderTest { @Mock private ResourceManager resourceManager; + @Mock + private PrimaryDataStoreDao storagePoolDao; + @Test public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException { Long hostId = 1L; @@ -94,7 +102,7 @@ public class NASBackupProviderTest { ReflectionTestUtils.setField(backup, "id", 1L); BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1024L); + "nfs", "address", "sync", 1024L, null); VMInstanceVO vm = mock(VMInstanceVO.class); Mockito.when(vm.getLastHostId()).thenReturn(hostId); @@ -113,7 +121,7 @@ public class NASBackupProviderTest { @Test public void testSyncBackupStorageStats() throws AgentUnavailableException, OperationTimedoutException { BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1024L); + "nfs", "address", "sync", 1024L, null); HostVO host = mock(HostVO.class); Mockito.when(resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, 1L)).thenReturn(host); @@ -132,7 +140,7 @@ public class NASBackupProviderTest { @Test public void testListBackupOfferings() { BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1024L); + "nfs", "address", "sync", 1024L, null); ReflectionTestUtils.setField(backupRepository, "uuid", "uuid"); Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")).thenReturn(Collections.singletonList(backupRepository)); @@ -146,11 +154,11 @@ public class NASBackupProviderTest { @Test public void testGetBackupStorageStats() { BackupRepositoryVO backupRepository1 = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1000L); + "nfs", "address", "sync", 1000L, null); backupRepository1.setUsedBytes(500L); BackupRepositoryVO backupRepository2 = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 2000L); + "nfs", "address", "sync", 2000L, null); backupRepository2.setUsedBytes(600L); Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")) @@ -227,4 +235,118 @@ public class NASBackupProviderTest { Mockito.verify(backupDao).update(Mockito.anyLong(), Mockito.any(BackupVO.class)); Mockito.verify(agentManager).send(anyLong(), Mockito.any(TakeBackupCommand.class)); } + + @Test + public void testGetVMHypervisorHost() { + Long hostId = 1L; + Long vmId = 1L; + Long zoneId = 1L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + + HostVO host = mock(HostVO.class); + Mockito.when(host.getId()).thenReturn(hostId); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(hostId, result.getId())); + Mockito.verify(hostDao).findById(hostId); + } + + @Test + public void testGetVMHypervisorHostWithHostDown() { + Long hostId = 1L; + Long clusterId = 2L; + Long vmId = 1L; + Long zoneId = 1L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + + HostVO downHost = mock(HostVO.class); + Mockito.when(downHost.getStatus()).thenReturn(Status.Down); + Mockito.when(downHost.getClusterId()).thenReturn(clusterId); + Mockito.when(hostDao.findById(hostId)).thenReturn(downHost); + + HostVO upHostInCluster = mock(HostVO.class); + Mockito.when(upHostInCluster.getId()).thenReturn(3L); + Mockito.when(upHostInCluster.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(List.of(upHostInCluster)); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(Long.valueOf(3L), result.getId())); + Mockito.verify(hostDao).findById(hostId); + Mockito.verify(hostDao).findHypervisorHostInCluster(clusterId); + } + + @Test + public void testGetVMHypervisorHostWithUpHostViaRootVolumeCluster() { + Long vmId = 1L; + Long zoneId = 1L; + Long clusterId = 2L; + Long poolId = 3L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(null); + Mockito.when(vm.getId()).thenReturn(vmId); + + VolumeVO rootVolume = mock(VolumeVO.class); + Mockito.when(rootVolume.getPoolId()).thenReturn(poolId); + Mockito.when(volumeDao.getInstanceRootVolume(vmId)).thenReturn(rootVolume); + + StoragePoolVO storagePool = mock(StoragePoolVO.class); + Mockito.when(storagePool.getClusterId()).thenReturn(clusterId); + Mockito.when(storagePoolDao.findById(poolId)).thenReturn(storagePool); + + HostVO upHostInCluster = mock(HostVO.class); + Mockito.when(upHostInCluster.getId()).thenReturn(4L); + Mockito.when(upHostInCluster.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(List.of(upHostInCluster)); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(Long.valueOf(4L), result.getId())); + Mockito.verify(volumeDao).getInstanceRootVolume(vmId); + Mockito.verify(storagePoolDao).findById(poolId); + Mockito.verify(hostDao).findHypervisorHostInCluster(clusterId); + } + + @Test + public void testGetVMHypervisorHostFallbackToZoneWideKVMHost() { + Long hostId = 1L; + Long clusterId = 2L; + Long vmId = 1L; + Long zoneId = 1L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + Mockito.when(vm.getDataCenterId()).thenReturn(zoneId); + + HostVO downHost = mock(HostVO.class); + Mockito.when(downHost.getStatus()).thenReturn(Status.Down); + Mockito.when(downHost.getClusterId()).thenReturn(clusterId); + Mockito.when(hostDao.findById(hostId)).thenReturn(downHost); + + Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(Collections.emptyList()); + + HostVO fallbackHost = mock(HostVO.class); + Mockito.when(fallbackHost.getId()).thenReturn(5L); + Mockito.when(resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId)) + .thenReturn(fallbackHost); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(Long.valueOf(5L), result.getId())); + Mockito.verify(hostDao).findById(hostId); + Mockito.verify(hostDao).findHypervisorHostInCluster(clusterId); + Mockito.verify(resourceManager).findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId); + } } diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index f39aedb55f2..66b633e11a9 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -158,6 +158,11 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid }; } + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return false; + } + @Override public String getName() { return "networker"; @@ -630,7 +635,7 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid public boolean willDeleteBackupsOnOfferingRemoval() { return false; } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { - return true; + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + return new Pair<>(true, null); } } diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index c81c5d34ea2..39970dab342 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -337,11 +337,11 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { final Long zoneId = backup.getZoneId(); final String restorePointId = backup.getExternalId(); final String restoreLocation = vm.getInstanceName(); - return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, restoreLocation, hostIp, dataStoreUuid).first(); + return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, restoreLocation, hostIp, dataStoreUuid); } @Override @@ -358,6 +358,11 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, public void syncBackupStorageStats(Long zoneId) { } + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return false; + } + @Override public String getConfigComponentName() { return BackupService.class.getSimpleName(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 0e5091ebcf4..243cf2efa03 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -61,42 +61,49 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper backedVolumeUUIDs = command.getBackupVolumesUUIDs(); List restoreVolumePaths = command.getRestoreVolumePaths(); String restoreVolumeUuid = command.getRestoreVolumeUUID(); + Integer mountTimeout = command.getMountTimeout() * 1000; String newVolumeId = null; try { + String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions, mountTimeout); if (Objects.isNull(vmExists)) { String volumePath = restoreVolumePaths.get(0); int lastIndex = volumePath.lastIndexOf("/"); newVolumeId = volumePath.substring(lastIndex + 1); - restoreVolume(backupPath, backupRepoType, backupRepoAddress, volumePath, diskType, restoreVolumeUuid, - new Pair<>(vmName, command.getVmState()), mountOptions); + restoreVolume(backupPath, volumePath, diskType, restoreVolumeUuid, + new Pair<>(vmName, command.getVmState()), mountDirectory); } else if (Boolean.TRUE.equals(vmExists)) { - restoreVolumesOfExistingVM(restoreVolumePaths, backedVolumeUUIDs, backupPath, backupRepoType, backupRepoAddress, mountOptions); + restoreVolumesOfExistingVM(restoreVolumePaths, backedVolumeUUIDs, backupPath, mountDirectory); } else { - restoreVolumesOfDestroyedVMs(restoreVolumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions); + restoreVolumesOfDestroyedVMs(restoreVolumePaths, vmName, backupPath, mountDirectory); } } catch (CloudRuntimeException e) { - String errorMessage = "Failed to restore backup for VM: " + vmName + "."; - if (e.getMessage() != null && !e.getMessage().isEmpty()) { - errorMessage += " Details: " + e.getMessage(); - } - logger.error(errorMessage); + String errorMessage = e.getMessage() != null ? e.getMessage() : ""; return new BackupAnswer(command, false, errorMessage); } return new BackupAnswer(command, true, newVolumeId); } - private void restoreVolumesOfExistingVM(List restoreVolumePaths, List backedVolumesUUIDs, String backupPath, - String backupRepoType, String backupRepoAddress, String mountOptions) { + private void verifyBackupFile(String backupPath, String volUuid) { + if (!checkBackupPathExists(backupPath)) { + throw new CloudRuntimeException(String.format("Backup file for the volume [%s] does not exist.", volUuid)); + } + if (!checkBackupFileImage(backupPath)) { + throw new CloudRuntimeException(String.format("Backup qcow2 file for the volume [%s] is corrupt.", volUuid)); + } + } + + private void restoreVolumesOfExistingVM(List restoreVolumePaths, List backedVolumesUUIDs, + String backupPath, String mountDirectory) { String diskType = "root"; - String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions); try { for (int idx = 0; idx < restoreVolumePaths.size(); idx++) { String restoreVolumePath = restoreVolumePaths.get(idx); String backupVolumeUuid = backedVolumesUUIDs.get(idx); Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, null, backupPath, diskType, backupVolumeUuid); diskType = "datadisk"; + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); if (!replaceVolumeWithBackup(restoreVolumePath, bkpPathAndVolUuid.first())) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } @@ -107,15 +114,14 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper volumePaths, String vmName, String backupPath, - String backupRepoType, String backupRepoAddress, String mountOptions) { - String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions); + private void restoreVolumesOfDestroyedVMs(List volumePaths, String vmName, String backupPath, String mountDirectory) { String diskType = "root"; try { for (int i = 0; i < volumePaths.size(); i++) { String volumePath = volumePaths.get(i); Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, null); diskType = "datadisk"; + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } @@ -126,12 +132,12 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper vmNameAndState, String mountOptions) { - String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions); + private void restoreVolume(String backupPath, String volumePath, String diskType, String volumeUUID, + Pair vmNameAndState, String mountDirectory) { Pair bkpPathAndVolUuid; try { bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, volumeUUID); + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } @@ -140,8 +146,6 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper(bkpPath, volUuid); } + private boolean checkBackupFileImage(String backupPath) { + int exitValue = Script.runSimpleBashScriptForExitValue(String.format("qemu-img check %s", backupPath)); + return exitValue == 0; + } + + private boolean checkBackupPathExists(String backupPath) { + int exitValue = Script.runSimpleBashScriptForExitValue(String.format("ls %s", backupPath)); + return exitValue == 0; + } + private boolean replaceVolumeWithBackup(String volumePath, String backupPath) { int exitValue = Script.runSimpleBashScriptForExitValue(String.format(RSYNC_COMMAND, backupPath, volumePath)); return exitValue == 0; diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java new file mode 100644 index 00000000000..d120abd0a1b --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java @@ -0,0 +1,499 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.utils.script.Script; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.backup.BackupAnswer; +import org.apache.cloudstack.backup.RestoreBackupCommand; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtRestoreBackupCommandWrapperTest { + + private LibvirtRestoreBackupCommandWrapper wrapper; + private LibvirtComputingResource libvirtComputingResource; + private RestoreBackupCommand command; + + @Before + public void setUp() { + wrapper = new LibvirtRestoreBackupCommandWrapper(); + libvirtComputingResource = Mockito.mock(LibvirtComputingResource.class); + command = Mockito.mock(RestoreBackupCommand.class); + } + + @Test + public void testExecuteWithVmExistsNull() throws Exception { + when(command.getVmName()).thenReturn("test-vm"); + when(command.getBackupPath()).thenReturn("backup/path"); + when(command.getBackupRepoAddress()).thenReturn("192.168.1.100:/backup"); + when(command.getBackupRepoType()).thenReturn("nfs"); + when(command.getMountOptions()).thenReturn("rw"); + when(command.isVmExists()).thenReturn(null); + when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); + when(command.getRestoreVolumeUUID()).thenReturn("volume-123"); + when(command.getVmState()).thenReturn(VirtualMachine.State.Running); + when(command.getMountTimeout()).thenReturn(30); + + try (MockedStatic filesMock = mockStatic(Files.class)) { + Path tempPath = Mockito.mock(Path.class); + when(tempPath.toString()).thenReturn("/tmp/csbackup.abc123"); + filesMock.when(() -> Files.createTempDirectory(anyString())).thenReturn(tempPath); + + try (MockedStatic