From 33bb92acce2dd2f3b8acccc4c0d81fcc37a4714f Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 19 Jan 2024 18:42:01 +0100 Subject: [PATCH 1/7] Veeam: Support Veeam 11 and 12 (#8241) This PR fixes several issues in the testing of Veeam 11 and Veeam12 - Import Veeam.Backup.PowerShell and silently ignore the warning messages - Fix issue when assign vm to backup offerings, which caused by separator (\r\n) - Fix authorization failure in veeam 12a, which is because v1_4 is not supported in veeam 12a any more - Fix exception if backup name has space - Fix backup metrics in veeam12, which is because powershell command does not return the values needed - Fix Incorrect datetime value, which is because powershell command returns a datetime which is not supported in Java - Fix issue during backup restoration if VM has both ROOT and DATA disks. This PR also has the following update - Add integration test test/integration/smoke/test_backup_recovery_veeam.py - Make some UI changes - Add zone setting backup.plugin.veeam.version. If it is not set, CloudStack will get veeam version via powershell commands. - Add zone setting backup.plugin.veeam.task.poll.interval and backup.plugin.veeam.task.poll.max.retry --- .../api/command/user/vm/ListVMsCmd.java | 4 +- .../PrepareForBackupRestorationCommand.java | 43 +++ .../src/main/java/com/cloud/vm/NicVO.java | 12 + .../main/java/com/cloud/vm/dao/NicDao.java | 2 + .../java/com/cloud/vm/dao/NicDaoImpl.java | 8 + .../apache/cloudstack/backup/BackupVO.java | 14 + plugins/backup/veeam/pom.xml | 15 + .../backup/VeeamBackupProvider.java | 67 +++- .../cloudstack/backup/veeam/VeeamClient.java | 285 ++++++++++++--- .../backup/veeam/api/BackupFile.java | 160 +++++++++ .../backup/veeam/api/BackupFiles.java | 39 +++ .../backup/veeam/api/VmRestorePoint.java | 149 ++++++++ .../backup/veeam/api/VmRestorePoints.java | 39 +++ .../backup/veeam/VeeamClientTest.java | 329 +++++++++++++++++- .../com/cloud/hypervisor/guru/VMwareGuru.java | 17 +- .../vmware/resource/VmwareResource.java | 32 ++ .../cloudstack/backup/BackupManagerImpl.java | 29 +- .../smoke/test_backup_recovery_veeam.py | 302 ++++++++++++++++ tools/marvin/marvin/lib/base.py | 66 +++- ui/src/components/view/ListView.vue | 2 +- ui/src/config/section/compute.js | 4 +- ui/src/config/section/storage.js | 6 +- .../views/compute/backup/BackupSchedule.vue | 8 + .../vmware/mo/VirtualMachineMO.java | 25 ++ .../vmware/mo/VmdkFileDescriptor.java | 59 ++++ 25 files changed, 1654 insertions(+), 62 deletions(-) create mode 100644 core/src/main/java/org/apache/cloudstack/backup/PrepareForBackupRestorationCommand.java create mode 100644 plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/BackupFile.java create mode 100644 plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/BackupFiles.java create mode 100644 plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/VmRestorePoint.java create mode 100644 plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/VmRestorePoints.java create mode 100644 test/integration/smoke/test_backup_recovery_veeam.py diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java index e609655c580..bd3b0623312 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java @@ -95,8 +95,8 @@ public class ListVMsCmd extends BaseListTaggedResourcesCmd implements UserCmd { @Parameter(name = ApiConstants.DETAILS, type = CommandType.LIST, collectionType = CommandType.STRING, - description = "comma separated list of host details requested, " - + "value can be a list of [all, group, nics, stats, secgrp, tmpl, servoff, diskoff, iso, volume, min, affgrp]." + description = "comma separated list of vm details requested, " + + "value can be a list of [all, group, nics, stats, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]." + " If no parameter is passed in, the details will be defaulted to all") private List viewDetails; diff --git a/core/src/main/java/org/apache/cloudstack/backup/PrepareForBackupRestorationCommand.java b/core/src/main/java/org/apache/cloudstack/backup/PrepareForBackupRestorationCommand.java new file mode 100644 index 00000000000..25306fb5f73 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/PrepareForBackupRestorationCommand.java @@ -0,0 +1,43 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class PrepareForBackupRestorationCommand extends Command { + + private String vmName; + + protected PrepareForBackupRestorationCommand() { + } + + public PrepareForBackupRestorationCommand(String vmName) { + this.vmName = vmName; + } + + public String getVmName() { + return vmName; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/engine/schema/src/main/java/com/cloud/vm/NicVO.java b/engine/schema/src/main/java/com/cloud/vm/NicVO.java index 8905ebf732b..fba7c966c44 100644 --- a/engine/schema/src/main/java/com/cloud/vm/NicVO.java +++ b/engine/schema/src/main/java/com/cloud/vm/NicVO.java @@ -30,6 +30,9 @@ import javax.persistence.Id; import javax.persistence.Table; import javax.persistence.Transient; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + import com.cloud.network.Networks.AddressFormat; import com.cloud.network.Networks.Mode; import com.cloud.utils.db.GenericDao; @@ -399,6 +402,15 @@ public class NicVO implements Nic { } @Override + public int hashCode() { + return new HashCodeBuilder(17, 31).append(id).toHashCode(); + } + + @Override + public boolean equals(Object obj) { + return EqualsBuilder.reflectionEquals(this, obj); + } + public Integer getMtu() { return mtu; } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java index 13eb04ba6b8..68f57329d77 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java @@ -91,6 +91,8 @@ public interface NicDao extends GenericDao { NicVO findByMacAddress(String macAddress); + NicVO findByNetworkIdAndMacAddressIncludingRemoved(long networkId, String mac); + List findNicsByIpv6GatewayIpv6CidrAndReserver(String ipv6Gateway, String ipv6Cidr, String reserverName); NicVO findByIpAddressAndVmType(String ip, VirtualMachine.Type vmType); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java index fdc36b4f918..59d2417b073 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java @@ -219,6 +219,14 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { return findOneBy(sc); } + @Override + public NicVO findByNetworkIdAndMacAddressIncludingRemoved(long networkId, String mac) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("network", networkId); + sc.setParameters("macAddress", mac); + return findOneIncludingRemovedBy(sc); + } + @Override public NicVO findDefaultNicForVM(long instanceId) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index dc47fcb6bb3..2ecbfd56460 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -17,6 +17,9 @@ package org.apache.cloudstack.backup; +import com.cloud.utils.db.GenericDao; + +import java.util.Date; import java.util.UUID; import javax.persistence.Column; @@ -51,6 +54,9 @@ public class BackupVO implements Backup { @Column(name = "date") private String date; + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + @Column(name = "size") private Long size; @@ -192,4 +198,12 @@ public class BackupVO implements Backup { public String getName() { return null; } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } } diff --git a/plugins/backup/veeam/pom.xml b/plugins/backup/veeam/pom.xml index 146476c46ed..f0f80d7e337 100644 --- a/plugins/backup/veeam/pom.xml +++ b/plugins/backup/veeam/pom.xml @@ -28,6 +28,21 @@ + + org.apache.cloudstack + cloud-core + ${project.version} + + + org.apache.cloudstack + cloud-engine-api + ${project.version} + + + org.apache.cloudstack + cloud-engine-components-api + ${project.version} + org.apache.cloudstack cloud-plugin-hypervisor-vmware diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 02f08d602bb..1a445080b5c 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -29,6 +29,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.dao.BackupDao; @@ -40,11 +41,17 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.log4j.Logger; +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; +import com.cloud.event.EventVO; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.vmware.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; +import com.cloud.user.User; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.db.Transaction; @@ -53,6 +60,7 @@ import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; public class VeeamBackupProvider extends AdapterBase implements BackupProvider, Configurable { @@ -64,6 +72,10 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, "backup.plugin.veeam.url", "https://localhost:9398/api/", "The Veeam backup and recovery URL.", true, ConfigKey.Scope.Zone); + public ConfigKey VeeamVersion = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.veeam.version", "0", + "The version of Veeam backup and recovery. CloudStack will get Veeam server version via PowerShell commands if it is 0 or not set", true, ConfigKey.Scope.Zone); + private ConfigKey VeeamUsername = new ConfigKey<>("Advanced", String.class, "backup.plugin.veeam.username", "administrator", "The Veeam backup and recovery username.", true, ConfigKey.Scope.Zone); @@ -81,6 +93,12 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, private static ConfigKey VeeamRestoreTimeout = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.restore.timeout", "600", "The Veeam B&R API restore backup timeout in seconds.", true, ConfigKey.Scope.Zone); + private static ConfigKey VeeamTaskPollInterval = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.task.poll.interval", "5", + "The time interval in seconds when the management server polls for Veeam task status.", true, ConfigKey.Scope.Zone); + + private static ConfigKey VeeamTaskPollMaxRetry = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.task.poll.max.retry", "120", + "The max number of retrying times when the management server polls for Veeam task status.", true, ConfigKey.Scope.Zone); + @Inject private VmwareDatacenterZoneMapDao vmwareDatacenterZoneMapDao; @Inject @@ -89,11 +107,16 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, private BackupDao backupDao; @Inject private VMInstanceDao vmInstanceDao; + @Inject + private AgentManager agentMgr; + @Inject + private VirtualMachineManager virtualMachineManager; protected VeeamClient getClient(final Long zoneId) { try { - return new VeeamClient(VeeamUrl.valueIn(zoneId), VeeamUsername.valueIn(zoneId), VeeamPassword.valueIn(zoneId), - VeeamValidateSSLSecurity.valueIn(zoneId), VeeamApiRequestTimeout.valueIn(zoneId), VeeamRestoreTimeout.valueIn(zoneId)); + return new VeeamClient(VeeamUrl.valueIn(zoneId), VeeamVersion.valueIn(zoneId), VeeamUsername.valueIn(zoneId), VeeamPassword.valueIn(zoneId), + VeeamValidateSSLSecurity.valueIn(zoneId), VeeamApiRequestTimeout.valueIn(zoneId), VeeamRestoreTimeout.valueIn(zoneId), + VeeamTaskPollInterval.valueIn(zoneId), VeeamTaskPollMaxRetry.valueIn(zoneId)); } catch (URISyntaxException e) { throw new CloudRuntimeException("Failed to parse Veeam API URL: " + e.getMessage()); } catch (NoSuchAlgorithmException | KeyManagementException e) { @@ -234,7 +257,36 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, @Override public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { final String restorePointId = backup.getExternalId(); - return getClient(vm.getDataCenterId()).restoreFullVM(vm.getInstanceName(), restorePointId); + try { + return getClient(vm.getDataCenterId()).restoreFullVM(vm.getInstanceName(), restorePointId); + } catch (Exception ex) { + LOG.error(String.format("Failed to restore Full VM due to: %s. Retrying after some preparation", ex.getMessage())); + prepareForBackupRestoration(vm); + return getClient(vm.getDataCenterId()).restoreFullVM(vm.getInstanceName(), restorePointId); + } + } + + private void prepareForBackupRestoration(VirtualMachine vm) { + if (!Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType())) { + return; + } + LOG.info("Preparing for restoring VM " + vm); + PrepareForBackupRestorationCommand command = new PrepareForBackupRestorationCommand(vm.getInstanceName()); + Long hostId = virtualMachineManager.findClusterAndHostIdForVm(vm.getId()).second(); + if (hostId == null) { + throw new CloudRuntimeException("Cannot find a host to prepare for restoring VM " + vm); + } + try { + Answer answer = agentMgr.easySend(hostId, command); + if (answer != null && answer.getResult()) { + LOG.info("Succeeded to prepare for restoring VM " + vm); + } else { + throw new CloudRuntimeException(String.format("Failed to prepare for restoring VM %s. details: %s", vm, + (answer != null ? answer.getDetails() : null))); + } + } catch (Exception e) { + throw new CloudRuntimeException(String.format("Failed to prepare for restoring VM %s due to exception %s", vm, e)); + } } @Override @@ -330,6 +382,10 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, + "domain_id: %s, zone_id: %s].", backup.getUuid(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), backup.getBackupOfferingId(), backup.getAccountId(), backup.getDomainId(), backup.getZoneId())); backupDao.persist(backup); + + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_CREATE, + String.format("Created backup %s for VM ID: %s", backup.getUuid(), vm.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); } } for (final Long backupIdToRemove : removeList) { @@ -349,11 +405,14 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ VeeamUrl, + VeeamVersion, VeeamUsername, VeeamPassword, VeeamValidateSSLSecurity, VeeamApiRequestTimeout, - VeeamRestoreTimeout + VeeamRestoreTimeout, + VeeamTaskPollInterval, + VeeamTaskPollMaxRetry }; } diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index 40fbe97028a..7d3bfc50d18 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -20,12 +20,15 @@ package org.apache.cloudstack.backup.veeam; import static org.apache.cloudstack.backup.VeeamBackupProvider.BACKUP_IDENTIFIER; import java.io.IOException; +import java.io.InputStream; import java.net.SocketTimeoutException; import java.net.URI; import java.net.URISyntaxException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -42,6 +45,8 @@ import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupOffering; +import org.apache.cloudstack.backup.veeam.api.BackupFile; +import org.apache.cloudstack.backup.veeam.api.BackupFiles; import org.apache.cloudstack.backup.veeam.api.BackupJobCloneInfo; import org.apache.cloudstack.backup.veeam.api.CreateObjectInJobSpec; import org.apache.cloudstack.backup.veeam.api.EntityReferences; @@ -55,7 +60,10 @@ import org.apache.cloudstack.backup.veeam.api.ObjectsInJob; import org.apache.cloudstack.backup.veeam.api.Ref; import org.apache.cloudstack.backup.veeam.api.RestoreSession; import org.apache.cloudstack.backup.veeam.api.Task; +import org.apache.cloudstack.backup.veeam.api.VmRestorePoint; +import org.apache.cloudstack.backup.veeam.api.VmRestorePoints; import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.http.HttpHeaders; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -71,6 +79,7 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.log4j.Logger; +import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.nio.TrustAllManager; @@ -90,18 +99,31 @@ public class VeeamClient { private final HttpClient httpClient; private static final String RESTORE_VM_SUFFIX = "CS-RSTR-"; private static final String SESSION_HEADER = "X-RestSvcSessionId"; + private static final String BACKUP_REFERENCE = "BackupReference"; + private static final String HIERARCHY_ROOT_REFERENCE = "HierarchyRootReference"; + private static final String REPOSITORY_REFERENCE = "RepositoryReference"; + private static final String RESTORE_POINT_REFERENCE = "RestorePointReference"; + private static final String BACKUP_FILE_REFERENCE = "BackupFileReference"; + private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + private static final SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private String veeamServerIp; + private final Integer veeamServerVersion; private String veeamServerUsername; private String veeamServerPassword; private String veeamSessionId = null; - private int restoreTimeout; + private final int restoreTimeout; private final int veeamServerPort = 22; + private final int taskPollInterval; + private final int taskPollMaxRetry; - public VeeamClient(final String url, final String username, final String password, final boolean validateCertificate, final int timeout, - final int restoreTimeout) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { + public VeeamClient(final String url, final Integer version, final String username, final String password, final boolean validateCertificate, final int timeout, + final int restoreTimeout, final int taskPollInterval, final int taskPollMaxRetry) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { this.apiURI = new URI(url); this.restoreTimeout = restoreTimeout; + this.taskPollInterval = taskPollInterval; + this.taskPollMaxRetry = taskPollMaxRetry; final RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) @@ -125,6 +147,7 @@ public class VeeamClient { authenticate(username, password); setVeeamSshCredentials(this.apiURI.getHost(), username, password); + this.veeamServerVersion = (version != null && version != 0) ? version : getVeeamServerVersion(); } protected void setVeeamSshCredentials(String hostIp, String username, String password) { @@ -135,7 +158,7 @@ public class VeeamClient { private void authenticate(final String username, final String password) { // https://helpcenter.veeam.com/docs/backup/rest/http_authentication.html?ver=95u4 - final HttpPost request = new HttpPost(apiURI.toString() + "/sessionMngr/?v=v1_4"); + final HttpPost request = new HttpPost(apiURI.toString() + "/sessionMngr/?v=latest"); request.setHeader(HttpHeaders.AUTHORIZATION, "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes())); try { final HttpResponse response = httpClient.execute(request); @@ -158,6 +181,26 @@ public class VeeamClient { } } + protected Integer getVeeamServerVersion() { + final List cmds = Arrays.asList( + "$InstallPath = Get-ItemProperty -Path 'HKLM:\\Software\\Veeam\\Veeam Backup and Replication\\' ^| Select -ExpandProperty CorePath", + "Add-Type -LiteralPath \\\"$InstallPath\\Veeam.Backup.Configuration.dll\\\"", + "$ProductData = [Veeam.Backup.Configuration.BackupProduct]::Create()", + "$Version = $ProductData.ProductVersion.ToString()", + "if ($ProductData.MarketName -ne '') {$Version += \\\" $($ProductData.MarketName)\\\"}", + "$Version" + ); + Pair response = executePowerShellCommands(cmds); + if (response == null || !response.first() || response.second() == null || StringUtils.isBlank(response.second().trim())) { + LOG.error("Failed to get veeam server version, using default version"); + return 0; + } else { + Integer majorVersion = NumbersUtil.parseInt(response.second().trim().split("\\.")[0], 0); + LOG.info(String.format("Veeam server full version is %s, major version is %s", response.second().trim(), majorVersion)); + return majorVersion; + } + } + private void checkResponseOK(final HttpResponse response) { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_NO_CONTENT) { LOG.debug("Requested Veeam resource does not exist"); @@ -238,7 +281,7 @@ public class VeeamClient { final ObjectMapper objectMapper = new XmlMapper(); final EntityReferences references = objectMapper.readValue(response.getEntity().getContent(), EntityReferences.class); for (final Ref ref : references.getRefs()) { - if (ref.getName().equals(vmwareDcName) && ref.getType().equals("HierarchyRootReference")) { + if (ref.getName().equals(vmwareDcName) && ref.getType().equals(HIERARCHY_ROOT_REFERENCE)) { return ref.getUid(); } } @@ -286,7 +329,7 @@ public class VeeamClient { private boolean checkTaskStatus(final HttpResponse response) throws IOException { final Task task = parseTaskResponse(response); - for (int i = 0; i < 120; i++) { + for (int i = 0; i < this.taskPollMaxRetry; i++) { final HttpResponse taskResponse = get("/tasks/" + task.getTaskId()); final Task polledTask = parseTaskResponse(taskResponse); if (polledTask.getState().equals("Finished")) { @@ -309,7 +352,7 @@ public class VeeamClient { throw new CloudRuntimeException("Failed to assign VM to backup offering due to: " + polledTask.getResult().getMessage()); } try { - Thread.sleep(5000); + Thread.sleep(this.taskPollInterval * 1000); } catch (InterruptedException e) { LOG.debug("Failed to sleep while polling for Veeam task status due to: ", e); } @@ -324,6 +367,10 @@ public class VeeamClient { if (session.getResult().equals("Success")) { return true; } + if (session.getResult().equalsIgnoreCase("Failed")) { + String sessionUid = session.getUid(); + throw new CloudRuntimeException(String.format("Restore job [%s] failed.", sessionUid)); + } try { Thread.sleep(1000); } catch (InterruptedException ignored) { @@ -355,7 +402,7 @@ public class VeeamClient { final ObjectMapper objectMapper = new XmlMapper(); final EntityReferences references = objectMapper.readValue(response.getEntity().getContent(), EntityReferences.class); for (final Ref ref : references.getRefs()) { - if (ref.getType().equals("RepositoryReference") && ref.getName().equals(repositoryName)) { + if (ref.getType().equals(REPOSITORY_REFERENCE) && ref.getName().equals(repositoryName)) { return ref; } } @@ -368,7 +415,7 @@ public class VeeamClient { protected String getRepositoryNameFromJob(String backupName) { final List cmds = Arrays.asList( - String.format("$Job = Get-VBRJob -name \"%s\"", backupName), + String.format("$Job = Get-VBRJob -name '%s'", backupName), "$Job.GetBackupTargetRepository() ^| select Name ^| Format-List" ); Pair result = executePowerShellCommands(cmds); @@ -376,7 +423,7 @@ public class VeeamClient { throw new CloudRuntimeException(String.format("Failed to get Repository Name from Job [name: %s].", backupName)); } - for (String block : result.second().split("\n\n")) { + for (String block : result.second().split("\r\n")) { if (block.matches("Name(\\s)+:(.)*")) { return block.split(":")[1].trim(); } @@ -553,7 +600,11 @@ public class VeeamClient { */ protected String transformPowerShellCommandList(List cmds) { StringJoiner joiner = new StringJoiner(";"); - joiner.add("PowerShell Add-PSSnapin VeeamPSSnapin"); + if (isLegacyServer()) { + joiner.add("PowerShell Add-PSSnapin VeeamPSSnapin"); + } else { + joiner.add("PowerShell Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue"); + } for (String cmd : cmds) { joiner.add(cmd); } @@ -584,22 +635,22 @@ public class VeeamClient { public boolean setJobSchedule(final String jobName) { Pair result = executePowerShellCommands(Arrays.asList( - String.format("$job = Get-VBRJob -Name \"%s\"", jobName), + String.format("$job = Get-VBRJob -Name '%s'", jobName), "if ($job) { Set-VBRJobSchedule -Job $job -Daily -At \"11:00\" -DailyKind Weekdays }" )); - return result.first() && !result.second().isEmpty() && !result.second().contains(FAILED_TO_DELETE); + return result != null && result.first() && !result.second().isEmpty() && !result.second().contains(FAILED_TO_DELETE); } public boolean deleteJobAndBackup(final String jobName) { Pair result = executePowerShellCommands(Arrays.asList( - String.format("$job = Get-VBRJob -Name \"%s\"", jobName), + String.format("$job = Get-VBRJob -Name '%s'", jobName), "if ($job) { Remove-VBRJob -Job $job -Confirm:$false }", - String.format("$backup = Get-VBRBackup -Name \"%s\"", jobName), + String.format("$backup = Get-VBRBackup -Name '%s'", jobName), "if ($backup) { Remove-VBRBackup -Backup $backup -FromDisk -Confirm:$false }", "$repo = Get-VBRBackupRepository", "Sync-VBRBackupRepository -Repository $repo" )); - return result.first() && !result.second().contains(FAILED_TO_DELETE); + return result != null && result.first() && !result.second().contains(FAILED_TO_DELETE); } public boolean deleteBackup(final String restorePointId) { @@ -610,40 +661,123 @@ public class VeeamClient { "$repo = Get-VBRBackupRepository", "Sync-VBRBackupRepository -Repository $repo", "} else { ", - " Write-Output \"Failed to delete\"", + " Write-Output 'Failed to delete'", " Exit 1", "}" )); - return result.first() && !result.second().contains(FAILED_TO_DELETE); + return result != null && result.first() && !result.second().contains(FAILED_TO_DELETE); } public Map getBackupMetrics() { + if (isLegacyServer()) { + return getBackupMetricsLegacy(); + } else { + return getBackupMetricsViaVeeamAPI(); + } + } + + public Map getBackupMetricsViaVeeamAPI() { + LOG.debug("Trying to get backup metrics via Veeam B&R API"); + + try { + final HttpResponse response = get(String.format("/backupFiles?format=Entity")); + checkResponseOK(response); + return processHttpResponseForBackupMetrics(response.getEntity().getContent()); + } catch (final IOException e) { + LOG.error("Failed to get backup metrics via Veeam B&R API due to:", e); + checkResponseTimeOut(e); + } + return new HashMap<>(); + } + + protected Map processHttpResponseForBackupMetrics(final InputStream content) { + Map metrics = new HashMap<>(); + try { + final ObjectMapper objectMapper = new XmlMapper(); + final BackupFiles backupFiles = objectMapper.readValue(content, BackupFiles.class); + if (backupFiles == null || CollectionUtils.isEmpty(backupFiles.getBackupFiles())) { + throw new CloudRuntimeException("Could not get backup metrics via Veeam B&R API"); + } + for (final BackupFile backupFile : backupFiles.getBackupFiles()) { + String vmUuid = null; + String backupName = null; + List links = backupFile.getLink(); + for (Link link : links) { + if (BACKUP_REFERENCE.equals(link.getType())) { + backupName = link.getName(); + break; + } + } + if (backupName != null && backupName.contains(BACKUP_IDENTIFIER)) { + final String[] names = backupName.split(BACKUP_IDENTIFIER); + if (names.length > 1) { + vmUuid = names[1]; + } + } + if (vmUuid == null) { + continue; + } + if (vmUuid.contains(" - ")) { + vmUuid = vmUuid.split(" - ")[0]; + } + Long usedSize = 0L; + Long dataSize = 0L; + if (metrics.containsKey(vmUuid)) { + usedSize = metrics.get(vmUuid).getBackupSize(); + dataSize = metrics.get(vmUuid).getDataSize(); + } + if (backupFile.getBackupSize() != null) { + usedSize += Long.valueOf(backupFile.getBackupSize()); + } + if (backupFile.getDataSize() != null) { + dataSize += Long.valueOf(backupFile.getDataSize()); + } + metrics.put(vmUuid, new Backup.Metric(usedSize, dataSize)); + } + } catch (final IOException e) { + LOG.error("Failed to process response to get backup metrics via Veeam B&R API due to:", e); + checkResponseTimeOut(e); + } + return metrics; + } + + public Map getBackupMetricsLegacy() { final String separator = "====="; final List cmds = Arrays.asList( - "$backups = Get-VBRBackup", - "foreach ($backup in $backups) {" + - "$backup.JobName;" + - "$storageGroups = $backup.GetStorageGroups();" + - "foreach ($group in $storageGroups) {" + - "$usedSize = 0;" + - "$dataSize = 0;" + - "$sizePerStorage = $group.GetStorages().Stats.BackupSize;" + - "$dataPerStorage = $group.GetStorages().Stats.DataSize;" + - "foreach ($size in $sizePerStorage) {" + - "$usedSize += $size;" + - "}" + - "foreach ($size in $dataPerStorage) {" + - "$dataSize += $size;" + - "}" + - "$usedSize;" + - "$dataSize;" + - "}" + - "echo \"" + separator + "\"" + - "}" + "$backups = Get-VBRBackup", + "foreach ($backup in $backups) {" + + " $backup.JobName;" + + " $storageGroups = $backup.GetStorageGroups();" + + " foreach ($group in $storageGroups) {" + + " $usedSize = 0;" + + " $dataSize = 0;" + + " $sizePerStorage = $group.GetStorages().Stats.BackupSize;" + + " $dataPerStorage = $group.GetStorages().Stats.DataSize;" + + " foreach ($size in $sizePerStorage) {" + + " $usedSize += $size;" + + " }" + + " foreach ($size in $dataPerStorage) {" + + " $dataSize += $size;" + + " }" + + " $usedSize;" + + " $dataSize;" + + " }" + + " echo \"" + separator + "\"" + + "}" ); Pair response = executePowerShellCommands(cmds); + if (response == null || !response.first()) { + throw new CloudRuntimeException("Failed to get backup metrics via PowerShell command"); + } + return processPowerShellResultForBackupMetrics(response.second()); + } + + protected Map processPowerShellResultForBackupMetrics(final String result) { + LOG.debug("Processing powershell result: " + result); + + final String separator = "====="; final Map sizes = new HashMap<>(); - for (final String block : response.second().split(separator + "\r\n")) { + for (final String block : result.split(separator + "\r\n")) { final String[] parts = block.split("\r\n"); if (parts.length != 3) { continue; @@ -677,9 +811,9 @@ public class VeeamClient { return new Backup.RestorePoint(id, created, type); } - public List listRestorePoints(String backupName, String vmInternalName) { + public List listRestorePointsLegacy(String backupName, String vmInternalName) { final List cmds = Arrays.asList( - String.format("$backup = Get-VBRBackup -Name \"%s\"", backupName), + String.format("$backup = Get-VBRBackup -Name '%s'", backupName), String.format("if ($backup) { $restore = (Get-VBRRestorePoint -Backup:$backup -Name \"%s\" ^| Where-Object {$_.IsConsistent -eq $true})", vmInternalName), "if ($restore) { $restore ^| Format-List } }" ); @@ -700,6 +834,71 @@ public class VeeamClient { return restorePoints; } + public List listRestorePoints(String backupName, String vmInternalName) { + if (isLegacyServer()) { + return listRestorePointsLegacy(backupName, vmInternalName); + } else { + return listVmRestorePointsViaVeeamAPI(vmInternalName); + } + } + + public List listVmRestorePointsViaVeeamAPI(String vmInternalName) { + LOG.debug(String.format("Trying to list VM restore points via Veeam B&R API for VM %s: ", vmInternalName)); + + try { + final HttpResponse response = get(String.format("/vmRestorePoints?format=Entity")); + checkResponseOK(response); + return processHttpResponseForVmRestorePoints(response.getEntity().getContent(), vmInternalName); + } catch (final IOException e) { + LOG.error("Failed to list VM restore points via Veeam B&R API due to:", e); + checkResponseTimeOut(e); + } + return new ArrayList<>(); + } + + public List processHttpResponseForVmRestorePoints(InputStream content, String vmInternalName) { + List vmRestorePointList = new ArrayList<>(); + try { + final ObjectMapper objectMapper = new XmlMapper(); + final VmRestorePoints vmRestorePoints = objectMapper.readValue(content, VmRestorePoints.class); + if (vmRestorePoints == null) { + throw new CloudRuntimeException("Could not get VM restore points via Veeam B&R API"); + } + for (final VmRestorePoint vmRestorePoint : vmRestorePoints.getVmRestorePoints()) { + LOG.debug(String.format("Processing VM restore point Name=%s, VmDisplayName=%s for vm name=%s", + vmRestorePoint.getName(), vmRestorePoint.getVmDisplayName(), vmInternalName)); + if (!vmInternalName.equals(vmRestorePoint.getVmDisplayName())) { + continue; + } + boolean isReady = true; + List links = vmRestorePoint.getLink(); + for (Link link : links) { + if (Arrays.asList(BACKUP_FILE_REFERENCE, RESTORE_POINT_REFERENCE).contains(link.getType()) && !link.getRel().equals("Up")) { + LOG.info(String.format("The VM restore point is not ready. Reference: %s, state: %s", link.getType(), link.getRel())); + isReady = false; + break; + } + } + if (!isReady) { + continue; + } + String vmRestorePointId = vmRestorePoint.getUid().substring(vmRestorePoint.getUid().lastIndexOf(':') + 1); + String created = formatDate(vmRestorePoint.getCreationTimeUtc()); + String type = vmRestorePoint.getPointType(); + LOG.debug(String.format("Adding restore point %s, %s, %s", vmRestorePointId, created, type)); + vmRestorePointList.add(new Backup.RestorePoint(vmRestorePointId, created, type)); + } + } catch (final IOException | ParseException e) { + LOG.error("Failed to process response to get VM restore points via Veeam B&R API due to:", e); + checkResponseTimeOut(e); + } + return vmRestorePointList; + } + + private String formatDate(String date) throws ParseException { + return newDateFormat.format(dateFormat.parse(StringUtils.substring(date, 0, 19))); + } + public Pair restoreVMToDifferentLocation(String restorePointId, String hostIp, String dataStoreUuid) { final String restoreLocation = RESTORE_VM_SUFFIX + UUID.randomUUID().toString(); final String datastoreId = dataStoreUuid.replace("-",""); @@ -717,4 +916,8 @@ public class VeeamClient { } return new Pair<>(result.first(), restoreLocation); } + + private boolean isLegacyServer() { + return this.veeamServerVersion != null && (this.veeamServerVersion > 0 && this.veeamServerVersion < 11); + } } diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/BackupFile.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/BackupFile.java new file mode 100644 index 00000000000..2b28793b1fb --- /dev/null +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/BackupFile.java @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.veeam.api; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +import java.util.List; + +@JacksonXmlRootElement(localName = "BackupFile") +public class BackupFile { + @JacksonXmlProperty(localName = "Type", isAttribute = true) + private String type; + + @JacksonXmlProperty(localName = "Href", isAttribute = true) + private String href; + + @JacksonXmlProperty(localName = "Name", isAttribute = true) + private String name; + + @JacksonXmlProperty(localName = "UID", isAttribute = true) + private String uid; + + @JacksonXmlProperty(localName = "Link") + @JacksonXmlElementWrapper(localName = "Links") + private List link; + + @JacksonXmlProperty(localName = "FilePath") + private String filePath; + + @JacksonXmlProperty(localName = "BackupSize") + private String backupSize; + + @JacksonXmlProperty(localName = "DataSize") + private String dataSize; + + @JacksonXmlProperty(localName = "DeduplicationRatio") + private String deduplicationRatio; + + @JacksonXmlProperty(localName = "CompressRatio") + private String compressRatio; + + @JacksonXmlProperty(localName = "CreationTimeUtc") + private String creationTimeUtc; + + @JacksonXmlProperty(localName = "FileType") + private String fileType; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUid() { + return uid; + } + + public void setUid(String uid) { + this.uid = uid; + } + + public List getLink() { + return link; + } + + public void setLink(List link) { + this.link = link; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public String getBackupSize() { + return backupSize; + } + + public void setBackupSize(String backupSize) { + this.backupSize = backupSize; + } + + public String getDataSize() { + return dataSize; + } + + public void setDataSize(String dataSize) { + this.dataSize = dataSize; + } + + public String getDeduplicationRatio() { + return deduplicationRatio; + } + + public void setDeduplicationRatio(String deduplicationRatio) { + this.deduplicationRatio = deduplicationRatio; + } + + public String getCompressRatio() { + return compressRatio; + } + + public void setCompressRatio(String compressRatio) { + this.compressRatio = compressRatio; + } + + public String getCreationTimeUtc() { + return creationTimeUtc; + } + + public void setCreationTimeUtc(String creationTimeUtc) { + this.creationTimeUtc = creationTimeUtc; + } + + public String getFileType() { + return fileType; + } + + public void setFileType(String fileType) { + this.fileType = fileType; + } +} diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/BackupFiles.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/BackupFiles.java new file mode 100644 index 00000000000..4ff7d0c088b --- /dev/null +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/BackupFiles.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.veeam.api; + +import java.util.List; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JacksonXmlRootElement(localName = "BackupFiles") +public class BackupFiles { + @JacksonXmlProperty(localName = "BackupFile") + @JacksonXmlElementWrapper(localName = "BackupFile", useWrapping = false) + private List backupFiles; + + public List getBackupFiles() { + return backupFiles; + } + + public void setBackupFiles(List backupFiles) { + this.backupFiles = backupFiles; + } +} diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/VmRestorePoint.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/VmRestorePoint.java new file mode 100644 index 00000000000..beaa11cd5d4 --- /dev/null +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/VmRestorePoint.java @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.veeam.api; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +import java.util.List; + +@JacksonXmlRootElement(localName = "VmRestorePoint") +public class VmRestorePoint { + @JacksonXmlProperty(localName = "Type", isAttribute = true) + private String type; + + @JacksonXmlProperty(localName = "Href", isAttribute = true) + private String href; + + @JacksonXmlProperty(localName = "Name", isAttribute = true) + private String name; + + @JacksonXmlProperty(localName = "UID", isAttribute = true) + private String uid; + + @JacksonXmlProperty(localName = "VmDisplayName", isAttribute = true) + private String vmDisplayName; + + @JacksonXmlProperty(localName = "Link") + @JacksonXmlElementWrapper(localName = "Links") + private List link; + + @JacksonXmlProperty(localName = "CreationTimeUTC") + private String creationTimeUtc; + + @JacksonXmlProperty(localName = "VmName") + private String vmName; + + @JacksonXmlProperty(localName = "Algorithm") + private String algorithm; + + @JacksonXmlProperty(localName = "PointType") + private String pointType; + + @JacksonXmlProperty(localName = "HierarchyObjRef") + private String hierarchyObjRef; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUid() { + return uid; + } + + public void setUid(String uid) { + this.uid = uid; + } + + public String getVmDisplayName() { + return vmDisplayName; + } + + public void setVmDisplayName(String vmDisplayName) { + this.vmDisplayName = vmDisplayName; + } + + public List getLink() { + return link; + } + + public void setLink(List link) { + this.link = link; + } + + public String getCreationTimeUtc() { + return creationTimeUtc; + } + + public void setCreationTimeUtc(String creationTimeUtc) { + this.creationTimeUtc = creationTimeUtc; + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public String getAlgorithm() { + return algorithm; + } + + public void setAlgorithm(String algorithm) { + this.algorithm = algorithm; + } + + public String getPointType() { + return pointType; + } + + public void setPointType(String pointType) { + this.pointType = pointType; + } + + public String getHierarchyObjRef() { + return hierarchyObjRef; + } + + public void setHierarchyObjRef(String hierarchyObjRef) { + this.hierarchyObjRef = hierarchyObjRef; + } +} diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/VmRestorePoints.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/VmRestorePoints.java new file mode 100644 index 00000000000..2b59a3ef23c --- /dev/null +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/api/VmRestorePoints.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.veeam.api; + +import java.util.List; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + +@JacksonXmlRootElement(localName = "VmRestorePoints") +public class VmRestorePoints { + @JacksonXmlProperty(localName = "VmRestorePoint") + @JacksonXmlElementWrapper(localName = "VmRestorePoint", useWrapping = false) + private List VmRestorePoints; + + public List getVmRestorePoints() { + return VmRestorePoints; + } + + public void setVmRestorePoints(List vmRestorePoints) { + VmRestorePoints = vmRestorePoints; + } +} diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index a155d351bc6..48d1f886b48 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -27,9 +27,13 @@ import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static org.junit.Assert.fail; import static org.mockito.Mockito.times; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.List; +import java.util.Map; +import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.veeam.api.RestoreSession; import org.apache.http.HttpResponse; @@ -62,9 +66,10 @@ public class VeeamClientTest { .withStatus(201) .withHeader("X-RestSvcSessionId", "some-session-auth-id") .withBody(""))); - client = new VeeamClient("http://localhost:9399/api/", adminUsername, adminPassword, true, 60, 600); + client = new VeeamClient("http://localhost:9399/api/", 12, adminUsername, adminPassword, true, 60, 600, 5, 120); mockClient = Mockito.mock(VeeamClient.class); Mockito.when(mockClient.getRepositoryNameFromJob(Mockito.anyString())).thenCallRealMethod(); + Mockito.when(mockClient.getVeeamServerVersion()).thenCallRealMethod(); } @Test @@ -139,7 +144,7 @@ public class VeeamClientTest { @Test public void getRepositoryNameFromJobTestSuccess() throws Exception { String backupName = "TEST-BACKUP3"; - Pair response = new Pair(Boolean.TRUE, "\n\nName : test"); + Pair response = new Pair(Boolean.TRUE, "\r\nName : test"); Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); String repositoryNameFromJob = mockClient.getRepositoryNameFromJob(backupName); Assert.assertEquals("test", repositoryNameFromJob); @@ -162,4 +167,324 @@ public class VeeamClientTest { } Mockito.verify(mockClient, times(10)).get(Mockito.anyString()); } + + private void verifyBackupMetrics(Map metrics) { + Assert.assertEquals(2, metrics.size()); + + Assert.assertTrue(metrics.containsKey("d1bd8abd-fc73-4b77-9047-7be98a2ecb72")); + Assert.assertEquals(537776128L, (long) metrics.get("d1bd8abd-fc73-4b77-9047-7be98a2ecb72").getBackupSize()); + Assert.assertEquals(2147506644L, (long) metrics.get("d1bd8abd-fc73-4b77-9047-7be98a2ecb72").getDataSize()); + + Assert.assertTrue(metrics.containsKey("0d752ca6-d628-4d85-a739-75275e4661e6")); + Assert.assertEquals(1268682752L, (long) metrics.get("0d752ca6-d628-4d85-a739-75275e4661e6").getBackupSize()); + Assert.assertEquals(15624049921L, (long) metrics.get("0d752ca6-d628-4d85-a739-75275e4661e6").getDataSize()); + } + + @Test + public void testProcessPowerShellResultForBackupMetrics() { + String result = "i-2-3-VM-CSBKP-d1bd8abd-fc73-4b77-9047-7be98a2ecb72\r\n" + + "537776128\r\n" + + "2147506644\r\n" + + "=====\r\n" + + "i-13-22-VM-CSBKP-b3b3cb75-cfbf-4496-9c63-a08a93347276\r\n" + + "=====\r\n" + + "backup-job-based-on-sla\r\n" + + "=====\r\n" + + "i-12-20-VM-CSBKP-9f292f11-00ec-4915-84f0-e3895828640e\r\n" + + "=====\r\n" + + "i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\r\n" + + "1268682752\r\n" + + "15624049921\r\n" + + "=====\r\n"; + + Map metrics = client.processPowerShellResultForBackupMetrics(result); + + verifyBackupMetrics(metrics); + } + + @Test + public void testProcessHttpResponseForBackupMetricsForV11() { + String xmlResponse = "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-10-28T000059_745D.vbk\n" + + " 579756032\n" + + " 7516219400\n" + + " 5.83\n" + + " 2.22\n" + + " 2023-10-27T23:00:13.74Z\n" + + " vbk\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-11-05T000022_7987.vib\n" + + " 12083200\n" + + " 69232800\n" + + " 1\n" + + " 6.67\n" + + " 2023-11-05T00:00:22.827Z\n" + + " vib\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-11-01T000035_BEBF.vib\n" + + " 12398592\n" + + " 71329948\n" + + " 1\n" + + " 6.67\n" + + " 2023-11-01T00:00:35.163Z\n" + + " vib\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-11-04T000109_2AC1.vbk\n" + + " 581083136\n" + + " 7516219404\n" + + " 5.82\n" + + " 2.22\n" + + " 2023-11-04T00:00:24.973Z\n" + + " vbk\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-10-29T000033_F468.vib\n" + + " 11870208\n" + + " 72378524\n" + + " 1\n" + + " 7.14\n" + + " 2023-10-28T23:00:33.233Z\n" + + " vib\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-10-30T000022_0CE3.vib\n" + + " 14409728\n" + + " 76572828\n" + + " 1\n" + + " 6.25\n" + + " 2023-10-30T00:00:22.7Z\n" + + " vib\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-11-06T000018_055B.vib\n" + + " 17883136\n" + + " 80767136\n" + + " 1\n" + + " 5\n" + + " 2023-11-06T00:00:18.253Z\n" + + " vib\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-11-02T000029_65BE.vib\n" + + " 12521472\n" + + " 72378525\n" + + " 1\n" + + " 6.67\n" + + " 2023-11-02T00:00:29.05Z\n" + + " vib\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-3-VM-CSBKP-d1bd8abd-fc73-4b77-9047-7be98a2ecb72\\i-2-3-VM-CSBKP-d1bd8abd-fc73-4b77-9047-7be98aD2023-10-25T145951_8062.vbk\n" + + " 537776128\n" + + " 2147506644\n" + + " 1.68\n" + + " 2.38\n" + + " 2023-10-25T13:59:51.76Z\n" + + " vbk\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-11-03T000024_7ACF.vib\n" + + " 14217216\n" + + " 76572832\n" + + " 1\n" + + " 6.25\n" + + " 2023-11-03T00:00:24.803Z\n" + + " vib\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-10-31T000015_4624.vib\n" + + " 12460032\n" + + " 72378524\n" + + " 1\n" + + " 6.67\n" + + " 2023-10-31T00:00:15.853Z\n" + + " vib\n" + + " \n" + + "\n"; + + InputStream inputStream = new ByteArrayInputStream(xmlResponse.getBytes()); + Map metrics = client.processHttpResponseForBackupMetrics(inputStream); + + verifyBackupMetrics(metrics); + } + + @Test + public void testGetBackupMetricsViaVeeamAPI() { + String xmlResponse = "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-4-VM-CSBKP-506760dc-ed77-40d6-a91d-e0914e7a1ad8\\i-2-4-VM.vm-1036D2023-11-03T162535_89D6.vbk\n" + + " 535875584\n" + + " 2147507235\n" + + " 1.68\n" + + " 2.38\n" + + " 2023-11-03T16:25:35.920773Z\n" + + " vbk\n" + + " \n" + + ""; + + wireMockRule.stubFor(get(urlMatching(".*/backupFiles\\?format=Entity")) + .willReturn(aResponse() + .withHeader("content-type", "application/xml") + .withStatus(200) + .withBody(xmlResponse))); + Map metrics = client.getBackupMetricsViaVeeamAPI(); + + Assert.assertEquals(1, metrics.size()); + Assert.assertTrue(metrics.containsKey("506760dc-ed77-40d6-a91d-e0914e7a1ad8")); + Assert.assertEquals(535875584L, (long) metrics.get("506760dc-ed77-40d6-a91d-e0914e7a1ad8").getBackupSize()); + Assert.assertEquals(2147507235L, (long) metrics.get("506760dc-ed77-40d6-a91d-e0914e7a1ad8").getDataSize()); + } + + @Test + public void testListVmRestorePointsViaVeeamAPI() { + String xmlResponse = "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 2023-11-03T16:26:12.209913Z\n" + + " i-2-4-VM\n" + + " Full\n" + + " Full\n" + + " urn:VMware:Vm:adb5423b-b578-4c26-8ab8-cde9c1faec55.vm-1036\n" + + " \n" + + "\n"; + String vmName = "i-2-4-VM"; + + wireMockRule.stubFor(get(urlMatching(".*/vmRestorePoints\\?format=Entity")) + .willReturn(aResponse() + .withHeader("content-type", "application/xml") + .withStatus(200) + .withBody(xmlResponse))); + List vmRestorePointList = client.listVmRestorePointsViaVeeamAPI(vmName); + + Assert.assertEquals(1, vmRestorePointList.size()); + Assert.assertEquals("f6d504cf-eafe-4cd2-8dfc-e9cfe2f1e977", vmRestorePointList.get(0).getId()); + Assert.assertEquals("2023-11-03 16:26:12", vmRestorePointList.get(0).getCreated()); + Assert.assertEquals("Full", vmRestorePointList.get(0).getType()); + } + + @Test + public void testGetVeeamServerVersionAllGood() { + Pair response = new Pair(Boolean.TRUE, "12.0.0.1"); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Assert.assertEquals(12, (int) mockClient.getVeeamServerVersion()); + } + + @Test + public void testGetVeeamServerVersionWithError() { + Pair response = new Pair(Boolean.FALSE, ""); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Assert.assertEquals(0, (int) mockClient.getVeeamServerVersion()); + } + + @Test + public void testGetVeeamServerVersionWithEmptyVersion() { + Pair response = new Pair(Boolean.TRUE, ""); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Assert.assertEquals(0, (int) mockClient.getVeeamServerVersion()); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index db41ab19d56..aa3b314fb3f 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -781,8 +781,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co volume = createVolume(disk, vmToImport, domainId, zoneId, accountId, instanceId, poolId, templateId, backup, true); operation = "created"; } - s_logger.debug(String.format("VM [id: %s, instanceName: %s] backup restore operation %s volume [id: %s].", instanceId, vmInstanceVO.getInstanceName(), - operation, volume.getUuid())); + s_logger.debug(String.format("Sync volumes to %s in backup restore operation: %s volume [id: %s].", vmInstanceVO, operation, volume.getUuid())); } } @@ -879,9 +878,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co String tag = parts[parts.length - 1]; String[] tagSplit = tag.split("-"); tag = tagSplit[tagSplit.length - 1]; + + s_logger.debug(String.format("Trying to find network with vlan: [%s].", vlan)); NetworkVO networkVO = networkDao.findByVlan(vlan); if (networkVO == null) { networkVO = createNetworkRecord(zoneId, tag, vlan, accountId, domainId); + s_logger.debug(String.format("Created new network record [id: %s] with details [zoneId: %s, tag: %s, vlan: %s, accountId: %s and domainId: %s].", + networkVO.getUuid(), zoneId, tag, vlan, accountId, domainId)); } return networkVO; } @@ -893,6 +896,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co Map mapping = new HashMap<>(); for (String networkName : vmNetworkNames) { NetworkVO networkVO = getGuestNetworkFromNetworkMorName(networkName, accountId, zoneId, domainId); + s_logger.debug(String.format("Mapping network name [%s] to networkVO [id: %s].", networkName, networkVO.getUuid())); mapping.put(networkName, networkVO); } return mapping; @@ -927,12 +931,19 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co String macAddress = pair.first(); String networkName = pair.second(); NetworkVO networkVO = networksMapping.get(networkName); - NicVO nicVO = nicDao.findByNetworkIdAndMacAddress(networkVO.getId(), macAddress); + NicVO nicVO = nicDao.findByNetworkIdAndMacAddressIncludingRemoved(networkVO.getId(), macAddress); if (nicVO != null) { + s_logger.warn(String.format("Find NIC in DB with networkId [%s] and MAC Address [%s], so this NIC will be removed from list of unmapped NICs of VM [id: %s, name: %s].", + networkVO.getId(), macAddress, vm.getUuid(), vm.getInstanceName())); allNics.remove(nicVO); + + if (nicVO.getRemoved() != null) { + nicDao.unremove(nicVO.getId()); + } } } for (final NicVO unMappedNic : allNics) { + s_logger.debug(String.format("Removing NIC [%s] from backup restored %s.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(unMappedNic, "uuid", "macAddress"), vm)); vmManager.removeNicFromVm(vm, unMappedNic); } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 6af074222fa..22d0a796e14 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -49,6 +49,7 @@ import javax.naming.ConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.backup.PrepareForBackupRestorationCommand; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.configdrive.ConfigDrive; @@ -606,6 +607,8 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes answer = execute((GetVmVncTicketCommand) cmd); } else if (clz == GetAutoScaleMetricsCommand.class) { answer = execute((GetAutoScaleMetricsCommand) cmd); + } else if (clz == PrepareForBackupRestorationCommand.class) { + answer = execute((PrepareForBackupRestorationCommand) cmd); } else { answer = Answer.createUnsupportedCommandAnswer(cmd); } @@ -7751,6 +7754,35 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes } } + private Answer execute(PrepareForBackupRestorationCommand command) { + try { + VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); + + String vmName = command.getVmName(); + VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName); + + if (vmMo == null) { + if (hyperHost instanceof HostMO) { + ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), ((HostMO) hyperHost).getParentMor()); + vmMo = clusterMo.findVmOnHyperHost(vmName); + } + } + + if (vmMo == null) { + String msg = "VM " + vmName + " no longer exists to execute PrepareForBackupRestorationCommand command"; + s_logger.error(msg); + throw new Exception(msg); + } + + vmMo.removeChangeTrackPathFromVmdkForDisks(); + + return new Answer(command, true, "success"); + } catch (Exception e) { + s_logger.error("Unexpected exception: ", e); + return new Answer(command, false, "Unable to execute PrepareForBackupRestorationCommand due to " + e.toString()); + } + } + private Integer getVmwareWindowTimeInterval() { Integer windowInterval = VmwareManager.VMWARE_STATS_TIME_WINDOW.value(); if (windowInterval == null || windowInterval < 20) { diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 4e18caa684b..bbdf730e06d 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -75,6 +75,7 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.ActionEvent; import com.cloud.event.ActionEventUtils; import com.cloud.event.EventTypes; +import com.cloud.event.EventVO; import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; @@ -477,6 +478,11 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { throw new CloudRuntimeException("The assigned backup offering does not allow ad-hoc user backup"); } + ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), + EventTypes.EVENT_VM_BACKUP_CREATE, "creating backup for VM ID:" + vm.getUuid(), + vmId, ApiCommandResourceType.VirtualMachine.toString(), + true, 0); + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); if (backupProvider != null && backupProvider.takeBackup(vm)) { return true; @@ -555,10 +561,21 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { } catch (final Exception e) { LOG.error(String.format("Failed to import VM [vmInternalName: %s] from backup restoration [%s] with hypervisor [type: %s] due to: [%s].", vmInternalName, ReflectionToStringBuilderUtils.reflectOnlySelectedFields(backup, "id", "uuid", "vmId", "externalId", "backupType"), hypervisorType, e.getMessage()), e); + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_ERROR, EventTypes.EVENT_VM_BACKUP_RESTORE, + String.format("Failed to import VM %s from backup %s with hypervisor [type: %s]", vmInternalName, backup.getUuid(), hypervisorType), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); throw new CloudRuntimeException("Error during vm backup restoration and import: " + e.getMessage()); } if (vm == null) { - LOG.error("Failed to import restored VM " + vmInternalName + " with hypervisor type " + hypervisorType + " using backup of VM ID " + backup.getVmId()); + String message = String.format("Failed to import restored VM %s with hypervisor type %s using backup of VM ID %s", + vmInternalName, hypervisorType, backup.getVmId()); + LOG.error(message); + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_ERROR, EventTypes.EVENT_VM_BACKUP_RESTORE, + message, vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); + } else { + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_RESTORE, + String.format("Restored VM %s from backup %s", vm.getUuid(), backup.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); } return vm != null; } @@ -588,9 +605,17 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { throw new CloudRuntimeException("Failed to find backup offering of the VM backup"); } + ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventTypes.EVENT_VM_BACKUP_RESTORE, + String.format("Restoring VM %s from backup %s", vm.getUuid(), backup.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), + true, 0); + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); if (!backupProvider.restoreVMFromBackup(vm, backup)) { - throw new CloudRuntimeException("Error restoring VM from backup ID " + backup.getId()); + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_ERROR, EventTypes.EVENT_VM_BACKUP_RESTORE, + String.format("Failed to restore VM %s from backup %s", vm.getInstanceName(), backup.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); + throw new CloudRuntimeException("Error restoring VM from backup with uuid " + backup.getUuid()); } return importRestoredVM(vm.getDataCenterId(), vm.getDomainId(), vm.getAccountId(), vm.getUserId(), vm.getInstanceName(), vm.getHypervisorType(), backup); diff --git a/test/integration/smoke/test_backup_recovery_veeam.py b/test/integration/smoke/test_backup_recovery_veeam.py new file mode 100644 index 00000000000..d0da66fa7c2 --- /dev/null +++ b/test/integration/smoke/test_backup_recovery_veeam.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import wait_until +from marvin.lib.base import (Account, ServiceOffering, DiskOffering, Volume, VirtualMachine, + BackupOffering, Configurations, Backup, BackupSchedule) +from marvin.lib.common import (get_domain, get_zone, get_template) +from nose.plugins.attrib import attr +from marvin.codes import FAILED + +import time + +class TestVeeamBackupAndRecovery(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + # Setup + + cls.testClient = super(TestVeeamBackupAndRecovery, cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + cls.services = cls.testClient.getParsedTestDataConfig() + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services["mode"] = cls.zone.networktype + cls.hypervisor = cls.testClient.getHypervisorInfo() + cls.domain = get_domain(cls.apiclient) + cls.template = get_template(cls.apiclient, cls.zone.id, cls.services["ostype"]) + if cls.template == FAILED: + assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = cls.template.id + cls._cleanup = [] + + # Check backup configuration values, set them to enable the veeam provider + backup_enabled_cfg = Configurations.list(cls.apiclient, name='backup.framework.enabled', zoneid=cls.zone.id) + backup_provider_cfg = Configurations.list(cls.apiclient, name='backup.framework.provider.plugin', zoneid=cls.zone.id) + cls.backup_enabled = backup_enabled_cfg[0].value + cls.backup_provider = backup_provider_cfg[0].value + + if cls.backup_enabled == "false": + Configurations.update(cls.apiclient, 'backup.framework.enabled', value='true', zoneid=cls.zone.id) + if cls.backup_provider != "veeam": + return + + if cls.hypervisor.lower() != 'vmware': + return + + cls.service_offering = ServiceOffering.create(cls.apiclient, cls.services["service_offerings"]["small"]) + cls._cleanup.append(cls.service_offering) + cls.disk_offering = DiskOffering.create(cls.apiclient, cls.services["disk_offering"]) + cls._cleanup.append(cls.disk_offering) + + @classmethod + def isBackupOfferingUsed(cls, existing_offerings, provider_offering): + if not existing_offerings: + return False + for existing_offering in existing_offerings: + if existing_offering.externalid == provider_offering.externalid: + return True + return False + + def waitForBackUp(self, vm): + def checkBackUp(): + backups = Backup.list(self.user_apiclient, vm.id) + if isinstance(backups, list) and len(backups) != 0: + return True, None + return False, None + + res, _ = wait_until(10, 60, checkBackUp) + if not res: + self.fail("Failed to wait for backup of VM %s to be Up" % vm.id) + + @classmethod + def tearDownClass(cls): + if cls.backup_enabled == "false": + Configurations.update(cls.apiclient, 'backup.framework.enabled', value=cls.backup_enabled, zoneid=cls.zone.id) + super(TestVeeamBackupAndRecovery, cls).tearDownClass() + + def setUp(self): + if self.backup_provider != "veeam": + raise self.skipTest("Skipping test cases which must only run for veeam") + if self.hypervisor.lower() != 'vmware': + raise self.skipTest("Skipping test cases which must only run for VMware") + self.cleanup = [] + + # Import backup offering + self.offering = None + existing_offerings = BackupOffering.listByZone(self.apiclient, self.zone.id) + provider_offerings = BackupOffering.listExternal(self.apiclient, self.zone.id) + if not provider_offerings: + self.skipTest("Skipping test cases as the provider offering is None") + for provider_offering in provider_offerings: + if not self.isBackupOfferingUsed(existing_offerings, provider_offering): + self.debug("Importing backup offering %s - %s" % (provider_offering.externalid, provider_offering.name)) + self.offering = BackupOffering.importExisting(self.apiclient, self.zone.id, provider_offering.externalid, + provider_offering.name, provider_offering.description) + if not self.offering: + self.fail("Failed to import backup offering %s" % provider_offering.name) + break + if not self.offering: + self.skipTest("Skipping test cases as there is no available provider offerings to import") + + # Create user account + self.account = Account.create(self.apiclient, self.services["account"], domainid=self.domain.id) + self.user_user = self.account.user[0] + self.user_apiclient = self.testClient.getUserApiClient( + self.user_user.username, self.domain.name + ) + self.cleanup.append(self.account) + + def tearDown(self): + super(TestVeeamBackupAndRecovery, self).tearDown() + + @attr(tags=["advanced", "backup"], required_hardware="false") + def test_01_import_list_delete_backup_offering(self): + """ + Import provider backup offering from Veeam Backup and Recovery Provider + """ + + # Verify offering is listed by user + imported_offering = BackupOffering.listByZone(self.user_apiclient, self.zone.id) + self.assertIsInstance(imported_offering, list, "List Backup Offerings should return a valid response") + self.assertNotEqual(len(imported_offering), 0, "Check if the list API returns a non-empty response") + matching_offerings = [x for x in imported_offering if x.id == self.offering.id] + self.assertNotEqual(len(matching_offerings), 0, "Check if there is a matching offering") + + # Delete backup offering + self.debug("Deleting backup offering %s" % self.offering.id) + self.offering.delete(self.apiclient) + + # Verify offering is not listed by user + imported_offering = BackupOffering.listByZone(self.user_apiclient, self.zone.id) + if imported_offering: + self.assertIsInstance(imported_offering, list, "List Backup Offerings should return a valid response") + matching_offerings = [x for x in imported_offering if x.id == self.offering.id] + self.assertEqual(len(matching_offerings), 0, "Check there is not a matching offering") + + @attr(tags=["advanced", "backup"], required_hardware="false") + def test_02_vm_backup_lifecycle(self): + """ + Test VM backup lifecycle + """ + + if self.offering: + self.cleanup.insert(0, self.offering) + + self.vm = VirtualMachine.create(self.user_apiclient, self.services["small"], accountid=self.account.name, + domainid=self.account.domainid, serviceofferingid=self.service_offering.id, + diskofferingid=self.disk_offering.id) + + # Verify there are no backups for the VM + backups = Backup.list(self.user_apiclient, self.vm.id) + self.assertEqual(backups, None, "There should not exist any backup for the VM") + + # Assign VM to offering and create ad-hoc backup + self.offering.assignOffering(self.user_apiclient, self.vm.id) + vms = VirtualMachine.list( + self.user_apiclient, + id=self.vm.id, + listall=True + ) + self.assertEqual( + isinstance(vms, list), + True, + "List virtual machines should return a valid list" + ) + self.assertEqual(1, len(vms), "List of the virtual machines should have 1 vm") + self.assertEqual(self.offering.id, vms[0].backupofferingid, "The virtual machine should have backup offering %s" % self.offering.id) + + # Create backup schedule on 01:00AM every Sunday + BackupSchedule.create(self.user_apiclient, self.vm.id, intervaltype="WEEKLY", timezone="CET", schedule="00:01:1") + backupSchedule = BackupSchedule.list(self.user_apiclient, self.vm.id) + self.assertIsNotNone(backupSchedule) + self.assertEqual("WEEKLY", backupSchedule.intervaltype) + self.assertEqual("00:01:1", backupSchedule.schedule) + self.assertEqual("CET", backupSchedule.timezone) + self.assertEqual(self.vm.id, backupSchedule.virtualmachineid) + self.assertEqual(self.vm.name, backupSchedule.virtualmachinename) + + # Update backup schedule on 02:00AM every 20th + BackupSchedule.update(self.user_apiclient, self.vm.id, intervaltype="MONTHLY", timezone="CET", schedule="00:02:20") + backupSchedule = BackupSchedule.list(self.user_apiclient, self.vm.id) + self.assertIsNotNone(backupSchedule) + self.assertEqual("MONTHLY", backupSchedule.intervaltype) + self.assertEqual("00:02:20", backupSchedule.schedule) + + # Delete backup schedule + BackupSchedule.delete(self.user_apiclient, self.vm.id) + + # Create backup + Backup.create(self.user_apiclient, self.vm.id) + + # Verify backup is created for the VM + self.waitForBackUp(self.vm) + backups = Backup.list(self.user_apiclient, self.vm.id) + self.assertEqual(len(backups), 1, "There should exist only one backup for the VM") + backup = backups[0] + + # Stop VM + self.vm.stop(self.user_apiclient, forced=True) + # Restore backup + Backup.restoreVM(self.user_apiclient, backup.id) + + # Delete backup + Backup.delete(self.user_apiclient, backup.id, forced=True) + + # Verify backup is deleted + backups = Backup.list(self.user_apiclient, self.vm.id) + self.assertEqual(backups, None, "There should not exist any backup for the VM") + + # Remove VM from offering + self.offering.removeOffering(self.user_apiclient, self.vm.id) + + @attr(tags=["advanced", "backup"], required_hardware="false") + def test_03_restore_volume_attach_vm(self): + """ + Test Volume Restore from Backup and Attach to VM + """ + + if self.offering: + self.cleanup.insert(0, self.offering) + + self.vm = VirtualMachine.create(self.user_apiclient, self.services["small"], accountid=self.account.name, + domainid=self.account.domainid, serviceofferingid=self.service_offering.id) + + self.vm_with_datadisk = VirtualMachine.create(self.user_apiclient, self.services["small"], accountid=self.account.name, + domainid=self.account.domainid, serviceofferingid=self.service_offering.id, + diskofferingid=self.disk_offering.id) + + # Assign VM to offering and create ad-hoc backup + self.offering.assignOffering(self.user_apiclient, self.vm_with_datadisk.id) + + # Create backup + Backup.create(self.user_apiclient, self.vm_with_datadisk.id) + + # Verify backup is created for the VM with datadisk + self.waitForBackUp(self.vm_with_datadisk) + backups = Backup.list(self.user_apiclient, self.vm_with_datadisk.id) + self.assertEqual(len(backups), 1, "There should exist only one backup for the VM with datadisk") + backup = backups[0] + + try: + volumes = Volume.list( + self.user_apiclient, + virtualmachineid=self.vm_with_datadisk.id, + listall=True + ) + rootDiskId = None + dataDiskId = None + for volume in volumes: + if volume.type == 'ROOT': + rootDiskId = volume.id + elif volume.type == 'DATADISK': + dataDiskId = volume.id + if rootDiskId: + # Restore ROOT volume of vm_with_datadisk and attach to vm + Backup.restoreVolumeFromBackupAndAttachToVM( + self.user_apiclient, + backupid=backup.id, + volumeid=rootDiskId, + virtualmachineid=self.vm.id + ) + vm_volumes = Volume.list( + self.user_apiclient, + virtualmachineid=self.vm.id, + listall=True + ) + self.assertTrue(isinstance(vm_volumes, list), "List volumes should return a valid list") + self.assertEqual(2, len(vm_volumes), "The number of volumes should be 2") + if dataDiskId: + # Restore DATADISK volume of vm_with_datadisk and attach to vm + Backup.restoreVolumeFromBackupAndAttachToVM( + self.user_apiclient, + backupid=backup.id, + volumeid=dataDiskId, + virtualmachineid=self.vm.id + ) + vm_volumes = Volume.list( + self.user_apiclient, + virtualmachineid=self.vm.id, + listall=True + ) + self.assertTrue(isinstance(vm_volumes, list), "List volumes should return a valid list") + self.assertEqual(3, len(vm_volumes), "The number of volumes should be 2") + finally: + # Delete backup + Backup.delete(self.user_apiclient, backup.id, forced=True) diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index bf8a6a761b5..68595a01058 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -5916,8 +5916,10 @@ class ResourceDetails: cmd.resourcetype = resourcetype return (apiclient.removeResourceDetail(cmd)) + # Backup and Recovery + class BackupOffering: def __init__(self, items): @@ -5982,6 +5984,7 @@ class BackupOffering: cmd.forced = forced return (apiclient.removeVirtualMachineFromBackupOffering(cmd)) + class Backup: def __init__(self, items): @@ -5993,14 +5996,16 @@ class Backup: cmd = createBackup.createBackupCmd() cmd.virtualmachineid = vmid - return (apiclient.createBackup(cmd)) + return Backup(apiclient.createBackup(cmd).__dict__) @classmethod - def delete(self, apiclient, id): + def delete(self, apiclient, id, forced=None): """Delete VM backup""" cmd = deleteBackup.deleteBackupCmd() cmd.id = id + if forced: + cmd.forced = forced return (apiclient.deleteBackup(cmd)) @classmethod @@ -6012,13 +6017,66 @@ class Backup: cmd.listall = True return (apiclient.listBackups(cmd)) - def restoreVM(self, apiclient): + @classmethod + def restoreVM(self, apiclient, backupid): """Restore VM from backup""" cmd = restoreBackup.restoreBackupCmd() - cmd.id = self.id + cmd.id = backupid return (apiclient.restoreBackup(cmd)) + @classmethod + def restoreVolumeFromBackupAndAttachToVM(self, apiclient, backupid, volumeid, virtualmachineid): + """Restore VM from backup""" + + cmd = restoreVolumeFromBackupAndAttachToVM.restoreVolumeFromBackupAndAttachToVMCmd() + cmd.backupid = backupid + cmd.volumeid = volumeid + cmd.virtualmachineid = virtualmachineid + return (apiclient.restoreVolumeFromBackupAndAttachToVM(cmd)) + + +class BackupSchedule: + + def __init__(self, items): + self.__dict__.update(items) + + @classmethod + def create(self, apiclient, vmid, **kwargs): + """Create VM backup schedule""" + + cmd = createBackupSchedule.createBackupScheduleCmd() + cmd.virtualmachineid = vmid + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + return BackupSchedule(apiclient.createBackupSchedule(cmd).__dict__) + + @classmethod + def delete(self, apiclient, vmid): + """Delete VM backup schedule""" + + cmd = deleteBackupSchedule.deleteBackupScheduleCmd() + cmd.virtualmachineid = vmid + return (apiclient.deleteBackupSchedule(cmd)) + + @classmethod + def list(self, apiclient, vmid): + """List VM backup schedule""" + + cmd = listBackupSchedule.listBackupScheduleCmd() + cmd.virtualmachineid = vmid + cmd.listall = True + return (apiclient.listBackupSchedule(cmd)) + + @classmethod + def update(self, apiclient, vmid, **kwargs): + """Update VM backup schedule""" + + cmd = updateBackupSchedule.updateBackupScheduleCmd() + cmd.virtualmachineid = vmid + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + return (apiclient.updateBackupSchedule(cmd)) + + class ProjectRole: def __init__(self, items): diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 286d9d16d2a..1afeae9c4a1 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -577,7 +577,7 @@ export default { }, enableGroupAction () { return ['vm', 'alert', 'vmgroup', 'ssh', 'userdata', 'affinitygroup', 'autoscalevmgroup', 'volume', 'snapshot', - 'vmsnapshot', 'guestnetwork', 'vpc', 'publicip', 'vpnuser', 'vpncustomergateway', + 'vmsnapshot', 'backup', 'guestnetwork', 'vpc', 'publicip', 'vpnuser', 'vpncustomergateway', 'project', 'account', 'systemvm', 'router', 'computeoffering', 'systemoffering', 'diskoffering', 'backupoffering', 'networkoffering', 'vpcoffering', 'ilbvm', 'kubernetes', 'comment' ].includes(this.$route.name) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 007bd8c3f9d..0ef53012ba0 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -32,9 +32,9 @@ export default { permission: ['listVirtualMachinesMetrics'], resourceType: 'UserVm', params: () => { - var params = { details: 'servoff,tmpl,nics' } + var params = { details: 'servoff,tmpl,nics,backoff' } if (store.getters.metrics) { - params = { details: 'servoff,tmpl,nics,stats' } + params = { details: 'servoff,tmpl,nics,backoff,stats' } } return params }, diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 1cf31350fb2..d73b989f74e 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -488,7 +488,11 @@ export default { label: 'label.delete.backup', message: 'message.delete.backup', dataView: true, - show: (record) => { return record.state !== 'Destroyed' } + show: (record) => { return record.state !== 'Destroyed' }, + groupAction: true, + popup: true, + groupMap: (selection, values) => { return selection.map(x => { return { id: x, forced: values.forced } }) }, + args: ['forced'] } ] } diff --git a/ui/src/views/compute/backup/BackupSchedule.vue b/ui/src/views/compute/backup/BackupSchedule.vue index 914a1121ffb..32da2d440a7 100644 --- a/ui/src/views/compute/backup/BackupSchedule.vue +++ b/ui/src/views/compute/backup/BackupSchedule.vue @@ -40,6 +40,9 @@ +