apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

NAS B&R Plugin enhancements (#9666)

Browse Source 
* NAS B&R Plugin enhancements

* Prevent printing mount opts which may include password by removing from response

* revert marvin change

* add sanity checks to validate minimum qemu and libvirt versions

* check is user running script is part of libvirt group

* revert changes of retore expunged VM

* add code coverage ignore file

* remove check

* issue with listing schedules and add defensive checks

* redirect logs to agent log file

* add some more debugging

* remove test file

* prevent deletion of cks cluster when vms associated to a backup offering

* delete all snapshot policies when bkp offering is disassociated from a VM

* Fix `updateTemplatePermission` when the UI is set to a language other than English (#9766)

* Fix updateTemplatePermission UI in non-english language

* Improve fix

---------

* Add nobrl in the mountopts for cifs file system

* Fix restoration of VM / volumes with cifs

* add cifs utils for el8

* add cifs-utils for ubuntu cloudstack-agent

* syntax error

* remove required constraint on both vmid and id params for the delete bkp schedule command
This commit is contained in:
Pearl Dsilva 2025-03-04 11:32:09 -05:00 committed by GitHub
parent a6b1403ca6
commit 7f4e6a9d51
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 156 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
api/src/main/java/org/apache/cloudstack/api/command/user/backup/DeleteBackupScheduleCmd.java
View File

@ -26,6 +26,7 @@ import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.BackupScheduleResponse;
import org.apache.cloudstack.api.response.SuccessResponse;
import org.apache.cloudstack.api.response.UserVmResponse;
import org.apache.cloudstack.backup.BackupManager;
@ -54,10 +55,16 @@ public class DeleteBackupScheduleCmd extends BaseCmd {
@Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID,
type = CommandType.UUID,
entityType = UserVmResponse.class,
required = true,
description = "ID of the VM")
private Long vmId;
@Parameter(name = ApiConstants.ID,
type = CommandType.UUID,
entityType = BackupScheduleResponse.class,
description = "ID of the schedule",
since = "4.20.1")
private Long id;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -66,6 +73,9 @@ public class DeleteBackupScheduleCmd extends BaseCmd {
return vmId;
}
public Long getId() { return id; }
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@ -73,7 +83,7 @@ public class DeleteBackupScheduleCmd extends BaseCmd {
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
try {
boolean result = backupManager.deleteBackupSchedule(getVmId());
boolean result = backupManager.deleteBackupSchedule(this);
if (result) {
SuccessResponse response = new SuccessResponse(getCommandName());
response.setResponseName(getCommandName());

12
api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java
View File

@ -57,10 +57,6 @@ public class BackupRepositoryResponse extends BaseResponse {
@Param(description = "backup type")
private String type;
@SerializedName(ApiConstants.MOUNT_OPTIONS)
@Param(description = "mount options for the backup repository")
private String mountOptions;
@SerializedName(ApiConstants.CAPACITY_BYTES)
@Param(description = "capacity of the backup repository")
private Long capacityBytes;
@ -112,14 +108,6 @@ public class BackupRepositoryResponse extends BaseResponse {
this.address = address;
}
public String getMountOptions() {
return mountOptions;
}
public void setMountOptions(String mountOptions) {
this.mountOptions = mountOptions;
}
public String getProviderName() {
return providerName;
}

5
api/src/main/java/org/apache/cloudstack/backup/BackupManager.java
View File

@ -22,6 +22,7 @@ import java.util.List;
import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd;
import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd;
import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd;
import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd;
import org.apache.cloudstack.api.command.user.backup.ListBackupOfferingsCmd;
import org.apache.cloudstack.api.command.user.backup.ListBackupsCmd;
import org.apache.cloudstack.framework.config.ConfigKey;
@ -111,10 +112,10 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer
/**
* Deletes VM backup schedule for a VM
* @param vmId
* @param cmd
* @return
*/
boolean deleteBackupSchedule(Long vmId);
boolean deleteBackupSchedule(DeleteBackupScheduleCmd cmd);
/**
* Creates backup of a VM

1
packaging/el8/cloud.spec
View File

@ -114,6 +114,7 @@ Requires: iproute
Requires: ipset
Requires: perl
Requires: rsync
Requires: cifs-utils
Requires: (python3-libvirt or python3-libvirt-python)
Requires: (qemu-img or qemu-tools)
Requires: qemu-kvm

6
plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java
View File

@ -221,6 +221,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co
restoreCommand.setBackupPath(backup.getExternalId());
restoreCommand.setBackupRepoType(backupRepository.getType());
restoreCommand.setBackupRepoAddress(backupRepository.getAddress());
restoreCommand.setMountOptions(backupRepository.getMountOptions());
restoreCommand.setVmName(vm.getName());
restoreCommand.setVolumePaths(getVolumePaths(volumes));
restoreCommand.setVmExists(vm.getRemoved() == null);
@ -289,6 +290,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co
restoreCommand.setVmName(vmNameAndState.first());
restoreCommand.setVolumePaths(Collections.singletonList(String.format("%s/%s", dataStore.getLocalPath(), volumeUUID)));
restoreCommand.setDiskType(volume.getVolumeType().name().toLowerCase(Locale.ROOT));
restoreCommand.setMountOptions(backupRepository.getMountOptions());
restoreCommand.setVmExists(null);
restoreCommand.setVmState(vmNameAndState.second());
restoreCommand.setRestoreVolumeUUID(volumeUuid);
@ -373,9 +375,13 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co
Long vmBackupSize = 0L;
Long vmBackupProtectedSize = 0L;
for (final Backup backup: backupDao.listByVmId(null, vm.getId())) {
if (Objects.nonNull(backup.getSize())) {
vmBackupSize += backup.getSize();
}
if (Objects.nonNull(backup.getProtectedSize())) {
vmBackupProtectedSize += backup.getProtectedSize();
}
}
Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize);
LOG.debug("Metrics for VM {} is [backup size: {}, data size: {}].", vm, vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize());
metrics.put(vm, vmBackupMetric);

22
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java
View File

@ -67,7 +67,7 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper<RestoreBa
int lastIndex = volumePath.lastIndexOf("/");
newVolumeId = volumePath.substring(lastIndex + 1);
restoreVolume(backupPath, backupRepoType, backupRepoAddress, volumePath, diskType, restoreVolumeUuid,
new Pair<>(vmName, command.getVmState()));
new Pair<>(vmName, command.getVmState()), mountOptions);
} else if (Boolean.TRUE.equals(vmExists)) {
restoreVolumesOfExistingVM(volumePaths, backupPath, backupRepoType, backupRepoAddress, mountOptions);
} else {
@ -80,7 +80,7 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper<RestoreBa
private void restoreVolumesOfExistingVM(List<String> volumePaths, String backupPath,
String backupRepoType, String backupRepoAddress, String mountOptions) {
String diskType = "root";
String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType);
String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions);
try {
for (int idx = 0; idx < volumePaths.size(); idx++) {
String volumePath = volumePaths.get(idx);
@ -101,7 +101,7 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper<RestoreBa
private void restoreVolumesOfDestroyedVMs(List<String> volumePaths, String vmName, String backupPath,
String backupRepoType, String backupRepoAddress, String mountOptions) {
String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType);
String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions);
String diskType = "root";
try {
for (int i = 0; i < volumePaths.size(); i++) {
@ -121,8 +121,8 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper<RestoreBa
}
private void restoreVolume(String backupPath, String backupRepoType, String backupRepoAddress, String volumePath,
String diskType, String volumeUUID, Pair<String, VirtualMachine.State> vmNameAndState) {
String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType);
String diskType, String volumeUUID, Pair<String, VirtualMachine.State> vmNameAndState, String mountOptions) {
String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions);
Pair<String, String> bkpPathAndVolUuid;
try {
bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, volumeUUID);
@ -145,12 +145,22 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper<RestoreBa
}
private String mountBackupDirectory(String backupRepoAddress, String backupRepoType) {
private String mountBackupDirectory(String backupRepoAddress, String backupRepoType, String mountOptions) {
String randomChars = RandomStringUtils.random(5, true, false);
String mountDirectory = String.format("%s.%s",BACKUP_TEMP_FILE_PREFIX , randomChars);
try {
mountDirectory = Files.createTempDirectory(mountDirectory).toString();
String mountOpts = null;
if (Objects.nonNull(mountOptions)) {
mountOpts = mountOptions;
if ("cifs".equals(backupRepoType)) {
mountOpts += ",nobrl";
}
}
String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory);
if (Objects.nonNull(mountOpts)) {
mount += " -o " + mountOpts;
}
Script.runSimpleBashScript(mount);
} catch (Exception e) {
throw new CloudRuntimeException(String.format("Failed to mount %s to %s", backupRepoType, backupRepoAddress), e);

14
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View File

@ -36,6 +36,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
@ -1467,6 +1468,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
List<KubernetesClusterVmMapVO> vmMapList = kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId);
List<VMInstanceVO> vms = vmMapList.stream().map(vmMap -> vmInstanceDao.findById(vmMap.getVmId())).collect(Collectors.toList());
if (checkIfVmsAssociatedWithBackupOffering(vms)) {
throw new CloudRuntimeException("Unable to delete Kubernetes cluster, as node(s) are associated to a backup offering");
}
for (KubernetesClusterVmMapVO vmMap : vmMapList) {
try {
userVmService.destroyVm(vmMap.getVmId(), expunge);
@ -1489,6 +1494,15 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
}
public static boolean checkIfVmsAssociatedWithBackupOffering(List<VMInstanceVO> vms) {
for(VMInstanceVO vm : vms) {
if (Objects.nonNull(vm.getBackupOfferingId())) {
return true;
}
}
return false;
}
@Override
public ListResponse<KubernetesClusterResponse> listKubernetesClusters(ListKubernetesClustersCmd cmd) {
if (!KubernetesServiceEnabled.value()) {

4
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java
View File

@ -245,6 +245,10 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod
init();
validateClusterSate();
this.clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
List<VMInstanceVO> vms = this.clusterVMs.stream().map(vmMap -> vmInstanceDao.findById(vmMap.getVmId())).collect(Collectors.toList());
if (KubernetesClusterManagerImpl.checkIfVmsAssociatedWithBackupOffering(vms)) {
throw new CloudRuntimeException("Unable to delete Kubernetes cluster, as node(s) are associated to a backup offering");
}
boolean cleanupNetwork = true;
final KubernetesClusterDetailsVO clusterDetails = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "networkCleanup");
if (clusterDetails != null) {

68
scripts/vm/hypervisor/kvm/nasbackup.sh
View File

@ -31,6 +31,58 @@ NAS_ADDRESS=""
MOUNT_OPTS=""
BACKUP_DIR=""
DISK_PATHS=""
logFile="/var/log/cloudstack/agent/agent.log"
log() {
[[ "$verb" -eq 1 ]] && builtin echo "$@"
if [[ "$1" == "-ne" || "$1" == "-e" || "$1" == "-n" ]]; then
builtin echo -e "$(date '+%Y-%m-%d %H-%M-%S>')" "${@: 2}" >> "$logFile"
else
builtin echo "$(date '+%Y-%m-%d %H-%M-%S>')" "$@" >> "$logFile"
fi
}
vercomp() {
local IFS=.
local i ver1=($1) ver2=($3)
# Compare each segment of the version numbers
for ((i=0; i<${#ver1[@]}; i++)); do
if [[ -z ${ver2[i]} ]]; then
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]})); then
return 0 # Version 1 is greater
elif ((10#${ver1[i]} < 10#${ver2[i]})); then
return 2 # Version 2 is greater
fi
done
return 0 # Versions are equal
}
sanity_checks() {
hvVersion=$(virsh version | grep hypervisor | awk '{print $(NF)}')
libvVersion=$(virsh version | grep libvirt | awk '{print $(NF)}' | tail -n 1)
apiVersion=$(virsh version | grep API | awk '{print $(NF)}')
# Compare qemu version (hvVersion >= 4.2.0)
vercomp "$hvVersion" ">=" "4.2.0"
hvStatus=$?
# Compare libvirt version (libvVersion >= 7.2.0)
vercomp "$libvVersion" ">=" "7.2.0"
libvStatus=$?
if [[ $hvStatus -eq 0 && $libvStatus -eq 0 ]]; then
log -ne "Success... [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]"
else
echo "Failure... Your QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0"
exit 1
fi
log -ne "Environment Sanity Checks successfully passed"
}
### Operation methods ###
@ -79,7 +131,7 @@ backup_stopped_vm() {
name="root"
for disk in $DISK_PATHS; do
volUuid="${disk##*/}"
qemu-img convert -O qcow2 $disk $dest/$name.$volUuid.qcow2
qemu-img convert -O qcow2 $disk $dest/$name.$volUuid.qcow2 | tee -a "$logFile"
name="datadisk"
done
sync
@ -99,7 +151,16 @@ delete_backup() {
mount_operation() {
mount_point=$(mktemp -d -t csbackup.XXXXX)
dest="$mount_point/${BACKUP_DIR}"
mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS})
if [ ${NAS_TYPE} == "cifs" ]; then
MOUNT_OPTS="${MOUNT_OPTS},nobrl"
fi
mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) | tee -a "$logFile"
if [ $? -eq 0 ]; then
log -ne "Successfully mounted ${NAS_TYPE} store"
else
echo "Failed to mount ${NAS_TYPE} store"
exit 1
fi
}
function usage {
@ -157,6 +218,9 @@ while [[ $# -gt 0 ]]; do
esac
done
# Perform Initial sanity checks
sanity_checks
if [ "$OP" = "backup" ]; then
STATE=$(virsh -c qemu:///system list | grep $VM | awk '{print $3}')
if [ "$STATE" = "running" ]; then

1
server/src/main/java/com/cloud/api/ApiResponseHelper.java
View File

@ -5440,7 +5440,6 @@ public class ApiResponseHelper implements ResponseGenerator {
response.setAddress(backupRepository.getAddress());
response.setProviderName(backupRepository.getProvider());
response.setType(backupRepository.getType());
response.setMountOptions(backupRepository.getMountOptions());
response.setCapacityBytes(backupRepository.getCapacityBytes());
response.setObjectName("backuprepository");
DataCenter zone = ApiDBUtils.findZoneById(backupRepository.getZoneId());

36
server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java
View File

@ -23,6 +23,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.TimeZone;
import java.util.Timer;
import java.util.TimerTask;
@ -61,7 +62,6 @@ import org.apache.cloudstack.backup.dao.BackupDao;
import org.apache.cloudstack.backup.dao.BackupOfferingDao;
import org.apache.cloudstack.backup.dao.BackupScheduleDao;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.jobs.AsyncJobDispatcher;
import org.apache.cloudstack.framework.jobs.AsyncJobManager;
@ -162,8 +162,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
private VirtualMachineManager virtualMachineManager;
@Inject
private VolumeApiService volumeApiService;
@Inject
private VolumeOrchestrationService volumeOrchestrationService;
private AsyncJobDispatcher asyncJobDispatcher;
private Timer backupTimer;
@ -396,8 +394,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(),
"Backup-" + vm.getHostName() + "-" + vm.getUuid(), vm.getBackupOfferingId(), null, null,
Backup.class.getSimpleName(), vm.getUuid());
final BackupSchedule backupSchedule = backupScheduleDao.findByVM(vm.getId());
if (backupSchedule != null) {
final List<BackupScheduleVO> backupSchedules = backupScheduleDao.listByVM(vm.getId());
for(BackupSchedule backupSchedule: backupSchedules) {
backupScheduleDao.remove(backupSchedule.getId());
}
result = true;
@ -455,7 +453,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
schedule.setTimezone(timezoneId);
schedule.setScheduledTimestamp(nextDateTime);
backupScheduleDao.update(schedule.getId(), schedule);
return backupScheduleDao.findByVM(vmId);
return backupScheduleDao.findById(schedule.getId());
}
@Override
@ -469,17 +467,34 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
@Override
@ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_SCHEDULE_DELETE, eventDescription = "deleting VM backup schedule")
public boolean deleteBackupSchedule(final Long vmId) {
public boolean deleteBackupSchedule(DeleteBackupScheduleCmd cmd) {
Long vmId = cmd.getVmId();
Long id = cmd.getId();
if (Objects.isNull(vmId) && Objects.isNull(id)) {
throw new InvalidParameterValueException("Either instance ID or ID of backup schedule needs to be specified");
}
if (Objects.nonNull(vmId)) {
final VMInstanceVO vm = findVmById(vmId);
validateForZone(vm.getDataCenterId());
accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm);
final BackupSchedule schedule = backupScheduleDao.findByVM(vmId);
return deleteAllVMBackupSchedules(vm.getId());
} else {
final BackupSchedule schedule = backupScheduleDao.findById(id);
if (schedule == null) {
throw new CloudRuntimeException("VM has no backup schedule defined, no need to delete anything.");
throw new CloudRuntimeException("Could not find the requested backup schedule.");
}
return backupScheduleDao.remove(schedule.getId());
}
}
private boolean deleteAllVMBackupSchedules(long vmId) {
List<BackupScheduleVO> vmBackupSchedules = backupScheduleDao.listByVM(vmId);
boolean success = true;
for (BackupScheduleVO vmBackupSchedule : vmBackupSchedules) {
success = success && backupScheduleDao.remove(vmBackupSchedule.getId());
}
return success;
}
@Override
@ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating VM backup", async = true)
@ -622,6 +637,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
!vm.getState().equals(VirtualMachine.State.Destroyed)) {
throw new CloudRuntimeException("Existing VM should be stopped before being restored from backup");
}
// This is done to handle historic backups if any with Veeam / Networker plugins
List<Backup.VolumeInfo> backupVolumes = CollectionUtils.isNullOrEmpty(backup.getBackedUpVolumes()) ?
vm.getBackupVolumeList() : backup.getBackedUpVolumes();

2
ui/src/config/section/config.js
View File

@ -151,7 +151,7 @@ export default {
],
mapping: {
type: {
options: ['nfs']
options: ['nfs', 'cifs']
},
provider: {
value: (record) => { return 'nas' }