From 68e504aff97dd7ffc0eb7d94f3acf9ea9d606d57 Mon Sep 17 00:00:00 2001 From: Peinthor Rene Date: Thu, 9 Nov 2023 05:08:10 +0100 Subject: [PATCH] Linstor backup snaphots (#8067) This PR adds an config option for the Linstor primary storage driver, that allows you to automatically backup volume snapshots to the secondary storage. Additionally it will not mangle the need java-linstor dependency into the client.jar, but instead just copy the java-linstor.jar into lib. Config option is called: lin.backup.snapshots and is default false The scope of this change should be limited, as it only touches the Linstor driver and a part of copyAsync was implemented with 2 new Linstor specific commands. --- client/pom.xml | 7 + plugins/storage/volume/linstor/pom.xml | 6 + .../storage/LinstorBackupSnapshotCommand.java | 28 ++ .../LinstorRevertBackupSnapshotCommand.java | 28 ++ .../LinstorBackupSnapshotCommandWrapper.java | 167 +++++++++ ...torRevertBackupSnapshotCommandWrapper.java | 92 +++++ .../kvm/storage/LinstorStorageAdaptor.java | 10 +- .../LinstorPrimaryDataStoreDriverImpl.java | 352 ++++++++++++++++-- .../LinstorPrimaryDatastoreProviderImpl.java | 4 +- .../util/LinstorConfigurationManager.java | 40 ++ .../storage/datastore/util/LinstorUtil.java | 89 +++++ .../spring-storage-volume-linstor-context.xml | 2 + pom.xml | 5 + 13 files changed, 791 insertions(+), 39 deletions(-) create mode 100644 plugins/storage/volume/linstor/src/main/java/com/cloud/api/storage/LinstorBackupSnapshotCommand.java create mode 100644 plugins/storage/volume/linstor/src/main/java/com/cloud/api/storage/LinstorRevertBackupSnapshotCommand.java create mode 100644 plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java create mode 100644 plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorRevertBackupSnapshotCommandWrapper.java create mode 100644 plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java diff --git a/client/pom.xml b/client/pom.xml index 22cca63e0c8..de9e910b978 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -837,6 +837,12 @@ false ${project.build.directory}/lib + + com.linbit.linstor.api + java-linstor + false + ${project.build.directory}/lib + org.bouncycastle bctls-jdk15on @@ -880,6 +886,7 @@ mysql:mysql-connector-java org.apache.cloudstack:cloud-plugin-storage-volume-storpool org.apache.cloudstack:cloud-plugin-storage-volume-linstor + com.linbit.linstor.api:java-linstor diff --git a/plugins/storage/volume/linstor/pom.xml b/plugins/storage/volume/linstor/pom.xml index 69de4ff3d67..44e79dbce51 100644 --- a/plugins/storage/volume/linstor/pom.xml +++ b/plugins/storage/volume/linstor/pom.xml @@ -43,6 +43,12 @@ cloud-plugin-hypervisor-kvm ${project.version} + + org.apache.cloudstack + cloud-engine-storage-snapshot + ${project.version} + compile + diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/api/storage/LinstorBackupSnapshotCommand.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/api/storage/LinstorBackupSnapshotCommand.java new file mode 100644 index 00000000000..8d887dbba21 --- /dev/null +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/api/storage/LinstorBackupSnapshotCommand.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.api.storage; + +import com.cloud.agent.api.to.DataTO; +import org.apache.cloudstack.storage.command.CopyCommand; + +public class LinstorBackupSnapshotCommand extends CopyCommand +{ + public LinstorBackupSnapshotCommand(DataTO srcData, DataTO destData, int timeout, boolean executeInSequence) + { + super(srcData, destData, timeout, executeInSequence); + } +} diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/api/storage/LinstorRevertBackupSnapshotCommand.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/api/storage/LinstorRevertBackupSnapshotCommand.java new file mode 100644 index 00000000000..1f1880d5da5 --- /dev/null +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/api/storage/LinstorRevertBackupSnapshotCommand.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.api.storage; + +import com.cloud.agent.api.to.DataTO; +import org.apache.cloudstack.storage.command.CopyCommand; + +public class LinstorRevertBackupSnapshotCommand extends CopyCommand +{ + public LinstorRevertBackupSnapshotCommand(DataTO srcData, DataTO destData, int timeout, boolean executeInSequence) + { + super(srcData, destData, timeout, executeInSequence); + } +} diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java new file mode 100644 index 00000000000..a210d53d7e7 --- /dev/null +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.io.IOException; + +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.api.storage.LinstorBackupSnapshotCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; +import org.joda.time.Duration; +import org.libvirt.LibvirtException; + +@ResourceWrapper(handles = LinstorBackupSnapshotCommand.class) +public final class LinstorBackupSnapshotCommandWrapper + extends CommandWrapper +{ + private static final Logger s_logger = Logger.getLogger(LinstorBackupSnapshotCommandWrapper.class); + + private String zfsSnapdev(boolean hide, String zfsUrl) { + Script script = new Script("/usr/bin/zfs", Duration.millis(5000)); + script.add("set"); + script.add("snapdev=" + (hide ? "hidden" : "visible")); + script.add(zfsUrl.substring(6)); // cutting zfs:// + return script.execute(); + } + + private String qemuShrink(String path, long sizeByte, long timeout) { + Script qemuImg = new Script("qemu-img", Duration.millis(timeout)); + qemuImg.add("resize"); + qemuImg.add("--shrink"); + qemuImg.add(path); + qemuImg.add("" + sizeByte); + return qemuImg.execute(); + } + + static void cleanupSecondaryPool(final KVMStoragePool secondaryPool) { + if (secondaryPool != null) { + try { + secondaryPool.delete(); + } catch (final Exception e) { + s_logger.debug("Failed to delete secondary storage", e); + } + } + } + + private String convertImageToQCow2( + final String srcPath, + final SnapshotObjectTO dst, + final KVMStoragePool secondaryPool, + int waitMilliSeconds + ) + throws LibvirtException, QemuImgException, IOException + { + final String dstDir = secondaryPool.getLocalPath() + File.separator + dst.getPath(); + FileUtils.forceMkdir(new File(dstDir)); + + final String dstPath = dstDir + File.separator + dst.getName(); + final QemuImgFile srcFile = new QemuImgFile(srcPath, QemuImg.PhysicalDiskFormat.RAW); + final QemuImgFile dstFile = new QemuImgFile(dstPath, QemuImg.PhysicalDiskFormat.QCOW2); + + // NOTE: the qemu img will also contain the drbd metadata at the end + final QemuImg qemu = new QemuImg(waitMilliSeconds); + qemu.convert(srcFile, dstFile); + s_logger.info("Backup snapshot " + srcFile + " to " + dstPath); + return dstPath; + } + + private SnapshotObjectTO setCorrectSnapshotSize(final SnapshotObjectTO dst, final String dstPath) { + final File snapFile = new File(dstPath); + final long size = snapFile.exists() ? snapFile.length() : 0; + + final SnapshotObjectTO snapshot = new SnapshotObjectTO(); + snapshot.setPath(dst.getPath() + File.separator + dst.getName()); + snapshot.setPhysicalSize(size); + return snapshot; + } + + @Override + public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingResource serverResource) + { + s_logger.debug("LinstorBackupSnapshotCommandWrapper: " + cmd.getSrcTO().getPath() + " -> " + cmd.getDestTO().getPath()); + final SnapshotObjectTO src = (SnapshotObjectTO) cmd.getSrcTO(); + final SnapshotObjectTO dst = (SnapshotObjectTO) cmd.getDestTO(); + KVMStoragePool secondaryPool = null; + final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + KVMStoragePool linstorPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.Linstor, src.getDataStore().getUuid()); + boolean zfsHidden = false; + String srcPath = src.getPath(); + + if (linstorPool == null) { + return new CopyCmdAnswer("Unable to get linstor storage pool from destination volume."); + } + + final DataStoreTO dstDataStore = dst.getDataStore(); + if (!(dstDataStore instanceof NfsTO)) { + return new CopyCmdAnswer("Backup Linstor snapshot: Only NFS secondary supported at present!"); + } + + try + { + // provide the linstor snapshot block device + // on lvm thin this should already be there in /dev/mapper/vg-snapshotname + // on zfs we need to unhide the snapshot block device + s_logger.info("Src: " + srcPath + " | " + src.getName()); + if (srcPath.startsWith("zfs://")) { + zfsHidden = true; + if (zfsSnapdev(false, srcPath) != null) { + return new CopyCmdAnswer("Unable to unhide zfs snapshot device."); + } + srcPath = "/dev/" + srcPath.substring(6); + } + + secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl()); + + String dstPath = convertImageToQCow2(srcPath, dst, secondaryPool, cmd.getWaitInMillSeconds()); + + // resize to real volume size, cutting of drbd metadata + String result = qemuShrink(dstPath, src.getVolume().getSize(), cmd.getWaitInMillSeconds()); + if (result != null) { + return new CopyCmdAnswer("qemu-img shrink failed: " + result); + } + s_logger.info("Backup shrunk " + dstPath + " to actual size " + src.getVolume().getSize()); + + SnapshotObjectTO snapshot = setCorrectSnapshotSize(dst, dstPath); + return new CopyCmdAnswer(snapshot); + } catch (final Exception e) { + final String error = String.format("Failed to backup snapshot with id [%s] with a pool %s, due to %s", + cmd.getSrcTO().getId(), cmd.getSrcTO().getDataStore().getUuid(), e.getMessage()); + s_logger.error(error); + return new CopyCmdAnswer(cmd, e); + } finally { + cleanupSecondaryPool(secondaryPool); + if (zfsHidden) { + zfsSnapdev(true, src.getPath()); + } + } + } +} diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorRevertBackupSnapshotCommandWrapper.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorRevertBackupSnapshotCommandWrapper.java new file mode 100644 index 00000000000..511b5a40ca8 --- /dev/null +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorRevertBackupSnapshotCommandWrapper.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; + +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.api.storage.LinstorRevertBackupSnapshotCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.log4j.Logger; +import org.libvirt.LibvirtException; + +@ResourceWrapper(handles = LinstorRevertBackupSnapshotCommand.class) +public final class LinstorRevertBackupSnapshotCommandWrapper + extends CommandWrapper +{ + private static final Logger s_logger = Logger.getLogger(LinstorRevertBackupSnapshotCommandWrapper.class); + + private void convertQCow2ToRAW(final String srcPath, final String dstPath, int waitMilliSeconds) + throws LibvirtException, QemuImgException + { + final QemuImgFile srcQemuFile = new QemuImgFile( + srcPath, QemuImg.PhysicalDiskFormat.QCOW2); + final QemuImg qemu = new QemuImg(waitMilliSeconds); + final QemuImgFile dstFile = new QemuImgFile(dstPath, QemuImg.PhysicalDiskFormat.RAW); + qemu.convert(srcQemuFile, dstFile); + } + + @Override + public CopyCmdAnswer execute(LinstorRevertBackupSnapshotCommand cmd, LibvirtComputingResource serverResource) + { + s_logger.debug("LinstorRevertBackupSnapshotCommandWrapper: " + cmd.getSrcTO().getPath() + " -> " + cmd.getDestTO().getPath()); + final SnapshotObjectTO src = (SnapshotObjectTO) cmd.getSrcTO(); + final VolumeObjectTO dst = (VolumeObjectTO) cmd.getDestTO(); + KVMStoragePool secondaryPool = null; + final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + KVMStoragePool linstorPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.Linstor, dst.getDataStore().getUuid()); + + if (linstorPool == null) { + return new CopyCmdAnswer("Unable to get linstor storage pool from destination volume."); + } + + try + { + final DataStoreTO srcDataStore = src.getDataStore(); + File srcFile = new File(src.getPath()); + secondaryPool = storagePoolMgr.getStoragePoolByURI( + srcDataStore.getUrl() + File.separator + srcFile.getParent()); + + convertQCow2ToRAW( + secondaryPool.getLocalPath() + File.separator + srcFile.getName(), + linstorPool.getPhysicalDisk(dst.getPath()).getPath(), + cmd.getWaitInMillSeconds()); + + final VolumeObjectTO dstVolume = new VolumeObjectTO(); + dstVolume.setPath(dst.getPath()); + return new CopyCmdAnswer(dstVolume); + } catch (final Exception e) { + final String error = String.format("Failed to revert snapshot with id [%s] with a pool %s, due to %s", + cmd.getSrcTO().getId(), cmd.getSrcTO().getDataStore().getUuid(), e.getMessage()); + s_logger.error(error); + return new CopyCmdAnswer(cmd, e); + } finally { + LinstorBackupSnapshotCommandWrapper.cleanupSecondaryPool(secondaryPool); + } + } +} diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 2847484e30e..92b617c7128 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -28,6 +28,7 @@ import java.util.StringJoiner; import javax.annotation.Nonnull; +import org.apache.cloudstack.storage.datastore.util.LinstorUtil; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; @@ -65,8 +66,8 @@ public class LinstorStorageAdaptor implements StorageAdaptor { return new DevelopersApi(client); } - private String getLinstorRscName(String name) { - return "cs-" + name; + private static String getLinstorRscName(String name) { + return LinstorUtil.RSC_PREFIX + name; } private String getHostname() { @@ -214,6 +215,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor { public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { + s_logger.debug(String.format("Linstor.createPhysicalDisk: %s;%s", name, format)); final String rscName = getLinstorRscName(name); LinstorStoragePool lpool = (LinstorStoragePool) pool; final DevelopersApi api = getLinstorAPI(pool); @@ -254,6 +256,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor { throw new CloudRuntimeException("Linstor: viewResources didn't return resources or volumes."); } } catch (ApiException apiEx) { + s_logger.error(String.format("Linstor.createPhysicalDisk: ApiException: %s", apiEx.getBestMessage())); throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); } } @@ -424,7 +427,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor { @Override public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPools, int timeout, byte[] srcPassphrase, byte[] destPassphrase, Storage.ProvisioningType provisioningType) { - s_logger.debug("Linstor: copyPhysicalDisk"); + s_logger.debug(String.format("Linstor.copyPhysicalDisk: %s -> %s", disk.getPath(), name)); final QemuImg.PhysicalDiskFormat sourceFormat = disk.getFormat(); final String sourcePath = disk.getPath(); @@ -433,6 +436,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor { final KVMPhysicalDisk dstDisk = destPools.createPhysicalDisk( name, QemuImg.PhysicalDiskFormat.RAW, provisioningType, disk.getVirtualSize(), null); + s_logger.debug(String.format("Linstor.copyPhysicalDisk: dstPath: %s", dstDisk.getPath())); final QemuImgFile destFile = new QemuImgFile(dstDisk.getPath()); destFile.setFormat(dstDisk.getFormat()); destFile.setSize(disk.getVirtualSize()); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index d67b8ab7d5b..c0f3cb4b459 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -28,6 +28,7 @@ import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted; import com.linbit.linstor.api.model.ResourceDefinitionCreate; import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroupSpawn; +import com.linbit.linstor.api.model.ResourceMakeAvailable; import com.linbit.linstor.api.model.ResourceWithVolumes; import com.linbit.linstor.api.model.Snapshot; import com.linbit.linstor.api.model.SnapshotRestore; @@ -43,18 +44,26 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.ResizeVolumeAnswer; import com.cloud.agent.api.storage.ResizeVolumeCommand; +import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.api.storage.LinstorBackupSnapshotCommand; +import com.cloud.api.storage.LinstorRevertBackupSnapshotCommand; +import com.cloud.configuration.Config; import com.cloud.host.Host; +import com.cloud.host.dao.HostDao; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.ResizeVolumePayload; import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; @@ -67,8 +76,10 @@ import com.cloud.storage.dao.SnapshotDetailsVO; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachineManager; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -80,10 +91,14 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.storage.RemoteHostEndPoint; import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.util.LinstorConfigurationManager; import org.apache.cloudstack.storage.datastore.util.LinstorUtil; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.volume.VolumeObject; @@ -98,6 +113,10 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver @Inject private SnapshotDao _snapshotDao; @Inject private SnapshotDetailsDao _snapshotDetailsDao; @Inject private StorageManager _storageMgr; + @Inject + ConfigurationDao _configDao; + @Inject + private HostDao _hostDao; public LinstorPrimaryDataStoreDriverImpl() { @@ -109,10 +128,12 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver Map mapCapabilities = new HashMap<>(); // Linstor will be restricted to only run on LVM-THIN and ZFS storage pools with ACS + // This enables template caching on our primary storage mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_VOLUME.toString(), Boolean.TRUE.toString()); // fetch if lvm-thin or ZFS - mapCapabilities.put(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString(), Boolean.TRUE.toString()); + boolean system_snapshot = !LinstorConfigurationManager.BackupSnapshots.value(); + mapCapabilities.put(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString(), Boolean.toString(system_snapshot)); // CAN_CREATE_VOLUME_FROM_SNAPSHOT see note from CAN_CREATE_VOLUME_FROM_VOLUME mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_SNAPSHOT.toString(), Boolean.TRUE.toString()); @@ -216,6 +237,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } throw new CloudRuntimeException("Linstor: Unable to delete snapshot: " + rscDefName); } + s_logger.info("Linstor: Deleted snapshot " + snapshotName + " for resource " + rscDefName); } catch (ApiException apiEx) { s_logger.error("Linstor: ApiEx - " + apiEx.getMessage()); @@ -402,27 +424,46 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } - private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO) - { - DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); - final String rscGrp = storagePoolVO.getUserInfo() != null && !storagePoolVO.getUserInfo().isEmpty() ? + private String getRscGrp(StoragePoolVO storagePoolVO) { + return storagePoolVO.getUserInfo() != null && !storagePoolVO.getUserInfo().isEmpty() ? storagePoolVO.getUserInfo() : "DfltRscGrp"; + } + private String createResourceBase( + String rscName, long sizeInBytes, String volName, String vmName, DevelopersApi api, String rscGrp) { ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn(); - final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid(); rscGrpSpawn.setResourceDefinitionName(rscName); - rscGrpSpawn.addVolumeSizesItem(vol.getSize() / 1024); + rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024); try { s_logger.info("Linstor: Spawn resource " + rscName); - ApiCallRcList answers = linstorApi.resourceGroupSpawn(rscGrp, rscGrpSpawn); + ApiCallRcList answers = api.resourceGroupSpawn(rscGrp, rscGrpSpawn); checkLinstorAnswersThrow(answers); - applyAuxProps(linstorApi, rscName, vol.getName(), vol.getAttachedVmName()); + applyAuxProps(api, rscName, volName, vmName); + + return getDeviceName(api, rscName); + } catch (ApiException apiEx) + { + s_logger.error("Linstor: ApiEx - " + apiEx.getMessage()); + throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); + } + } + + private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO) { + DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); + final String rscGrp = getRscGrp(storagePoolVO); + + final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid(); + String deviceName = createResourceBase( + rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), linstorApi, rscGrp); + + try + { applyQoSSettings(storagePoolVO, linstorApi, rscName, vol.getMaxIops()); - return getDeviceName(linstorApi, rscName); + return deviceName; } catch (ApiException apiEx) { s_logger.error("Linstor: ApiEx - " + apiEx.getMessage()); @@ -487,8 +528,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } private String createResourceFromSnapshot(long csSnapshotId, String rscName, StoragePoolVO storagePoolVO) { - final String rscGrp = storagePoolVO.getUserInfo() != null && !storagePoolVO.getUserInfo().isEmpty() ? - storagePoolVO.getUserInfo() : "DfltRscGrp"; + final String rscGrp = getRscGrp(storagePoolVO); final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); SnapshotVO snapshotVO = _snapshotDao.findById(csSnapshotId); @@ -654,6 +694,59 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } + private String revertSnapshotFromImageStore( + final SnapshotInfo snapshot, + final VolumeInfo volumeInfo, + final DevelopersApi linstorApi, + final String rscName) + throws ApiException { + String resultMsg = null; + String value = _configDao.getValue(Config.BackupSnapshotWait.toString()); + int _backupsnapshotwait = NumbersUtil.parseInt( + value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue())); + + LinstorRevertBackupSnapshotCommand cmd = new LinstorRevertBackupSnapshotCommand( + snapshot.getTO(), + volumeInfo.getTO(), + _backupsnapshotwait, + VirtualMachineManager.ExecuteInSequence.value()); + + Optional optEP = getDiskfullEP(linstorApi, rscName); + if (optEP.isPresent()) { + Answer answer = optEP.get().sendMessage(cmd); + if (!answer.getResult()) { + resultMsg = answer.getDetails(); + } + } else { + resultMsg = "Unable to get matching Linstor endpoint."; + } + return resultMsg; + } + + private String doRevertSnapshot(final SnapshotInfo snapshot, final VolumeInfo volumeInfo) { + final StoragePool pool = (StoragePool) volumeInfo.getDataStore(); + final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(pool.getHostAddress()); + final String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getUuid(); + String resultMsg; + try { + if (snapshot.getDataStore().getRole() == DataStoreRole.Primary) { + final String snapName = LinstorUtil.RSC_PREFIX + snapshot.getUuid(); + + ApiCallRcList answers = linstorApi.resourceSnapshotRollback(rscName, snapName); + resultMsg = checkLinstorAnswers(answers); + } else if (snapshot.getDataStore().getRole() == DataStoreRole.Image) { + resultMsg = revertSnapshotFromImageStore(snapshot, volumeInfo, linstorApi, rscName); + } else { + resultMsg = "Linstor: Snapshot revert datastore not supported"; + } + } catch (ApiException apiEx) { + s_logger.error("Linstor: ApiEx - " + apiEx.getMessage()); + resultMsg = apiEx.getBestMessage(); + } + + return resultMsg; + } + @Override public void revertSnapshot( SnapshotInfo snapshot, @@ -670,19 +763,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver return; } - String resultMsg; - try { - final StoragePool pool = (StoragePool) snapshot.getDataStore(); - final String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getUuid(); - final String snapName = LinstorUtil.RSC_PREFIX + snapshot.getUuid(); - final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(pool.getHostAddress()); - - ApiCallRcList answers = linstorApi.resourceSnapshotRollback(rscName, snapName); - resultMsg = checkLinstorAnswers(answers); - } catch (ApiException apiEx) { - s_logger.error("Linstor: ApiEx - " + apiEx.getMessage()); - resultMsg = apiEx.getBestMessage(); - } + String resultMsg = doRevertSnapshot(snapshot, volumeInfo); if (callback != null) { @@ -692,24 +773,211 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } + private static boolean canCopySnapshotCond(DataObject srcData, DataObject dstData) { + return srcData.getType() == DataObjectType.SNAPSHOT && dstData.getType() == DataObjectType.SNAPSHOT + && (dstData.getDataStore().getRole() == DataStoreRole.Image + || dstData.getDataStore().getRole() == DataStoreRole.ImageCache); + } + + private static boolean canCopyTemplateCond(DataObject srcData, DataObject dstData) { + return srcData.getType() == DataObjectType.TEMPLATE && dstData.getType() == DataObjectType.TEMPLATE + && dstData.getDataStore().getRole() == DataStoreRole.Primary + && (srcData.getDataStore().getRole() == DataStoreRole.Image + || srcData.getDataStore().getRole() == DataStoreRole.ImageCache); + } + @Override - public boolean canCopy(DataObject srcData, DataObject destData) + public boolean canCopy(DataObject srcData, DataObject dstData) { + s_logger.debug("LinstorPrimaryDataStoreDriverImpl.canCopy: " + srcData.getType() + " -> " + dstData.getType()); + + if (canCopySnapshotCond(srcData, dstData)) { + SnapshotInfo sinfo = (SnapshotInfo) srcData; + VolumeInfo volume = sinfo.getBaseVolume(); + StoragePoolVO storagePool = _storagePoolDao.findById(volume.getPoolId()); + return storagePool.getStorageProviderName().equals(LinstorUtil.PROVIDER_NAME); + } else if (canCopyTemplateCond(srcData, dstData)) { + TemplateInfo tInfo = (TemplateInfo) dstData; + StoragePoolVO storagePoolVO = _storagePoolDao.findById(dstData.getDataStore().getId()); + return storagePoolVO != null + && storagePoolVO.getPoolType() == Storage.StoragePoolType.Linstor + && tInfo.getSize() != null; + } return false; } @Override - public void copyAsync(DataObject srcData, DataObject destData, AsyncCompletionCallback callback) + public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCallback callback) { - // as long as canCopy is false, this isn't called - s_logger.debug("Linstor: copyAsync with srcdata: " + srcData.getUuid()); + s_logger.debug("LinstorPrimaryDataStoreDriverImpl.copyAsync: " + + srcData.getType() + " -> " + dstData.getType()); + + final CopyCommandResult res; + if (canCopySnapshotCond(srcData, dstData)) { + String errMsg = null; + Answer answer = copySnapshot(srcData, dstData); + if (answer != null && !answer.getResult()) { + errMsg = answer.getDetails(); + } else { + // delete primary storage snapshot + SnapshotInfo sinfo = (SnapshotInfo) srcData; + VolumeInfo volume = sinfo.getBaseVolume(); + deleteSnapshot( + srcData.getDataStore(), + LinstorUtil.RSC_PREFIX + volume.getUuid(), + LinstorUtil.RSC_PREFIX + sinfo.getUuid()); + } + res = new CopyCommandResult(null, answer); + res.setResult(errMsg); + } else if (canCopyTemplateCond(srcData, dstData)) { + Answer answer = copyTemplate(srcData, dstData); + res = new CopyCommandResult(null, answer); + } else { + Answer answer = new Answer(null, false, "noimpl"); + res = new CopyCommandResult(null, answer); + res.setResult("Not implemented yet"); + } + callback.complete(res); + } + + private Optional getLinstorEP(DevelopersApi api, String rscName) throws ApiException { + List linstorNodeNames = LinstorUtil.getLinstorNodeNames(api); + Collections.shuffle(linstorNodeNames); // do not always pick the first linstor node + + Host host = null; + for (String nodeName : linstorNodeNames) { + host = _hostDao.findByName(nodeName); + if (host != null) { + s_logger.info(String.format("Linstor: Make resource %s available on node %s ...", rscName, nodeName)); + ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, nodeName, new ResourceMakeAvailable()); + if (!answers.hasError()) { + break; // found working host + } else { + s_logger.error( + String.format("Linstor: Unable to make resource %s on node %s available: %s", + rscName, + nodeName, + LinstorUtil.getBestErrorMessage(answers))); + } + } + } + + if (host == null) + { + s_logger.error("Linstor: Couldn't create a resource on any cloudstack host."); + return Optional.empty(); + } + else + { + return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host)); + } + } + + private Optional getDiskfullEP(DevelopersApi api, String rscName) throws ApiException { + com.linbit.linstor.api.model.StoragePool linSP = + LinstorUtil.getDiskfulStoragePool(api, rscName); + if (linSP != null) + { + Host host = _hostDao.findByName(linSP.getNodeName()); + if (host == null) + { + s_logger.error("Linstor: Host '" + linSP.getNodeName() + "' not found."); + return Optional.empty(); + } + else + { + return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host)); + } + } + return Optional.empty(); + } + + private Answer copyTemplate(DataObject srcData, DataObject dstData) { + TemplateInfo tInfo = (TemplateInfo) dstData; + final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId()); + final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress()); + final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid(); + createResourceBase( + LinstorUtil.RSC_PREFIX + dstData.getUuid(), + tInfo.getSize(), + tInfo.getName(), + "", + api, + getRscGrp(pool)); + + int nMaxExecutionMinutes = NumbersUtil.parseInt( + _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); + CopyCommand cmd = new CopyCommand( + srcData.getTO(), + dstData.getTO(), + nMaxExecutionMinutes * 60 * 1000, + VirtualMachineManager.ExecuteInSequence.value()); + Answer answer; + + try { + Optional optEP = getLinstorEP(api, rscName); + if (optEP.isPresent()) { + answer = optEP.get().sendMessage(cmd); + } + else { + answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint."); + } + } catch (ApiException exc) { + s_logger.error("copy template failed: ", exc); + throw new CloudRuntimeException(exc.getBestMessage()); + } + return answer; + } + + protected Answer copySnapshot(DataObject srcData, DataObject destData) { + String value = _configDao.getValue(Config.BackupSnapshotWait.toString()); + int _backupsnapshotwait = NumbersUtil.parseInt( + value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue())); + + SnapshotInfo snapshotInfo = (SnapshotInfo)srcData; + Boolean snapshotFullBackup = snapshotInfo.getFullBackup(); + final StoragePoolVO pool = _storagePoolDao.findById(srcData.getDataStore().getId()); + final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress()); + boolean fullSnapshot = true; + if (snapshotFullBackup != null) { + fullSnapshot = snapshotFullBackup; + } + Map options = new HashMap<>(); + options.put("fullSnapshot", fullSnapshot + ""); + options.put(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key(), + String.valueOf(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value())); + options.put("volumeSize", snapshotInfo.getBaseVolume().getSize() + ""); + + try { + CopyCommand cmd = new LinstorBackupSnapshotCommand( + srcData.getTO(), + destData.getTO(), + _backupsnapshotwait, + VirtualMachineManager.ExecuteInSequence.value()); + cmd.setOptions(options); + + Optional optEP = getDiskfullEP( + api, LinstorUtil.RSC_PREFIX + snapshotInfo.getBaseVolume().getUuid()); + Answer answer; + if (optEP.isPresent()) { + answer = optEP.get().sendMessage(cmd); + } else { + answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint."); + } + return answer; + } catch (Exception e) { + s_logger.debug("copy snapshot failed: ", e); + throw new CloudRuntimeException(e.toString()); + } + } @Override public void copyAsync(DataObject srcData, DataObject destData, Host destHost, AsyncCompletionCallback callback) { // as long as canCopy is false, this isn't called - s_logger.debug("Linstor: copyAsync with srcdata: " + srcData.getUuid()); + s_logger.debug("Linstor: copyAsync with host"); + copyAsync(srcData, destData, callback); } private CreateCmdResult notifyResize( @@ -794,6 +1062,23 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver s_logger.debug("Linstor: handleQualityOfServiceForVolumeMigration"); } + private Answer createAnswerAndPerstistDetails(DevelopersApi api, SnapshotInfo snapshotInfo, String rscName) + throws ApiException { + SnapshotObjectTO snapshotTO = (SnapshotObjectTO)snapshotInfo.getTO(); + com.linbit.linstor.api.model.StoragePool linStoragePool = LinstorUtil.getDiskfulStoragePool(api, rscName); + if (linStoragePool == null) { + throw new CloudRuntimeException("Linstor: Unable to find storage pool for resource " + rscName); + } + + final String path = LinstorUtil.getSnapshotPath(linStoragePool, rscName, LinstorUtil.RSC_PREFIX + snapshotInfo.getUuid()); + snapshotTO.setPath(path); + SnapshotDetailsVO details = new SnapshotDetailsVO( + snapshotInfo.getId(), snapshotInfo.getUuid(), path, false); + _snapshotDetailsDao.persist(details); + + return new CreateObjectAnswer(snapshotTO); + } + @Override public void takeSnapshot(SnapshotInfo snapshotInfo, AsyncCompletionCallback callback) { @@ -823,12 +1108,11 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver result.setResult(errMsg); } else { - s_logger.info(String.format("Successfully took snapshot from %s", rscName)); + s_logger.info(String.format("Successfully took snapshot %s from %s", snapshot.getName(), rscName)); - SnapshotObjectTO snapshotObjectTo = (SnapshotObjectTO)snapshotInfo.getTO(); - snapshotObjectTo.setPath(rscName + "-" + snapshotInfo.getName()); + Answer answer = createAnswerAndPerstistDetails(api, snapshotInfo, rscName); - result = new CreateCmdResult(null, new CreateObjectAnswer(snapshotObjectTo)); + result = new CreateCmdResult(null, answer); result.setResult(null); } } catch (ApiException apiExc) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorPrimaryDatastoreProviderImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorPrimaryDatastoreProviderImpl.java index de0a16986c5..563f542db37 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorPrimaryDatastoreProviderImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorPrimaryDatastoreProviderImpl.java @@ -27,16 +27,16 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreProvider; import org.apache.cloudstack.storage.datastore.driver.LinstorPrimaryDataStoreDriverImpl; import org.apache.cloudstack.storage.datastore.lifecycle.LinstorPrimaryDataStoreLifeCycleImpl; +import org.apache.cloudstack.storage.datastore.util.LinstorUtil; public class LinstorPrimaryDatastoreProviderImpl implements PrimaryDataStoreProvider { - private final static String PROVIDER_NAME = "Linstor"; protected PrimaryDataStoreDriver driver; protected HypervisorHostListener listener; protected DataStoreLifeCycle lifecycle; @Override public String getName() { - return PROVIDER_NAME; + return LinstorUtil.PROVIDER_NAME; } @Override diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java new file mode 100644 index 00000000000..16cc24a78d4 --- /dev/null +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.util; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; + +public class LinstorConfigurationManager implements Configurable +{ + public static final ConfigKey BackupSnapshots = new ConfigKey<>(Boolean.class, "lin.backup.snapshots", "Advanced", "false", + "Backup Linstor primary storage snapshots to secondary storage (deleting ps snapshot), only works on hyperconverged setups.", true, ConfigKey.Scope.Global, null); + + public static final ConfigKey[] CONFIG_KEYS = new ConfigKey[] { BackupSnapshots }; + + @Override + public String getConfigComponentName() + { + return LinstorConfigurationManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() + { + return CONFIG_KEYS; + } +} diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index f1760a003ab..cd2191a6ef1 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -20,12 +20,20 @@ import com.linbit.linstor.api.ApiClient; import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.Configuration; import com.linbit.linstor.api.DevelopersApi; +import com.linbit.linstor.api.model.ApiCallRc; +import com.linbit.linstor.api.model.ApiCallRcList; +import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.ProviderKind; import com.linbit.linstor.api.model.ResourceGroup; +import com.linbit.linstor.api.model.ResourceWithVolumes; import com.linbit.linstor.api.model.StoragePool; +import com.linbit.linstor.api.model.Volume; + +import javax.annotation.Nonnull; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.log4j.Logger; @@ -33,6 +41,7 @@ import org.apache.log4j.Logger; public class LinstorUtil { private static final Logger s_logger = Logger.getLogger(LinstorUtil.class); + public final static String PROVIDER_NAME = "Linstor"; public static final String RSC_PREFIX = "cs-"; public static final String RSC_GROUP = "resourceGroup"; @@ -47,6 +56,86 @@ public class LinstorUtil { return new DevelopersApi(client); } + public static String getBestErrorMessage(ApiCallRcList answers) { + return answers != null && !answers.isEmpty() ? + answers.stream() + .filter(ApiCallRc::isError) + .findFirst() + .map(ApiCallRc::getMessage) + .orElse((answers.get(0)).getMessage()) : null; + } + + public static List getLinstorNodeNames(@Nonnull DevelopersApi api) throws ApiException + { + List nodes = api.nodeList( + Collections.emptyList(), + Collections.emptyList(), + null, + null + ); + + return nodes.stream().map(Node::getName).collect(Collectors.toList()); + } + + public static com.linbit.linstor.api.model.StoragePool + getDiskfulStoragePool(@Nonnull DevelopersApi api, @Nonnull String rscName) throws ApiException + { + List resources = api.viewResources( + Collections.emptyList(), + Collections.singletonList(rscName), + Collections.emptyList(), + Collections.emptyList(), + null, + null); + + String nodeName = null; + String storagePoolName = null; + for (ResourceWithVolumes rwv : resources) { + if (rwv.getVolumes().isEmpty()) { + continue; + } + Volume vol = rwv.getVolumes().get(0); + if (vol.getProviderKind() != ProviderKind.DISKLESS) { + nodeName = rwv.getNodeName(); + storagePoolName = vol.getStoragePoolName(); + break; + } + } + + if (nodeName == null) { + return null; + } + + List sps = api.viewStoragePools( + Collections.singletonList(nodeName), + Collections.singletonList(storagePoolName), + Collections.emptyList(), + null, + null + ); + return !sps.isEmpty() ? sps.get(0) : null; + } + + public static String getSnapshotPath(com.linbit.linstor.api.model.StoragePool sp, String rscName, String snapshotName) { + final String suffix = "00000"; + final String backingPool = sp.getProps().get("StorDriver/StorPoolName"); + final String path; + switch (sp.getProviderKind()) { + case LVM_THIN: + path = String.format("/dev/mapper/%s-%s_%s_%s", + backingPool.split("/")[0], rscName.replace("-", "--"), suffix, snapshotName.replace("-", "--")); + break; + case ZFS: + case ZFS_THIN: + path = String.format("zfs://%s/%s_%s@%s", backingPool.split("/")[0], rscName, suffix, snapshotName); + break; + default: + throw new CloudRuntimeException( + String.format("Linstor: storage pool type %s doesn't support snapshots.", sp.getProviderKind())); + } + return path; + } + public static long getCapacityBytes(String linstorUrl, String rscGroupName) { DevelopersApi linstorApi = getLinstorAPI(linstorUrl); try { diff --git a/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml b/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml index e4fcb456263..48913cdb3f2 100644 --- a/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml +++ b/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml @@ -29,4 +29,6 @@ + diff --git a/pom.xml b/pom.xml index 365b13a3fbf..6200b8cabe4 100644 --- a/pom.xml +++ b/pom.xml @@ -699,6 +699,11 @@ xml-apis 2.0.2 + + com.linbit.linstor.api + java-linstor + ${cs.java-linstor.version} +