Linstor backup snaphots (#8067)

This PR adds an config option for the Linstor primary storage driver, that allows you to automatically backup
volume snapshots to the secondary storage.
Additionally it will not mangle the need java-linstor dependency into the client.jar, but instead just copy
the java-linstor.jar into lib.

Config option is called: lin.backup.snapshots and is default false

The scope of this change should be limited, as it only touches the Linstor driver and a part of copyAsync
was implemented with 2 new Linstor specific commands.

client/pom.xml

@@ -837,6 +837,12 @@
                                     <overWrite>false</overWrite>
                                     <outputDirectory>${project.build.directory}/lib</outputDirectory>
                                 </artifactItem>
+                                <artifactItem>
+                                    <groupId>com.linbit.linstor.api</groupId>
+                                    <artifactId>java-linstor</artifactId>
+                                    <overWrite>false</overWrite>
+                                    <outputDirectory>${project.build.directory}/lib</outputDirectory>
+                                </artifactItem>
                                 <artifactItem>
                                     <groupId>org.bouncycastle</groupId>
                                     <artifactId>bctls-jdk15on</artifactId>
@@ -880,6 +886,7 @@
                                     <exclude>mysql:mysql-connector-java</exclude>
                                     <exclude>org.apache.cloudstack:cloud-plugin-storage-volume-storpool</exclude>
                                     <exclude>org.apache.cloudstack:cloud-plugin-storage-volume-linstor</exclude>
+                                    <exclude>com.linbit.linstor.api:java-linstor</exclude>
                                 </excludes>
                             </artifactSet>
                             <transformers>

plugins/storage/volume/linstor/pom.xml

@@ -43,6 +43,12 @@
             <artifactId>cloud-plugin-hypervisor-kvm</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-engine-storage-snapshot</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

plugins/storage/volume/linstor/src/main/java/com/cloud/api/storage/LinstorBackupSnapshotCommand.java

@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.api.storage;
+
+import com.cloud.agent.api.to.DataTO;
+import org.apache.cloudstack.storage.command.CopyCommand;
+
+public class LinstorBackupSnapshotCommand extends CopyCommand
+{
+    public LinstorBackupSnapshotCommand(DataTO srcData, DataTO destData, int timeout, boolean executeInSequence)
+    {
+        super(srcData, destData, timeout, executeInSequence);
+    }
+}

plugins/storage/volume/linstor/src/main/java/com/cloud/api/storage/LinstorRevertBackupSnapshotCommand.java

@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.api.storage;
+
+import com.cloud.agent.api.to.DataTO;
+import org.apache.cloudstack.storage.command.CopyCommand;
+
+public class LinstorRevertBackupSnapshotCommand extends CopyCommand
+{
+    public LinstorRevertBackupSnapshotCommand(DataTO srcData, DataTO destData, int timeout, boolean executeInSequence)
+    {
+        super(srcData, destData, timeout, executeInSequence);
+    }
+}

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java

@@ -0,0 +1,167 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import java.io.File;
+import java.io.IOException;
+
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.api.storage.LinstorBackupSnapshotCommand;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.cloud.storage.Storage;
+import com.cloud.utils.script.Script;
+import org.apache.cloudstack.storage.command.CopyCmdAnswer;
+import org.apache.cloudstack.storage.to.SnapshotObjectTO;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+import org.joda.time.Duration;
+import org.libvirt.LibvirtException;
+
+@ResourceWrapper(handles = LinstorBackupSnapshotCommand.class)
+public final class LinstorBackupSnapshotCommandWrapper
+    extends CommandWrapper<LinstorBackupSnapshotCommand, CopyCmdAnswer, LibvirtComputingResource>
+{
+    private static final Logger s_logger = Logger.getLogger(LinstorBackupSnapshotCommandWrapper.class);
+
+    private String zfsSnapdev(boolean hide, String zfsUrl) {
+        Script script = new Script("/usr/bin/zfs", Duration.millis(5000));
+        script.add("set");
+        script.add("snapdev=" + (hide ? "hidden" : "visible"));
+        script.add(zfsUrl.substring(6));  // cutting zfs://
+        return script.execute();
+    }
+
+    private String qemuShrink(String path, long sizeByte, long timeout) {
+        Script qemuImg = new Script("qemu-img", Duration.millis(timeout));
+        qemuImg.add("resize");
+        qemuImg.add("--shrink");
+        qemuImg.add(path);
+        qemuImg.add("" + sizeByte);
+        return qemuImg.execute();
+    }
+
+    static void cleanupSecondaryPool(final KVMStoragePool secondaryPool) {
+        if (secondaryPool != null) {
+            try {
+                secondaryPool.delete();
+            } catch (final Exception e) {
+                s_logger.debug("Failed to delete secondary storage", e);
+            }
+        }
+    }
+
+    private String convertImageToQCow2(
+        final String srcPath,
+        final SnapshotObjectTO dst,
+        final KVMStoragePool secondaryPool,
+        int waitMilliSeconds
+        )
+        throws LibvirtException, QemuImgException, IOException
+    {
+        final String dstDir = secondaryPool.getLocalPath() + File.separator + dst.getPath();
+        FileUtils.forceMkdir(new File(dstDir));
+
+        final String dstPath = dstDir + File.separator + dst.getName();
+        final QemuImgFile srcFile = new QemuImgFile(srcPath, QemuImg.PhysicalDiskFormat.RAW);
+        final QemuImgFile dstFile = new QemuImgFile(dstPath, QemuImg.PhysicalDiskFormat.QCOW2);
+
+        // NOTE: the qemu img will also contain the drbd metadata at the end
+        final QemuImg qemu = new QemuImg(waitMilliSeconds);
+        qemu.convert(srcFile, dstFile);
+        s_logger.info("Backup snapshot " + srcFile + " to " + dstPath);
+        return dstPath;
+    }
+
+    private SnapshotObjectTO setCorrectSnapshotSize(final SnapshotObjectTO dst, final String dstPath) {
+        final File snapFile = new File(dstPath);
+        final long size = snapFile.exists() ? snapFile.length() : 0;
+
+        final SnapshotObjectTO snapshot = new SnapshotObjectTO();
+        snapshot.setPath(dst.getPath() + File.separator + dst.getName());
+        snapshot.setPhysicalSize(size);
+        return snapshot;
+    }
+
+    @Override
+    public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingResource serverResource)
+    {
+        s_logger.debug("LinstorBackupSnapshotCommandWrapper: " + cmd.getSrcTO().getPath() + " -> " + cmd.getDestTO().getPath());
+        final SnapshotObjectTO src = (SnapshotObjectTO) cmd.getSrcTO();
+        final SnapshotObjectTO dst = (SnapshotObjectTO) cmd.getDestTO();
+        KVMStoragePool secondaryPool = null;
+        final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr();
+        KVMStoragePool linstorPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.Linstor, src.getDataStore().getUuid());
+        boolean zfsHidden = false;
+        String srcPath = src.getPath();
+
+        if (linstorPool == null) {
+            return new CopyCmdAnswer("Unable to get linstor storage pool from destination volume.");
+        }
+
+        final DataStoreTO dstDataStore = dst.getDataStore();
+        if (!(dstDataStore instanceof NfsTO)) {
+            return new CopyCmdAnswer("Backup Linstor snapshot: Only NFS secondary supported at present!");
+        }
+
+        try
+        {
+            // provide the linstor snapshot block device
+            // on lvm thin this should already be there in /dev/mapper/vg-snapshotname
+            // on zfs we need to unhide the snapshot block device
+            s_logger.info("Src: " + srcPath + " | " + src.getName());
+            if (srcPath.startsWith("zfs://")) {
+                zfsHidden = true;
+                if (zfsSnapdev(false, srcPath) != null) {
+                    return new CopyCmdAnswer("Unable to unhide zfs snapshot device.");
+                }
+                srcPath = "/dev/" + srcPath.substring(6);
+            }
+
+            secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl());
+
+            String dstPath = convertImageToQCow2(srcPath, dst, secondaryPool, cmd.getWaitInMillSeconds());
+
+            // resize to real volume size, cutting of drbd metadata
+            String result = qemuShrink(dstPath, src.getVolume().getSize(), cmd.getWaitInMillSeconds());
+            if (result != null) {
+                return new CopyCmdAnswer("qemu-img shrink failed: " + result);
+            }
+            s_logger.info("Backup shrunk " + dstPath + " to actual size " + src.getVolume().getSize());
+
+            SnapshotObjectTO snapshot = setCorrectSnapshotSize(dst, dstPath);
+            return new CopyCmdAnswer(snapshot);
+        } catch (final Exception e) {
+            final String error = String.format("Failed to backup snapshot with id [%s] with a pool %s, due to %s",
+                cmd.getSrcTO().getId(), cmd.getSrcTO().getDataStore().getUuid(), e.getMessage());
+            s_logger.error(error);
+            return new CopyCmdAnswer(cmd, e);
+        } finally {
+            cleanupSecondaryPool(secondaryPool);
+            if (zfsHidden) {
+                zfsSnapdev(true, src.getPath());
+            }
+        }
+    }
+}

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorRevertBackupSnapshotCommandWrapper.java

@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import java.io.File;
+
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.api.storage.LinstorRevertBackupSnapshotCommand;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.cloud.storage.Storage;
+import org.apache.cloudstack.storage.command.CopyCmdAnswer;
+import org.apache.cloudstack.storage.to.SnapshotObjectTO;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+@ResourceWrapper(handles = LinstorRevertBackupSnapshotCommand.class)
+public final class LinstorRevertBackupSnapshotCommandWrapper
+    extends CommandWrapper<LinstorRevertBackupSnapshotCommand, CopyCmdAnswer, LibvirtComputingResource>
+{
+    private static final Logger s_logger = Logger.getLogger(LinstorRevertBackupSnapshotCommandWrapper.class);
+
+    private void convertQCow2ToRAW(final String srcPath, final String dstPath, int waitMilliSeconds)
+        throws LibvirtException, QemuImgException
+    {
+        final QemuImgFile srcQemuFile = new QemuImgFile(
+            srcPath, QemuImg.PhysicalDiskFormat.QCOW2);
+        final QemuImg qemu = new QemuImg(waitMilliSeconds);
+        final QemuImgFile dstFile = new QemuImgFile(dstPath, QemuImg.PhysicalDiskFormat.RAW);
+        qemu.convert(srcQemuFile, dstFile);
+    }
+
+    @Override
+    public CopyCmdAnswer execute(LinstorRevertBackupSnapshotCommand cmd, LibvirtComputingResource serverResource)
+    {
+        s_logger.debug("LinstorRevertBackupSnapshotCommandWrapper: " + cmd.getSrcTO().getPath() + " -> " + cmd.getDestTO().getPath());
+        final SnapshotObjectTO src = (SnapshotObjectTO) cmd.getSrcTO();
+        final VolumeObjectTO dst = (VolumeObjectTO) cmd.getDestTO();
+        KVMStoragePool secondaryPool = null;
+        final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr();
+        KVMStoragePool linstorPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.Linstor, dst.getDataStore().getUuid());
+
+        if (linstorPool == null) {
+            return new CopyCmdAnswer("Unable to get linstor storage pool from destination volume.");
+        }
+
+        try
+        {
+            final DataStoreTO srcDataStore = src.getDataStore();
+            File srcFile = new File(src.getPath());
+            secondaryPool = storagePoolMgr.getStoragePoolByURI(
+                srcDataStore.getUrl() + File.separator + srcFile.getParent());
+
+            convertQCow2ToRAW(
+                secondaryPool.getLocalPath() + File.separator + srcFile.getName(),
+                linstorPool.getPhysicalDisk(dst.getPath()).getPath(),
+                cmd.getWaitInMillSeconds());
+
+            final VolumeObjectTO dstVolume = new VolumeObjectTO();
+            dstVolume.setPath(dst.getPath());
+            return new CopyCmdAnswer(dstVolume);
+        } catch (final Exception e) {
+            final String error = String.format("Failed to revert snapshot with id [%s] with a pool %s, due to %s",
+                cmd.getSrcTO().getId(), cmd.getSrcTO().getDataStore().getUuid(), e.getMessage());
+            s_logger.error(error);
+            return new CopyCmdAnswer(cmd, e);
+        } finally {
+            LinstorBackupSnapshotCommandWrapper.cleanupSecondaryPool(secondaryPool);
+        }
+    }
+}

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java

@@ -28,6 +28,7 @@ import java.util.StringJoiner;
 
 import javax.annotation.Nonnull;
 
+import org.apache.cloudstack.storage.datastore.util.LinstorUtil;
 import org.apache.cloudstack.utils.qemu.QemuImg;
 import org.apache.cloudstack.utils.qemu.QemuImgException;
 import org.apache.cloudstack.utils.qemu.QemuImgFile;
@@ -65,8 +66,8 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
         return new DevelopersApi(client);
     }
 
-    private String getLinstorRscName(String name) {
-        return "cs-" + name;
+    private static String getLinstorRscName(String name) {
+        return LinstorUtil.RSC_PREFIX + name;
     }
 
     private String getHostname() {
@@ -214,6 +215,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
     public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format,
                                               Storage.ProvisioningType provisioningType, long size, byte[] passphrase)
     {
+        s_logger.debug(String.format("Linstor.createPhysicalDisk: %s;%s", name, format));
         final String rscName = getLinstorRscName(name);
         LinstorStoragePool lpool = (LinstorStoragePool) pool;
         final DevelopersApi api = getLinstorAPI(pool);
@@ -254,6 +256,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
                 throw new CloudRuntimeException("Linstor: viewResources didn't return resources or volumes.");
             }
         } catch (ApiException apiEx) {
+            s_logger.error(String.format("Linstor.createPhysicalDisk: ApiException: %s", apiEx.getBestMessage()));
             throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
         }
     }
@@ -424,7 +427,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
     @Override
     public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPools, int timeout, byte[] srcPassphrase, byte[] destPassphrase, Storage.ProvisioningType provisioningType)
     {
-        s_logger.debug("Linstor: copyPhysicalDisk");
+        s_logger.debug(String.format("Linstor.copyPhysicalDisk: %s -> %s", disk.getPath(), name));
         final QemuImg.PhysicalDiskFormat sourceFormat = disk.getFormat();
         final String sourcePath = disk.getPath();
 
@@ -433,6 +436,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
         final KVMPhysicalDisk dstDisk = destPools.createPhysicalDisk(
             name, QemuImg.PhysicalDiskFormat.RAW, provisioningType, disk.getVirtualSize(), null);
 
+        s_logger.debug(String.format("Linstor.copyPhysicalDisk: dstPath: %s", dstDisk.getPath()));
         final QemuImgFile destFile = new QemuImgFile(dstDisk.getPath());
         destFile.setFormat(dstDisk.getFormat());
         destFile.setSize(disk.getVirtualSize());

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java

@@ -28,6 +28,7 @@ import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted;
 import com.linbit.linstor.api.model.ResourceDefinitionCreate;
 import com.linbit.linstor.api.model.ResourceDefinitionModify;
 import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceMakeAvailable;
 import com.linbit.linstor.api.model.ResourceWithVolumes;
 import com.linbit.linstor.api.model.Snapshot;
 import com.linbit.linstor.api.model.SnapshotRestore;
@@ -43,18 +44,26 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.ResizeVolumeAnswer;
 import com.cloud.agent.api.storage.ResizeVolumeCommand;
+import com.cloud.agent.api.to.DataObjectType;
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.api.to.StorageFilerTO;
+import com.cloud.api.storage.LinstorBackupSnapshotCommand;
+import com.cloud.api.storage.LinstorRevertBackupSnapshotCommand;
+import com.cloud.configuration.Config;
 import com.cloud.host.Host;
+import com.cloud.host.dao.HostDao;
+import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.ResizeVolumePayload;
 import com.cloud.storage.SnapshotVO;
 import com.cloud.storage.Storage.StoragePoolType;
+import com.cloud.storage.Storage;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VMTemplateStoragePoolVO;
@@ -67,8 +76,10 @@ import com.cloud.storage.dao.SnapshotDetailsVO;
 import com.cloud.storage.dao.VMTemplatePoolDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.dao.VolumeDetailsDao;
+import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.VirtualMachineManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
@@ -80,10 +91,14 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.storage.RemoteHostEndPoint;
 import org.apache.cloudstack.storage.command.CommandResult;
+import org.apache.cloudstack.storage.command.CopyCommand;
 import org.apache.cloudstack.storage.command.CreateObjectAnswer;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.datastore.util.LinstorConfigurationManager;
 import org.apache.cloudstack.storage.datastore.util.LinstorUtil;
 import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.volume.VolumeObject;
@@ -98,6 +113,10 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
     @Inject private SnapshotDao _snapshotDao;
     @Inject private SnapshotDetailsDao _snapshotDetailsDao;
     @Inject private StorageManager _storageMgr;
+    @Inject
+    ConfigurationDao _configDao;
+    @Inject
+    private HostDao _hostDao;
 
     public LinstorPrimaryDataStoreDriverImpl()
     {
@@ -109,10 +128,12 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
         Map<String, String> mapCapabilities = new HashMap<>();
 
         // Linstor will be restricted to only run on LVM-THIN and ZFS storage pools with ACS
+        // This enables template caching on our primary storage
         mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_VOLUME.toString(), Boolean.TRUE.toString());
 
         // fetch if lvm-thin or ZFS
-        mapCapabilities.put(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString(), Boolean.TRUE.toString());
+        boolean system_snapshot = !LinstorConfigurationManager.BackupSnapshots.value();
+        mapCapabilities.put(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString(), Boolean.toString(system_snapshot));
 
         // CAN_CREATE_VOLUME_FROM_SNAPSHOT see note from CAN_CREATE_VOLUME_FROM_VOLUME
         mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_SNAPSHOT.toString(), Boolean.TRUE.toString());
@@ -216,6 +237,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
                 }
                 throw new CloudRuntimeException("Linstor: Unable to delete snapshot: " + rscDefName);
             }
+            s_logger.info("Linstor: Deleted snapshot " + snapshotName + " for resource " + rscDefName);
         } catch (ApiException apiEx)
         {
             s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
@@ -402,27 +424,46 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
         }
     }
 
-    private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO)
-    {
-        DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());
-        final String rscGrp = storagePoolVO.getUserInfo() != null && !storagePoolVO.getUserInfo().isEmpty() ?
+    private String getRscGrp(StoragePoolVO storagePoolVO) {
+        return storagePoolVO.getUserInfo() != null && !storagePoolVO.getUserInfo().isEmpty() ?
             storagePoolVO.getUserInfo() : "DfltRscGrp";
+    }
 
+    private String createResourceBase(
+        String rscName, long sizeInBytes, String volName, String vmName, DevelopersApi api, String rscGrp) {
         ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn();
-        final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid();
         rscGrpSpawn.setResourceDefinitionName(rscName);
-        rscGrpSpawn.addVolumeSizesItem(vol.getSize() / 1024);
+        rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024);
 
         try
         {
             s_logger.info("Linstor: Spawn resource " + rscName);
-            ApiCallRcList answers = linstorApi.resourceGroupSpawn(rscGrp, rscGrpSpawn);
+            ApiCallRcList answers = api.resourceGroupSpawn(rscGrp, rscGrpSpawn);
             checkLinstorAnswersThrow(answers);
 
-            applyAuxProps(linstorApi, rscName, vol.getName(), vol.getAttachedVmName());
+            applyAuxProps(api, rscName, volName, vmName);
+
+            return getDeviceName(api, rscName);
+        } catch (ApiException apiEx)
+        {
+            s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
+            throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+        }
+    }
+
+    private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO) {
+        DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());
+        final String rscGrp = getRscGrp(storagePoolVO);
+
+        final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid();
+        String deviceName = createResourceBase(
+            rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), linstorApi, rscGrp);
+
+        try
+        {
             applyQoSSettings(storagePoolVO, linstorApi, rscName, vol.getMaxIops());
 
-            return getDeviceName(linstorApi, rscName);
+            return deviceName;
         } catch (ApiException apiEx)
         {
             s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
@@ -487,8 +528,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
     }
 
     private String createResourceFromSnapshot(long csSnapshotId, String rscName, StoragePoolVO storagePoolVO) {
-        final String rscGrp = storagePoolVO.getUserInfo() != null && !storagePoolVO.getUserInfo().isEmpty() ?
-            storagePoolVO.getUserInfo() : "DfltRscGrp";
+        final String rscGrp = getRscGrp(storagePoolVO);
         final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());
 
         SnapshotVO snapshotVO = _snapshotDao.findById(csSnapshotId);
@@ -654,6 +694,59 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
         }
     }
 
+    private String revertSnapshotFromImageStore(
+        final SnapshotInfo snapshot,
+        final VolumeInfo volumeInfo,
+        final DevelopersApi linstorApi,
+        final String rscName)
+    throws ApiException {
+        String resultMsg = null;
+        String value = _configDao.getValue(Config.BackupSnapshotWait.toString());
+        int _backupsnapshotwait = NumbersUtil.parseInt(
+            value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue()));
+
+        LinstorRevertBackupSnapshotCommand cmd = new LinstorRevertBackupSnapshotCommand(
+            snapshot.getTO(),
+            volumeInfo.getTO(),
+            _backupsnapshotwait,
+            VirtualMachineManager.ExecuteInSequence.value());
+
+        Optional<RemoteHostEndPoint> optEP = getDiskfullEP(linstorApi, rscName);
+        if (optEP.isPresent()) {
+            Answer answer = optEP.get().sendMessage(cmd);
+            if (!answer.getResult()) {
+                resultMsg = answer.getDetails();
+            }
+        } else {
+            resultMsg = "Unable to get matching Linstor endpoint.";
+        }
+        return resultMsg;
+    }
+
+    private String doRevertSnapshot(final SnapshotInfo snapshot, final VolumeInfo volumeInfo) {
+        final StoragePool pool = (StoragePool) volumeInfo.getDataStore();
+        final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(pool.getHostAddress());
+        final String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getUuid();
+        String resultMsg;
+        try {
+            if (snapshot.getDataStore().getRole() == DataStoreRole.Primary) {
+                final String snapName = LinstorUtil.RSC_PREFIX + snapshot.getUuid();
+
+                ApiCallRcList answers = linstorApi.resourceSnapshotRollback(rscName, snapName);
+                resultMsg = checkLinstorAnswers(answers);
+            } else if (snapshot.getDataStore().getRole() == DataStoreRole.Image) {
+                resultMsg = revertSnapshotFromImageStore(snapshot, volumeInfo, linstorApi, rscName);
+            } else {
+                resultMsg = "Linstor: Snapshot revert datastore not supported";
+            }
+        } catch (ApiException apiEx) {
+            s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
+            resultMsg = apiEx.getBestMessage();
+        }
+
+        return resultMsg;
+    }
+
     @Override
     public void revertSnapshot(
         SnapshotInfo snapshot,
@@ -670,19 +763,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
             return;
         }
 
-        String resultMsg;
-        try {
-            final StoragePool pool = (StoragePool) snapshot.getDataStore();
-            final String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getUuid();
-            final String snapName = LinstorUtil.RSC_PREFIX + snapshot.getUuid();
-            final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(pool.getHostAddress());
-
-            ApiCallRcList answers = linstorApi.resourceSnapshotRollback(rscName, snapName);
-            resultMsg = checkLinstorAnswers(answers);
-        } catch (ApiException apiEx) {
-            s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
-            resultMsg = apiEx.getBestMessage();
-        }
+        String resultMsg = doRevertSnapshot(snapshot, volumeInfo);
 
         if (callback != null)
         {
@@ -692,24 +773,211 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
         }
     }
 
+    private static boolean canCopySnapshotCond(DataObject srcData, DataObject dstData) {
+        return srcData.getType() == DataObjectType.SNAPSHOT && dstData.getType() == DataObjectType.SNAPSHOT
+            && (dstData.getDataStore().getRole() == DataStoreRole.Image
+            || dstData.getDataStore().getRole() == DataStoreRole.ImageCache);
+    }
+
+    private static boolean canCopyTemplateCond(DataObject srcData, DataObject dstData) {
+        return srcData.getType() == DataObjectType.TEMPLATE && dstData.getType() == DataObjectType.TEMPLATE
+            && dstData.getDataStore().getRole() == DataStoreRole.Primary
+            && (srcData.getDataStore().getRole() == DataStoreRole.Image
+            || srcData.getDataStore().getRole() == DataStoreRole.ImageCache);
+    }
+
     @Override
-    public boolean canCopy(DataObject srcData, DataObject destData)
+    public boolean canCopy(DataObject srcData, DataObject dstData)
     {
+        s_logger.debug("LinstorPrimaryDataStoreDriverImpl.canCopy: " + srcData.getType() + " -> " + dstData.getType());
+
+        if (canCopySnapshotCond(srcData, dstData)) {
+            SnapshotInfo sinfo = (SnapshotInfo) srcData;
+            VolumeInfo volume = sinfo.getBaseVolume();
+            StoragePoolVO storagePool = _storagePoolDao.findById(volume.getPoolId());
+            return storagePool.getStorageProviderName().equals(LinstorUtil.PROVIDER_NAME);
+        } else if (canCopyTemplateCond(srcData, dstData)) {
+            TemplateInfo tInfo = (TemplateInfo) dstData;
+            StoragePoolVO storagePoolVO = _storagePoolDao.findById(dstData.getDataStore().getId());
+            return storagePoolVO != null
+                && storagePoolVO.getPoolType() == Storage.StoragePoolType.Linstor
+                && tInfo.getSize() != null;
+        }
         return false;
     }
 
     @Override
-    public void copyAsync(DataObject srcData, DataObject destData, AsyncCompletionCallback<CopyCommandResult> callback)
+    public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCallback<CopyCommandResult> callback)
     {
-        // as long as canCopy is false, this isn't called
-        s_logger.debug("Linstor: copyAsync with srcdata: " + srcData.getUuid());
+        s_logger.debug("LinstorPrimaryDataStoreDriverImpl.copyAsync: "
+            + srcData.getType() + " -> " + dstData.getType());
+
+        final CopyCommandResult res;
+        if (canCopySnapshotCond(srcData, dstData)) {
+            String errMsg = null;
+            Answer answer = copySnapshot(srcData, dstData);
+            if (answer != null && !answer.getResult()) {
+                errMsg = answer.getDetails();
+            } else {
+                // delete primary storage snapshot
+                SnapshotInfo sinfo = (SnapshotInfo) srcData;
+                VolumeInfo volume = sinfo.getBaseVolume();
+                deleteSnapshot(
+                    srcData.getDataStore(),
+                    LinstorUtil.RSC_PREFIX + volume.getUuid(),
+                    LinstorUtil.RSC_PREFIX + sinfo.getUuid());
+            }
+            res = new CopyCommandResult(null, answer);
+            res.setResult(errMsg);
+        } else if (canCopyTemplateCond(srcData, dstData)) {
+            Answer answer = copyTemplate(srcData, dstData);
+            res = new CopyCommandResult(null, answer);
+        } else {
+            Answer answer = new Answer(null, false, "noimpl");
+            res = new CopyCommandResult(null, answer);
+            res.setResult("Not implemented yet");
+        }
+        callback.complete(res);
+    }
+
+    private Optional<RemoteHostEndPoint> getLinstorEP(DevelopersApi api, String rscName) throws ApiException {
+        List<String> linstorNodeNames = LinstorUtil.getLinstorNodeNames(api);
+        Collections.shuffle(linstorNodeNames);  // do not always pick the first linstor node
+
+        Host host = null;
+        for (String nodeName : linstorNodeNames) {
+            host = _hostDao.findByName(nodeName);
+            if (host != null) {
+                s_logger.info(String.format("Linstor: Make resource %s available on node %s ...", rscName, nodeName));
+                ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, nodeName, new ResourceMakeAvailable());
+                if (!answers.hasError()) {
+                    break; // found working host
+                } else {
+                    s_logger.error(
+                        String.format("Linstor: Unable to make resource %s on node %s available: %s",
+                            rscName,
+                            nodeName,
+                            LinstorUtil.getBestErrorMessage(answers)));
+                }
+            }
+        }
+
+        if (host == null)
+        {
+            s_logger.error("Linstor: Couldn't create a resource on any cloudstack host.");
+            return Optional.empty();
+        }
+        else
+        {
+            return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
+        }
+    }
+
+    private Optional<RemoteHostEndPoint> getDiskfullEP(DevelopersApi api, String rscName) throws ApiException {
+        com.linbit.linstor.api.model.StoragePool linSP =
+            LinstorUtil.getDiskfulStoragePool(api, rscName);
+        if (linSP != null)
+        {
+            Host host = _hostDao.findByName(linSP.getNodeName());
+            if (host == null)
+            {
+                s_logger.error("Linstor: Host '" + linSP.getNodeName() + "' not found.");
+                return Optional.empty();
+            }
+            else
+            {
+                return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
+            }
+        }
+        return Optional.empty();
+    }
+
+    private Answer copyTemplate(DataObject srcData, DataObject dstData) {
+        TemplateInfo tInfo = (TemplateInfo) dstData;
+        final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId());
+        final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress());
+        final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid();
+        createResourceBase(
+            LinstorUtil.RSC_PREFIX + dstData.getUuid(),
+            tInfo.getSize(),
+            tInfo.getName(),
+            "",
+            api,
+            getRscGrp(pool));
+
+        int nMaxExecutionMinutes = NumbersUtil.parseInt(
+            _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30);
+        CopyCommand cmd = new CopyCommand(
+            srcData.getTO(),
+            dstData.getTO(),
+            nMaxExecutionMinutes * 60 * 1000,
+            VirtualMachineManager.ExecuteInSequence.value());
+        Answer answer;
+
+        try {
+            Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, rscName);
+            if (optEP.isPresent()) {
+                answer = optEP.get().sendMessage(cmd);
+            }
+            else {
+                answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint.");
+            }
+        } catch (ApiException exc) {
+            s_logger.error("copy template failed: ", exc);
+            throw new CloudRuntimeException(exc.getBestMessage());
+        }
+        return answer;
+    }
+
+    protected Answer copySnapshot(DataObject srcData, DataObject destData) {
+        String value = _configDao.getValue(Config.BackupSnapshotWait.toString());
+        int _backupsnapshotwait = NumbersUtil.parseInt(
+            value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue()));
+
+        SnapshotInfo snapshotInfo = (SnapshotInfo)srcData;
+        Boolean snapshotFullBackup = snapshotInfo.getFullBackup();
+        final StoragePoolVO pool = _storagePoolDao.findById(srcData.getDataStore().getId());
+        final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress());
+        boolean fullSnapshot = true;
+        if (snapshotFullBackup != null) {
+            fullSnapshot = snapshotFullBackup;
+        }
+        Map<String, String> options = new HashMap<>();
+        options.put("fullSnapshot", fullSnapshot + "");
+        options.put(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key(),
+            String.valueOf(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value()));
+        options.put("volumeSize", snapshotInfo.getBaseVolume().getSize() + "");
+
+        try {
+            CopyCommand cmd = new LinstorBackupSnapshotCommand(
+                srcData.getTO(),
+                destData.getTO(),
+                _backupsnapshotwait,
+                VirtualMachineManager.ExecuteInSequence.value());
+            cmd.setOptions(options);
+
+            Optional<RemoteHostEndPoint> optEP = getDiskfullEP(
+                api, LinstorUtil.RSC_PREFIX + snapshotInfo.getBaseVolume().getUuid());
+            Answer answer;
+            if (optEP.isPresent()) {
+                answer = optEP.get().sendMessage(cmd);
+            } else {
+                answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint.");
+            }
+            return answer;
+        } catch (Exception e) {
+            s_logger.debug("copy snapshot failed: ", e);
+            throw new CloudRuntimeException(e.toString());
+        }
+
     }
 
     @Override
     public void copyAsync(DataObject srcData, DataObject destData, Host destHost, AsyncCompletionCallback<CopyCommandResult> callback)
     {
         // as long as canCopy is false, this isn't called
-        s_logger.debug("Linstor: copyAsync with srcdata: " + srcData.getUuid());
+        s_logger.debug("Linstor: copyAsync with host");
+        copyAsync(srcData, destData, callback);
     }
 
     private CreateCmdResult notifyResize(
@@ -794,6 +1062,23 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
         s_logger.debug("Linstor: handleQualityOfServiceForVolumeMigration");
     }
 
+    private Answer createAnswerAndPerstistDetails(DevelopersApi api, SnapshotInfo snapshotInfo, String rscName)
+        throws ApiException {
+        SnapshotObjectTO snapshotTO = (SnapshotObjectTO)snapshotInfo.getTO();
+        com.linbit.linstor.api.model.StoragePool linStoragePool = LinstorUtil.getDiskfulStoragePool(api, rscName);
+        if (linStoragePool == null) {
+            throw new CloudRuntimeException("Linstor: Unable to find storage pool for resource " + rscName);
+        }
+
+        final String path = LinstorUtil.getSnapshotPath(linStoragePool, rscName, LinstorUtil.RSC_PREFIX + snapshotInfo.getUuid());
+        snapshotTO.setPath(path);
+        SnapshotDetailsVO details = new SnapshotDetailsVO(
+            snapshotInfo.getId(), snapshotInfo.getUuid(), path, false);
+        _snapshotDetailsDao.persist(details);
+
+        return new CreateObjectAnswer(snapshotTO);
+    }
+
     @Override
     public void takeSnapshot(SnapshotInfo snapshotInfo, AsyncCompletionCallback<CreateCmdResult> callback)
     {
@@ -823,12 +1108,11 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
                 result.setResult(errMsg);
             } else
             {
-                s_logger.info(String.format("Successfully took snapshot from %s", rscName));
+                s_logger.info(String.format("Successfully took snapshot %s from %s", snapshot.getName(), rscName));
 
-                SnapshotObjectTO snapshotObjectTo = (SnapshotObjectTO)snapshotInfo.getTO();
-                snapshotObjectTo.setPath(rscName + "-" + snapshotInfo.getName());
+                Answer answer = createAnswerAndPerstistDetails(api, snapshotInfo, rscName);
 
-                result = new CreateCmdResult(null, new CreateObjectAnswer(snapshotObjectTo));
+                result = new CreateCmdResult(null, answer);
                 result.setResult(null);
             }
         } catch (ApiException apiExc)

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorPrimaryDatastoreProviderImpl.java

@@ -27,16 +27,16 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreProvider;
 import org.apache.cloudstack.storage.datastore.driver.LinstorPrimaryDataStoreDriverImpl;
 import org.apache.cloudstack.storage.datastore.lifecycle.LinstorPrimaryDataStoreLifeCycleImpl;
+import org.apache.cloudstack.storage.datastore.util.LinstorUtil;
 
 public class LinstorPrimaryDatastoreProviderImpl implements PrimaryDataStoreProvider {
-    private final static String PROVIDER_NAME = "Linstor";
     protected PrimaryDataStoreDriver driver;
     protected HypervisorHostListener listener;
     protected DataStoreLifeCycle lifecycle;
 
     @Override
     public String getName() {
-        return PROVIDER_NAME;
+        return LinstorUtil.PROVIDER_NAME;
     }
 
     @Override

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.datastore.util;
+
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
+
+public class LinstorConfigurationManager implements Configurable
+{
+    public static final ConfigKey<Boolean> BackupSnapshots = new ConfigKey<>(Boolean.class, "lin.backup.snapshots", "Advanced", "false",
+        "Backup Linstor primary storage snapshots to secondary storage (deleting ps snapshot), only works on hyperconverged setups.", true, ConfigKey.Scope.Global, null);
+
+    public static final ConfigKey<?>[] CONFIG_KEYS = new ConfigKey<?>[] { BackupSnapshots };
+
+    @Override
+    public String getConfigComponentName()
+    {
+        return LinstorConfigurationManager.class.getSimpleName();
+    }
+
+    @Override
+    public ConfigKey<?>[] getConfigKeys()
+    {
+        return CONFIG_KEYS;
+    }
+}

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java

@@ -20,12 +20,20 @@ import com.linbit.linstor.api.ApiClient;
 import com.linbit.linstor.api.ApiException;
 import com.linbit.linstor.api.Configuration;
 import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Node;
 import com.linbit.linstor.api.model.ProviderKind;
 import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
 import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.Volume;
+
+import javax.annotation.Nonnull;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.log4j.Logger;
@@ -33,6 +41,7 @@ import org.apache.log4j.Logger;
 public class LinstorUtil {
     private static final Logger s_logger = Logger.getLogger(LinstorUtil.class);
 
+    public final static String PROVIDER_NAME = "Linstor";
     public static final String RSC_PREFIX = "cs-";
     public static final String RSC_GROUP = "resourceGroup";
 
@@ -47,6 +56,86 @@ public class LinstorUtil {
         return new DevelopersApi(client);
     }
 
+    public static String getBestErrorMessage(ApiCallRcList answers) {
+        return answers != null && !answers.isEmpty() ?
+            answers.stream()
+                .filter(ApiCallRc::isError)
+                .findFirst()
+                .map(ApiCallRc::getMessage)
+                .orElse((answers.get(0)).getMessage()) : null;
+    }
+
+    public static List<String> getLinstorNodeNames(@Nonnull DevelopersApi api) throws ApiException
+    {
+        List<Node> nodes = api.nodeList(
+            Collections.emptyList(),
+            Collections.emptyList(),
+            null,
+            null
+        );
+
+        return nodes.stream().map(Node::getName).collect(Collectors.toList());
+    }
+
+    public static com.linbit.linstor.api.model.StoragePool
+    getDiskfulStoragePool(@Nonnull DevelopersApi api, @Nonnull String rscName) throws ApiException
+    {
+        List<ResourceWithVolumes> resources = api.viewResources(
+            Collections.emptyList(),
+            Collections.singletonList(rscName),
+            Collections.emptyList(),
+            Collections.emptyList(),
+            null,
+            null);
+
+        String nodeName = null;
+        String storagePoolName = null;
+        for (ResourceWithVolumes rwv : resources) {
+            if (rwv.getVolumes().isEmpty()) {
+                continue;
+            }
+            Volume vol = rwv.getVolumes().get(0);
+            if (vol.getProviderKind() != ProviderKind.DISKLESS) {
+                nodeName = rwv.getNodeName();
+                storagePoolName = vol.getStoragePoolName();
+                break;
+            }
+        }
+
+        if (nodeName == null) {
+            return null;
+        }
+
+        List<com.linbit.linstor.api.model.StoragePool> sps = api.viewStoragePools(
+            Collections.singletonList(nodeName),
+            Collections.singletonList(storagePoolName),
+            Collections.emptyList(),
+            null,
+            null
+        );
+        return !sps.isEmpty() ? sps.get(0) : null;
+    }
+
+    public static String getSnapshotPath(com.linbit.linstor.api.model.StoragePool sp, String rscName, String snapshotName) {
+        final String suffix = "00000";
+        final String backingPool = sp.getProps().get("StorDriver/StorPoolName");
+        final String path;
+        switch (sp.getProviderKind()) {
+            case LVM_THIN:
+                path = String.format("/dev/mapper/%s-%s_%s_%s",
+                    backingPool.split("/")[0], rscName.replace("-", "--"), suffix, snapshotName.replace("-", "--"));
+                break;
+            case ZFS:
+            case ZFS_THIN:
+                path = String.format("zfs://%s/%s_%s@%s", backingPool.split("/")[0], rscName, suffix, snapshotName);
+                break;
+            default:
+                throw new CloudRuntimeException(
+                    String.format("Linstor: storage pool type %s doesn't support snapshots.", sp.getProviderKind()));
+        }
+        return path;
+    }
+
     public static long getCapacityBytes(String linstorUrl, String rscGroupName) {
         DevelopersApi linstorApi = getLinstorAPI(linstorUrl);
         try {

plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml

@@ -29,4 +29,6 @@
 
     <bean id="linstorPrimaryDataStoreProviderImpl"
         class="org.apache.cloudstack.storage.datastore.provider.LinstorPrimaryDatastoreProviderImpl" />
+    <bean id="linstorConfigManager"
+          class="org.apache.cloudstack.storage.datastore.util.LinstorConfigurationManager" />
 </beans>

pom.xml

@@ -699,6 +699,11 @@
                 <artifactId>xml-apis</artifactId>
                 <version>2.0.2</version>
             </dependency>
+            <dependency>
+                <groupId>com.linbit.linstor.api</groupId>
+                <artifactId>java-linstor</artifactId>
+                <version>${cs.java-linstor.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>