linstor: support QoS(IOPs) and small improvements (#6682)

This PR has 3 improvements for the Linstor primary storage driver:

- Create a separate jar of it and move all Linstor related classes into the correct project (similar to the storpool plugin)
- Add aux properties for Cloudstack volumes in Linstor to make it easier to identify them in Linstor
- Add support for IOPs settings with the Linstor storage plugin

client/pom.xml

@@ -767,6 +767,12 @@
                                     <overWrite>false</overWrite>
                                     <outputDirectory>${project.build.directory}/lib</outputDirectory>
                                 </artifactItem>
+                                <artifactItem>
+                                    <groupId>org.apache.cloudstack</groupId>
+                                    <artifactId>cloud-plugin-storage-volume-linstor</artifactId>
+                                    <overWrite>false</overWrite>
+                                    <outputDirectory>${project.build.directory}/lib</outputDirectory>
+                                </artifactItem>
                                 <artifactItem>
                                     <groupId>org.bouncycastle</groupId>
                                     <artifactId>bctls-jdk15on</artifactId>
@@ -811,6 +817,7 @@
                                     <exclude>org.bouncycastle:bctls-jdk15on</exclude>
                                     <exclude>mysql:mysql-connector-java</exclude>
                                     <exclude>org.apache.cloudstack:cloud-plugin-storage-volume-storpool</exclude>
+                                    <exclude>org.apache.cloudstack:cloud-plugin-storage-volume-linstor</exclude>
                                 </excludes>
                             </artifactSet>
                             <transformers>

debian/rules

@@ -42,6 +42,7 @@ override_dh_auto_install:
 	install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
 	install -D plugins/hypervisors/kvm/target/dependencies/* $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
 	install -D plugins/storage/volume/storpool/target/cloud-plugin-storage-volume-storpool-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
+	install -D plugins/storage/volume/linstor/target/cloud-plugin-storage-volume-linstor-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
 
 	install -d -m0755 debian/$(PACKAGE)-agent/lib/systemd/system
 	install -m0644 packaging/systemd/$(PACKAGE)-agent.service debian/$(PACKAGE)-agent/lib/systemd/system/$(PACKAGE)-agent.service

packaging/centos7/cloud.spec

@@ -355,6 +355,7 @@ install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}
 install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar
 cp plugins/hypervisors/kvm/target/dependencies/*  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
 cp plugins/storage/volume/storpool/target/*.jar  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
+cp plugins/storage/volume/linstor/target/*.jar  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
 
 # Usage server
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage

packaging/centos8/cloud.spec

@@ -348,6 +348,7 @@ install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}
 install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar
 cp plugins/hypervisors/kvm/target/dependencies/*  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
 cp plugins/storage/volume/storpool/target/*.jar  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
+cp plugins/storage/volume/linstor/target/*.jar  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
 
 # Usage server
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage

packaging/suse15/cloud.spec

@@ -350,6 +350,7 @@ install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}
 install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar
 cp plugins/hypervisors/kvm/target/dependencies/*  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
 cp plugins/storage/volume/storpool/target/*.jar  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
+cp plugins/storage/volume/linstor/target/*.jar  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
 
 # Usage server
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage

plugins/storage/volume/linstor/pom.xml

@@ -38,6 +38,11 @@
             <artifactId>java-linstor</artifactId>
             <version>${cs.java-linstor.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-plugin-hypervisor-kvm</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java

@@ -71,7 +71,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
 
     private String getHostname() {
         // either there is already some function for that in the agent or a better way.
-        ProcessBuilder pb = new ProcessBuilder("hostname");
+        ProcessBuilder pb = new ProcessBuilder("/usr/bin/hostname");
         try
         {
             String result;
@@ -86,7 +86,8 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
             p.destroy();
             return result.trim();
         } catch (IOException | InterruptedException exc) {
-            throw new CloudRuntimeException("Unable to run 'hostname' command.");
+            Thread.currentThread().interrupt();
+            throw new CloudRuntimeException("Unable to run '/usr/bin/hostname' command.");
         }
     }
 
@@ -310,11 +311,11 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
     public boolean disconnectPhysicalDiskByPath(String localPath)
     {
         // get first storage pool from the map, as we don't know any better:
-        if (!MapStorageUuidToStoragePool.isEmpty())
+        Optional<KVMStoragePool> optFirstPool = MapStorageUuidToStoragePool.values().stream().findFirst();
+        if (optFirstPool.isPresent())
         {
             s_logger.debug("Linstor: disconnectPhysicalDiskByPath " + localPath);
-            String firstKey = MapStorageUuidToStoragePool.keySet().stream().findFirst().get();
+            final KVMStoragePool pool = optFirstPool.get();
-            final KVMStoragePool pool = MapStorageUuidToStoragePool.get(firstKey);
 
             s_logger.debug("Linstor: Using storpool: " + pool.getUuid());
             final DevelopersApi api = getLinstorAPI(pool);

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java

@@ -21,22 +21,28 @@ import com.linbit.linstor.api.CloneWaiter;
 import com.linbit.linstor.api.DevelopersApi;
 import com.linbit.linstor.api.model.ApiCallRc;
 import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Properties;
 import com.linbit.linstor.api.model.ResourceDefinition;
 import com.linbit.linstor.api.model.ResourceDefinitionCloneRequest;
 import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted;
 import com.linbit.linstor.api.model.ResourceDefinitionCreate;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
 import com.linbit.linstor.api.model.ResourceGroupSpawn;
 import com.linbit.linstor.api.model.ResourceWithVolumes;
 import com.linbit.linstor.api.model.Snapshot;
 import com.linbit.linstor.api.model.SnapshotRestore;
+import com.linbit.linstor.api.model.VolumeDefinition;
 import com.linbit.linstor.api.model.VolumeDefinitionModify;
 
 import javax.annotation.Nonnull;
 import javax.inject.Inject;
+
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.ResizeVolumeAnswer;
@@ -242,14 +248,17 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
                 deleteResourceDefinition(storagePool, rscName);
 
                 long usedBytes = storagePool.getUsedBytes();
-                long capacityIops = storagePool.getCapacityIops();
+                Long capacityIops = storagePool.getCapacityIops();
+
+                if (capacityIops != null)
+                {
+                    if (volumeInfo.getMaxIops() != null)
+                        capacityIops += volumeInfo.getMaxIops();
+                    storagePool.setCapacityIops(Math.max(0, capacityIops));
+                }
 
                 usedBytes -= volumeInfo.getSize();
-                if (volumeInfo.getMaxIops() != null)
-                    capacityIops += volumeInfo.getMaxIops();
-
                 storagePool.setUsedBytes(Math.max(0, usedBytes));
-                storagePool.setCapacityIops(Math.max(0, capacityIops));
 
                 _storagePoolDao.update(storagePoolId, storagePool);
             }
@@ -325,6 +334,72 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
         }
     }
 
+    private void applyQoSSettings(StoragePoolVO storagePool, DevelopersApi api, String rscName, Long maxIops)
+        throws ApiException
+    {
+        Long currentQosIops = null;
+        List<VolumeDefinition> vlmDfns = api.volumeDefinitionList(rscName, null, null);
+        if (!vlmDfns.isEmpty())
+        {
+            Properties props = vlmDfns.get(0).getProps();
+            long iops = Long.parseLong(props.getOrDefault("sys/fs/blkio_throttle_write_iops", "0"));
+            currentQosIops = iops > 0 ? iops : null;
+        }
+
+        if (!Objects.equals(maxIops, currentQosIops))
+        {
+            VolumeDefinitionModify vdm = new VolumeDefinitionModify();
+            if (maxIops != null)
+            {
+                Properties props = new Properties();
+                props.put("sys/fs/blkio_throttle_read_iops", "" + maxIops);
+                props.put("sys/fs/blkio_throttle_write_iops", "" + maxIops);
+                vdm.overrideProps(props);
+                s_logger.info("Apply qos setting: " + maxIops + " to " + rscName);
+            }
+            else
+            {
+                s_logger.info("Remove QoS setting for " + rscName);
+                vdm.deleteProps(Arrays.asList("sys/fs/blkio_throttle_read_iops", "sys/fs/blkio_throttle_write_iops"));
+            }
+            ApiCallRcList answers = api.volumeDefinitionModify(rscName, 0, vdm);
+            checkLinstorAnswersThrow(answers);
+
+            Long capacityIops = storagePool.getCapacityIops();
+            if (capacityIops != null)
+            {
+                long vcIops = currentQosIops != null ? currentQosIops * -1 : 0;
+                long vMaxIops = maxIops != null ? maxIops : 0;
+                long newIops = vcIops + vMaxIops;
+                capacityIops -= newIops;
+                s_logger.info("Current storagepool " + storagePool.getName() + " iops capacity:  " + capacityIops);
+                storagePool.setCapacityIops(Math.max(0, capacityIops));
+                _storagePoolDao.update(storagePool.getId(), storagePool);
+            }
+        }
+    }
+
+    private void applyAuxProps(DevelopersApi api, String rscName, String dispName, String vmName)
+        throws ApiException
+    {
+        ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+        Properties props = new Properties();
+        if (dispName != null)
+        {
+            props.put("Aux/cs-name", dispName);
+        }
+        if (vmName != null)
+        {
+            props.put("Aux/cs-vm-name", vmName);
+        }
+        if (!props.isEmpty())
+        {
+            rdm.setOverrideProps(props);
+            ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
+            checkLinstorAnswersThrow(answers);
+        }
+    }
+
     private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO)
     {
         DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());
@@ -338,10 +413,13 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
 
         try
         {
-            s_logger.debug("Linstor: Spawn resource " + rscName);
+            s_logger.info("Linstor: Spawn resource " + rscName);
             ApiCallRcList answers = linstorApi.resourceGroupSpawn(rscGrp, rscGrpSpawn);
             checkLinstorAnswersThrow(answers);
 
+            applyAuxProps(linstorApi, rscName, vol.getName(), vol.getAttachedVmName());
+            applyQoSSettings(storagePoolVO, linstorApi, rscName, vol.getMaxIops());
+
             return getDeviceName(linstorApi, rscName);
         } catch (ApiException apiEx)
         {
@@ -361,7 +439,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
             final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());
 
             try {
-                s_logger.debug("Clone resource definition " + cloneRes + " to " + rscName);
+                s_logger.info("Clone resource definition " + cloneRes + " to " + rscName);
                 ResourceDefinitionCloneRequest cloneRequest = new ResourceDefinitionCloneRequest();
                 cloneRequest.setName(rscName);
                 ResourceDefinitionCloneStarted cloneStarted = linstorApi.resourceDefinitionClone(
@@ -373,6 +451,10 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
                     throw new CloudRuntimeException("Clone for resource " + rscName + " failed.");
                 }
 
+                s_logger.info("Clone resource definition " + cloneRes + " to " + rscName + " finished");
+                applyAuxProps(linstorApi, rscName, volumeInfo.getName(), volumeInfo.getAttachedVmName());
+                applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeInfo.getMaxIops());
+
                 return getDeviceName(linstorApi, rscName);
             } catch (ApiException apiEx) {
                 s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
@@ -413,10 +495,13 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
             checkLinstorAnswersThrow(answers);
 
             // restore snapshot to new resource
-            s_logger.debug("Restore resource from snapshot: " + cloneRes + ":" + snapName);
+            s_logger.info("Restore resource from snapshot: " + cloneRes + ":" + snapName);
             answers = linstorApi.resourceSnapshotRestore(cloneRes, snapName, snapshotRestore);
             checkLinstorAnswersThrow(answers);
 
+            applyAuxProps(linstorApi, rscName, volumeVO.getName(), null);
+            applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeVO.getMaxIops());
+
             return getDeviceName(linstorApi, rscName);
         } catch (ApiException apiEx) {
             s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
@@ -608,12 +693,11 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
     }
 
     private CreateCmdResult notifyResize(
-        DataObject data,
+        VolumeObject vol,
         long oldSize,
         ResizeVolumePayload resizeParameter)
     {
-        VolumeObject vol = (VolumeObject) data;
+        StoragePool pool = (StoragePool) vol.getDataStore();
-        StoragePool pool = (StoragePool) data.getDataStore();
 
         ResizeVolumeCommand resizeCmd =
             new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, resizeParameter.newSize, resizeParameter.shrinkOk,
@@ -642,7 +726,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
     public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> callback)
     {
         final VolumeObject vol = (VolumeObject) data;
-        final StoragePool pool = (StoragePool) data.getDataStore();
+        final StoragePoolVO pool = _storagePoolDao.findById(data.getDataStore().getId());
         final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress());
         final ResizeVolumePayload resizeParameter = (ResizeVolumePayload) vol.getpayload();
 
@@ -654,6 +738,14 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
         dfm.setSizeKib(resizeParameter.newSize / 1024);
         try
         {
+            applyQoSSettings(pool, api, rscName, resizeParameter.newMaxIops);
+            {
+                final VolumeVO volume = _volumeDao.findById(vol.getId());
+                volume.setMinIops(resizeParameter.newMinIops);
+                volume.setMaxIops(resizeParameter.newMaxIops);
+                _volumeDao.update(volume.getId(), volume);
+            }
+
             ApiCallRcList answers = api.volumeDefinitionModify(rscName, 0, dfm);
             if (answers.hasError())
             {
@@ -680,7 +772,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
         } else
         {
             // notify guests
-            result = notifyResize(data, oldSize, resizeParameter);
+            result = notifyResize(vol, oldSize, resizeParameter);
         }
 
         callback.complete(result);

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java

@@ -89,6 +89,7 @@ public class LinstorPrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLif
         Long clusterId = (Long) dsInfos.get("clusterId");
         String storagePoolName = (String) dsInfos.get("name");
         String providerName = (String) dsInfos.get("providerName");
+        Long capacityIops = (Long) dsInfos.get("capacityIops");
         String tags = (String) dsInfos.get("tags");
         @SuppressWarnings("unchecked")
         Map<String, String> details = (Map<String, String>) dsInfos.get("details");
@@ -145,11 +146,14 @@ public class LinstorPrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLif
         }
 
         long capacityBytes = LinstorUtil.getCapacityBytes(url, resourceGroup);
-
         if (capacityBytes <= 0) {
             throw new IllegalArgumentException("'capacityBytes' must be present and greater than 0.");
         }
 
+        if (capacityIops != null) {
+            parameters.setCapacityIops(capacityIops);
+        }
+
         parameters.setHost(url);
         parameters.setPort(port);
         parameters.setPath(resourceGroup);
@@ -161,7 +165,7 @@ public class LinstorPrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLif
         parameters.setManaged(false);
         parameters.setCapacityBytes(capacityBytes);
         parameters.setUsedBytes(0);
-        parameters.setCapacityIops(0L);
+        parameters.setCapacityIops(capacityIops);
         parameters.setHypervisorType(HypervisorType.KVM);
         parameters.setTags(tags);
         parameters.setDetails(details);

ui/src/views/infra/AddPrimaryStorage.vue

@@ -308,6 +308,12 @@
           </a-form-item>
         </div>
         <div v-if="form.protocol === 'Linstor'">
+          <a-form-item name="capacityIops" ref="capacityIops">
+            <template #label>
+              <tooltip-label :title="$t('label.capacityiops')" :tooltip="apiParams.capacityiops.description"/>
+            </template>
+            <a-input v-model:value="form.capacityIops" :placeholder="apiParams.capacityiops.description" />
+          </a-form-item>
           <a-form-item name="resourcegroup" ref="resourcegroup">
             <template #label>
               <tooltip-label :title="$t('label.resourcegroup')" :tooltip="$t('message.linstor.resourcegroup.description')"/>
@@ -749,6 +755,9 @@ export default {
           params.provider = 'Linstor'
           values.managed = false
           params['details[0].resourceGroup'] = values.resourcegroup
+          if (values.capacityIops && values.capacityIops.length > 0) {
+            params.capacityIops = values.capacityIops.split(',').join('')
+          }
         }
         params.url = url
         if (values.provider !== 'DefaultPrimary' && values.provider !== 'PowerFlex') {