Added support for system VMs to make use of managed storage

plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java

@@ -490,7 +490,10 @@ public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         } else if (dataObject.getType() == DataObjectType.TEMPLATE) {
             TemplateInfo templateInfo = (TemplateInfo)dataObject;
 
-            volumeSize = (long)(templateInfo.getSize() + templateInfo.getSize() * (LOWEST_HYPERVISOR_SNAPSHOT_RESERVE / 100f));
+            // TemplateInfo sometimes has a size equal to null.
+            long templateSize = templateInfo.getSize() != null ? templateInfo.getSize() : 0;
+
+            volumeSize = (long)(templateSize + templateSize * (LOWEST_HYPERVISOR_SNAPSHOT_RESERVE / 100f));
         }
 
         return volumeSize;

plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java

@@ -21,6 +21,7 @@ package org.apache.cloudstack.storage.datastore.lifecycle;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
 import javax.inject.Inject;
 
@@ -40,7 +41,6 @@ import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper;
 
 import com.cloud.agent.api.StoragePoolInfo;
 import com.cloud.capacity.CapacityManager;
-import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.host.HostVO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
@@ -88,10 +88,6 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC
         String storageVip = SolidFireUtil.getStorageVip(url);
         int storagePort = SolidFireUtil.getStoragePort(url);
 
-        DataCenterVO zone = _zoneDao.findById(zoneId);
-
-        String uuid = SolidFireUtil.PROVIDER_NAME + "_" + zone.getUuid() + "_" + storageVip;
-
         if (capacityBytes == null || capacityBytes <= 0) {
             throw new IllegalArgumentException("'capacityBytes' must be present and greater than 0.");
         }
@@ -106,7 +102,7 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC
         parameters.setPort(storagePort);
         parameters.setPath(SolidFireUtil.getModifiedUrl(url));
         parameters.setType(StoragePoolType.Iscsi);
-        parameters.setUuid(uuid);
+        parameters.setUuid(UUID.randomUUID().toString());
         parameters.setZoneId(zoneId);
         parameters.setName(storagePoolName);
         parameters.setProviderName(providerName);

server/src/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -1969,6 +1969,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         final String vmTypeString = cmd.getSystemVmType();
         VirtualMachine.Type vmType = null;
         boolean allowNetworkRate = false;
+
+        Boolean isCustomizedIops;
+
         if (cmd.getIsSystem()) {
             if (vmTypeString == null || VirtualMachine.Type.DomainRouter.toString().toLowerCase().equals(vmTypeString)) {
                 vmType = VirtualMachine.Type.DomainRouter;
@@ -1983,8 +1986,19 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 throw new InvalidParameterValueException("Invalid systemVmType. Supported types are: " + VirtualMachine.Type.DomainRouter + ", " + VirtualMachine.Type.ConsoleProxy
                         + ", " + VirtualMachine.Type.SecondaryStorageVm);
             }
+
+            if (cmd.isCustomizedIops() != null) {
+                throw new InvalidParameterValueException("Customized IOPS is not a valid parameter for a system VM.");
+            }
+
+            isCustomizedIops = false;
+
+            if (cmd.getHypervisorSnapshotReserve() != null) {
+                throw new InvalidParameterValueException("Hypervisor snapshot reserve is not a valid parameter for a system VM.");
+            }
         } else {
             allowNetworkRate = true;
+            isCustomizedIops = cmd.isCustomizedIops();
         }
 
         if (cmd.getNetworkRate() != null) {
@@ -2009,7 +2023,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
 
         return createServiceOffering(userId, cmd.getIsSystem(), vmType, cmd.getServiceOfferingName(), cpuNumber, memory, cpuSpeed, cmd.getDisplayText(),
                 cmd.getProvisioningType(), localStorageRequired, offerHA, limitCpuUse, volatileVm, cmd.getTags(), cmd.getDomainId(), cmd.getHostTag(),
-                cmd.getNetworkRate(), cmd.getDeploymentPlanner(), cmd.getDetails(), cmd.isCustomizedIops(), cmd.getMinIops(), cmd.getMaxIops(),
+                cmd.getNetworkRate(), cmd.getDeploymentPlanner(), cmd.getDetails(), isCustomizedIops, cmd.getMinIops(), cmd.getMaxIops(),
                 cmd.getBytesReadRate(), cmd.getBytesWriteRate(), cmd.getIopsReadRate(), cmd.getIopsWriteRate(), cmd.getHypervisorSnapshotReserve());
     }
 

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -82,6 +82,7 @@ import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.storage.command.DettachCommand;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
@@ -1113,7 +1114,16 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
                     cleanupSecondaryStorage(recurring);
 
                     List<VolumeVO> vols = _volsDao.listVolumesToBeDestroyed(new Date(System.currentTimeMillis() - ((long) StorageCleanupDelay.value() << 10)));
+
                     for (VolumeVO vol : vols) {
+                        try {
+                            // If this fails, just log a warning. It's ideal if we clean up the host-side clustered file
+                            // system, but not necessary.
+                            handleManagedStorage(vol);
+                        } catch (Exception e) {
+                            s_logger.warn("Unable to destroy host-side clustered file system " + vol.getUuid(), e);
+                        }
+
                         try {
                             volService.expungeVolumeAsync(volFactory.getVolume(vol.getId()));
                         } catch (Exception e) {
@@ -1233,6 +1243,47 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         }
     }
 
+    private void handleManagedStorage(Volume volume) {
+        Long instanceId = volume.getInstanceId();
+
+        // The idea of this "if" statement is to see if we need to remove an SR/datastore before
+        // deleting the volume that supports it on a SAN. This only applies for managed storage.
+        if (instanceId != null) {
+            StoragePoolVO storagePool = _storagePoolDao.findById(volume.getPoolId());
+
+            if (storagePool != null && storagePool.isManaged()) {
+                DataTO volTO = volFactory.getVolume(volume.getId()).getTO();
+                DiskTO disk = new DiskTO(volTO, volume.getDeviceId(), volume.getPath(), volume.getVolumeType());
+
+                DettachCommand cmd = new DettachCommand(disk, null);
+
+                cmd.setManaged(true);
+
+                cmd.setStorageHost(storagePool.getHostAddress());
+                cmd.setStoragePort(storagePool.getPort());
+
+                cmd.set_iScsiName(volume.get_iScsiName());
+
+                VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(instanceId);
+
+                Long lastHostId = vmInstanceVO.getLastHostId();
+
+                if (lastHostId != null) {
+                    Answer answer = _agentMgr.easySend(lastHostId, cmd);
+
+                    if (answer != null && answer.getResult()) {
+                        VolumeInfo volumeInfo = volFactory.getVolume(volume.getId());
+                        HostVO host = _hostDao.findById(lastHostId);
+
+                        volService.revokeAccess(volumeInfo, host, volumeInfo.getDataStore());
+                    } else {
+                        s_logger.warn("Unable to remove host-side clustered file system for the following volume: " + volume.getUuid());
+                    }
+                }
+            }
+        }
+    }
+
     @DB
     List<Long> findAllVolumeIdInSnapshotTable(Long storeId) {
         String sql = "SELECT volume_id from snapshots, snapshot_store_ref WHERE snapshots.id = snapshot_store_ref.snapshot_id and store_id=? GROUP BY volume_id";

test/integration/plugins/solidfire/TestManagedSystemVMs.py

@@ -394,10 +394,10 @@ class TestManagedSystemVMs(cloudstackTestCase):
                 self.apiClient,
                 self.testdata[TestData.systemOfferingFailure]
             )
-
-            self.assert_(True, "The service offering was created, but should not have been.")
         except:
-            pass
+            return
+
+        self.assert_(False, "The service offering was created, but should not have been.")
 
     def _prepare_to_use_managed_storage_for_system_vms(self):
         self._update_system_vm_unique_name(TestManagedSystemVMs._secondary_storage_unique_name, TestManagedSystemVMs._secondary_storage_temp_unique_name)

ui/scripts/configuration.js

@@ -1146,6 +1146,82 @@
                                             number: true
                                         }
                                     },
+                                    qosType: {
+                                        label: 'label.qos.type',
+                                        docID: 'helpDiskOfferingQoSType',
+                                        select: function(args) {
+                                            var items = [];
+                                            items.push({
+                                                id: '',
+                                                description: ''
+                                            });
+                                            items.push({
+                                                id: 'hypervisor',
+                                                description: 'hypervisor'
+                                            });
+                                            items.push({
+                                                id: 'storage',
+                                                description: 'storage'
+                                            });
+                                            args.response.success({
+                                                data: items
+                                            });
+
+                                            args.$select.change(function() {
+                                                var $form = $(this).closest('form');
+                                                var $isCustomizedIops = $form.find('.form-item[rel=isCustomizedIops]');
+                                                var $minIops = $form.find('.form-item[rel=minIops]');
+                                                var $maxIops = $form.find('.form-item[rel=maxIops]');
+                                                var $diskBytesReadRate = $form.find('.form-item[rel=diskBytesReadRate]');
+                                                var $diskBytesWriteRate = $form.find('.form-item[rel=diskBytesWriteRate]');
+                                                var $diskIopsReadRate = $form.find('.form-item[rel=diskIopsReadRate]');
+                                                var $diskIopsWriteRate = $form.find('.form-item[rel=diskIopsWriteRate]');
+
+                                                var qosId = $(this).val();
+
+                                                if (qosId == 'storage') { // Storage QoS
+                                                    $diskBytesReadRate.hide();
+                                                    $diskBytesWriteRate.hide();
+                                                    $diskIopsReadRate.hide();
+                                                    $diskIopsWriteRate.hide();
+
+                                                    $minIops.css('display', 'inline-block');
+                                                    $maxIops.css('display', 'inline-block');
+                                                } else if (qosId == 'hypervisor') { // Hypervisor Qos
+                                                    $minIops.hide();
+                                                    $maxIops.hide();
+
+                                                    $diskBytesReadRate.css('display', 'inline-block');
+                                                    $diskBytesWriteRate.css('display', 'inline-block');
+                                                    $diskIopsReadRate.css('display', 'inline-block');
+                                                    $diskIopsWriteRate.css('display', 'inline-block');
+                                                } else { // No Qos
+                                                    $diskBytesReadRate.hide();
+                                                    $diskBytesWriteRate.hide();
+                                                    $diskIopsReadRate.hide();
+                                                    $diskIopsWriteRate.hide();
+                                                    $minIops.hide();
+                                                    $maxIops.hide();
+                                                }
+                                            });
+                                        }
+                                    },
+                                    minIops: {
+                                        label: 'label.disk.iops.min',
+                                        docID: 'helpDiskOfferingDiskIopsMin',
+                                        validation: {
+                                            required: false,
+                                            number: true
+                                        }
+                                    },
+                                    maxIops: {
+                                        label: 'label.disk.iops.max',
+                                        docID: 'helpDiskOfferingDiskIopsMax',
+                                        validation: {
+                                            required: false,
+                                            number: true
+                                        }
+                                    },
                                     diskBytesReadRate: {
                                         label: 'label.disk.bytes.read.rate',
                                         docID: 'helpSystemOfferingDiskBytesReadRate',
@@ -1255,26 +1331,44 @@
                                         networkrate: args.data.networkRate
                                     });
                                 }
+
+                                if (args.data.qosType == 'storage') {
+                                    if (args.data.minIops != null && args.data.minIops.length > 0) {
+                                        $.extend(data, {
+                                            miniops: args.data.minIops
+                                        });
+                                    }
+
+                                    if (args.data.maxIops != null && args.data.maxIops.length > 0) {
+                                        $.extend(data, {
+                                            maxiops: args.data.maxIops
+                                        });
+                                    }
+                                } else if (args.data.qosType == 'hypervisor') {
                                     if (args.data.diskBytesReadRate != null && args.data.diskBytesReadRate.length > 0) {
                                         $.extend(data, {
                                             bytesreadrate: args.data.diskBytesReadRate
                                         });
                                     }
+
                                     if (args.data.diskBytesWriteRate != null && args.data.diskBytesWriteRate.length > 0) {
                                         $.extend(data, {
                                             byteswriterate: args.data.diskBytesWriteRate
                                         });
                                     }
+
                                     if (args.data.diskIopsReadRate != null && args.data.diskIopsReadRate.length > 0) {
                                         $.extend(data, {
                                             iopsreadrate: args.data.diskIopsReadRate
                                         });
                                     }
+
                                     if (args.data.diskIopsWriteRate != null && args.data.diskIopsWriteRate.length > 0) {
                                         $.extend(data, {
                                             iopswriterate: args.data.diskIopsWriteRate
                                         });
                                     }
+                                }
 
                                 $.extend(data, {
                                     offerha: (args.data.offerHA == "on")
@@ -1476,6 +1570,24 @@
                                     networkrate: {
                                         label: 'label.network.rate'
                                     },
+                                    miniops: {
+                                        label: 'label.disk.iops.min',
+                                        converter: function(args) {
+                                            if (args > 0)
+                                                return args;
+                                            else
+                                                return "N/A";
+                                        }
+                                    },
+                                    maxiops: {
+                                        label: 'label.disk.iops.max',
+                                        converter: function(args) {
+                                            if (args > 0)
+                                                return args;
+                                            else
+                                                return "N/A";
+                                        }
+                                    },
                                     diskBytesReadRate: {
                                         label: 'label.disk.bytes.read.rate'
                                     },