CLOUDSTACK-6092: Storage OverProvisioning as a Per Primary Basis Allow storage.overprovisioning.factor to be specified at storape pool level.

Signed-off-by: Sateesh Chodapuneedi <sateesh@apache.org>

api/src/org/apache/cloudstack/api/response/StoragePoolResponse.java

@@ -107,6 +107,10 @@ public class StoragePoolResponse extends BaseResponse {
     @Param(description = "the scope of the storage pool")
     private String scope;
 
+    @SerializedName("overprovisionfactor")
+    @Param(description = "the overprovisionfactor for the storage pool")
+    private String overprovisionfactor;
+
     @SerializedName(ApiConstants.HYPERVISOR)
     @Param(description = "the hypervisor type of the storage pool")
     private String hypervisor;
@@ -301,4 +305,8 @@ public class StoragePoolResponse extends BaseResponse {
     public void setSuitableForMigration(Boolean suitableForMigration) {
         this.suitableForMigration = suitableForMigration;
     }
+
+    public void setOverProvisionFactor(String overProvisionFactor) {
+        this.overprovisionfactor = overProvisionFactor;
+    }
 }

engine/components-api/src/com/cloud/capacity/CapacityManager.java

@@ -44,7 +44,7 @@ public interface CapacityManager {
         "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", true,
         ConfigKey.Scope.Zone);
     static final ConfigKey<Double> StorageOverprovisioningFactor = new ConfigKey<Double>("Storage", Double.class, StorageOverprovisioningFactorCK, "2",
-        "Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)", true, ConfigKey.Scope.Zone);
+        "Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)", true, ConfigKey.Scope.StoragePool);
     static final ConfigKey<Double> StorageAllocatedCapacityDisableThreshold =
         new ConfigKey<Double>(
             "Alert",

server/src/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java

@@ -24,13 +24,13 @@ import javax.inject.Inject;
 
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
-
 import org.apache.cloudstack.api.response.StoragePoolResponse;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.query.vo.StoragePoolJoinVO;
 import com.cloud.capacity.Capacity;
+import com.cloud.capacity.CapacityManager;
 import com.cloud.storage.ScopeType;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.StorageStats;
@@ -103,6 +103,7 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase<StoragePoolJoinVO, Lo
         poolResponse.setClusterId(pool.getClusterUuid());
         poolResponse.setClusterName(pool.getClusterName());
         poolResponse.setTags(pool.getTag());
+        poolResponse.setOverProvisionFactor(Double.toString(CapacityManager.StorageOverprovisioningFactor.valueIn(pool.getId())));
 
         // set async job
         if (pool.getJobId() != null) {
@@ -156,6 +157,7 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase<StoragePoolJoinVO, Lo
         poolResponse.setDiskSizeTotal(pool.getCapacityBytes());
         poolResponse.setDiskSizeAllocated(allocatedSize);
         poolResponse.setCapacityIops(pool.getCapacityIops());
+        poolResponse.setOverProvisionFactor(Double.toString(CapacityManager.StorageOverprovisioningFactor.valueIn(pool.getId())));
 
         // TODO: StatsCollector does not persist data
         StorageStats stats = ApiDBUtils.getStoragePoolStatistics(pool.getId());

server/src/com/cloud/capacity/StorageCapacityListener.java

@@ -67,7 +67,7 @@ public class StorageCapacityListener implements Listener {
 
         StartupStorageCommand ssCmd = (StartupStorageCommand)startup;
         if (ssCmd.getResourceType() == Storage.StorageResourceType.STORAGE_HOST) {
-            BigDecimal overProvFactor = _storageMgr.getStorageOverProvisioningFactor(server.getDataCenterId());
+            BigDecimal overProvFactor = BigDecimal.valueOf(CapacityManager.StorageOverprovisioningFactor.value());
             CapacityVO capacity =
                 new CapacityVO(server.getId(), server.getDataCenterId(), server.getPodId(), server.getClusterId(), 0L, (overProvFactor.multiply(new BigDecimal(
                     server.getTotalSize()))).longValue(), Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED);

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -854,8 +854,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
     }
 
     @Override
-    public BigDecimal getStorageOverProvisioningFactor(Long dcId) {
+    public BigDecimal getStorageOverProvisioningFactor(Long poolId) {
-        return new BigDecimal(CapacityManager.StorageOverprovisioningFactor.valueIn(dcId));
+        return new BigDecimal(CapacityManager.StorageOverprovisioningFactor.valueIn(poolId));
     }
 
     @Override
@@ -869,13 +869,18 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
         long totalOverProvCapacity;
         if (storagePool.getPoolType() == StoragePoolType.NetworkFilesystem || storagePool.getPoolType() == StoragePoolType.VMFS) {
-            BigDecimal overProvFactor = getStorageOverProvisioningFactor(storagePool.getDataCenterId());
-            totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(storagePool.getCapacityBytes())).longValue();
             // All this is for the inaccuracy of floats for big number multiplication.
+            BigDecimal overProvFactor = getStorageOverProvisioningFactor(storagePool.getId());
+            totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(storagePool.getCapacityBytes())).longValue();
+            s_logger.debug("Found storage pool " + storagePool.getName() + " of type " + storagePool.getPoolType().toString() + " with overprovisioning factor "
+                    + overProvFactor.toString());
+            s_logger.debug("Total over provisioned capacity calculated is " + overProvFactor + " * " + storagePool.getCapacityBytes());
         } else {
+            s_logger.debug("Found storage pool " + storagePool.getName() + " of type " + storagePool.getPoolType().toString());
             totalOverProvCapacity = storagePool.getCapacityBytes();
         }
 
+        s_logger.debug("Total over provisioned capacity of the pool " + storagePool.getName() + " id: " + storagePool.getId() + " is " + totalOverProvCapacity);
         if (capacities.size() == 0) {
             CapacityVO capacity =
                 new CapacityVO(storagePool.getId(), storagePool.getDataCenterId(), storagePool.getPodId(), storagePool.getClusterId(), allocated, totalOverProvCapacity,
@@ -1552,16 +1557,22 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
         long totalOverProvCapacity;
         if (pool.getPoolType() == StoragePoolType.NetworkFilesystem || pool.getPoolType() == StoragePoolType.VMFS) {
-            totalOverProvCapacity = getStorageOverProvisioningFactor(pool.getDataCenterId()).multiply(new BigDecimal(pool.getCapacityBytes())).longValue();
+            BigDecimal overProvFactor = getStorageOverProvisioningFactor(pool.getId());
+            totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(pool.getCapacityBytes())).longValue();
+            s_logger.debug("Found storage pool " + poolVO.getName() + " of type " + pool.getPoolType().toString() + " with overprovisioning factor "
+                    + overProvFactor.toString());
+            s_logger.debug("Total over provisioned capacity calculated is " + overProvFactor + " * " + pool.getCapacityBytes());
         } else {
             totalOverProvCapacity = pool.getCapacityBytes();
+            s_logger.debug("Found storage pool " + poolVO.getName() + " of type " + pool.getPoolType().toString());
         }
 
+        s_logger.debug("Total capacity of the pool " + poolVO.getName() + " id: " + pool.getId() + " is " + totalOverProvCapacity);
         double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getDataCenterId());
         if (s_logger.isDebugEnabled()) {
             s_logger.debug("Checking pool: " + pool.getId() + " for volume allocation " + volumes.toString() + ", maxSize : " + totalOverProvCapacity +
-                ", totalAllocatedSize : " + allocatedSizeWithtemplate + ", askingSize : " + totalAskingSize + ", allocated disable threshold: " +
+                    ", totalAllocatedSize : " + allocatedSizeWithtemplate + ", askingSize : " + totalAskingSize + ", allocated disable threshold: " +
-                storageAllocatedThreshold);
+                    storageAllocatedThreshold);
         }
 
         double usedPercentage = (allocatedSizeWithtemplate + totalAskingSize) / (double)(totalOverProvCapacity);

setup/db/db/schema-430to440.sql

@@ -742,3 +742,6 @@ UPDATE `cloud`.`guest_os` SET `created` = now();
 ALTER TABLE `cloud`.`vm_reservation` ADD COLUMN `deployment_planner` varchar(40) DEFAULT NULL COMMENT 'Preferred deployment planner for the vm';
 ALTER TABLE `cloud`.`vpc_offerings` ADD COLUMN supports_distributed_router boolean default false;
 ALTER TABLE `cloud`.`vpc` ADD COLUMN uses_distributed_router  boolean default false;
+
+INSERT INTO `cloud`.`storage_pool_details` (pool_id,name,value,display) SELECT storage_pool.id,data_center_details.name,data_center_details.value,data_center_details.display FROM `cloud`.`storage_pool` JOIN `cloud`.`data_center_details` ON data_center_details.dc_id=storage_pool.data_center_id WHERE data_center_details.name = "storage.overprovisioning.factor";
+DELETE FROM `cloud`.`data_center_details` WHERE name="storage.overprovisioning.factor";

test/integration/smoke/test_over_provisioning.py

@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+""" Test for storage.overprovisioning.factor update
+"""
+#Import Local Modules
+from marvin.cloudstackTestCase import *
+from marvin.cloudstackAPI import *
+from marvin.integration.lib.utils import *
+from marvin.integration.lib.base import *
+from marvin.integration.lib.common import *
+from nose.plugins.attrib import attr
+#Import System modules
+
+class TestUpdateOverProvision(cloudstackTestCase):
+    """
+    Test to update a storage.overprovisioning.factor
+    """
+    def setUp(self):
+        self.apiClient = self.testClient.getApiClient()
+
+    @attr(tags=["simulator", "devcloud", "basic", "advanced"])
+    def test_UpdateStorageOverProvisioningFactor(self):
+        """
+        test update configuration setting at storage scope
+        @return:
+        """
+
+        """ 1. list storagepools for id """
+        """ 2. list overprovisioning factor for storage pool """
+        """ 3. update setting for the pool"""
+        """ 4. list overprovisioning factor for storage pool and assert"""
+
+        """ list storagepools """
+        storage_pools = StoragePool.list(
+                                self.apiClient,
+                                listall=True
+                                )
+        self.assertEqual(
+                         isinstance(storage_pools, list),
+                         True,
+                         "List storage pools should not return empty response"
+                         )
+
+        if len(storage_pools) < 1:
+            raise self.skipTest(
+                            "The environment don't have storage pools required for test")
+
+        for pool in storage_pools:
+            if pool.type == "NetworkFilesystem" or pool.type == "VMFS":
+                break
+        if pool.type != "NetworkFilesystem" and pool.type != "VMFS":
+            raise self.skipTest("Storage overprovisioning currently not supported on " + pool.type + " pools")
+
+        self.poolId = pool.id
+        """ list overprovisioning factor for storage pool """
+        factorOld = float(pool.overprovisionfactor)
+        factorNew = str(factorOld + 1.0)
+
+        """ update setting for the pool"""
+        updateConfigurationCmd = updateConfiguration.updateConfigurationCmd()
+        updateConfigurationCmd.name = "storage.overprovisioning.factor"
+        updateConfigurationCmd.value = factorNew
+        updateConfigurationCmd.storageid = pool.id
+
+        updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd)
+
+        self.debug("updated the parameter %s with value %s"%(updateConfigurationResponse.name, updateConfigurationResponse.value))
+
+        storage_pools = StoragePool.list(
+                                self.apiClient,
+                                id = self.poolId
+                                )
+        pool = storage_pools[0]
+        factorNew = float(pool.overprovisionfactor)
+        self.assertNotEqual(int(factorNew), int(factorOld)," Check if overprovision factor of storage pool has changed")
+        self.assertEqual(int(factorNew), int(factorOld + 1.0)," Check if overprovision factor of storage pool has increased by 1")
+
+    def tearDown(self):
+        """Reset the storage.overprovisioning.factor back to its original value
+        @return:
+        """
+        storage_pools = StoragePool.list(
+                                self.apiClient,
+                                id = self.poolId
+                                )
+        pool = storage_pools[0]
+        updateConfigurationCmd = updateConfiguration.updateConfigurationCmd()
+        updateConfigurationCmd.name = "storage.overprovisioning.factor"
+        factorOld = float(pool.overprovisionfactor)
+        factorNew = str(factorOld - 1.0)
+        updateConfigurationCmd.value = factorNew
+        updateConfigurationCmd.storageid = pool.id
+        updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd)