Cloudstack 10064: Secondary storage Usage for uploadedVolume is not collected (#2258)

Description: For Volumes on Secondary Storage, (Uploaded Volume) the usage is not accounted for.

The fix is implemented as follows:

A new Usage Type is added for the Volume on secondary storage : VOLUME_SECONDARY (id=26)
A new storage type, 'Volume' is defined.
When a volume is uploaded and the usage server executes next,entry will be added to the usage_storage helper table for all the volumes uploaded since the Usage server executed last.
When the uploaded volume is attached, the 'deleted' column in the usage_storage table is set to the time-stamp when the volume was deleted
2 entries will be added to the cloud_usage table with usage_type=26 and usage_type=6 (Volume usage on primary). One for the duration the volume was on primary and other for the duration it was on secondary.
Entry is added to the helper table volume_usage for accounting for the primary storage.Next execution of the usage server and on-wards, usage entry for usage_type=6 only will be added.

api/src/org/apache/cloudstack/usage/UsageTypes.java

@@ -42,6 +42,7 @@ public class UsageTypes {
     public static final int VM_DISK_BYTES_READ = 23;
     public static final int VM_DISK_BYTES_WRITE = 24;
     public static final int VM_SNAPSHOT = 25;
+    public static final int VOLUME_SECONDARY = 26;
 
     public static List<UsageTypeResponse> listUsageTypes() {
         List<UsageTypeResponse> responseList = new ArrayList<UsageTypeResponse>();

test/integration/component/test_ss_volume_usage.py

@@ -0,0 +1,203 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+""" Test cases for checking that the secondary Storage usage is accounted. This is verified by checking the usage_event table
+for a volume in 'Uploaded' state.
+    This test case does the following:
+    1.Creates an account and uploads a volume.
+    2.After the volume is uploaded successfully, connects to the database
+    3.From the database verifies that an entry is added to cloud.events table for the uploaded volume.
+    4.Cleans up the resources.
+"""
+
+from marvin.cloudstackTestCase import *
+from marvin.cloudstackAPI import *
+from marvin.lib.utils import *
+from marvin.lib.base import *
+from marvin.lib.common import *
+from nose.plugins.attrib import attr
+from marvin.sshClient import SshClient
+from marvin.codes import (BACKED_UP, PASS, FAIL)
+import time
+
+
+def verify_vm(self, vmid, state):
+    list_vm = list_virtual_machines(self.userapiclient,
+                                    account=self.account.name,
+                                    domainid=self.account.domainid,
+                                    id=vmid
+                                    )
+    self.assertEqual(
+        validateList(list_vm)[0],
+        PASS,
+        "Check List vm response for vmid: %s" %
+        vmid)
+    self.assertGreater(
+        len(list_vm),
+        0,
+        "Check the list vm response for vm id:  %s" %
+        vmid)
+    vm = list_vm[0]
+    self.assertEqual(
+        vm.id,
+        str(vmid),
+        "Vm deployed is different from the test")
+    self.assertEqual(vm.state, state, "VM is in %s state" %state)
+
+
+def uploadVolume(self):
+    # upload a volume
+    self.debug("Upload volume format is '%s'" %self.uploadVolumeformat)
+    self.testdata["configurableData"]["upload_volume"]["format"] = self.uploadVolumeformat
+    self.testdata["configurableData"]["upload_volume"]["url"] = self.uploadvolumeUrl
+    upload_volume = Volume.upload(
+        self.apiclient,
+        self.testdata["configurableData"]["upload_volume"],
+        account=self.account.name,
+        domainid=self.domain.id,
+        zoneid=self.zone.id
+    )
+    upload_volume.wait_for_upload(self.apiclient)
+    return upload_volume.id
+
+def restartUsageServer(self):
+    #Restart usage server
+
+    sshClient = SshClient(
+        self.mgtSvrDetails["mgtSvrIp"],
+        22,
+        self.mgtSvrDetails["user"],
+        self.mgtSvrDetails["passwd"]
+    )
+    command = "service cloudstack-usage restart"
+    sshClient.execute(command)
+    return
+
+def checkUsage(self, uuid_upload_volume_id):
+    volume_id = self.dbclient.execute("SELECT id from cloud.volumes where uuid='%s';" % uuid_upload_volume_id)
+    self.debug("Volume id of uploaded volume is= %s" %volume_id[0]);
+    qryresult_after_usageServerExecution = self.dbclient.execute(
+        "SELECT type FROM cloud.usage_event where resource_id = '%s';" % (volume_id[0]))
+    self.debug("Usage Type is %s " % qryresult_after_usageServerExecution[0][0])
+    self.assertEqual(qryresult_after_usageServerExecution[0][0], 'VOLUME.UPLOAD')
+
+class TestSecondaryVolumeUsage(cloudstackTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        testClient = super(TestSecondaryVolumeUsage, cls).getClsTestClient()
+        cls.apiclient = testClient.getApiClient()
+        cls.dbclient = testClient.getDbConnection()
+        cls.testdata = testClient.getParsedTestDataConfig()
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+        cls.storagetype = 'shared'
+        # Get Zone, Domain and templates
+        cls.domain = get_domain(cls.apiclient)
+        cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
+        cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__
+        cls._cleanup = []
+
+        # Create an account
+        cls.account = Account.create(
+            cls.apiclient,
+            cls.testdata["account"],
+            domainid=cls.domain.id
+        )
+        cls._cleanup.append(cls.account)
+
+        # Create user api client of the account
+        cls.userapiclient = testClient.getUserApiClient(
+            UserName=cls.account.name,
+            DomainName=cls.account.domain
+        )
+
+        # Create Service offering
+        cls.service_offering = ServiceOffering.create(
+            cls.apiclient,
+            cls.testdata["service_offering"],
+        )
+        cls._cleanup.append(cls.service_offering)
+
+        cls.disk_offering = DiskOffering.create(
+            cls.apiclient,
+            cls.testdata["disk_offering"],
+        )
+
+        cls._cleanup.append(cls.disk_offering)
+
+        cls.skip = 0
+        hosts = list_hosts(
+            cls.apiclient,
+            type="Routing"
+        )
+
+        for hypervisorhost in hosts:
+            if hypervisorhost.hypervisor.lower() in ["xenserver"]:
+                cls.uploadVolumeformat = "VHD"
+                cls.uploadvolumeUrl = "http://download.cloudstack.org/releases/2.0.0/systemvm.vhd.bz2"
+                break
+            elif hypervisorhost.hypervisor.lower() in ["vmware"]:
+                cls.uploadVolumeformat = "OVA"
+                cls.uploadvolumeUrl = "http://download.cloudstack.org/releases/2.2.0/systemvm-redundant-router.ova"
+                break
+            elif hypervisorhost.hypervisor == "KVM":
+                cls.uploadVolumeformat = "QCOW2"
+                cls.uploadvolumeUrl = "http://download.cloudstack.org/releases/2.0.0/UbuntuServer-10-04-64bit.qcow2.bz2"
+                break
+            elif hypervisorhost.hypervisor == "LXC":
+                cls.uploadvolumeformat = "QCOW2"
+                cls.uploadvolumeUrl = "http://download.cloudstack.org/releases/2.0.0/UbuntuServer-10-04-64bit.qcow2.bz2"
+                break
+            else:
+                break
+
+        cls.template = get_template(
+            cls.apiclient,
+            cls.zone.id,
+            cls.testdata["ostype"])
+
+        try:
+            cls.vm = VirtualMachine.create(
+                cls.userapiclient,
+                cls.testdata["small"],
+                templateid=cls.template.id,
+                accountid=cls.account.name,
+                domainid=cls.account.domainid,
+                serviceofferingid=cls.service_offering.id,
+                zoneid=cls.zone.id
+            )
+
+        except Exception as e:
+            cls.tearDownClass()
+            raise e
+        return
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls._cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+
+    @attr(tags=["basic", "advanced"], required_hardware="true")
+    def test_01_SecondaryUsageUploadedVolume(self):
+        try:
+            uploaded_volume_id_uuid = uploadVolume(self)
+            checkUsage(self, uploaded_volume_id_uuid)
+        except Exception as e:
+            self.tearDown()
+            raise e
+        return

usage/src/com/cloud/usage/StorageTypes.java

@@ -20,4 +20,5 @@ public class StorageTypes {
     public static final int TEMPLATE = 1;
     public static final int ISO = 2;
     public static final int SNAPSHOT = 3;
+    public static final int VOLUME = 4;
 }

usage/src/com/cloud/usage/UsageManagerImpl.java

@@ -985,7 +985,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
 
     private boolean isVolumeEvent(String eventType) {
         return eventType != null &&
-                (eventType.equals(EventTypes.EVENT_VOLUME_CREATE) || eventType.equals(EventTypes.EVENT_VOLUME_DELETE) || eventType.equals(EventTypes.EVENT_VOLUME_RESIZE));
+                (eventType.equals(EventTypes.EVENT_VOLUME_CREATE) || eventType.equals(EventTypes.EVENT_VOLUME_DELETE) || eventType.equals(EventTypes.EVENT_VOLUME_RESIZE) || eventType.equals(EventTypes.EVENT_VOLUME_UPLOAD));
     }
 
     private boolean isTemplateEvent(String eventType) {
@@ -1390,6 +1390,21 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
 
         long volId = event.getResourceId();
 
+        if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType())) {
+            //For volumes which are 'attached' successfully, set the 'deleted' column in the usage_storage table,
+            //so that the secondary storage should stop accounting and only primary will be accounted.
+            SearchCriteria<UsageStorageVO> sc = _usageStorageDao.createSearchCriteria();
+            sc.addAnd("id", SearchCriteria.Op.EQ, volId);
+            sc.addAnd("storageType", SearchCriteria.Op.EQ, StorageTypes.VOLUME);
+            List<UsageStorageVO> volumesVOs = _usageStorageDao.search(sc, null);
+            if (volumesVOs != null) {
+                if (volumesVOs.size() == 1) {
+                    s_logger.debug("Setting the volume with id: " + volId + " to 'deleted' in the usage_storage table.");
+                    volumesVOs.get(0).setDeleted(event.getCreateDate());
+                    _usageStorageDao.update(volumesVOs.get(0));
+                }
+            }
+        }
         if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType()) || EventTypes.EVENT_VOLUME_RESIZE.equals(event.getType())) {
             SearchCriteria<UsageVolumeVO> sc = _usageVolumeDao.createSearchCriteria();
             sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
@@ -1430,6 +1445,32 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
                 volumesVO.setDeleted(event.getCreateDate()); // there really shouldn't be more than one
                 _usageVolumeDao.update(volumesVO);
             }
+        } else if (EventTypes.EVENT_VOLUME_UPLOAD.equals(event.getType())) {
+            //For Upload event add an entry to the usage_storage table.
+            SearchCriteria<UsageStorageVO> sc = _usageStorageDao.createSearchCriteria();
+            sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
+            sc.addAnd("id", SearchCriteria.Op.EQ, volId);
+            sc.addAnd("deleted", SearchCriteria.Op.NULL);
+            List<UsageStorageVO> volumesVOs = _usageStorageDao.search(sc, null);
+
+            if (volumesVOs.size() > 0) {
+                //This is a safeguard to avoid double counting of volumes.
+                s_logger.error("Found duplicate usage entry for volume: " + volId + " assigned to account: " + event.getAccountId() + "; marking as deleted...");
+            }
+            for (UsageStorageVO volumesVO : volumesVOs) {
+                if (s_logger.isDebugEnabled()) {
+                    s_logger.debug("deleting volume: " + volumesVO.getId() + " from account: " + volumesVO.getAccountId());
+                }
+                volumesVO.setDeleted(event.getCreateDate());
+                _usageStorageDao.update(volumesVO);
+            }
+
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("create volume with id : " + volId + " for account: " + event.getAccountId());
+            }
+            Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId());
+            UsageStorageVO volumeVO = new UsageStorageVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), StorageTypes.VOLUME, event.getTemplateId(), event.getSize(), event.getCreateDate(), null);
+            _usageStorageDao.persist(volumeVO);
         }
     }
 

usage/src/com/cloud/usage/parser/StorageUsageParser.java

@@ -180,6 +180,10 @@ public class StorageUsageParser {
                 usage_type = UsageTypes.SNAPSHOT;
                 usageDesc += "Snapshot ";
                 break;
+            case StorageTypes.VOLUME:
+                usage_type = UsageTypes.VOLUME_SECONDARY;
+                usageDesc += "Volume ";
+                break;
         }
         //Create the usage record
         usageDesc += "Id:" + storageId + " Size:" + size;