Merge remote-tracking branch 'origin/4.11'

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

.dockerignore

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cloudstack/tools/docker/Dockerfile
+.dockerignore
+.idea
+.git
+venv

.gitignore

@@ -98,3 +98,4 @@ plugins/hypervisors/kvm/.pydevproject
 scripts/.pydevproject
 *.qcow2
 *.raw
+venv

.travis.yml

@@ -86,6 +86,7 @@ env:
              smoke/test_pvlan
              smoke/test_regions
              smoke/test_reset_vm_on_reboot
+             smoke/test_resource_accounting
              smoke/test_resource_detail
              smoke/test_router_dhcphosts
              smoke/test_router_dns

engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java

@@ -248,9 +248,10 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
             + "        ELSE CONVERT(vmd.value, UNSIGNED INTEGER) "
             + "    END)) as total "
             + " from vm_instance vm "
-            + " join service_offering_view so on so.id = vm.service_offering_id "
+            + " join service_offering so on so.id = vm.service_offering_id "
             + " left join user_vm_details vmd on vmd.vm_id = vm.id and vmd.name = '%s' "
             + " where vm.type = 'User' and state not in ('Destroyed', 'Error', 'Expunging') and display_vm = true and account_id = ? ";
+
     @Override
     public long countCpuNumberAllocatedToAccount(long accountId) {
         String sqlCountCpuNumberAllocatedToAccount = String.format(baseSqlCountComputingResourceAllocatedToAccount, ResourceType.cpu, ResourceType.cpu, "cpuNumber");

systemvm/debian/opt/cloud/bin/cs/CsAddress.py

@@ -524,15 +524,16 @@ class CsIP:
                     CsHelper.execute("sudo ip route add throw " + self.config.address().dbag['eth1'][0]['network'] + " table " + tableName + " proto static")
 
                 # add 'defaul via gateway' rule in the device specific routing table
-                if "gateway" in self.address and self.address["gateway"] != "None":
+                if "gateway" in self.address and self.address["gateway"] and self.address["gateway"] != "None":
                     route.add_route(self.dev, self.address["gateway"])
+                if "network" in self.address and self.address["network"]:
                     route.add_network_route(self.dev, str(self.address["network"]))
 
                 if self.get_type() in ["public"]:
                     CsRule(self.dev).addRule("from " + str(self.address["network"]))
 
             if self.config.is_vpc():
-                if self.get_type() in ["public"] and "gateway" in self.address and self.address["gateway"] != "None":
+                if self.get_type() in ["public"] and "gateway" in self.address and self.address["gateway"] and self.address["gateway"] != "None":
                     route.add_route(self.dev, self.address["gateway"])
                     for inf, addresses in self.config.address().dbag.iteritems():
                         if not inf.startswith("eth"):

systemvm/debian/opt/cloud/bin/cs/CsRoute.py

@@ -50,9 +50,13 @@ class CsRoute:
         """ Wrapper method that adds table name and device to route statement """
         # ip route add dev eth1 table Table_eth1 10.0.2.0/24
         table = self.get_tablename(dev)
+
+        if not table or not address:
+            empty_param = "table" if not table else "address"
+            logging.info("Empty parameter received %s while trying to add route, skipping" % empty_param)
+        else:
             logging.info("Adding route: dev " + dev + " table: " +
                          table + " network: " + address + " if not present")
-        cmd = "dev %s table %s %s" % (dev, table, address)
             cmd = "default via %s table %s proto static" % (address, table)
             self.set_route(cmd)
 
@@ -60,6 +64,11 @@ class CsRoute:
         """ Wrapper method that adds table name and device to route statement """
         # ip route add dev eth1 table Table_eth1 10.0.2.0/24
         table = self.get_tablename(dev)
+
+        if not table or not address:
+            empty_param = "table" if not table else "address"
+            logging.info("Empty parameter received %s while trying to add network route, skipping" % empty_param)
+        else:
             logging.info("Adding route: dev " + dev + " table: " +
                          table + " network: " + address + " if not present")
             cmd = "throw %s table %s proto static" % (address, table)

test/integration/component/test_browse_templates.py

@@ -230,7 +230,7 @@ class TestBrowseUploadVolume(cloudstackTestCase):
 
         return(totaltemplates)
 
-    def getstoragelimts(self,rtype):
+    def getstoragelimits(self, rtype):
 
         cmd=updateResourceCount.updateResourceCountCmd()
         cmd.account=self.account.name
@@ -1652,7 +1652,7 @@ class TestBrowseUploadVolume(cloudstackTestCase):
 
             self.debug("========================= Test 22 Upload template and verify secondary storage limits========================")
 
-            initialsecondarystoragelimit=self.getstoragelimts(11)
+            initialsecondarystoragelimit=self.getstoragelimits(11)
             browseup_template1=self.browse_upload_template()
 
             tmpldetails=Template.list(
@@ -1662,7 +1662,7 @@ class TestBrowseUploadVolume(cloudstackTestCase):
                                      zoneid=self.zone.id)
 
 
-            afteruploadsecondarystoragelimit=self.getstoragelimts(11)
+            afteruploadsecondarystoragelimit=self.getstoragelimits(11)
 
             if afteruploadsecondarystoragelimit!=(initialsecondarystoragelimit+tmpldetails[0].size):
                 self.fail("Secondary Storage Resouce Count is not updated")
@@ -1709,10 +1709,10 @@ class TestBrowseUploadVolume(cloudstackTestCase):
                                      templatefilter="all",
                                      zoneid=self.zone.id)
 
-            initialuploadprimarystoragelimit=self.getstoragelimts(11)
+            initialuploadprimarystoragelimit=self.getstoragelimits(11)
             self.delete_template(browseup_template1)
 
-            afteruploadprimarystoragelimit=self.getstoragelimts(11)
+            afteruploadprimarystoragelimit=self.getstoragelimits(11)
 
             if afteruploadprimarystoragelimit!=(initialuploadprimarystoragelimit-tmpldetails[0].size):
                 self.fail("Secondary Storage Resource Count is not updated after deletion")

test/integration/smoke/test_resource_accounting.py

@@ -0,0 +1,243 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from nose.plugins.attrib import attr
+
+# Import Local Modules
+from marvin.cloudstackTestCase import cloudstackTestCase
+from marvin.lib.base import (Domain,
+                             Account,
+                             ServiceOffering,
+                             VirtualMachine, updateResourceCount)
+from marvin.lib.common import (get_zone,
+                               get_test_template)
+from marvin.lib.utils import (cleanup_resources)
+
+
+class Services:
+    """Test Account Services
+    """
+
+    def __init__(self):
+        self.services = {
+            "domain": {
+                "name": "Domain",
+            },
+            "account": {
+                "email": "test@test.com",
+                "firstname": "Test",
+                "lastname": "User",
+                "username": "test",
+                # Random characters are appended for unique
+                # username
+                "password": "fr3sca",
+            },
+            "user": {
+                "email": "user@test.com",
+                "firstname": "User",
+                "lastname": "User",
+                "username": "User",
+                # Random characters are appended for unique
+                # username
+                "password": "fr3sca",
+            },
+            "service_offering_it_1": {
+                "name": "InstanceType-1",
+                "displaytext": "Tiny Instance",
+                "cpunumber": 1,
+                "cpuspeed": 100,
+                "memory": 128,
+            },
+            "service_offering_it_2": {
+                "name": "InstanceType-2",
+                "displaytext": "Tiny Instance",
+                "cpunumber": 2,
+                "cpuspeed": 100,
+                "memory": 512,
+            },
+            "virtual_machine_1": {
+                "displayname": "Test VM1",
+                "username": "root",
+                "password": "password",
+                "ssh_port": 22,
+                "hypervisor": 'XenServer',
+                "privateport": 22,
+                "publicport": 22,
+                "protocol": 'TCP',
+            },
+            "virtual_machine_2": {
+                "displayname": "Test VM2",
+                "username": "root",
+                "password": "password",
+                "ssh_port": 22,
+                "hypervisor": 'XenServer',
+                "privateport": 22,
+                "publicport": 22,
+                "protocol": 'TCP',
+            },
+            "template": {
+                "displaytext": "Public Template",
+                "name": "Public template",
+                "ostype": 'CentOS 5.6 (64-bit)',
+                "url": "",
+                "hypervisor": '',
+                "format": '',
+                "isfeatured": True,
+                "ispublic": True,
+                "isextractable": True,
+                "templatefilter": "self"
+            },
+            "natrule": {
+                "publicport": 22,
+                "privateport": 22,
+                "protocol": 'TCP',
+            },
+            "ostype": 'CentOS 5.6 (64-bit)',
+            "sleep": 60,
+            "timeout": 10,
+        }
+
+
+class TestRAMCPUResourceAccounting(cloudstackTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.testClient = super(
+            TestRAMCPUResourceAccounting,
+            cls).getClsTestClient()
+        cls.api_client = cls.testClient.getApiClient()
+
+        cls.services = Services().services
+        cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests())
+        cls.services['mode'] = cls.zone.networktype
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+        # Create an account, domain etc
+        cls.domain = Domain.create(
+            cls.api_client,
+            cls.services["domain"],
+        )
+        cls.account = Account.create(
+            cls.api_client,
+            cls.services["account"],
+            admin=True,
+            domainid=cls.domain.id
+        )
+
+        cls.template = get_test_template(
+            cls.api_client,
+            cls.zone.id,
+            cls.hypervisor)
+
+        cls.services["virtual_machine_1"]["zoneid"] = cls.zone.id
+        cls.services["virtual_machine_1"]["template"] = cls.template.id
+
+        cls.services["virtual_machine_2"]["zoneid"] = cls.zone.id
+        cls.services["virtual_machine_2"]["template"] = cls.template.id
+
+        cls._cleanup = [
+            cls.account,
+            cls.domain
+        ]
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            # Cleanup resources used
+            cleanup_resources(cls.api_client, cls._cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.dbclient = self.testClient.getDbConnection()
+        self.cleanup = []
+
+    def tearDown(self):
+        try:
+            # Clean up, terminate the created accounts
+            cleanup_resources(self.apiclient, self.cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+
+    def get_resource_amount(self, resource_type):
+        cmd = updateResourceCount.updateResourceCountCmd()
+        cmd.account = self.account.name
+        cmd.domainid = self.domain.id
+        cmd.resourcetype = resource_type
+        response = self.apiclient.updateResourceCount(cmd)
+        amount = response[0].resourcecount
+        return amount
+
+    @attr(tags=["advanced", "basic"], required_hardware="false")
+    def test_01_so_removal_resource_update(self):
+
+        self.service_offering_it_1 = ServiceOffering.create(
+            self.api_client,
+            self.services["service_offering_it_1"],
+            domainid=self.domain.id
+        )
+
+        self.cleanup.append(self.service_offering_it_1)
+
+        self.service_offering_it_2 = ServiceOffering.create(
+            self.api_client,
+            self.services["service_offering_it_2"],
+            domainid=self.domain.id
+        )
+
+        self.cleanup.append(self.service_offering_it_2)
+
+        vm_1 = VirtualMachine.create(
+            self.apiclient,
+            self.services["virtual_machine_1"],
+            accountid=self.account.name,
+            domainid=self.account.domainid,
+            serviceofferingid=self.service_offering_it_1.id
+        )
+
+        self.debug("Deployed VM in account: %s, ID: %s" % (self.account.name, vm_1.id))
+        self.cleanup.append(vm_1)
+
+        vm_2 = VirtualMachine.create(
+            self.apiclient,
+            self.services["virtual_machine_2"],
+            accountid=self.account.name,
+            domainid=self.account.domainid,
+            serviceofferingid=self.service_offering_it_2.id
+        )
+
+        self.debug("Deployed VM in account: %s, ID: %s" % (self.account.name, vm_2.id))
+        self.cleanup.append(vm_2)
+
+        CPU_RESOURCE_ID = 8
+        RAM_RESOURCE_ID = 9
+
+        cores = int(self.get_resource_amount(CPU_RESOURCE_ID))
+        ram = int(self.get_resource_amount(RAM_RESOURCE_ID))
+
+        self.assertEqual(cores, self.services['service_offering_it_1']['cpunumber'] + self.services['service_offering_it_2']['cpunumber'])
+        self.assertEqual(ram, self.services['service_offering_it_1']['memory'] + self.services['service_offering_it_2']['memory'])
+
+        self.service_offering_it_2.delete(self.apiclient)
+
+        self.cleanup = self.cleanup[0:-1]
+
+        cores = int(self.get_resource_amount(CPU_RESOURCE_ID))
+        ram = int(self.get_resource_amount(RAM_RESOURCE_ID))
+
+        self.assertEqual(cores, self.services['service_offering_it_1']['cpunumber'] + self.services['service_offering_it_2']['cpunumber'])
+        self.assertEqual(ram, self.services['service_offering_it_1']['memory'] + self.services['service_offering_it_2']['memory'])

tools/appliance/systemvmtemplate/template.json

@@ -38,8 +38,8 @@
       "disk_interface": "virtio",
       "net_device": "virtio-net",
 
-      "iso_url": "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-9.5.0-amd64-netinst.iso",
-      "iso_checksum": "efe75000c066506326c74a97257163b3050d656a5be8708a6826b0f810208d0a58f413c446de09919c580de8fac6d0a47774534725dd9fdd00c94859e370f373",
+      "iso_url": "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-9.6.0-amd64-netinst.iso",
+      "iso_checksum": "fcd77acbd46f33e0a266faf284acc1179ab0a3719e4b8abebac555307aa978aa242d7052c8d41e1a5fc6d1b30bc6ca6d62269e71526b71c9d5199b13339f0e25",
       "iso_checksum_type": "sha512",
 
       "vm_name": "systemvmtemplate",