diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..6ca3ad4aa23 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cloudstack/tools/docker/Dockerfile +.dockerignore +.idea +.git +venv diff --git a/.gitignore b/.gitignore index bdf411897b7..7b1874991c8 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,4 @@ plugins/hypervisors/kvm/.pydevproject scripts/.pydevproject *.qcow2 *.raw +venv diff --git a/.travis.yml b/.travis.yml index 788f02d70ed..f176f5788ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,6 +86,7 @@ env: smoke/test_pvlan smoke/test_regions smoke/test_reset_vm_on_reboot + smoke/test_resource_accounting smoke/test_resource_detail smoke/test_router_dhcphosts smoke/test_router_dns diff --git a/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java b/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java index dbf2228183b..4bd8302a345 100644 --- a/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java @@ -248,9 +248,10 @@ public class ResourceCountDaoImpl extends GenericDaoBase + " ELSE CONVERT(vmd.value, UNSIGNED INTEGER) " + " END)) as total " + " from vm_instance vm " - + " join service_offering_view so on so.id = vm.service_offering_id " + + " join service_offering so on so.id = vm.service_offering_id " + " left join user_vm_details vmd on vmd.vm_id = vm.id and vmd.name = '%s' " + " where vm.type = 'User' and state not in ('Destroyed', 'Error', 'Expunging') and display_vm = true and account_id = ? "; + @Override public long countCpuNumberAllocatedToAccount(long accountId) { String sqlCountCpuNumberAllocatedToAccount = String.format(baseSqlCountComputingResourceAllocatedToAccount, ResourceType.cpu, ResourceType.cpu, "cpuNumber"); diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index 10b6d3cfdbe..ab0cee60039 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -524,15 +524,16 @@ class CsIP: CsHelper.execute("sudo ip route add throw " + self.config.address().dbag['eth1'][0]['network'] + " table " + tableName + " proto static") # add 'defaul via gateway' rule in the device specific routing table - if "gateway" in self.address and self.address["gateway"] != "None": + if "gateway" in self.address and self.address["gateway"] and self.address["gateway"] != "None": route.add_route(self.dev, self.address["gateway"]) - route.add_network_route(self.dev, str(self.address["network"])) + if "network" in self.address and self.address["network"]: + route.add_network_route(self.dev, str(self.address["network"])) if self.get_type() in ["public"]: CsRule(self.dev).addRule("from " + str(self.address["network"])) if self.config.is_vpc(): - if self.get_type() in ["public"] and "gateway" in self.address and self.address["gateway"] != "None": + if self.get_type() in ["public"] and "gateway" in self.address and self.address["gateway"] and self.address["gateway"] != "None": route.add_route(self.dev, self.address["gateway"]) for inf, addresses in self.config.address().dbag.iteritems(): if not inf.startswith("eth"): diff --git a/systemvm/debian/opt/cloud/bin/cs/CsRoute.py b/systemvm/debian/opt/cloud/bin/cs/CsRoute.py index 47d3d2a91af..ce87bb0533c 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsRoute.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsRoute.py @@ -50,20 +50,29 @@ class CsRoute: """ Wrapper method that adds table name and device to route statement """ # ip route add dev eth1 table Table_eth1 10.0.2.0/24 table = self.get_tablename(dev) - logging.info("Adding route: dev " + dev + " table: " + - table + " network: " + address + " if not present") - cmd = "dev %s table %s %s" % (dev, table, address) - cmd = "default via %s table %s proto static" % (address, table) - self.set_route(cmd) + + if not table or not address: + empty_param = "table" if not table else "address" + logging.info("Empty parameter received %s while trying to add route, skipping" % empty_param) + else: + logging.info("Adding route: dev " + dev + " table: " + + table + " network: " + address + " if not present") + cmd = "default via %s table %s proto static" % (address, table) + self.set_route(cmd) def add_network_route(self, dev, address): """ Wrapper method that adds table name and device to route statement """ # ip route add dev eth1 table Table_eth1 10.0.2.0/24 table = self.get_tablename(dev) - logging.info("Adding route: dev " + dev + " table: " + - table + " network: " + address + " if not present") - cmd = "throw %s table %s proto static" % (address, table) - self.set_route(cmd) + + if not table or not address: + empty_param = "table" if not table else "address" + logging.info("Empty parameter received %s while trying to add network route, skipping" % empty_param) + else: + logging.info("Adding route: dev " + dev + " table: " + + table + " network: " + address + " if not present") + cmd = "throw %s table %s proto static" % (address, table) + self.set_route(cmd) def set_route(self, cmd, method="add"): """ Add a route if it is not already defined """ diff --git a/test/integration/component/test_browse_templates.py b/test/integration/component/test_browse_templates.py index 80e9a135b80..4340092b9a0 100644 --- a/test/integration/component/test_browse_templates.py +++ b/test/integration/component/test_browse_templates.py @@ -230,7 +230,7 @@ class TestBrowseUploadVolume(cloudstackTestCase): return(totaltemplates) - def getstoragelimts(self,rtype): + def getstoragelimits(self, rtype): cmd=updateResourceCount.updateResourceCountCmd() cmd.account=self.account.name @@ -1652,7 +1652,7 @@ class TestBrowseUploadVolume(cloudstackTestCase): self.debug("========================= Test 22 Upload template and verify secondary storage limits========================") - initialsecondarystoragelimit=self.getstoragelimts(11) + initialsecondarystoragelimit=self.getstoragelimits(11) browseup_template1=self.browse_upload_template() tmpldetails=Template.list( @@ -1662,7 +1662,7 @@ class TestBrowseUploadVolume(cloudstackTestCase): zoneid=self.zone.id) - afteruploadsecondarystoragelimit=self.getstoragelimts(11) + afteruploadsecondarystoragelimit=self.getstoragelimits(11) if afteruploadsecondarystoragelimit!=(initialsecondarystoragelimit+tmpldetails[0].size): self.fail("Secondary Storage Resouce Count is not updated") @@ -1709,10 +1709,10 @@ class TestBrowseUploadVolume(cloudstackTestCase): templatefilter="all", zoneid=self.zone.id) - initialuploadprimarystoragelimit=self.getstoragelimts(11) + initialuploadprimarystoragelimit=self.getstoragelimits(11) self.delete_template(browseup_template1) - afteruploadprimarystoragelimit=self.getstoragelimts(11) + afteruploadprimarystoragelimit=self.getstoragelimits(11) if afteruploadprimarystoragelimit!=(initialuploadprimarystoragelimit-tmpldetails[0].size): self.fail("Secondary Storage Resource Count is not updated after deletion") diff --git a/test/integration/smoke/test_resource_accounting.py b/test/integration/smoke/test_resource_accounting.py new file mode 100644 index 00000000000..043a7af86df --- /dev/null +++ b/test/integration/smoke/test_resource_accounting.py @@ -0,0 +1,243 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from nose.plugins.attrib import attr + +# Import Local Modules +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.base import (Domain, + Account, + ServiceOffering, + VirtualMachine, updateResourceCount) +from marvin.lib.common import (get_zone, + get_test_template) +from marvin.lib.utils import (cleanup_resources) + + +class Services: + """Test Account Services + """ + + def __init__(self): + self.services = { + "domain": { + "name": "Domain", + }, + "account": { + "email": "test@test.com", + "firstname": "Test", + "lastname": "User", + "username": "test", + # Random characters are appended for unique + # username + "password": "fr3sca", + }, + "user": { + "email": "user@test.com", + "firstname": "User", + "lastname": "User", + "username": "User", + # Random characters are appended for unique + # username + "password": "fr3sca", + }, + "service_offering_it_1": { + "name": "InstanceType-1", + "displaytext": "Tiny Instance", + "cpunumber": 1, + "cpuspeed": 100, + "memory": 128, + }, + "service_offering_it_2": { + "name": "InstanceType-2", + "displaytext": "Tiny Instance", + "cpunumber": 2, + "cpuspeed": 100, + "memory": 512, + }, + "virtual_machine_1": { + "displayname": "Test VM1", + "username": "root", + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + }, + "virtual_machine_2": { + "displayname": "Test VM2", + "username": "root", + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + }, + "template": { + "displaytext": "Public Template", + "name": "Public template", + "ostype": 'CentOS 5.6 (64-bit)', + "url": "", + "hypervisor": '', + "format": '', + "isfeatured": True, + "ispublic": True, + "isextractable": True, + "templatefilter": "self" + }, + "natrule": { + "publicport": 22, + "privateport": 22, + "protocol": 'TCP', + }, + "ostype": 'CentOS 5.6 (64-bit)', + "sleep": 60, + "timeout": 10, + } + + +class TestRAMCPUResourceAccounting(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + cls.testClient = super( + TestRAMCPUResourceAccounting, + cls).getClsTestClient() + cls.api_client = cls.testClient.getApiClient() + + cls.services = Services().services + cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + cls.hypervisor = cls.testClient.getHypervisorInfo() + # Create an account, domain etc + cls.domain = Domain.create( + cls.api_client, + cls.services["domain"], + ) + cls.account = Account.create( + cls.api_client, + cls.services["account"], + admin=True, + domainid=cls.domain.id + ) + + cls.template = get_test_template( + cls.api_client, + cls.zone.id, + cls.hypervisor) + + cls.services["virtual_machine_1"]["zoneid"] = cls.zone.id + cls.services["virtual_machine_1"]["template"] = cls.template.id + + cls.services["virtual_machine_2"]["zoneid"] = cls.zone.id + cls.services["virtual_machine_2"]["template"] = cls.template.id + + cls._cleanup = [ + cls.account, + cls.domain + ] + + @classmethod + def tearDownClass(cls): + try: + # Cleanup resources used + cleanup_resources(cls.api_client, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + + def tearDown(self): + try: + # Clean up, terminate the created accounts + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def get_resource_amount(self, resource_type): + cmd = updateResourceCount.updateResourceCountCmd() + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.resourcetype = resource_type + response = self.apiclient.updateResourceCount(cmd) + amount = response[0].resourcecount + return amount + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_01_so_removal_resource_update(self): + + self.service_offering_it_1 = ServiceOffering.create( + self.api_client, + self.services["service_offering_it_1"], + domainid=self.domain.id + ) + + self.cleanup.append(self.service_offering_it_1) + + self.service_offering_it_2 = ServiceOffering.create( + self.api_client, + self.services["service_offering_it_2"], + domainid=self.domain.id + ) + + self.cleanup.append(self.service_offering_it_2) + + vm_1 = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine_1"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering_it_1.id + ) + + self.debug("Deployed VM in account: %s, ID: %s" % (self.account.name, vm_1.id)) + self.cleanup.append(vm_1) + + vm_2 = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine_2"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering_it_2.id + ) + + self.debug("Deployed VM in account: %s, ID: %s" % (self.account.name, vm_2.id)) + self.cleanup.append(vm_2) + + CPU_RESOURCE_ID = 8 + RAM_RESOURCE_ID = 9 + + cores = int(self.get_resource_amount(CPU_RESOURCE_ID)) + ram = int(self.get_resource_amount(RAM_RESOURCE_ID)) + + self.assertEqual(cores, self.services['service_offering_it_1']['cpunumber'] + self.services['service_offering_it_2']['cpunumber']) + self.assertEqual(ram, self.services['service_offering_it_1']['memory'] + self.services['service_offering_it_2']['memory']) + + self.service_offering_it_2.delete(self.apiclient) + + self.cleanup = self.cleanup[0:-1] + + cores = int(self.get_resource_amount(CPU_RESOURCE_ID)) + ram = int(self.get_resource_amount(RAM_RESOURCE_ID)) + + self.assertEqual(cores, self.services['service_offering_it_1']['cpunumber'] + self.services['service_offering_it_2']['cpunumber']) + self.assertEqual(ram, self.services['service_offering_it_1']['memory'] + self.services['service_offering_it_2']['memory']) diff --git a/tools/appliance/systemvmtemplate/template.json b/tools/appliance/systemvmtemplate/template.json index 4810840dbbe..1c62b1e0942 100644 --- a/tools/appliance/systemvmtemplate/template.json +++ b/tools/appliance/systemvmtemplate/template.json @@ -38,8 +38,8 @@ "disk_interface": "virtio", "net_device": "virtio-net", - "iso_url": "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-9.5.0-amd64-netinst.iso", - "iso_checksum": "efe75000c066506326c74a97257163b3050d656a5be8708a6826b0f810208d0a58f413c446de09919c580de8fac6d0a47774534725dd9fdd00c94859e370f373", + "iso_url": "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-9.6.0-amd64-netinst.iso", + "iso_checksum": "fcd77acbd46f33e0a266faf284acc1179ab0a3719e4b8abebac555307aa978aa242d7052c8d41e1a5fc6d1b30bc6ca6d62269e71526b71c9d5199b13339f0e25", "iso_checksum_type": "sha512", "vm_name": "systemvmtemplate",