From ab580de38d6517598571034cb5caa56ba5cdbdf1 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Tue, 9 Dec 2014 16:45:04 +0530 Subject: [PATCH 01/39] CLOUDSTACK-8053: Fix loop termination - test_vpc_host_maintenance.py Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/maint/test_vpc_host_maintenance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/component/maint/test_vpc_host_maintenance.py b/test/integration/component/maint/test_vpc_host_maintenance.py index 702746918cf..74d59e736b9 100644 --- a/test/integration/component/maint/test_vpc_host_maintenance.py +++ b/test/integration/component/maint/test_vpc_host_maintenance.py @@ -221,6 +221,7 @@ class TestVMLifeCycleHostmaintenance(cloudstackTestCase): cls.hosts = Host.list(cls.api_client, clusterid=cluster.id) if len(cls.hosts) >= 2: clusterWithSufficientHosts = cluster + break if clusterWithSufficientHosts is None: raise unittest.SkipTest("No Cluster with 2 hosts found") From b23fd5ee53c228681697359b0ad007ba5b539416 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Tue, 9 Dec 2014 12:29:17 +0530 Subject: [PATCH 02/39] CLOUDSTACK-8049: Tagging test cases which cannot be run on simulator accordingly Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_ps_resize_volume.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/component/test_ps_resize_volume.py b/test/integration/component/test_ps_resize_volume.py index 5df4a6279f0..bff16725f5b 100644 --- a/test/integration/component/test_ps_resize_volume.py +++ b/test/integration/component/test_ps_resize_volume.py @@ -159,7 +159,7 @@ class TestResizeVolume(cloudstackTestCase): return [FAIL, e] return [PASS, None] - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_01_increase_volume_size_within_account_limit(self): """Test increasing volume size within the account limit and verify primary storage usage @@ -223,7 +223,7 @@ class TestResizeVolume(cloudstackTestCase): self.fail("Failed with exception: %s" % e) return - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_02_increase_volume_size_above_account_limit(self): """Test increasing volume size above the account limit @@ -281,7 +281,7 @@ class TestResizeVolume(cloudstackTestCase): volume.resize(apiclient, diskofferingid=self.disk_offering_20_GB.id) return - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_03_increase_volume_size_above_domain_limit(self): """Test increasing volume size above the domain limit From e22cc6e9409a54858f3ffd803fcdde55e75ee8c6 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Tue, 9 Dec 2014 10:56:03 +0530 Subject: [PATCH 03/39] CLOUDSTACK-8048: Fix test_ss_limits.py Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_ss_limits.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/integration/component/test_ss_limits.py b/test/integration/component/test_ss_limits.py index 4d6efc4eb33..c83a7f58f44 100644 --- a/test/integration/component/test_ss_limits.py +++ b/test/integration/component/test_ss_limits.py @@ -233,7 +233,14 @@ class TestSecondaryStorageLimits(cloudstackTestCase): except Exception as e: self.fail("Failed to create template: %s" % e) - templateSize = (template.size / (1024**3)) + templates = Template.list(apiclient, + templatefilter=\ + self.services["template_2"]["templatefilter"], + id=template.id) + self.assertEqual(validateList(templates)[0],PASS,\ + "templates list validation failed") + + templateSize = (templates[0].size / (1024**3)) response = matchResourceCount(self.apiclient, templateSize, resourceType=RESOURCE_SECONDARY_STORAGE, accountid=self.account.id) From 8278d88f76ee129af75cd585b916bd6719e34e4c Mon Sep 17 00:00:00 2001 From: Jayapal Date: Wed, 3 Dec 2014 16:49:24 +0530 Subject: [PATCH 04/39] CLOUDSTACK-8030: Updated router to come up egress default ALLOW On default iptables rules are updated to add ACCEPT egress traffic. If the network egress default policy is false, CS remove ACCEPT and adds the DROP rule which is egress default rule when there are no other egress rules. If the CS network egress default policy is true, CS won't configure any default rule for egress because router already came up to accept egress traffic. If there are already egress rules for network then the egress rules get applied on VR. For isolated network with out firewall service, VR default allows egress traffic (guestnetwork --> public network) --- .../orchestration/NetworkOrchestrator.java | 30 ++++++++++--------- .../resource/CitrixResourceBase.java | 1 + .../network/element/VirtualRouterElement.java | 5 ++-- .../VirtualNetworkApplianceManagerImpl.java | 5 +++- .../config/etc/iptables/iptables-router | 3 ++ .../config/opt/cloud/bin/firewall_egress.sh | 8 ++--- 6 files changed, 31 insertions(+), 21 deletions(-) diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 17a2a9b7269..b28a26b846b 100755 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -1147,6 +1147,22 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra // This method re-programs the rules/ips for existing network protected boolean reprogramNetworkRules(long networkId, Account caller, Network network) throws ResourceUnavailableException { boolean success = true; + + //Apply egress rules first to effect the egress policy early on the guest traffic + List firewallEgressRulesToApply = _firewallDao.listByNetworkPurposeTrafficType(networkId, Purpose.Firewall, FirewallRule.TrafficType.Egress); + NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId()); + DataCenter zone = _dcDao.findById(network.getDataCenterId()); + if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) && _networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) + && (network.getGuestType() == Network.GuestType.Isolated || (network.getGuestType() == Network.GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced))) { + // add default egress rule to accept the traffic + _firewallMgr.applyDefaultEgressFirewallRule(network.getId(), offering.getEgressDefaultPolicy(), true); + } + if (!_firewallMgr.applyFirewallRules(firewallEgressRulesToApply, false, caller)) { + s_logger.warn("Failed to reapply firewall Egress rule(s) as a part of network id=" + networkId + " restart"); + success = false; + } + + // associate all ip addresses if (!_ipAddrMgr.applyIpAssociations(network, false)) { s_logger.warn("Failed to apply ip addresses as a part of network id" + networkId + " restart"); @@ -1166,20 +1182,6 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra success = false; } - List firewallEgressRulesToApply = _firewallDao.listByNetworkPurposeTrafficType(networkId, Purpose.Firewall, FirewallRule.TrafficType.Egress); - NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId()); - //there are no egress rules then apply the default egress rule - DataCenter zone = _dcDao.findById(network.getDataCenterId()); - if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) && _networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) - && (network.getGuestType() == Network.GuestType.Isolated || (network.getGuestType() == Network.GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced))) { - // add default egress rule to accept the traffic - _firewallMgr.applyDefaultEgressFirewallRule(network.getId(), offering.getEgressDefaultPolicy(), true); - } - if (!_firewallMgr.applyFirewallRules(firewallEgressRulesToApply, false, caller)) { - s_logger.warn("Failed to reapply firewall Egress rule(s) as a part of network id=" + networkId + " restart"); - success = false; - } - // apply port forwarding rules if (!_rulesMgr.applyPortForwardingRulesForNetwork(networkId, false, caller)) { s_logger.warn("Failed to reapply port forwarding rule(s) as a part of network id=" + networkId + " restart"); diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index 9b282d8b33b..a090b1106ac 100644 --- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -573,6 +573,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe public ExecutionResult createFileInVR(String routerIp, String path, String filename, String content) { Connection conn = getConnection(); String rc = callHostPlugin(conn, "vmops", "createFileInDomr", "domrip", routerIp, "filepath", path + filename, "filecontents", content); + s_logger.debug ("VR Config file " + filename + " got created in VR with ip " + routerIp + " with content \n" + content); // Fail case would be start with "fail#" return new ExecutionResult(rc.startsWith("succ#"), rc.substring(5)); } diff --git a/server/src/com/cloud/network/element/VirtualRouterElement.java b/server/src/com/cloud/network/element/VirtualRouterElement.java index dc6ffc8b223..ae4c933922b 100755 --- a/server/src/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/com/cloud/network/element/VirtualRouterElement.java @@ -251,9 +251,10 @@ NetworkMigrationResponder, AggregatedCommandExecutor { } if (rules != null && rules.size() == 1) { - // for VR no need to add default egress rule to DENY traffic + // for VR no need to add default egress rule to ALLOW traffic + //The default allow rule is added from the router defalut iptables rules iptables-router if (rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System && - !_networkMdl.getNetworkEgressDefaultPolicy(config.getId())) + _networkMdl.getNetworkEgressDefaultPolicy(config.getId())) return true; } diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index fb39c0a2998..6c5ff9c8815 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -2789,7 +2789,7 @@ VirtualMachineGuru, Listener, Configurable, StateListener sourceCidr = new ArrayList(); @@ -2799,6 +2799,9 @@ VirtualMachineGuru, Listener, Configurable, StateListener Date: Wed, 10 Dec 2014 18:04:25 +0530 Subject: [PATCH 05/39] CLOUDSTACK-7974: remove old hostname entry for a VM when adding a VM When adding a VM, it adds an entry to /etc/hosts file on the VR but does not clear up any older entries for the VM with a same name. The fix uncomments the command that removes any old entries in the VM. Signed-off-by: Rohit Yadav (cherry picked from commit 63298d9b742811919717ffd6303c8a2e9d37a3dd) Signed-off-by: Rohit Yadav --- systemvm/patches/debian/config/opt/cloud/bin/edithosts.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/edithosts.sh b/systemvm/patches/debian/config/opt/cloud/bin/edithosts.sh index 4912bcecc78..f0d9c00ddf4 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/edithosts.sh +++ b/systemvm/patches/debian/config/opt/cloud/bin/edithosts.sh @@ -121,8 +121,7 @@ then sed -i /$ipv6],/d $DHCP_HOSTS fi # don't want to do this in the future, we can have same VM with multiple nics/entries -#sed -i /$host,/d $DHCP_HOSTS - +sed -i /$host,/d $DHCP_HOSTS #put in the new entry if [ $ipv4 ] From 75f920185308307fe4b3abf1b60b071346656ab7 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Wed, 10 Dec 2014 14:58:49 +0530 Subject: [PATCH 06/39] CLOUDSTACK-8057: test_dedicate_guest_vlan_range.py - Marking a test as invalid Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_dedicate_guest_vlan_ranges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/component/test_dedicate_guest_vlan_ranges.py b/test/integration/component/test_dedicate_guest_vlan_ranges.py index 29bc280dcbb..9a0e7f09ff3 100644 --- a/test/integration/component/test_dedicate_guest_vlan_ranges.py +++ b/test/integration/component/test_dedicate_guest_vlan_ranges.py @@ -434,7 +434,7 @@ class TestDedicateGuestVLANRange(cloudstackTestCase): return - @attr(tags = ["advanced", "selfservice"], required_hardware="false") + @attr(tags = ["invalid"]) def test_04_dedicate_guest_vlan_in_project(self): """Dedicate guest vlan range project owner account and test guest network vlan in project From 40f26142b38bda5b10b01b2a2aad77d1b0b5f36e Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Wed, 10 Dec 2014 16:14:49 +0530 Subject: [PATCH 07/39] CLOUDSTACK-8059: test_host_high_availability.py - Adding necessary code to check the hosts in cluster and update the host with appropriate tags to make it HA enabled Signed-off-by: SrikanteswaraRao Talluri --- .../maint/test_host_high_availability.py | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/test/integration/component/maint/test_host_high_availability.py b/test/integration/component/maint/test_host_high_availability.py index 0d76f817024..1824e75ad3a 100644 --- a/test/integration/component/maint/test_host_high_availability.py +++ b/test/integration/component/maint/test_host_high_availability.py @@ -19,14 +19,17 @@ """ #Import Local Modules from nose.plugins.attrib import attr -from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackTestCase import cloudstackTestCase, unittest from marvin.cloudstackAPI import (migrateVirtualMachine, prepareHostForMaintenance, cancelHostMaintenance) from marvin.lib.utils import cleanup_resources from marvin.lib.base import (Account, VirtualMachine, - ServiceOffering) + ServiceOffering, + Cluster, + Host, + Configurations) from marvin.lib.common import (get_zone, get_domain, get_template, @@ -99,6 +102,33 @@ class TestHostHighAvailability(cloudstackTestCase): cls.zone.id, cls.services["ostype"] ) + + + clusterWithSufficientHosts = None + clusters = Cluster.list(cls.api_client, zoneid=cls.zone.id) + for cluster in clusters: + cls.hosts = Host.list(cls.api_client, clusterid=cluster.id, type="Routing") + if len(cls.hosts) >= 3: + clusterWithSufficientHosts = cluster + break + + if clusterWithSufficientHosts is None: + raise unittest.SkipTest("No Cluster with 3 hosts found") + + configs = Configurations.list( + cls.api_client, + name='ha.tag' + ) + + assert isinstance(configs, list), "Config list not\ + retrieved for ha.tag" + + if configs[0].value != "ha": + raise unittest.SkipTest("Please set the global config\ + value for ha.tag as 'ha'") + + Host.update(cls.api_client, id=cls.hosts[2].id, hosttags="ha") + cls.services["virtual_machine"]["zoneid"] = cls.zone.id cls.services["virtual_machine"]["template"] = cls.template.id @@ -123,6 +153,8 @@ class TestHostHighAvailability(cloudstackTestCase): @classmethod def tearDownClass(cls): try: + # Remove the host from HA + Host.update(cls.api_client, id=cls.hosts[2].id, hosttags="") #Cleanup resources used cleanup_resources(cls.api_client, cls._cleanup) except Exception as e: From e59dac201dc66ac80728632d7218b9a787142243 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 10 Dec 2014 19:08:26 +0530 Subject: [PATCH 08/39] CLOUDSTACK-8014: Fix NPE searching including removed templates Steps to reproduce if you have this issue: - Create a VM's volume snapshot - Remove VM's template and mark the template as removed with timestamp in DB - Restart mgmt server and create a volume out of snapshot you should get NPE Fix: In `storagePoolHasEnoughSpace`, we're only searching for a VM's volume's snapshot's template by Id and not including removed templates. This is a corner case and NPE hits when template has been marked removed for a VM's volume's template so we should search including removed templates. Signed-off-by: Rohit Yadav (cherry picked from commit f189c105d8dde5491697b171b969e757f8f15858) Signed-off-by: Rohit Yadav --- server/src/com/cloud/storage/StorageManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 0b8a43b3dd4..1feea016202 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -1576,8 +1576,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C long totalAskingSize = 0; for (Volume volume : volumes) { if (volume.getTemplateId() != null) { - VMTemplateVO tmpl = _templateDao.findById(volume.getTemplateId()); - if (tmpl.getFormat() != ImageFormat.ISO) { + VMTemplateVO tmpl = _templateDao.findByIdIncludingRemoved(volume.getTemplateId()); + if (tmpl != null && tmpl.getFormat() != ImageFormat.ISO) { allocatedSizeWithtemplate = _capacityMgr.getAllocatedPoolCapacity(poolVO, tmpl); } } From 9bfb8e571934f70d00a69c3aa58cf848d016afb9 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 8 Dec 2014 15:45:29 +0100 Subject: [PATCH 09/39] CLOUDSTACK-2823: pass cmdline info to system vms for 30 times (cherry picked from commit 4eedfe53fcbab1d47b09eacaca1d803b67b6c4d2) --- .../resource/LibvirtComputingResource.java | 29 +++++++++---------- .../config/etc/init.d/cloud-early-config | 4 +-- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 576abe6d0f7..68c776ae248 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3831,24 +3831,21 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv // pass cmdline info to system vms if (vmSpec.getType() != VirtualMachine.Type.User) { - if ((conn.getVersion() < 1001000)) { // CLOUDSTACK-2823: try passCmdLine some times if kernel < 2.6.34 and qemu < 1.1.0 on hypervisor (for instance, CentOS 6.4) - //wait for 5 minutes at most - String controlIp = null; - for (NicTO nic : nics) { - if (nic.getType() == TrafficType.Control) { - controlIp = nic.getIp(); - } + //wait and try passCmdLine for 5 minutes at most for CLOUDSTACK-2823 + String controlIp = null; + for (NicTO nic : nics) { + if (nic.getType() == TrafficType.Control) { + controlIp = nic.getIp(); + break; } - for (int count = 0; count < 30; count++) { - passCmdLine(vmName, vmSpec.getBootArgs()); - //check router is up? - boolean result = _virtRouterResource.connect(controlIp, 1, 5000); - if (result) { - break; - } - } - } else { + } + for (int count = 0; count < 30; count++) { passCmdLine(vmName, vmSpec.getBootArgs()); + //check router is up? + boolean result = _virtRouterResource.connect(controlIp, 1, 5000); + if (result) { + break; + } } } diff --git a/systemvm/patches/debian/config/etc/init.d/cloud-early-config b/systemvm/patches/debian/config/etc/init.d/cloud-early-config index 294ae5fbeb2..f7d752db7e8 100755 --- a/systemvm/patches/debian/config/etc/init.d/cloud-early-config +++ b/systemvm/patches/debian/config/etc/init.d/cloud-early-config @@ -117,7 +117,7 @@ get_boot_params() { if [ ! -e /dev/vport0p1 ]; then log_it "/dev/vport0p1 not loaded, perhaps guest kernel is too old." && exit 2 fi - while [$cmd -eq ""]; do + while [ -z "$cmd" ]; do while read line; do if [[ $line == cmdline:* ]]; then cmd=${line//cmdline:/} @@ -128,7 +128,7 @@ get_boot_params() { echo $pubkey > /root/.ssh/authorized_keys fi done < /dev/vport0p1 - done + done chmod go-rwx /root/.ssh/authorized_keys ;; vmware) From 7ff118c90b80a51efea73f87ac70dc686d92753b Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Thu, 11 Dec 2014 12:41:46 +0530 Subject: [PATCH 10/39] CLOUDSTACK-8062: test_multiple_ip_ranges.py - Fix the test case to avoid using same vlan IP range in each test case Signed-off-by: SrikanteswaraRao Talluri --- .../component/test_multiple_ip_ranges.py | 521 +++++++++--------- tools/marvin/marvin/config/test_data.py | 8 + 2 files changed, 281 insertions(+), 248 deletions(-) diff --git a/test/integration/component/test_multiple_ip_ranges.py b/test/integration/component/test_multiple_ip_ranges.py index 32c39f7c6a0..1beea4b830a 100644 --- a/test/integration/component/test_multiple_ip_ranges.py +++ b/test/integration/component/test_multiple_ip_ranges.py @@ -16,109 +16,41 @@ # under the License. """ Tests for Multiple IP Ranges feature """ -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * +from marvin.cloudstackTestCase import cloudstackTestCase from marvin.cloudstackException import CloudstackAPIException -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * -from netaddr import * - +from marvin.lib.utils import cleanup_resources +from marvin.lib.base import (PublicIpRange) +from marvin.lib.common import (get_domain, + get_zone, + get_pod) +from netaddr import IPAddress, IPNetwork +import random from nose.plugins.attrib import attr -class Services: - """Test Multiple IP Ranges - """ - def __init__(self): - self.services = { - "account": { - "email": "test@test.com", - "firstname": "Test", - "lastname": "User", - "username": "test", - # Random characters are appended for unique - # username - "password": "password", - }, - "service_offering": { - "name": "Tiny Instance", - "displaytext": "Tiny Instance", - "cpunumber": 1, - "cpuspeed": 200, # in MHz - "memory": 256, # In MBs - }, - "disk_offering": { - "displaytext": "Small Disk", - "name": "Small Disk", - "disksize": 1 - }, - "templates": { - "displaytext": 'Template', - "name": 'Template', - "ostype": "CentOS 5.3 (64-bit)", - "templatefilter": 'self', - }, - "vlan_ip_range": { - "startip": "", - "endip": "", - "netmask": "", - "gateway": "", - "forvirtualnetwork": "false", - "vlan": "untagged", - }, - "ostype": "CentOS 5.3 (64-bit)", - "sleep": 60, - "timeout": 10, - } class TestMultipleIpRanges(cloudstackTestCase): + """Test Multiple IP Ranges for guest network """ - @classmethod def setUpClass(cls): cls.testClient = super(TestMultipleIpRanges, cls).getClsTestClient() cls.api_client = cls.testClient.getApiClient() - cls.services = Services().services + cls.services = cls.testClient.getParsedTestDataConfig() # Get Zone, Domain and templates cls.domain = get_domain(cls.api_client) cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) cls.pod = get_pod(cls.api_client, cls.zone.id) - cls.services['mode'] = cls.zone.networktype - cls.services["domainid"] = cls.domain.id cls.services["zoneid"] = cls.zone.id - cls.account = Account.create( - cls.api_client, - cls.services["account"], - domainid=cls.domain.id - ) - cls.services["account"] = cls.account.name - cls.disk_offering = DiskOffering.create( - cls.api_client, - cls.services["disk_offering"] - ) - cls.service_offering = ServiceOffering.create( - cls.api_client, - cls.services["service_offering"] - ) - cls.template = get_template( - cls.api_client, - cls.zone.id, - cls.services["ostype"] - ) - cls.services["templates"]["ostypeid"] = cls.template.ostypeid - cls.services["diskoffering"] = cls.disk_offering.id - cls._cleanup = [ - cls.account, - ] + cls._cleanup = [] return @classmethod def tearDownClass(cls): try: - #Cleanup resources used + # Cleanup resources used cleanup_resources(cls.api_client, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) @@ -127,12 +59,12 @@ class TestMultipleIpRanges(cloudstackTestCase): def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() - self.cleanup = [ ] + self.cleanup = [] return def tearDown(self): try: - #Clean up, terminate the resources created + # Clean up, terminate the resources created cleanup_resources(self.apiclient, self.cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) @@ -146,86 +78,102 @@ class TestMultipleIpRanges(cloudstackTestCase): self.endIp = publicIpRange[0].endip self.gateway = publicIpRange[0].gateway self.netmask = publicIpRange[0].netmask - #Pass ip address and mask length to IPNetwork to findout the CIDR - ip = IPNetwork(self.startIp+"/"+self.netmask) - new_cidr = ip.__iadd__(1) + # Pass ip address and mask length to IPNetwork to findout the CIDR + ip = IPNetwork(self.startIp + "/" + self.netmask) + # Take random increment factor to avoid adding the same vlan ip range + # in each test case + networkIncrementFactor = random.randint(1,255) + new_cidr = ip.__iadd__(networkIncrementFactor) ip2 = IPNetwork(new_cidr) return ip2 - def verify_vlan_range(self,vlan,services): - #compare vlan_list response with configured values + def verify_vlan_range(self, vlan, services): + # compare vlan_list response with configured values self.assertEqual( - isinstance(vlan, list), - True, - "Check list response returned a valid list" - ) + isinstance(vlan, list), + True, + "Check list response returned a valid list" + ) self.assertNotEqual( - len(vlan), - 0, - "check list vlan response" - ) + len(vlan), + 0, + "check list vlan response" + ) self.assertEqual( - str(vlan[0].startip), - str(services["startip"]), - "Start IP in vlan ip range is not matched with the configured start ip" - ) + str(vlan[0].startip), + str(services["startip"]), + "Start IP in vlan ip range is not matched with\ + the configured start ip" + ) self.assertEqual( - str(vlan[0].endip), - str(services["endip"]), - "End IP in vlan ip range is not matched with the configured end ip" - ) + str(vlan[0].endip), + str(services["endip"]), + "End IP in vlan ip range is not matched with\ + the configured end ip" + ) self.assertEqual( - str(vlan[0].gateway), - str(services["gateway"]), - "gateway in vlan ip range is not matched with the configured gateway" - ) + str(vlan[0].gateway), + str(services["gateway"]), + "gateway in vlan ip range is not matched with\ + the configured gateway" + ) self.assertEqual( - str(vlan[0].netmask), - str(services["netmask"]), - "netmask in vlan ip range is not matched with the configured netmask" - ) + str(vlan[0].netmask), + str(services["netmask"]), + "netmask in vlan ip range is not matched with\ + the configured netmask" + ) return @attr(tags=["sg"]) def test_01_add_ip_same_cidr(self): """Test add guest ip range in the existing cidr """ - #call increment_cidr function to get exiting cidr from the setup and increment it + # call increment_cidr function to get exiting cidr from the setup and + # increment it ip2 = self.increment_cidr() test_nw = ip2.network ip = IPAddress(test_nw) - #Add IP range(5 IPs) in the new CIDR + # Add IP range(5 IPs) in the new CIDR test_gateway = ip.__add__(1) test_startIp = ip.__add__(3) test_endIp = ip.__add__(10) - test_startIp2= ip.__add__(11) + test_startIp2 = ip.__add__(11) test_endIp2 = ip.__add__(15) - #Populating services with new IP range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp self.services["vlan_ip_range"]["endip"] = test_endIp self.services["vlan_ip_range"]["gateway"] = test_gateway self.services["vlan_ip_range"]["netmask"] = self.netmask self.services["vlan_ip_range"]["zoneid"] = self.zone.id self.services["vlan_ip_range"]["podid"] = self.pod.id - #create new vlan ip range - self.debug("Creating new ip range with new cidr in the same vlan") - new_vlan = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp,test_endIp)) + # create new vlan ip range + self.debug("Creating new ip range with new cidr in the same vlan") + new_vlan = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp, test_endIp)) self.cleanup.append(new_vlan) - new_vlan_res = new_vlan.list(self.apiclient,id=new_vlan.vlan.id) - #Compare list output with configured values - self.verify_vlan_range(new_vlan_res,self.services["vlan_ip_range"]) - #Add few more ips in the same CIDR + new_vlan_res = new_vlan.list(self.apiclient, id=new_vlan.vlan.id) + # Compare list output with configured values + self.verify_vlan_range(new_vlan_res, self.services["vlan_ip_range"]) + # Add few more ips in the same CIDR self.services["vlan_ip_range"]["startip"] = test_startIp2 self.services["vlan_ip_range"]["endip"] = test_endIp2 - self.debug("Creating new ip range in the existing CIDR") - new_vlan2 = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp2,test_endIp2)) + self.debug("Creating new ip range in the existing CIDR") + new_vlan2 = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp2, test_endIp2)) self.cleanup.append(new_vlan2) - #list new vlan ip range - new_vlan2_res = new_vlan2.list(self.apiclient,id=new_vlan2.vlan.id) - #Compare list output with configured values - self.verify_vlan_range(new_vlan2_res,self.services["vlan_ip_range"]) + # list new vlan ip range + new_vlan2_res = new_vlan2.list(self.apiclient, id=new_vlan2.vlan.id) + # Compare list output with configured values + self.verify_vlan_range(new_vlan2_res, self.services["vlan_ip_range"]) return @attr(tags=["sg"]) @@ -236,29 +184,34 @@ class TestMultipleIpRanges(cloudstackTestCase): 1.Get public vlan range (guest cidr) from the setup 2.Add IP range to a new cidr """ - #call increment_cidr function to get exiting cidr from the setup and increment it + # call increment_cidr function to get exiting cidr from the setup and + # increment it ip2 = self.increment_cidr() test_nw = ip2.network ip = IPAddress(test_nw) - #Add IP range(5 IPs) in the new CIDR + # Add IP range(5 IPs) in the new CIDR test_gateway = ip.__add__(1) test_startIp = ip.__add__(3) test_endIp = ip.__add__(10) - #Populating services with new IP range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp self.services["vlan_ip_range"]["endip"] = test_endIp self.services["vlan_ip_range"]["gateway"] = test_gateway self.services["vlan_ip_range"]["netmask"] = self.netmask self.services["vlan_ip_range"]["zoneid"] = self.zone.id self.services["vlan_ip_range"]["podid"] = self.pod.id - #create new vlan ip range - self.debug("Adding new ip range in different CIDR in same vlan") - new_vlan = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp,test_endIp)) + # create new vlan ip range + self.debug("Adding new ip range in different CIDR in same vlan") + new_vlan = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp, test_endIp)) self.cleanup.append(new_vlan) - new_vlan_res = new_vlan.list(self.apiclient,id=new_vlan.vlan.id) - #Compare list output with configured values - self.verify_vlan_range(new_vlan_res,self.services["vlan_ip_range"]) + new_vlan_res = new_vlan.list(self.apiclient, id=new_vlan.vlan.id) + # Compare list output with configured values + self.verify_vlan_range(new_vlan_res, self.services["vlan_ip_range"]) return @attr(tags=["sg"]) @@ -270,37 +223,45 @@ class TestMultipleIpRanges(cloudstackTestCase): 2.delete the ip range added at step1 3.Verify the ip range deletion using list APIs """ - #call increment_cidr function to get exiting cidr from the setup and increment it + # call increment_cidr function to get exiting cidr from the setup and + # increment it ip2 = self.increment_cidr() test_nw = ip2.network ip = IPAddress(test_nw) - #Add IP range(5 IPs) in the new CIDR + # Add IP range(5 IPs) in the new CIDR test_gateway = ip.__add__(1) test_startIp = ip.__add__(3) test_endIp = ip.__add__(10) - #Populating services with new IP range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp self.services["vlan_ip_range"]["endip"] = test_endIp self.services["vlan_ip_range"]["gateway"] = test_gateway self.services["vlan_ip_range"]["netmask"] = self.netmask self.services["vlan_ip_range"]["zoneid"] = self.zone.id self.services["vlan_ip_range"]["podid"] = self.pod.id - #create new vlan ip range - self.debug("Creating new ip range in the new cidr") - new_vlan = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp,test_endIp)) - new_vlan_res = new_vlan.list(self.apiclient,id=new_vlan.vlan.id) - #Compare list output with configured values - self.verify_vlan_range(new_vlan_res,self.services["vlan_ip_range"]) - #Delete the above IP range - self.debug("Deleting new ip range added in new cidr") + # create new vlan ip range + self.debug("Creating new ip range in the new cidr") + new_vlan = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp, test_endIp)) + new_vlan_res = new_vlan.list(self.apiclient, id=new_vlan.vlan.id) + # Compare list output with configured values + self.verify_vlan_range(new_vlan_res, self.services["vlan_ip_range"]) + # Delete the above IP range + self.debug("Deleting new ip range added in new cidr") new_vlan.delete(self.apiclient) - #listing vlan ip ranges with the id should through exception , if not mark the test case as failed + # listing vlan ip ranges with the id should through exception , if not + # mark the test case as failed try: new_vlan.list(self.apiclient, id=new_vlan.vlan.id) except CloudstackAPIException as cs: self.debug(cs.errorMsg) - self.assertTrue(cs.errorMsg.find("entity does not exist")>0, msg="Failed to delete IP range") + self.assertTrue( + cs.errorMsg.find("entity does not exist") > 0, + msg="Failed to delete IP range") return @attr(tags=["sg"]) @@ -311,42 +272,51 @@ class TestMultipleIpRanges(cloudstackTestCase): 1.Add non-contigous ip range in cidr added at step1 2.Verify the ip range using list APIs """ - #call increment_cidr function to get exiting cidr from the setup and increment it + # call increment_cidr function to get exiting cidr from the setup and + # increment it ip2 = self.increment_cidr() test_nw = ip2.network ip = IPAddress(test_nw) - #Add IP range(5 IPs) in the new CIDR + # Add IP range(5 IPs) in the new CIDR test_gateway = ip.__add__(1) test_startIp = ip.__add__(50) test_endIp = ip.__add__(60) - #Populating services with new IP range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp self.services["vlan_ip_range"]["endip"] = test_endIp self.services["vlan_ip_range"]["gateway"] = test_gateway self.services["vlan_ip_range"]["netmask"] = self.netmask self.services["vlan_ip_range"]["zoneid"] = self.zone.id self.services["vlan_ip_range"]["podid"] = self.pod.id - #create new vlan ip range - new_vlan = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp,test_endIp)) + # create new vlan ip range + new_vlan = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp, test_endIp)) self.cleanup.append(new_vlan) - new_vlan_res = new_vlan.list(self.apiclient,id=new_vlan.vlan.id) - #Compare list output with configured values - self.verify_vlan_range(new_vlan_res,self.services["vlan_ip_range"]) - #Add non-contiguous ip range in exiting cidr + new_vlan_res = new_vlan.list(self.apiclient, id=new_vlan.vlan.id) + # Compare list output with configured values + self.verify_vlan_range(new_vlan_res, self.services["vlan_ip_range"]) + # Add non-contiguous ip range in exiting cidr test_startIp2 = ip.__add__(10) test_endIp2 = ip.__add__(20) - #Populating services with new IP range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp2 self.services["vlan_ip_range"]["endip"] = test_endIp2 - #create new vlan ip range - self.debug("Adding non contiguous ip range") - new_vlan = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp,test_endIp)) + # create new vlan ip range + self.debug("Adding non contiguous ip range") + new_vlan = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp, test_endIp)) self.cleanup.append(new_vlan) - new_vlan_res = new_vlan.list(self.apiclient,id=new_vlan.vlan.id) - #Compare list output with configured values - self.verify_vlan_range(new_vlan_res,self.services["vlan_ip_range"]) + new_vlan_res = new_vlan.list(self.apiclient, id=new_vlan.vlan.id) + # Compare list output with configured values + self.verify_vlan_range(new_vlan_res, self.services["vlan_ip_range"]) return @attr(tags=["sg"]) @@ -354,48 +324,64 @@ class TestMultipleIpRanges(cloudstackTestCase): """Test adding overlapped ip range in existing cidr 1.Add ip range in new cidr e.g:10.147.40.10-10.147.40.100 - 2.Add ip range overlapped with the ip range in step1 e.g.10.147.40.90-150 + 2.Add ip range overlapped with the ip range in + step1 e.g.10.147.40.90-150 """ - #call increment_cidr function to get exiting cidr from the setup and increment it + # call increment_cidr function to get exiting cidr from the setup and + # increment it ip2 = self.increment_cidr() test_nw = ip2.network ip = IPAddress(test_nw) - #Add IP range in the new CIDR + # Add IP range in the new CIDR test_gateway = ip.__add__(1) test_startIp = ip.__add__(10) test_endIp = ip.__add__(30) test_startIp2 = ip.__add__(20) test_endIp2 = ip.__add__(40) - #Populating services with new IP range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp self.services["vlan_ip_range"]["endip"] = test_endIp self.services["vlan_ip_range"]["gateway"] = test_gateway self.services["vlan_ip_range"]["netmask"] = self.netmask self.services["vlan_ip_range"]["zoneid"] = self.zone.id self.services["vlan_ip_range"]["podid"] = self.pod.id - #create new vlan ip range - self.debug("Creating new ip range with startip:%s and endip: %s".format(test_startIp,test_endIp)) - new_vlan = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp,test_endIp)) + # create new vlan ip range + self.debug( + "Creating new ip range with startip:%s and endip: %s".format( + test_startIp, + test_endIp)) + new_vlan = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp, test_endIp)) self.cleanup.append(new_vlan) - new_vlan_res = new_vlan.list(self.apiclient,id=new_vlan.vlan.id) - #Compare list output with configured values - self.verify_vlan_range(new_vlan_res,self.services["vlan_ip_range"]) - #Add overlapped ip range - #Populating services with new IP range + new_vlan_res = new_vlan.list(self.apiclient, id=new_vlan.vlan.id) + # Compare list output with configured values + self.verify_vlan_range(new_vlan_res, self.services["vlan_ip_range"]) + # Add overlapped ip range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp2 self.services["vlan_ip_range"]["endip"] = test_endIp2 - #Try to create ip range overlapped with exiting ip range - self.debug("Adding overlapped ip range") + # Try to create ip range overlapped with exiting ip range + self.debug("Adding overlapped ip range") try: - new_vlan2 = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) + new_vlan2 = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) except CloudstackAPIException as cs: self.debug(cs.errorMsg) - self.assertTrue(cs.errorMsg.find("already has IPs that overlap with the new range")>0, msg="Fail:CS allowed adding overlapped ip ranges in guest cidr") + self.assertTrue( + cs.errorMsg.find("already has IPs that overlap with the new range") > 0, + msg="Fail:CS allowed adding overlapped ip\ + ranges in guest cidr") return - #Test will reach here if there is a bug in overlap ip range checking + # Test will reach here if there is a bug in overlap ip range checking self.cleanup.append(new_vlan2) - self.fail("CS should not accept overlapped ip ranges in guest traffic, but it allowed") + self.fail( + "CS should not accept overlapped ip ranges in\ + guest traffic, but it allowed") return @attr(tags=["sg"]) @@ -403,14 +389,17 @@ class TestMultipleIpRanges(cloudstackTestCase): """Test adding overlapped ip range with two existing cidr 1.Add ip range in new cidr e.g:10.147.40.2-10.147.40.10 - 2.Add another ip range in the same cidr e.g:10.147.40.20-10.147.40.30 - 2.Add ip range overlapped with both the ip ranges e.g.10.147.40.10-20 + 2.Add another ip range in the same cidr + e.g:10.147.40.20-10.147.40.30 + 3.Add ip range overlapped with both the ip ranges + e.g.10.147.40.10-20 """ - #call increment_cidr function to get exiting cidr from the setup and increment it + # call increment_cidr function to get exiting cidr from the setup and + # increment it ip2 = self.increment_cidr() test_nw = ip2.network ip = IPAddress(test_nw) - #Add IP range in the new CIDR + # Add IP range in the new CIDR test_gateway = ip.__add__(1) test_startIp = ip.__add__(2) test_endIp = ip.__add__(5) @@ -418,40 +407,55 @@ class TestMultipleIpRanges(cloudstackTestCase): test_endIp2 = ip.__add__(10) test_startIp3 = ip.__add__(5) test_endIp3 = ip.__add__(7) - #Populating services with new IP range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp self.services["vlan_ip_range"]["endip"] = test_endIp self.services["vlan_ip_range"]["gateway"] = test_gateway self.services["vlan_ip_range"]["netmask"] = self.netmask self.services["vlan_ip_range"]["zoneid"] = self.zone.id self.services["vlan_ip_range"]["podid"] = self.pod.id - #create new vlan ip range - new_vlan = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp,test_endIp)) + # create new vlan ip range + new_vlan = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp, test_endIp)) self.cleanup.append(new_vlan) - new_vlan_res = new_vlan.list(self.apiclient,id=new_vlan.vlan.id) - #Compare list output with configured values - self.verify_vlan_range(new_vlan_res,self.services["vlan_ip_range"]) - #Add 2nd IP range in the same CIDR + new_vlan_res = new_vlan.list(self.apiclient, id=new_vlan.vlan.id) + # Compare list output with configured values + self.verify_vlan_range(new_vlan_res, self.services["vlan_ip_range"]) + # Add 2nd IP range in the same CIDR self.services["vlan_ip_range"]["startip"] = test_startIp2 self.services["vlan_ip_range"]["endip"] = test_endIp2 - new_vlan2 = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp2,test_endIp2)) + new_vlan2 = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp2, test_endIp2)) self.cleanup.append(new_vlan2) - #Populating services with new IP range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp3 self.services["vlan_ip_range"]["endip"] = test_endIp3 - #Try to create ip range overlapped with exiting ip range - self.debug("Adding ip range overlapped with two cidrs") + # Try to create ip range overlapped with exiting ip range + self.debug("Adding ip range overlapped with two cidrs") try: - new_vlan3 = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) + new_vlan3 = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) except CloudstackAPIException as cs: self.debug(cs.errorMsg) - self.assertTrue(cs.errorMsg.find("already has IPs that overlap with the new range")>0, msg="Fail:CS allowed adding overlapped ip ranges in guest cidr") + self.assertTrue( + cs.errorMsg.find("already has IPs that overlap with the new range") > 0, + msg="Fail:CS allowed adding overlapped ip\ + ranges in guest cidr") return - #Test will reach here if there is a bug in overlap ip range checking + # Test will reach here if there is a bug in overlap ip range checking self.cleanup.append(new_vlan3) - self.fail("CS should not accept overlapped ip ranges in guest traffic, but it allowed") + self.fail( + "CS should not accept overlapped ip ranges in guest\ + traffic, but it allowed") return @attr(tags=["sg"]) @@ -461,50 +465,61 @@ class TestMultipleIpRanges(cloudstackTestCase): 1.Add IP range in new CIDR 2.Try to add ip range superset to CIDR added in step1 """ - #call increment_cidr function to get exiting cidr from the setup and increment it + # call increment_cidr function to get exiting cidr from the setup and + # increment it ip2 = self.increment_cidr() test_nw = ip2.network ip = IPAddress(test_nw) - #Add IP range in the new CIDR + # Add IP range in the new CIDR test_gateway = ip.__add__(1) test_startIp = ip.__add__(2) test_endIp = ip.__add__(10) test_startIp2 = ip.__add__(20) test_endIp2 = ip.__add__(30) - #Populating services with new IP range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp self.services["vlan_ip_range"]["endip"] = test_endIp self.services["vlan_ip_range"]["gateway"] = test_gateway self.services["vlan_ip_range"]["netmask"] = self.netmask self.services["vlan_ip_range"]["zoneid"] = self.zone.id self.services["vlan_ip_range"]["podid"] = self.pod.id - #create new vlan ip range - new_vlan = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp,test_endIp)) + # create new vlan ip range + new_vlan = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp, test_endIp)) self.cleanup.append(new_vlan) - new_vlan_res = new_vlan.list(self.apiclient,id=new_vlan.vlan.id) - #Compare list output with configured values - self.verify_vlan_range(new_vlan_res,self.services["vlan_ip_range"]) - #Add ip range superset to the existing CIDR - #Following code finds the netmask superset to existing CIDR + new_vlan_res = new_vlan.list(self.apiclient, id=new_vlan.vlan.id) + # Compare list output with configured values + self.verify_vlan_range(new_vlan_res, self.services["vlan_ip_range"]) + # Add ip range superset to the existing CIDR + # Following code finds the netmask superset to existing CIDR cidr = ip2.cidr - mask_len = 2**(32-cidr.prefixlen) + mask_len = 2 ** (32 - cidr.prefixlen) netmask = IPAddress(self.netmask) superset = netmask.__isub__(mask_len) - #Add this superset netmask to services + # Add this superset netmask to services self.services["vlan_ip_range"]["netmask"] = superset self.services["vlan_ip_range"]["startip"] = test_startIp2 self.services["vlan_ip_range"]["endip"] = test_endIp2 - self.debug("Adding IP range super set to existing CIDR") + self.debug("Adding IP range super set to existing CIDR") try: - new_vlan2 = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) + new_vlan2 = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) except CloudstackAPIException as cs: self.debug(cs.errorMsg) - self.assertTrue(cs.errorMsg.find("superset")>0, msg="Fail: CS allowed adding ip range superset to existing CIDR") + self.assertTrue( + cs.errorMsg.find("superset") > 0, + msg="Fail: CS allowed adding ip range\ + superset to existing CIDR") return - #Test will reach here if there is a bug in allowing superset ip range + # Test will reach here if there is a bug in allowing superset ip range self.cleanup.append(new_vlan2) - self.fail("CS should not allow adding ip range superset to existing CIDR") + self.fail( + "CS should not allow adding ip range superset to existing CIDR") return @attr(tags=["sg"]) @@ -514,48 +529,58 @@ class TestMultipleIpRanges(cloudstackTestCase): 1.Add IP range in new CIDR 2.Try to add ip range subset to CIDR added in step1 """ - #call increment_cidr function to get exiting cidr from the setup and increment it + # call increment_cidr function to get exiting cidr from the setup and + # increment it ip2 = self.increment_cidr() test_nw = ip2.network ip = IPAddress(test_nw) - #Add IP range in the new CIDR + # Add IP range in the new CIDR test_gateway = ip.__add__(1) test_startIp = ip.__add__(2) test_endIp = ip.__add__(10) test_startIp2 = ip.__add__(20) test_endIp2 = ip.__add__(30) - #Populating services with new IP range + # Populating services with new IP range self.services["vlan_ip_range"]["startip"] = test_startIp self.services["vlan_ip_range"]["endip"] = test_endIp self.services["vlan_ip_range"]["gateway"] = test_gateway self.services["vlan_ip_range"]["netmask"] = self.netmask self.services["vlan_ip_range"]["zoneid"] = self.zone.id self.services["vlan_ip_range"]["podid"] = self.pod.id - #create new vlan ip range - new_vlan = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) - self.debug("Created new vlan range with startip:%s and endip:%s" %(test_startIp,test_endIp)) + # create new vlan ip range + new_vlan = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) + self.debug( + "Created new vlan range with startip:%s and endip:%s" % + (test_startIp, test_endIp)) self.cleanup.append(new_vlan) - new_vlan_res = new_vlan.list(self.apiclient,id=new_vlan.vlan.id) - #Compare list output with configured values - self.verify_vlan_range(new_vlan_res,self.services["vlan_ip_range"]) - #Add ip range superset to the existing CIDR - #Following code finds the netmask superset to existing CIDR + new_vlan_res = new_vlan.list(self.apiclient, id=new_vlan.vlan.id) + # Compare list output with configured values + self.verify_vlan_range(new_vlan_res, self.services["vlan_ip_range"]) + # Add ip range superset to the existing CIDR + # Following code finds the netmask superset to existing CIDR cidr = ip2.cidr - mask_len = 2**(32-(cidr.prefixlen+1)) + mask_len = 2 ** (32 - (cidr.prefixlen + 1)) netmask = IPAddress(self.netmask) subset = netmask.__iadd__(mask_len) - #Add this superset netmask to services + # Add this superset netmask to services self.services["vlan_ip_range"]["netmask"] = subset self.services["vlan_ip_range"]["startip"] = test_startIp2 self.services["vlan_ip_range"]["endip"] = test_endIp2 - self.debug("Adding ip range subset to existing cidr") + self.debug("Adding ip range subset to existing cidr") try: - new_vlan2 = PublicIpRange.create(self.apiclient, self.services["vlan_ip_range"]) + new_vlan2 = PublicIpRange.create( + self.apiclient, + self.services["vlan_ip_range"]) except CloudstackAPIException as cs: self.debug(cs.errorMsg) - self.assertTrue(cs.errorMsg.find("subset")>0, msg="Fail: CS allowed adding ip range subset to existing CIDR") + self.assertTrue( + cs.errorMsg.find("subset") > 0, + msg="Fail: CS allowed adding ip range subset to existing CIDR") return - #Test will reach here if there is a bug in allowing superset ip range + # Test will reach here if there is a bug in allowing superset ip range self.cleanup.append(new_vlan2) - self.fail("CS should not allow adding ip range subset to existing CIDR") + self.fail( + "CS should not allow adding ip range subset to existing CIDR") return diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 4b50afa4bdd..5a468fe9a8a 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -829,6 +829,14 @@ test_data = { "epslifetime": "3600", "dpd": "false" }, + "vlan_ip_range": { + "startip": "", + "endip": "", + "netmask": "", + "gateway": "", + "forvirtualnetwork": "false", + "vlan": "untagged", + }, "ostype": "CentOS 5.6 (64-bit)", "sleep": 90, "timeout": 10, From bbebac733969ffa0125befdba28c49b48c44ee50 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Thu, 11 Dec 2014 17:12:50 +0530 Subject: [PATCH 11/39] CLOUDSTACK:8018: Fix test_redundant_router.test_redundantVR_internals - The SSH to router was being done from wrong host Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/maint/test_redundant_router.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/integration/component/maint/test_redundant_router.py b/test/integration/component/maint/test_redundant_router.py index 099ae2c47dd..01d635b7e6c 100644 --- a/test/integration/component/maint/test_redundant_router.py +++ b/test/integration/component/maint/test_redundant_router.py @@ -720,8 +720,7 @@ class TestRVRInternals(cloudstackTestCase): hosts = Host.list( self.apiclient, - id=master_router.hostid, - listall=True + id=master_router.hostid ) self.assertEqual( isinstance(hosts, list), @@ -734,8 +733,7 @@ class TestRVRInternals(cloudstackTestCase): hosts = Host.list( self.apiclient, - id=backup_router.hostid, - listall=True + id=backup_router.hostid ) self.assertEqual( isinstance(hosts, list), @@ -796,7 +794,7 @@ class TestRVRInternals(cloudstackTestCase): ) else: result = get_process_status( - master_host.ipaddress, + backup_host.ipaddress, 22, self.testdata['configurableData']['host']["username"], self.testdata['configurableData']['host']["password"], From 9aefd9bf4f580dd5954520b6b53909b1f732b7f8 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Thu, 11 Dec 2014 10:16:25 +0530 Subject: [PATCH 12/39] CLOUDSTACK:8058: Fix test_reset_ssh_keypair.py for EIP setup Signed-off-by: SrikanteswaraRao Talluri --- .../component/test_reset_ssh_keypair.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/integration/component/test_reset_ssh_keypair.py b/test/integration/component/test_reset_ssh_keypair.py index 370e9a8dfa6..a2e743a88b2 100644 --- a/test/integration/component/test_reset_ssh_keypair.py +++ b/test/integration/component/test_reset_ssh_keypair.py @@ -107,6 +107,15 @@ def wait_vm_start(apiclient, vmid, timeout, sleep): return timeout +def SetPublicIpForVM(apiclient, vm): + """ List VM and set the publicip (if available) of VM + to ssh_ip attribute""" + + vms = VirtualMachine.list(apiclient, id=vm.id, listall=True) + virtual_machine = vms[0] + if hasattr(vm, "publicip"): + vm.ssh_ip = virtual_machine.publicip + return vm class TestResetSSHKeypair(cloudstackTestCase): @@ -353,6 +362,12 @@ class TestResetSSHKeypair(cloudstackTestCase): % (virtual_machine.name, self.services["timeout"])) self.debug("SSH key path: %s" % str(keyPairFilePath)) + + # In case of EIP setup, public IP changes after VM start operation + # Assign the new publicip of the VM to its ssh_ip attribute + # so that correct IP address is used for getting the ssh client of VM + virtual_machine = SetPublicIpForVM(self.apiclient, virtual_machine) + try: virtual_machine.get_ssh_client(keyPairFileLocation=str(keyPairFilePath)) except Exception as e: @@ -462,6 +477,11 @@ class TestResetSSHKeypair(cloudstackTestCase): self.fail("The virtual machine %s failed to start even after %s minutes" % (virtual_machine.name, self.services["timeout"])) + # In case of EIP setup, public IP changes after VM start operation + # Assign the new publicip of the VM to its ssh_ip attribute + # so that correct IP address is used for getting the ssh client of VM + virtual_machine = SetPublicIpForVM(self.apiclient, virtual_machine) + self.debug("SSHing with new keypair") try: virtual_machine.get_ssh_client( @@ -572,6 +592,11 @@ class TestResetSSHKeypair(cloudstackTestCase): self.fail("The virtual machine %s failed to start even after %s minutes" % (virtual_machine.name, self.services["timeout"])) + # In case of EIP setup, public IP changes after VM start operation + # Assign the new publicip of the VM to its ssh_ip attribute + # so that correct IP address is used for getting the ssh client of VM + virtual_machine = SetPublicIpForVM(self.apiclient, virtual_machine) + self.debug("SSHing with new keypair") try: virtual_machine.get_ssh_client( @@ -683,6 +708,11 @@ class TestResetSSHKeypair(cloudstackTestCase): self.fail("The virtual machine %s failed to start even after %s minutes" % (virtual_machine.name, self.services["timeout"])) + # In case of EIP setup, public IP changes after VM start operation + # Assign the new publicip of the VM to its ssh_ip attribute + # so that correct IP address is used for getting the ssh client of VM + virtual_machine = SetPublicIpForVM(self.apiclient, virtual_machine) + self.debug("SSHing with new keypair") try: virtual_machine.get_ssh_client( @@ -1172,6 +1202,11 @@ class TestResetSSHKeyUserRights(cloudstackTestCase): self.fail("The virtual machine %s failed to start even after %s minutes" % (vms[0].name, self.services["timeout"])) + # In case of EIP setup, public IP changes after VM start operation + # Assign the new publicip of the VM to its ssh_ip attribute + # so that correct IP address is used for getting the ssh client of VM + virtual_machine = SetPublicIpForVM(self.apiclient, virtual_machine) + self.debug("SSHing with new keypair") try: virtual_machine.get_ssh_client( @@ -1311,6 +1346,11 @@ class TestResetSSHKeyUserRights(cloudstackTestCase): self.fail("The virtual machine %s failed to start even after %s minutes" % (virtual_machine.name, self.services["timeout"])) + # In case of EIP setup, public IP changes after VM start operation + # Assign the new publicip of the VM to its ssh_ip attribute + # so that correct IP address is used for getting the ssh client of VM + virtual_machine = SetPublicIpForVM(self.apiclient, virtual_machine) + self.debug("SSHing with new keypair") try: virtual_machine.get_ssh_client( @@ -1451,6 +1491,11 @@ class TestResetSSHKeyUserRights(cloudstackTestCase): self.fail("The virtual machine %s failed to start even after %s minutes" % (virtual_machine.name, self.services["timeout"])) + # In case of EIP setup, public IP changes after VM start operation + # Assign the new publicip of the VM to its ssh_ip attribute + # so that correct IP address is used for getting the ssh client of VM + virtual_machine = SetPublicIpForVM(self.apiclient, virtual_machine) + self.debug("SSHing with new keypair") try: virtual_machine.get_ssh_client( From fbe54974c514572d1c224528446618c31952e25a Mon Sep 17 00:00:00 2001 From: Min Chen Date: Wed, 10 Dec 2014 14:16:00 -0800 Subject: [PATCH 13/39] CLOUDSTACK-8061: Extracting volume when it is in migrating state causes both the operations to fail. --- .../src/com/cloud/vm/VmWorkExtractVolume.java | 38 ++++++ .../cloud/storage/VolumeApiServiceImpl.java | 124 +++++++++++++++++- 2 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 engine/components-api/src/com/cloud/vm/VmWorkExtractVolume.java diff --git a/engine/components-api/src/com/cloud/vm/VmWorkExtractVolume.java b/engine/components-api/src/com/cloud/vm/VmWorkExtractVolume.java new file mode 100644 index 00000000000..82b5db4f0af --- /dev/null +++ b/engine/components-api/src/com/cloud/vm/VmWorkExtractVolume.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm; + +public class VmWorkExtractVolume extends VmWork { + private static final long serialVersionUID = -565778516928408602L; + + private long volumeId; + private long zoneId; + + public VmWorkExtractVolume(long userId, long accountId, long vmId, String handlerName, long volumeId, long zoneId) { + super(userId, accountId, vmId, handlerName); + this.volumeId = volumeId; + this.zoneId = zoneId; + } + + public long getVolumeId() { + return volumeId; + } + + public long getZoneId() { + return zoneId; + } +} diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index 9deaae22917..0914c57132c 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -141,6 +141,7 @@ import com.cloud.vm.VmWork; import com.cloud.vm.VmWorkAttachVolume; import com.cloud.vm.VmWorkConstants; import com.cloud.vm.VmWorkDetachVolume; +import com.cloud.vm.VmWorkExtractVolume; import com.cloud.vm.VmWorkJobHandler; import com.cloud.vm.VmWorkJobHandlerProxy; import com.cloud.vm.VmWorkMigrateVolume; @@ -2043,14 +2044,70 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return volumeStoreRef.getExtractUrl(); } - dataStoreMgr.getPrimaryDataStore(volume.getPoolId()); - ImageStoreEntity secStore = (ImageStoreEntity)dataStoreMgr.getImageStore(zoneId); - secStore.getUri(); + VMInstanceVO vm = null; + if (volume.getInstanceId() != null) { + vm = _vmInstanceDao.findById(volume.getInstanceId()); + } + if (vm != null) { + // serialize VM operation + AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); + if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { + // avoid re-entrance + + VmWorkJobVO placeHolder = null; + placeHolder = createPlaceHolderWork(vm.getId()); + try { + return orchestrateExtractVolume(volume.getId(), zoneId); + } finally { + _workJobDao.expunge(placeHolder.getId()); + } + + } else { + Outcome outcome = extractVolumeThroughJobQueue(vm.getId(), volume.getId(), zoneId); + + try { + outcome.get(); + } catch (InterruptedException e) { + throw new RuntimeException("Operation is interrupted", e); + } catch (java.util.concurrent.ExecutionException e) { + throw new RuntimeException("Execution excetion", e); + } + + Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); + if (jobResult != null) { + if (jobResult instanceof ConcurrentOperationException) + throw (ConcurrentOperationException)jobResult; + else if (jobResult instanceof RuntimeException) + throw (RuntimeException)jobResult; + else if (jobResult instanceof Throwable) + throw new RuntimeException("Unexpected exception", (Throwable)jobResult); + } + + // retrieve the entity url from job result + if (jobResult != null && jobResult instanceof String) { + return (String)jobResult; + } + return null; + } + } + + return orchestrateExtractVolume(volume.getId(), zoneId); + } + + private String orchestrateExtractVolume(long volumeId, long zoneId) { + // get latest volume state to make sure that it is not updated by other parallel operations + VolumeVO volume = _volsDao.findById(volumeId); + if (volume == null || volume.getState() != Volume.State.Ready) { + throw new InvalidParameterValueException("Volume to be extracted has been removed or not in right state!"); + } + // perform extraction + ImageStoreEntity secStore = (ImageStoreEntity)dataStoreMgr.getImageStore(zoneId); String value = _configDao.getValue(Config.CopyVolumeWait.toString()); NumbersUtil.parseInt(value, Integer.parseInt(Config.CopyVolumeWait.getDefaultValue())); + // Copy volume from primary to secondary storage - VolumeInfo srcVol = volFactory.getVolume(volume.getId()); + VolumeInfo srcVol = volFactory.getVolume(volumeId); AsyncCallFuture cvAnswer = volService.copyVolume(srcVol, secStore); // Check if you got a valid answer. VolumeApiResult cvResult = null; @@ -2071,11 +2128,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic VolumeInfo vol = cvResult.getVolume(); String extractUrl = secStore.createEntityExtractUrl(vol.getPath(), vol.getFormat(), vol); - volumeStoreRef = _volumeStoreDao.findByVolume(volumeId); + VolumeDataStoreVO volumeStoreRef = _volumeStoreDao.findByVolume(volumeId); volumeStoreRef.setExtractUrl(extractUrl); volumeStoreRef.setExtractUrlCreated(DateUtil.now()); _volumeStoreDao.update(volumeStoreRef.getId(), volumeStoreRef); - return extractUrl; } @@ -2350,6 +2406,23 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic _storagePoolAllocators = storagePoolAllocators; } + public class VmJobVolumeUrlOutcome extends OutcomeImpl { + + public VmJobVolumeUrlOutcome(final AsyncJob job) { + super(String.class, job, VmJobCheckInterval.value(), new Predicate() { + @Override + public boolean checkCondition() { + AsyncJobVO jobVo = _entityMgr.findById(AsyncJobVO.class, job.getId()); + assert (jobVo != null); + if (jobVo == null || jobVo.getStatus() != JobInfo.Status.IN_PROGRESS) + return true; + + return false; + } + }, AsyncJob.Topics.JOB_STATE); + } + } + public class VmJobVolumeOutcome extends OutcomeImpl { private long _volumeId; @@ -2498,6 +2571,39 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return new VmJobVolumeOutcome(workJob,volumeId); } + public Outcome extractVolumeThroughJobQueue(final Long vmId, final long volumeId, + final long zoneId) { + + final CallContext context = CallContext.current(); + final User callingUser = context.getCallingUser(); + final Account callingAccount = context.getCallingAccount(); + + final VMInstanceVO vm = _vmInstanceDao.findById(vmId); + + VmWorkJobVO workJob = new VmWorkJobVO(context.getContextId()); + + workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); + workJob.setCmd(VmWorkExtractVolume.class.getName()); + + workJob.setAccountId(callingAccount.getId()); + workJob.setUserId(callingUser.getId()); + workJob.setStep(VmWorkJobVO.Step.Starting); + workJob.setVmType(VirtualMachine.Type.Instance); + workJob.setVmInstanceId(vm.getId()); + workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); + + // save work context info (there are some duplications) + VmWorkExtractVolume workInfo = new VmWorkExtractVolume(callingUser.getId(), callingAccount.getId(), vm.getId(), + VolumeApiServiceImpl.VM_WORK_JOB_HANDLER, volumeId, zoneId); + workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); + + _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + + AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); + + return new VmJobVolumeUrlOutcome(workJob); + } + public Outcome migrateVolumeThroughJobQueue(final Long vmId, final long volumeId, final long destPoolId, final boolean liveMigrate) { @@ -2565,6 +2671,12 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return new VmJobSnapshotOutcome(workJob,snapshotId); } + @ReflectionUse + private Pair orchestrateExtractVolume(VmWorkExtractVolume work) throws Exception { + String volUrl = orchestrateExtractVolume(work.getVolumeId(), work.getZoneId()); + return new Pair(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(volUrl)); + } + @ReflectionUse private Pair orchestrateAttachVolumeToVM(VmWorkAttachVolume work) throws Exception { Volume vol = orchestrateAttachVolumeToVM(work.getVmId(), work.getVolumeId(), work.getDeviceId()); From 76dc20a0ee199d0b2d15f7ffe21fc9c45c72e9cc Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 12 Dec 2014 11:13:18 +0100 Subject: [PATCH 14/39] CLOUDSTACK-255: fix NPE in createPortForwardingRule due to no PortForwarding provider (cherry picked from commit c9d492d7140dd53f1630bc5ea14b948184233612) --- server/src/com/cloud/network/NetworkModelImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/network/NetworkModelImpl.java b/server/src/com/cloud/network/NetworkModelImpl.java index 2edf9708ad0..1fd354df6d3 100755 --- a/server/src/com/cloud/network/NetworkModelImpl.java +++ b/server/src/com/cloud/network/NetworkModelImpl.java @@ -424,14 +424,14 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { // We only support one provider for one service now Map> serviceToProviders = getServiceProvidersMap(networkId); - Set oldProviders = serviceToProviders.get(services.toArray()[0]); - Provider oldProvider = (Provider)oldProviders.toArray()[0]; // Since IP already has service to bind with, the oldProvider can't be null Set newProviders = serviceToProviders.get(service); if (newProviders == null || newProviders.isEmpty()) { throw new InvalidParameterException("There is no new provider for IP " + publicIp.getAddress() + " of service " + service.getName() + "!"); } Provider newProvider = (Provider)newProviders.toArray()[0]; + Set oldProviders = serviceToProviders.get(services.toArray()[0]); + Provider oldProvider = (Provider)oldProviders.toArray()[0]; Network network = _networksDao.findById(networkId); NetworkElement oldElement = getElementImplementingProvider(oldProvider.getName()); NetworkElement newElement = getElementImplementingProvider(newProvider.getName()); From 47ee7c0319351267ff5370a20ec5e1a66583f688 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Wed, 10 Dec 2014 12:31:51 +0530 Subject: [PATCH 15/39] CLOUDSTACK-8055: Tagging test cases which can't be run on simulator accordingly Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_ss_domain_limits.py | 8 ++++---- test/integration/component/test_ss_limits.py | 8 ++++---- test/integration/component/test_ss_max_limits.py | 6 +++--- test/integration/component/test_ss_project_limits.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/test/integration/component/test_ss_domain_limits.py b/test/integration/component/test_ss_domain_limits.py index 998bb8ba360..2591d2f42a9 100644 --- a/test/integration/component/test_ss_domain_limits.py +++ b/test/integration/component/test_ss_domain_limits.py @@ -148,7 +148,7 @@ class TestMultipleChildDomain(cloudstackTestCase): return [FAIL, e, None] return [PASS, None, users] - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_01_multiple_domains_secondary_storage_limits(self): """Test secondary storage limit of domain and its sub-domains @@ -261,7 +261,7 @@ class TestMultipleChildDomain(cloudstackTestCase): self.assertTrue(result[2], "Resource count does not match") return - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_02_multiple_domains_secondary_storage_counts(self): """Test secondary storage counts in multiple child domains # Steps @@ -325,7 +325,7 @@ class TestMultipleChildDomain(cloudstackTestCase): self.fail("Failed to get zone list: %s" % e) return - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_03_copy_template(self): """Test secondary storage counts in multiple child domains # Steps @@ -485,7 +485,7 @@ class TestDeleteAccount(cloudstackTestCase): return [FAIL, e, None] return [PASS, None, users] - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_04_create_template_delete_account(self): """Test secondary storage limit of domain and its sub-domains diff --git a/test/integration/component/test_ss_limits.py b/test/integration/component/test_ss_limits.py index c83a7f58f44..c498302c003 100644 --- a/test/integration/component/test_ss_limits.py +++ b/test/integration/component/test_ss_limits.py @@ -128,7 +128,7 @@ class TestSecondaryStorageLimits(cloudstackTestCase): return [PASS, None] @data(ROOT_DOMAIN_ADMIN, CHILD_DOMAIN_ADMIN) - @attr(tags = ["advanced"], required_hardware="false") + @attr(tags = ["advanced"], required_hardware="true") def test_01_register_template(self, value): """Test register template # Validate the following: @@ -189,7 +189,7 @@ class TestSecondaryStorageLimits(cloudstackTestCase): return @data(ROOT_DOMAIN_ADMIN, CHILD_DOMAIN_ADMIN) - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_02_create_template_snapshot(self, value): """Test create snapshot and templates from volume @@ -248,7 +248,7 @@ class TestSecondaryStorageLimits(cloudstackTestCase): return @data(ROOT_DOMAIN_ADMIN, CHILD_DOMAIN_ADMIN) - @attr(tags = ["advanced"], required_hardware="false") + @attr(tags = ["advanced"], required_hardware="true") def test_03_register_iso(self, value): """Test register iso Steps and validations: @@ -310,7 +310,7 @@ class TestSecondaryStorageLimits(cloudstackTestCase): return @data(ROOT_DOMAIN_ADMIN, CHILD_DOMAIN_ADMIN) - @attr(tags = ["advanced"], required_hardware="false") + @attr(tags = ["advanced"], required_hardware="true") def test_04_copy_template(self, value): """Test copy template between zones diff --git a/test/integration/component/test_ss_max_limits.py b/test/integration/component/test_ss_max_limits.py index 49a1e4f17cb..dbbfb45ba34 100644 --- a/test/integration/component/test_ss_max_limits.py +++ b/test/integration/component/test_ss_max_limits.py @@ -170,7 +170,7 @@ class TestMaxSecondaryStorageLimits(cloudstackTestCase): return [FAIL, e] return [PASS, None] - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_01_deploy_vm_domain_limit_reached(self): """Test Try to deploy VM with admin account where account has not used the resources but @ domain they are not available @@ -207,7 +207,7 @@ class TestMaxSecondaryStorageLimits(cloudstackTestCase): template.delete(self.userapiclient) return - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_02_deploy_vm_account_limit_reached(self): """Test Try to deploy VM with admin account where account has used the resources but @ domain they are available @@ -244,7 +244,7 @@ class TestMaxSecondaryStorageLimits(cloudstackTestCase): template.delete(self.userapiclient) return - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_03_deploy_vm_project_limit_reached(self): """Test TTry to deploy VM with admin account where account has not used the resources but @ project they are not available diff --git a/test/integration/component/test_ss_project_limits.py b/test/integration/component/test_ss_project_limits.py index 0668f07cb26..ca58dbd7dcb 100644 --- a/test/integration/component/test_ss_project_limits.py +++ b/test/integration/component/test_ss_project_limits.py @@ -129,7 +129,7 @@ class TestProjectsVolumeLimits(cloudstackTestCase): return [FAIL, e] return [PASS, None] - @attr(tags=["advanced"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="true") def test_01_register_template_with_project(self): """Test register template # Validate the following: @@ -194,7 +194,7 @@ class TestProjectsVolumeLimits(cloudstackTestCase): self.assertEqual(response[0], PASS, response[1]) return - @attr(tags = ["advanced"], required_hardware="false") + @attr(tags = ["advanced"], required_hardware="true") def test_02_register_iso(self): """Test register iso Steps and validations: From 44d295e5c0ae2ecf8a6b0e6abf32fa665db8e83e Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Fri, 12 Dec 2014 13:45:12 +0530 Subject: [PATCH 16/39] CLOUDSTACK-8068: Fixing test script issue in test_ps_project_limits.py Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_ps_project_limits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/component/test_ps_project_limits.py b/test/integration/component/test_ps_project_limits.py index 9420e301685..bab37f6cab8 100644 --- a/test/integration/component/test_ps_project_limits.py +++ b/test/integration/component/test_ps_project_limits.py @@ -190,7 +190,7 @@ class TestProjectsVolumeLimits(cloudstackTestCase): host = hosts[0] self.vm.migrate(self.apiclient, host.id) except Exception as e: - self.fail("Exception occured" % e) + self.fail("Exception occured: %s" % e) expectedCount = self.initialResourceCount response = matchResourceCount( From ecbeb003ec94149adfeb8234f11979b5d99fba7b Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Fri, 12 Dec 2014 14:32:08 +0530 Subject: [PATCH 17/39] CLOUDSTACK-8055: Tagging test cases which can't be run on simulator accordingly Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_ip_reservation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/component/test_ip_reservation.py b/test/integration/component/test_ip_reservation.py index 37b783f7169..14b6f5c8cd6 100644 --- a/test/integration/component/test_ip_reservation.py +++ b/test/integration/component/test_ip_reservation.py @@ -305,7 +305,7 @@ class TestIpReservation(cloudstackTestCase): return @data(NAT_RULE, STATIC_NAT_RULE) - @attr(tags=["advanced"]) + @attr(tags=["advanced"], required_hardware="true") def test_nat_rules(self, value): """ Test NAT rules working with IP reservation # steps From e791e8ec6d6484c5ea7853bf7e8184103846a75e Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Fri, 12 Dec 2014 17:00:26 +0530 Subject: [PATCH 18/39] CLOUDSTACK-8069: Fixed test script in test_tags.py Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/component/test_tags.py b/test/integration/component/test_tags.py index c5a8ced8d0d..824cf011415 100644 --- a/test/integration/component/test_tags.py +++ b/test/integration/component/test_tags.py @@ -1711,7 +1711,7 @@ class TestResourceTags(cloudstackTestCase): self.apiclient, listall=True, resourceType='project', - projectid=project.id, + resourceIds=project.id, key='region', ) self.debug("tags = %s" % tags) From b07c9b7a9c0c4bfff63c722a7d619a96280102f1 Mon Sep 17 00:00:00 2001 From: Anshul Gangwar Date: Thu, 11 Dec 2014 11:12:18 +0530 Subject: [PATCH 19/39] CLOUDSTACK-8067: Fixed NPEs in MS log related to console proxy VM (cherry picked from commit 1115bc9cc6ebf2b321bb5a3ff6958d649150cfe5) Signed-off-by: Rohit Yadav --- .../src/com/cloud/consoleproxy/AgentHookBase.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/server/src/com/cloud/consoleproxy/AgentHookBase.java b/server/src/com/cloud/consoleproxy/AgentHookBase.java index 9864cae301f..8a5bae5db57 100644 --- a/server/src/com/cloud/consoleproxy/AgentHookBase.java +++ b/server/src/com/cloud/consoleproxy/AgentHookBase.java @@ -208,12 +208,13 @@ public abstract class AgentHookBase implements AgentHook { HostVO consoleProxyHost = findConsoleProxyHost(startupCmd); assert (consoleProxyHost != null); - - Answer answer = _agentMgr.send(consoleProxyHost.getId(), cmd); - if (answer == null || !answer.getResult()) { - s_logger.error("Console proxy agent reported that it failed to execute http handling startup command"); - } else { - s_logger.info("Successfully sent out command to start HTTP handling in console proxy agent"); + if (consoleProxyHost != null) { + Answer answer = _agentMgr.send(consoleProxyHost.getId(), cmd); + if (answer == null || !answer.getResult()) { + s_logger.error("Console proxy agent reported that it failed to execute http handling startup command"); + } else { + s_logger.info("Successfully sent out command to start HTTP handling in console proxy agent"); + } } }catch (NoSuchAlgorithmException e) { s_logger.error("Unexpected exception in SecureRandom Algorithm selection ", e); From 6321a29e4336de9ffe96e27968f896ec5a8bf37d Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Sat, 13 Dec 2014 00:15:40 +0530 Subject: [PATCH 20/39] CLOUDSTACK-8070: Upgrade router.ram.size using encryption util Upgrade fails if value is set using plain text encoding, the value needs to be encrypted (if a key was provided during db was setup). Signed-off-by: Rohit Yadav --- .../cloud/upgrade/dao/Upgrade442to450.java | 25 +++++++++++++++++++ setup/db/db/schema-442to450.sql | 2 -- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade442to450.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade442to450.java index aeb44a16028..e8a517b236b 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade442to450.java +++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade442to450.java @@ -18,6 +18,7 @@ package com.cloud.upgrade.dao; import java.io.File; +import java.io.UnsupportedEncodingException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -30,6 +31,7 @@ import java.util.Map; import java.util.Set; import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.crypt.DBEncryptionUtil; import org.apache.log4j.Logger; import com.cloud.utils.exception.CloudRuntimeException; @@ -68,10 +70,33 @@ public class Upgrade442to450 implements DbUpgrade { updateSystemVmTemplates(conn); dropInvalidKeyFromStoragePoolTable(conn); dropDuplicatedForeignKeyFromAsyncJobTable(conn); + updateMaxRouterSizeConfig(conn); upgradeMemoryOfVirtualRoutervmOffering(conn); upgradeMemoryOfInternalLoadBalancervmOffering(conn); } + private void updateMaxRouterSizeConfig(Connection conn) { + PreparedStatement updatePstmt = null; + try { + String encryptedValue = DBEncryptionUtil.encrypt("256"); + updatePstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value=? WHERE name='router.ram.size' AND category='Hidden'"); + updatePstmt.setBytes(1, encryptedValue.getBytes("UTF-8")); + updatePstmt.executeUpdate(); + } catch (SQLException e) { + throw new CloudRuntimeException("Unable to upgrade max ram size of router in config.", e); + } catch (UnsupportedEncodingException e) { + throw new CloudRuntimeException("Unable encrypt configuration values ", e); + } finally { + try { + if (updatePstmt != null) { + updatePstmt.close(); + } + } catch (SQLException e) { + } + } + s_logger.debug("Done updating router.ram.size config to 256"); + } + private void upgradeMemoryOfVirtualRoutervmOffering(Connection conn) { PreparedStatement updatePstmt = null; PreparedStatement selectPstmt = null; diff --git a/setup/db/db/schema-442to450.sql b/setup/db/db/schema-442to450.sql index 3981415f5d5..c51bd00cb37 100644 --- a/setup/db/db/schema-442to450.sql +++ b/setup/db/db/schema-442to450.sql @@ -448,8 +448,6 @@ UPDATE configuration SET value='KVM,XenServer,VMware,BareMetal,Ovm,LXC,Hyperv' W UPDATE `cloud`.`configuration` SET description="If set to true, will set guest VM's name as it appears on the hypervisor, to its hostname. The flag is supported for VMware hypervisor only" WHERE name='vm.instancename.flag'; INSERT IGNORE INTO `cloud`.`configuration`(category, instance, component, name, value, description, default_value) VALUES ('Advanced', 'DEFAULT', 'management-server', 'implicit.host.tags', 'GPU', 'Tag hosts at the time of host disovery based on the host properties/capabilities ', 'GPU'); -UPDATE `cloud`.`configuration` SET value='256' WHERE name='router.ram.size'; - DROP VIEW IF EXISTS `cloud`.`domain_router_view`; CREATE VIEW `cloud`.`domain_router_view` AS select From aca0f7959274d4cc3d6a8a727c65977952d66dce Mon Sep 17 00:00:00 2001 From: Jayapal Date: Mon, 15 Dec 2014 15:41:30 +0530 Subject: [PATCH 21/39] CLOUDSTACK-8072: Fixed NPE in deleting default ACL items in default ACL --- .../src/com/cloud/network/vpc/NetworkACLServiceImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/com/cloud/network/vpc/NetworkACLServiceImpl.java b/server/src/com/cloud/network/vpc/NetworkACLServiceImpl.java index 51bb71f7eb4..bee4018145d 100644 --- a/server/src/com/cloud/network/vpc/NetworkACLServiceImpl.java +++ b/server/src/com/cloud/network/vpc/NetworkACLServiceImpl.java @@ -612,13 +612,14 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); + if((aclItem.getAclId() == NetworkACL.DEFAULT_ALLOW) || (aclItem.getAclId() == NetworkACL.DEFAULT_DENY)){ + throw new InvalidParameterValueException("ACL Items in default ACL cannot be deleted"); + } + Account caller = CallContext.current().getCallingAccount(); _accountMgr.checkAccess(caller, null, true, vpc); - if((aclItem.getAclId() == NetworkACL.DEFAULT_ALLOW) || (aclItem.getAclId() == NetworkACL.DEFAULT_DENY)){ - throw new InvalidParameterValueException("ACL Items in default ACL cannot be deleted"); - } } return _networkAclMgr.revokeNetworkACLItem(ruleId); } From 453c5b3ec0365603a38468eb7de5d3dec0029ace Mon Sep 17 00:00:00 2001 From: Mike Tutkowski Date: Mon, 15 Dec 2014 14:11:31 -0700 Subject: [PATCH 22/39] When deleting primary storage that's based on the SolidFireShared plug-in (which uses a shared-IOPS approach), remove the volume from the volume access group it's in --- ...idFireSharedPrimaryDataStoreLifeCycle.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java index b70333d265c..4fa49ef9daa 100644 --- a/plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java @@ -52,6 +52,7 @@ import com.cloud.agent.api.CreateStoragePoolCommand; import com.cloud.agent.api.DeleteStoragePoolCommand; import com.cloud.agent.api.StoragePoolInfo; import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; import com.cloud.dc.ClusterVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; @@ -491,6 +492,8 @@ public class SolidFireSharedPrimaryDataStoreLifeCycle implements PrimaryDataStor _tmpltMgr.evictTemplateFromStoragePool(templatePoolVO); } + Long clusterId = null; + for (StoragePoolHostVO host : hostPoolRecords) { DeleteStoragePoolCommand deleteCmd = new DeleteStoragePoolCommand(storagePool); @@ -523,6 +526,12 @@ public class SolidFireSharedPrimaryDataStoreLifeCycle implements PrimaryDataStor if (answer != null && answer.getResult()) { s_logger.info("Successfully deleted storage pool using Host ID " + host.getHostId()); + HostVO hostVO = this._hostDao.findById(host.getHostId()); + + if (hostVO != null) { + clusterId = hostVO.getClusterId(); + } + break; } else { @@ -530,11 +539,35 @@ public class SolidFireSharedPrimaryDataStoreLifeCycle implements PrimaryDataStor } } + if (clusterId != null) { + removeVolumeFromVag(storagePool.getId(), clusterId); + } + deleteSolidFireVolume(storagePool.getId()); return _primaryDataStoreHelper.deletePrimaryDataStore(dataStore); } + private void removeVolumeFromVag(long storagePoolId, long clusterId) { + long sfVolumeId = getVolumeId(storagePoolId); + ClusterDetailsVO clusterDetail = _clusterDetailsDao.findDetail(clusterId, SolidFireUtil.getVagKey(storagePoolId)); + + String vagId = clusterDetail != null ? clusterDetail.getValue() : null; + + if (vagId != null) { + List hosts = _hostDao.findByClusterId(clusterId); + + SolidFireUtil.SolidFireConnection sfConnection = SolidFireUtil.getSolidFireConnection(storagePoolId, _storagePoolDetailsDao); + + SolidFireUtil.SolidFireVag sfVag = SolidFireUtil.getSolidFireVag(sfConnection, Long.parseLong(vagId)); + + String[] hostIqns = SolidFireUtil.getNewHostIqns(sfVag.getInitiators(), SolidFireUtil.getIqnsFromHosts(hosts)); + long[] volumeIds = SolidFireUtil.getNewVolumeIds(sfVag.getVolumeIds(), sfVolumeId, false); + + SolidFireUtil.modifySolidFireVag(sfConnection, sfVag.getId(), hostIqns, volumeIds); + } + } + private void deleteSolidFireVolume(long storagePoolId) { SolidFireUtil.SolidFireConnection sfConnection = SolidFireUtil.getSolidFireConnection(storagePoolId, _storagePoolDetailsDao); From 66b77380d06a38e24e3bd84e4804d3ff8d19ff95 Mon Sep 17 00:00:00 2001 From: Remi Bergsma Date: Mon, 15 Dec 2014 16:36:35 +0100 Subject: [PATCH 23/39] use directIO flags when dd'ing template This makes sure dom0 in xenserver doesn't get hammered when copying templates. It doesn't make sense to use the cache of dom0 as the template does not fit in memory. The directIO flags prevent it from trying. (cherry picked from commit 4e1527e87aaaa87d14d3c7d3a6782b80cbf36a8c) --- .../vm/hypervisor/xenserver/copy_vhd_from_secondarystorage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vm/hypervisor/xenserver/copy_vhd_from_secondarystorage.sh b/scripts/vm/hypervisor/xenserver/copy_vhd_from_secondarystorage.sh index 098015a419d..61c65ea45a0 100755 --- a/scripts/vm/hypervisor/xenserver/copy_vhd_from_secondarystorage.sh +++ b/scripts/vm/hypervisor/xenserver/copy_vhd_from_secondarystorage.sh @@ -104,7 +104,7 @@ copyvhd() exit 0 fi if [ "${parent##*vhd has}" = " no parent" ]; then - dd if=$srcvhd of=$desvhd bs=2M + dd if=$srcvhd of=$desvhd bs=2M oflag=direct iflag=direct if [ $? -ne 0 ]; then echo "31#failed to dd $srcvhd to $desvhd" cleanup From 6a8dcf5f204d51541b77d363567ad045b22071ae Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Tue, 16 Dec 2014 14:38:50 -0800 Subject: [PATCH 24/39] CLOUDSTACK-8076: UI > Templates menu > Template/ISO > "Filter by" dropdown > add a new option, "shared", which will list shared templates when being selected. --- ui/scripts/templates.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ui/scripts/templates.js b/ui/scripts/templates.js index 1ca3214ca13..dc0f055bf01 100644 --- a/ui/scripts/templates.js +++ b/ui/scripts/templates.js @@ -43,6 +43,9 @@ mine: { label: 'ui.listView.filters.mine' }, + shared: { + label: 'Shared' + }, featured: { label: 'label.featured' }, @@ -589,6 +592,11 @@ templatefilter: 'self' }); break; + case "shared": + $.extend(data, { + templatefilter: 'shared' + }); + break; case "featured": ignoreProject = true; $.extend(data, { @@ -1452,6 +1460,9 @@ mine: { label: 'ui.listView.filters.mine' }, + shared: { + label: 'Shared' + }, featured: { label: 'label.featured' }, @@ -1717,6 +1728,11 @@ isofilter: 'self' }); break; + case "shared": + $.extend(data, { + isofilter: 'shared' + }); + break; case "featured": ignoreProject = true; $.extend(data, { From e7a96fde790fe812843e1efe9d05814cfb76d7cc Mon Sep 17 00:00:00 2001 From: Brian Federle Date: Tue, 16 Dec 2014 14:53:20 -0800 Subject: [PATCH 25/39] CLOUDSTACK-8075: Front-end support for shared templates/ISOs --- ui/css/cloudstack3.css | 4 ++-- ui/index.jsp | 14 ++++++++++++-- ui/scripts/instanceWizard.js | 10 ++++++---- ui/scripts/ui-custom/instanceWizard.js | 11 ++++++++--- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/ui/css/cloudstack3.css b/ui/css/cloudstack3.css index 29148348736..b3f9b069f80 100644 --- a/ui/css/cloudstack3.css +++ b/ui/css/cloudstack3.css @@ -5847,14 +5847,14 @@ label.error { .multi-wizard .ui-tabs li.ui-state-default a { float: left; - padding: 15px; - width: 110px; text-align: center; font-size: 11px; margin-right: 1px; color: #4E6070; text-decoration: none; background: #DEE3E5; + padding-left: 0; + padding-right: 0; } .multi-wizard .ui-tabs li.ui-state-active a { diff --git a/ui/index.jsp b/ui/index.jsp index bab00eb2cea..83d5d54987f 100644 --- a/ui/index.jsp +++ b/ui/index.jsp @@ -148,7 +148,8 @@
    • -
    • +
    • +
@@ -166,6 +167,10 @@
+
+
+
+
@@ -184,7 +189,8 @@
    • -
    • +
    • +
diff --git a/ui/scripts/instanceWizard.js b/ui/scripts/instanceWizard.js index 2a02ad5b573..99e79be62db 100644 --- a/ui/scripts/instanceWizard.js +++ b/ui/scripts/instanceWizard.js @@ -231,14 +231,16 @@ templatesObj = { featuredtemplates: featuredTemplateObjs, communitytemplates: communityTemplateObjs, - mytemplates: myTemplateObjs - } + mytemplates: myTemplateObjs, + sharedtemplates: [] // ** Needs implementation ** + }; } else if (selectedTemplateOrIso == 'select-iso') { templatesObj = { featuredisos: featuredIsoObjs, communityisos: communityIsoObjs, - myisos: myIsoObjs - } + myisos: myIsoObjs, + sharedisos: [] // ** Needs implementation ** + }; } args.response.success({ hypervisor: { diff --git a/ui/scripts/ui-custom/instanceWizard.js b/ui/scripts/ui-custom/instanceWizard.js index 527dc105474..a4adc18807d 100644 --- a/ui/scripts/ui-custom/instanceWizard.js +++ b/ui/scripts/ui-custom/instanceWizard.js @@ -342,7 +342,7 @@ var $templateHypervisor = $step.find('input[type=hidden][wizard-field=hypervisor]'); // Get hypervisor from template - if (type == 'featuredtemplates' || type == 'communitytemplates' || type == 'mytemplates') { + if (type == 'featuredtemplates' || type == 'communitytemplates' || type == 'mytemplates' || type == 'sharedtemplates') { $selects.each(function() { var $select = $(this); var template = $.grep(args.data.templates[type], function(tmpl, v) { @@ -359,7 +359,7 @@ $templateHypervisor.attr('disabled', 'disabled'); } - if (type == 'featuredisos' || type == 'communityisos' || type == 'myisos') { + if (type == 'featuredisos' || type == 'communityisos' || type == 'myisos' || type == 'sharedisos') { // Create hypervisor select $selects.find('input').bind('click', function() { var $select = $(this).closest('.select'); @@ -368,6 +368,7 @@ $("#instance-wizard-featured-isos .select-container div.selected").removeClass('selected').find('div.hypervisor').remove(); $("#instance-wizard-community-isos .select-container div.selected").removeClass('selected').find('div.hypervisor').remove(); $("#instance-wizard-my-isos .select-container div.selected").removeClass('selected').find('div.hypervisor').remove(); + $("#instance-wizard-shared-isos .select-container div.selected").removeClass('selected').find('div.hypervisor').remove(); $select.addClass('selected').append( $('
').addClass('hypervisor') @@ -396,13 +397,17 @@ // Featured ISOs $( [ + // Templates ['featuredtemplates', 'instance-wizard-featured-templates'], ['communitytemplates', 'instance-wizard-community-templates'], ['mytemplates', 'instance-wizard-my-templates'], + ['sharedtemplates', 'instance-wizard-shared-templates'], + // ISOs ['featuredisos', 'instance-wizard-featured-isos'], ['communityisos', 'instance-wizard-community-isos'], - ['myisos', 'instance-wizard-my-isos'] + ['myisos', 'instance-wizard-my-isos'], + ['sharedisos', 'instance-wizard-shared-isos'], //['isos', 'instance-wizard-all-isos'] ] ).each(function() { From 20777ec3b4733301e2069e82e7eb231e4f3a2757 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Tue, 16 Dec 2014 16:15:34 -0800 Subject: [PATCH 26/39] CLOUDSTACK-8075: UI > Instances menu > Add Instance > Select template/ISO > add a new tab "shared" which will list shared templates/ISOs when being clicked. --- ui/scripts/instanceWizard.js | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/ui/scripts/instanceWizard.js b/ui/scripts/instanceWizard.js index 99e79be62db..a067d7d7a69 100644 --- a/ui/scripts/instanceWizard.js +++ b/ui/scripts/instanceWizard.js @@ -16,7 +16,7 @@ // under the License. (function($, cloudStack) { - var zoneObjs, hypervisorObjs, featuredTemplateObjs, communityTemplateObjs, myTemplateObjs, featuredIsoObjs, serviceOfferingObjs, community, networkObjs; + var zoneObjs, hypervisorObjs, featuredTemplateObjs, communityTemplateObjs, myTemplateObjs, sharedTemplateObjs, featuredIsoObjs, communityIsoObjs, myIsoObjs, sharedIsoObjs, serviceOfferingObjs, community, networkObjs; var selectedZoneObj, selectedTemplateObj, selectedHypervisor, selectedDiskOfferingObj; var selectedTemplateOrIso; //'select-template', 'select-iso' var step6ContainerType = 'nothing-to-select'; //'nothing-to-select', 'select-network', 'select-security-group', 'select-advanced-sg'(advanced sg-enabled zone) @@ -184,7 +184,22 @@ }); } } - }); + }); + $.ajax({ + url: createURL("listTemplates&templatefilter=sharedexecutable&zoneid=" + args.currentData.zoneid), + dataType: "json", + async: false, + success: function(json) { + if (json.listtemplatesresponse.template == null) { + sharedTemplateObjs = null; + } else { + sharedTemplateObjs = $.grep(json.listtemplatesresponse.template, function(item, index) { + if ($.inArray(item.hypervisor, hypervisorArray) > -1) + return true; + }); + } + } + }); } else if (selectedTemplateOrIso == 'select-iso') { $.ajax({ url: createURL("listIsos&isofilter=featured&zoneid=" + args.currentData.zoneid + "&bootable=true"), @@ -222,6 +237,18 @@ } } }); + $.ajax({ + url: createURL("listIsos&isofilter=sharedexecutable&zoneid=" + args.currentData.zoneid + "&bootable=true"), + dataType: "json", + async: false, + success: function(json) { + if (json.listisosresponse.iso == null) { + sharedIsoObjs = null; + } else { + sharedIsoObjs = json.listisosresponse.iso; + } + } + }); } //***** get templates/ISOs (end) ***** @@ -232,14 +259,14 @@ featuredtemplates: featuredTemplateObjs, communitytemplates: communityTemplateObjs, mytemplates: myTemplateObjs, - sharedtemplates: [] // ** Needs implementation ** + sharedtemplates: sharedTemplateObjs }; } else if (selectedTemplateOrIso == 'select-iso') { templatesObj = { featuredisos: featuredIsoObjs, communityisos: communityIsoObjs, myisos: myIsoObjs, - sharedisos: [] // ** Needs implementation ** + sharedisos: sharedIsoObjs }; } args.response.success({ From 9fb654cbad3ecd5e4a393f41ee5a6ad44c15680f Mon Sep 17 00:00:00 2001 From: Min Chen Date: Tue, 16 Dec 2014 15:54:45 -0800 Subject: [PATCH 27/39] CLOUDSTACK-8077: Not able to deploy VM using a shared template. --- server/src/com/cloud/vm/UserVmManagerImpl.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 7c60d4a35a7..4e64524fe71 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -2784,10 +2784,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } // Check templates permissions - if (!template.isPublicTemplate()) { - Account templateOwner = _accountMgr.getAccount(template.getAccountId()); - _accountMgr.checkAccess(owner, null, true, templateOwner); - } + _accountMgr.checkAccess(owner, AccessType.UseEntry, false, template); // check if the user data is correct validateUserData(userData, httpmethod); From c3508f61a18384e9201d3d1dc87c35c67e200039 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Tue, 16 Dec 2014 17:13:05 +0530 Subject: [PATCH 28/39] CLOUDSTACK-8074: Fixed maint/test_multiple_ip_ranges.py Signed-off-by: SrikanteswaraRao Talluri --- .../maint/test_multiple_ip_ranges.py | 972 +++++++++--------- 1 file changed, 510 insertions(+), 462 deletions(-) diff --git a/test/integration/component/maint/test_multiple_ip_ranges.py b/test/integration/component/maint/test_multiple_ip_ranges.py index 982dd7c85bc..f73c470d003 100644 --- a/test/integration/component/maint/test_multiple_ip_ranges.py +++ b/test/integration/component/maint/test_multiple_ip_ranges.py @@ -16,117 +16,72 @@ # under the License. """ Tests for Multiple IP Ranges feature """ -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.cloudstackException import CloudstackAPIException -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * -# from netaddr import * +from marvin.cloudstackTestCase import cloudstackTestCase, unittest +from marvin.lib.utils import cleanup_resources, get_process_status +from marvin.lib.base import (Account, + DiskOffering, + VirtualMachine, + Router, + ServiceOffering, + PublicIpRange) +from marvin.lib.common import (get_domain, + get_zone, + list_routers, + list_hosts, + get_pod, + get_template) import netaddr from nose.plugins.attrib import attr +from netaddr import IPNetwork, IPAddress +from marvin.sshClient import SshClient +import random -class Services: - """Test Multiple IP Ranges - """ - def __init__(self): - self.services = { - "account": { - "email": "test@test.com", - "firstname": "Test", - "lastname": "User", - "username": "test", - # Random characters are appended for unique - # username - "password": "password", - }, - "service_offering": { - "name": "Tiny Instance", - "displaytext": "Tiny Instance", - "cpunumber": 1, - "cpuspeed": 200, # in MHz - "memory": 256, # In MBs - }, - "disk_offering": { - "displaytext": "Small Disk", - "name": "Small Disk", - "disksize": 1 - }, - "templates": { - "displaytext": 'Template', - "name": 'Template', - "ostype": "CentOS 5.3 (64-bit)", - "templatefilter": 'self', - }, - "vlan_ip_range": { - "startip": "10.147.43.130", - "endip": "10.147.43.135", - "netmask": "255.255.255.192", - "gateway": "10.147.43.129", - "forvirtualnetwork": "false", - "vlan": "untagged", - }, - "server_without_disk": { - "displayname": "Test VM-No Disk", - "username": "root", - "password": "password", - "hypervisor": 'XenServer', - }, - "host": { - "publicport": 22, - "username": "root", # Host creds for SSH - "password": "password", - }, - "ostype": "CentOS 5.3 (64-bit)", - "sleep": 60, - "timeout": 10, - } class TestMultipleIpRanges(cloudstackTestCase): + """Test Multiple IP Ranges for guest network """ @classmethod def setUpClass(cls): - cls.testClient = super(TestEgressAfterHostMaintenance, cls).getClsTestClient() + cls.testClient = super(TestMultipleIpRanges, cls).getClsTestClient() cls.api_client = cls.testClient.getApiClient() cls.dbclient = cls.testClient.getDbConnection() - cls.services = Services().services + cls.testdata = cls.testClient.getParsedTestDataConfig() # Get Zone, Domain and templates cls.domain = get_domain(cls.api_client) cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) cls.pod = get_pod(cls.api_client, cls.zone.id) - cls.services['mode'] = cls.zone.networktype - cls.services["domainid"] = cls.domain.id - cls.services["zoneid"] = cls.zone.id + cls.testdata['mode'] = cls.zone.networktype + cls.testdata["domainid"] = cls.domain.id + cls.testdata["zoneid"] = cls.zone.id cls.account = Account.create( - cls.api_client, - cls.services["account"], - domainid=cls.domain.id - ) - cls.services["account"] = cls.account.name + cls.api_client, + cls.testdata["account"], + domainid=cls.domain.id + ) + cls.testdata["account"] = cls.account.name cls.disk_offering = DiskOffering.create( - cls.api_client, - cls.services["disk_offering"] - ) + cls.api_client, + cls.testdata["disk_offering"] + ) cls.service_offering = ServiceOffering.create( - cls.api_client, - cls.services["service_offering"] - ) + cls.api_client, + cls.testdata["service_offering"] + ) cls.template = get_template( - cls.api_client, - cls.zone.id, - cls.services["ostype"] - ) - cls.services["templates"]["ostypeid"] = cls.template.ostypeid - cls.services["diskoffering"] = cls.disk_offering.id + cls.api_client, + cls.zone.id, + cls.testdata["ostype"] + ) + cls.testdata["diskoffering"] = cls.disk_offering.id cls.dc_id = cls.dbclient.execute( - "select id from data_center where uuid = '%s';" % str(cls.services["zoneid"]) - ) + "select id from data_center where uuid = '%s';" % str( + cls.testdata["zoneid"])) cls.dc_id = cls.dc_id[0][0] cls.ids = cls.dbclient.execute( - "select id from user_ip_address where allocated is null and data_center_id = '%s';" % str(cls.dc_id) - ) + "select id from user_ip_address where allocated is null and data_center_id = '%s';" % str( + cls.dc_id)) cls.id_list = [] for i in range(len(cls.ids)): cls.id_list.append(cls.ids[i][0]) @@ -142,45 +97,76 @@ class TestMultipleIpRanges(cloudstackTestCase): cls.id_list = cls.id_list[:-2] for id in cls.id_list: cls.dbclient.execute( - "update user_ip_address set allocated=now() where id = '%s';" % str(id) - ) - # Add IP range in the new CIDR - cls.services["vlan_ip_range"]["zoneid"] = cls.zone.id - cls.services["vlan_ip_range"]["podid"] = cls.pod.id + "update user_ip_address set allocated=now() where id = '%s';" % + str(id)) # create new vlan ip range # Before creating ip range check the zone's network type - if cls.zone.networktype == 'Basic': - cls.new_vlan = PublicIpRange.create(cls.api_client, cls.services["vlan_ip_range"]) + if cls.zone.networktype.lower() == 'basic': + cls.new_vlan = cls.createNewVlanRange() else: - raise unittest.SkipTest("These tests can be run only on basic zone. So skipping the tests") + raise unittest.SkipTest( + "These tests can be run only on basic zone.\ + So skipping the tests") # Deploy vm in existing subnet if VR is not present - if cls.vr_state is False : + if cls.vr_state is False: cls.vm_res = VirtualMachine.create( - cls.api_client, - cls.services["server_without_disk"], - templateid=cls.template.id, - accountid=cls.account.name, - domainid=cls.services["domainid"], - zoneid=cls.services["zoneid"], - serviceofferingid=cls.service_offering.id, - mode=cls.services["mode"], - ) + cls.api_client, + cls.testdata["server_without_disk"], + templateid=cls.template.id, + accountid=cls.account.name, + domainid=cls.testdata["domainid"], + zoneid=cls.testdata["zoneid"], + serviceofferingid=cls.service_offering.id, + mode=cls.testdata["mode"], + ) cls._cleanup = [ - cls.new_vlan, - cls.account, - ] + cls.new_vlan, + cls.account, + ] return + @classmethod + def createNewVlanRange(cls): + """ Increment current cidr of vlan range present in network + and create new range + """ + publicIpRange = PublicIpRange.list(cls.api_client) + cls.startIp = publicIpRange[0].startip + cls.endIp = publicIpRange[0].endip + cls.gateway = publicIpRange[0].gateway + cls.netmask = publicIpRange[0].netmask + # Pass ip address and mask length to IPNetwork to findout the CIDR + ip = IPNetwork(cls.startIp + "/" + cls.netmask) + # Take random increment factor to avoid adding the same vlan ip range + # in each test case + networkIncrementFactor = random.randint(1,255) + new_cidr = ip.__iadd__(networkIncrementFactor) + ip2 = IPNetwork(new_cidr) + test_nw = ip2.network + ip = IPAddress(test_nw) + # Add IP range(5 IPs) in the new CIDR + test_gateway = ip.__add__(1) + test_startIp = ip.__add__(3) + test_endIp = ip.__add__(10) + # Populating services with new IP range + cls.testdata["vlan_ip_range"]["startip"] = test_startIp + cls.testdata["vlan_ip_range"]["endip"] = test_endIp + cls.testdata["vlan_ip_range"]["gateway"] = test_gateway + cls.testdata["vlan_ip_range"]["netmask"] = cls.netmask + cls.testdata["vlan_ip_range"]["zoneid"] = cls.zone.id + cls.testdata["vlan_ip_range"]["podid"] = cls.pod.id + + return PublicIpRange.create( + cls.api_client, + cls.testdata["vlan_ip_range"]) + @classmethod def tearDownClass(cls): try: for id in cls.id_list: cls.dbclient.execute( - "update user_ip_address set allocated=default where id = '%s';" % str(id) - ) - # Wait for expunge interval to cleanup VMs - wait_for_cleanup(cls.api_client, ["expunge.delay", "expunge.interval"]) - time.sleep(30) + "update user_ip_address set allocated=default where id = '%s';" % + str(id)) # Cleanup resources used cleanup_resources(cls.api_client, cls._cleanup) except Exception as e: @@ -190,48 +176,55 @@ class TestMultipleIpRanges(cloudstackTestCase): def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() - self.cleanup = [ ] + self.cleanup = [] # Deploy guest vm - try : + try: self.virtual_machine = VirtualMachine.create( - self.apiclient, - self.services["server_without_disk"], - templateid=self.template.id, - accountid=self.account.name, - domainid=self.services["domainid"], - zoneid=self.services["zoneid"], - serviceofferingid=self.service_offering.id, - mode=self.services["mode"], - ) - except Exception as e : - raise Exception("Warning: Exception during vm deployment: {}".format(e)) + self.apiclient, + self.testdata["server_without_disk"], + templateid=self.template.id, + accountid=self.account.name, + domainid=self.testdata["domainid"], + zoneid=self.testdata["zoneid"], + serviceofferingid=self.service_offering.id, + mode=self.testdata["mode"], + ) + except Exception as e: + raise Exception( + "Warning: Exception during vm deployment: {}".format(e)) self.vm_response = VirtualMachine.list( - self.apiclient, - id=self.virtual_machine.id - ) + self.apiclient, + id=self.virtual_machine.id + ) self.assertEqual( isinstance(self.vm_response, list), True, "Check VM list response returned a valid list" - ) - self.ip_range = list(netaddr.iter_iprange(unicode(self.services["vlan_ip_range"]["startip"]), unicode(self.services["vlan_ip_range"]["endip"]))) - self.nic_ip = netaddr.IPAddress(unicode(self.vm_response[0].nic[0].ipaddress)) + ) + self.ip_range = list( + netaddr.iter_iprange( + unicode( + self.testdata["vlan_ip_range"]["startip"]), unicode( + self.testdata["vlan_ip_range"]["endip"]))) + self.nic_ip = netaddr.IPAddress( + unicode( + self.vm_response[0].nic[0].ipaddress)) self.debug("vm got {} as ip address".format(self.nic_ip)) self.assertIn( - self.nic_ip, - self.ip_range, - "VM did not get the ip address from the new ip range" - ) + self.nic_ip, + self.ip_range, + "VM did not get the ip address from the new ip range" + ) ip_alias = self.dbclient.execute( - "select ip4_address from nic_ip_alias;" - ) + "select ip4_address from nic_ip_alias;" + ) self.alias_ip = str(ip_alias[0][0]) self.debug("alias ip : %s" % self.alias_ip) self.assertNotEqual( - self.alias_ip, - None, - "Error in creating ip alias. Please check MS logs" - ) + self.alias_ip, + None, + "Error in creating ip alias. Please check MS logs" + ) self.cleanup.append(self.virtual_machine) return @@ -246,35 +239,38 @@ class TestMultipleIpRanges(cloudstackTestCase): def verify_vlan_range(self, vlan, services): # compare vlan_list response with configured values self.assertEqual( - isinstance(vlan, list), - True, - "Check list response returned a valid list" - ) + isinstance(vlan, list), + True, + "Check list response returned a valid list" + ) self.assertNotEqual( - len(vlan), - 0, - "check list vlan response" - ) + len(vlan), + 0, + "check list vlan response" + ) self.assertEqual( - str(vlan[0].startip), - str(services["startip"]), - "Start IP in vlan ip range is not matched with the configured start ip" - ) + str(vlan[0].startip), + str(services["startip"]), + "Start IP in vlan ip range is not matched with the\ + configured start ip" + ) self.assertEqual( - str(vlan[0].endip), - str(services["endip"]), - "End IP in vlan ip range is not matched with the configured end ip" - ) + str(vlan[0].endip), + str(services["endip"]), + "End IP in vlan ip range is not matched with the configured end ip" + ) self.assertEqual( - str(vlan[0].gateway), - str(services["gateway"]), - "gateway in vlan ip range is not matched with the configured gateway" - ) + str(vlan[0].gateway), + str(services["gateway"]), + "gateway in vlan ip range is not matched with the\ + configured gateway" + ) self.assertEqual( - str(vlan[0].netmask), - str(services["netmask"]), - "netmask in vlan ip range is not matched with the configured netmask" - ) + str(vlan[0].netmask), + str(services["netmask"]), + "netmask in vlan ip range is not matched with\ + the configured netmask" + ) return @attr(tags=["sg"]) @@ -283,14 +279,20 @@ class TestMultipleIpRanges(cloudstackTestCase): 1.Deploy guest vm 2.Verify vm gets the ip address from new cidr """ - self.ip_range = list(netaddr.iter_iprange(unicode(self.services["vlan_ip_range"]["startip"]), unicode(self.services["vlan_ip_range"]["endip"]))) - self.nic_ip = netaddr.IPAddress(unicode(self.vm_response[0].nic[0].ipaddress)) + self.ip_range = list( + netaddr.iter_iprange( + unicode( + self.testdata["vlan_ip_range"]["startip"]), unicode( + self.testdata["vlan_ip_range"]["endip"]))) + self.nic_ip = netaddr.IPAddress( + unicode( + self.vm_response[0].nic[0].ipaddress)) self.debug("vm got {} as ip address".format(self.nic_ip)) self.assertIn( - self.nic_ip, - self.ip_range, - "VM did not get the ip address from the new ip range" - ) + self.nic_ip, + self.ip_range, + "VM did not get the ip address from the new ip range" + ) return @attr(tags=["sg"]) @@ -300,51 +302,60 @@ class TestMultipleIpRanges(cloudstackTestCase): 2.Verify dns service listens on alias ip in VR """ list_router_response = list_routers( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) + self.apiclient, + zoneid=self.zone.id, + listall=True + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] hosts = list_hosts( - self.apiclient, - zoneid=router.zoneid, - type='Routing', - state='Up', - id=router.hostid - ) + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list host returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list host returns a valid list" + ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" - ) + router.state, + 'Running', + "Check list router response for router state" + ) + + port = self.testdata['configurableData']['host']["publicport"] + username = self.testdata['configurableData']['host']["username"] + password = self.testdata['configurableData']['host']["password"] + + # SSH to host so that host key is saved in first + # attempt + SshClient(host.ipaddress, port, username, password) + proc = self.alias_ip + ":53" result = get_process_status( - host.ipaddress, - self.services['host']["publicport"], - self.services['host']["username"], - self.services['host']["password"], - router.linklocalip, - "netstat -atnp | grep %s" % proc - ) + host.ipaddress, + port, + username, + password, + router.linklocalip, + "netstat -atnp | grep %s" % proc + ) res = str(result) self.debug("Dns process status on alias ip: %s" % res) self.assertNotEqual( - res.find(proc) - - 1, - "dnsmasq service is not running on alias ip" - ) + res.find(proc) + - 1, + "dnsmasq service is not running on alias ip" + ) return @attr(tags=["sg"]) @@ -354,51 +365,60 @@ class TestMultipleIpRanges(cloudstackTestCase): 2.Verify password service(socat) listens on alias ip in VR """ list_router_response = list_routers( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) + self.apiclient, + zoneid=self.zone.id, + listall=True + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] hosts = list_hosts( - self.apiclient, - zoneid=router.zoneid, - type='Routing', - state='Up', - id=router.hostid - ) + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list host returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list host returns a valid list" + ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" - ) + router.state, + 'Running', + "Check list router response for router state" + ) + + port = self.testdata['configurableData']['host']["publicport"] + username = self.testdata['configurableData']['host']["username"] + password = self.testdata['configurableData']['host']["password"] + + # SSH to host so that host key is saved in first + # attempt + SshClient(host.ipaddress, port, username, password) + proc = "socat" result = get_process_status( - host.ipaddress, - self.services['host']["publicport"], - self.services['host']["username"], - self.services['host']["password"], - router.linklocalip, - "netstat -atnp | grep %s" % proc - ) + host.ipaddress, + port, + username, + password, + router.linklocalip, + "netstat -atnp | grep %s" % proc + ) res = str(result) self.debug("password process status on VR: %s" % res) self.assertNotEqual( - res.find(self.alias_ip) - - 1, - "password service is not running on alias ip" - ) + res.find(self.alias_ip) + - 1, + "password service is not running on alias ip" + ) return @attr(tags=["sg"]) @@ -408,51 +428,60 @@ class TestMultipleIpRanges(cloudstackTestCase): 2.Verify userdata service(apache2) listens on alias ip in VR """ list_router_response = list_routers( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) + self.apiclient, + zoneid=self.zone.id, + listall=True + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] hosts = list_hosts( - self.apiclient, - zoneid=router.zoneid, - type='Routing', - state='Up', - id=router.hostid - ) + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list host returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list host returns a valid list" + ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" - ) + router.state, + 'Running', + "Check list router response for router state" + ) + + port = self.testdata['configurableData']['host']["publicport"] + username = self.testdata['configurableData']['host']["username"] + password = self.testdata['configurableData']['host']["password"] + + # SSH to host so that host key is saved in first + # attempt + SshClient(host.ipaddress, port, username, password) + proc = "apache2" result = get_process_status( - host.ipaddress, - self.services['host']["publicport"], - self.services['host']["username"], - self.services['host']["password"], - router.linklocalip, - "netstat -atnp | grep %s" % proc - ) + host.ipaddress, + port, + username, + password, + router.linklocalip, + "netstat -atnp | grep %s" % proc + ) res = str(result) self.debug("userdata process status on VR: %s" % res) self.assertNotEqual( - res.find(self.alias_ip + ":80 ") - - 1, - "password service is not running on alias ip" - ) + res.find(self.alias_ip + ":80 ") + - 1, + "password service is not running on alias ip" + ) return @attr(tags=["sg"]) @@ -464,93 +493,94 @@ class TestMultipleIpRanges(cloudstackTestCase): 4.Verify ip alias removal after vm expunge """ list_router_response = list_routers( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) + self.apiclient, + zoneid=self.zone.id, + listall=True + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] hosts = list_hosts( - self.apiclient, - zoneid=router.zoneid, - type='Routing', - state='Up', - id=router.hostid - ) + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list host returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list host returns a valid list" + ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" - ) + router.state, + 'Running', + "Check list router response for router state" + ) + + port = self.testdata['configurableData']['host']["publicport"] + username = self.testdata['configurableData']['host']["username"] + password = self.testdata['configurableData']['host']["password"] + + # SSH to host so that host key is saved in first + # attempt + SshClient(host.ipaddress, port, username, password) + proc = "ip addr show eth0" result = get_process_status( - host.ipaddress, - self.services['host']["publicport"], - self.services['host']["username"], - self.services['host']["password"], - router.linklocalip, - proc - ) + host.ipaddress, + port, + username, + password, + router.linklocalip, + proc + ) res = str(result) self.debug("ip alias configuration on VR: %s" % res) self.assertNotEqual( - res.find(self.alias_ip) - - 1, - "ip alias is not created on VR eth0" - ) + res.find(self.alias_ip) + - 1, + "ip alias is not created on VR eth0" + ) self.virtual_machine.delete(self.apiclient) - expunge_del = Configurations.list( - self.apiclient, - name='expunge.delay' - ) - expunge_int = Configurations.list( - self.apiclient, - name='expunge.interval' - ) - wait_time = int(expunge_del[0].value) + int(expunge_int[0].value) + int(30) - self.debug("Waiting for {} seconds for the vm to expunge".format(wait_time)) - # wait for the vm to expunge - time.sleep(wait_time) - self.debug("Verify that expunging the last vm in the CIDR should delete the ip alias from VR") + self.debug( + "Verify that expunging the last vm in the CIDR should\ + delete the ip alias from VR") ip_alias2 = self.dbclient.execute( - "select ip4_address from nic_ip_alias;" - ) + "select ip4_address from nic_ip_alias;" + ) self.assertEqual( - isinstance(ip_alias2, list), - True, - "Error in sql query" - ) + isinstance(ip_alias2, list), + True, + "Error in sql query" + ) self.assertEqual( - len(ip_alias2), - 0, - "Failure in clearing ip alias entry from cloud db" - ) + len(ip_alias2), + 0, + "Failure in clearing ip alias entry from cloud db" + ) + proc = "ip addr show eth0" result = get_process_status( - host.ipaddress, - self.services['host']["publicport"], - self.services['host']["username"], - self.services['host']["password"], - router.linklocalip, - proc - ) + host.ipaddress, + port, + username, + password, + router.linklocalip, + proc + ) res = str(result) self.assertEqual( - res.find(self.alias_ip), - - 1, - "Failed to clean up ip alias from VR even after last vm expunge in the CIDR" - ) + res.find( + self.alias_ip), + - 1, + "Failed to clean up ip alias from VR even after\ + last vm expunge in the CIDR") self.debug("IP alias got deleted from VR successfully.") self.cleanup.remove(self.virtual_machine) return @@ -564,86 +594,95 @@ class TestMultipleIpRanges(cloudstackTestCase): 4.Verify ip alias on VR """ list_router_response = list_routers( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) + self.apiclient, + zoneid=self.zone.id, + listall=True + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] hosts = list_hosts( - self.apiclient, - zoneid=router.zoneid, - type='Routing', - state='Up', - id=router.hostid - ) + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list host returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list host returns a valid list" + ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" - ) + router.state, + 'Running', + "Check list router response for router state" + ) + + port = self.testdata['configurableData']['host']["publicport"] + username = self.testdata['configurableData']['host']["username"] + password = self.testdata['configurableData']['host']["password"] + + # SSH to host so that host key is saved in first + # attempt + SshClient(host.ipaddress, port, username, password) + proc = "ip addr show eth0" result = get_process_status( - host.ipaddress, - self.services['host']["publicport"], - self.services['host']["username"], - self.services['host']["password"], - router.linklocalip, - proc - ) + host.ipaddress, + port, + username, + password, + router.linklocalip, + proc + ) res = str(result) self.debug("ip alias configuration on VR: %s" % res) self.assertNotEqual( - res.find(self.alias_ip) - - 1, - "ip alias is not created on VR eth0" - ) + res.find(self.alias_ip) + - 1, + "ip alias is not created on VR eth0" + ) resp = Router.reboot( - self.apiclient, - router.id - ) + self.apiclient, + router.id + ) self.debug("Reboot router api response: %s" % resp) list_router_response = list_routers( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) + self.apiclient, + zoneid=self.zone.id, + listall=True + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] self.assertEqual( - router.state, - 'Running', - "Router is not in running state after reboot" - ) + router.state, + 'Running', + "Router is not in running state after reboot" + ) result = get_process_status( - host.ipaddress, - self.services['host']["publicport"], - self.services['host']["username"], - self.services['host']["password"], - router.linklocalip, - proc - ) + host.ipaddress, + port, + username, + password, + router.linklocalip, + proc + ) res = str(result) self.assertNotEqual( - res.find(self.alias_ip), - - 1, - "IP alias not present on VR after VR reboot" - ) + res.find(self.alias_ip), + - 1, + "IP alias not present on VR after VR reboot" + ) return @attr(tags=["sg"]) @@ -655,90 +694,99 @@ class TestMultipleIpRanges(cloudstackTestCase): 4.Verify ip alias on VR """ list_router_response = list_routers( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) + self.apiclient, + zoneid=self.zone.id, + listall=True + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] hosts = list_hosts( - self.apiclient, - zoneid=router.zoneid, - type='Routing', - state='Up', - id=router.hostid - ) + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list host returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list host returns a valid list" + ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" - ) + router.state, + 'Running', + "Check list router response for router state" + ) + + port = self.testdata['configurableData']['host']["publicport"] + username = self.testdata['configurableData']['host']["username"] + password = self.testdata['configurableData']['host']["password"] + + # SSH to host so that host key is saved in first + # attempt + SshClient(host.ipaddress, port, username, password) + proc = "ip addr show eth0" result = get_process_status( - host.ipaddress, - self.services['host']["publicport"], - self.services['host']["username"], - self.services['host']["password"], - router.linklocalip, - proc - ) + host.ipaddress, + port, + username, + password, + router.linklocalip, + proc + ) res = str(result) self.debug("ip alias configuration on VR: %s" % res) self.assertNotEqual( - res.find(self.alias_ip) - - 1, - "ip alias is not created on VR eth0" - ) + res.find(self.alias_ip) + - 1, + "ip alias is not created on VR eth0" + ) self.debug("Stopping VR") - stop_res = Router.stop( - self.apiclient, - router.id, - ) + Router.stop( + self.apiclient, + router.id, + ) self.debug("Starting VR") - start_res = Router.start( - self.apiclient, - router.id - ) + Router.start( + self.apiclient, + router.id + ) list_router_response = list_routers( - self.apiclient, - zoneid=self.zone.id, - listall=True - ) + self.apiclient, + zoneid=self.zone.id, + listall=True + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] self.assertEqual( - router.state, - 'Running', - "Router is not in running state after reboot" - ) + router.state, + 'Running', + "Router is not in running state after reboot" + ) self.debug("VR is up and Running") result = get_process_status( - host.ipaddress, - self.services['host']["publicport"], - self.services['host']["username"], - self.services['host']["password"], - router.linklocalip, - proc - ) + host.ipaddress, + port, + username, + password, + router.linklocalip, + proc + ) res = str(result) self.assertNotEqual( - res.find(self.alias_ip), - - 1, - "IP alias not present on VR after VR stop and start" - ) + res.find(self.alias_ip), + - 1, + "IP alias not present on VR after VR stop and start" + ) return From daabe92a992f072c6305b14b88fcda99e19a7190 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Wed, 17 Dec 2014 11:32:49 +0530 Subject: [PATCH 29/39] CLOUDSTACK-8081: Fixed VM snapshot test cases in test_escalation_instances.py and also dealt cleanup issues Signed-off-by: SrikanteswaraRao Talluri --- .../component/test_escalations_instances.py | 61 +++++++------------ 1 file changed, 22 insertions(+), 39 deletions(-) diff --git a/test/integration/component/test_escalations_instances.py b/test/integration/component/test_escalations_instances.py index 9f053a1e2fc..e79cd3d258a 100644 --- a/test/integration/component/test_escalations_instances.py +++ b/test/integration/component/test_escalations_instances.py @@ -2113,26 +2113,6 @@ class TestInstances(cloudstackTestCase): cls.api_client, cls.services["service_offerings"]["tiny"] ) - cls.account = Account.create( - cls.api_client, - cls.services["account"], - domainid=cls.domain.id - ) - # Getting authentication for user in newly created Account - cls.user = cls.account.user[0] - cls.userapiclient = cls.testClient.getUserApiClient( - cls.user.username, - cls.domain.name) - # Updating resource Limits - for i in range(0, 8): - Resources.updateLimit( - cls.api_client, - account=cls.account.name, - domainid=cls.domain.id, - max=-1, - resourcetype=i - ) - cls._cleanup.append(cls.account) cls._cleanup.append(cls.service_offering) cls._cleanup.append(cls.disk_offering) except Exception as e: @@ -2144,12 +2124,29 @@ class TestInstances(cloudstackTestCase): self.apiClient = self.testClient.getApiClient() self.cleanup = [] - self.cleanup_vm = [] + self.account = Account.create( + self.apiClient, + self.services["account"], + domainid=self.domain.id + ) + # Getting authentication for user in newly created Account + self.user = self.account.user[0] + self.userapiclient = self.testClient.getUserApiClient( + self.user.username, + self.domain.name) + # Updating resource Limits + for i in range(0, 8): + Resources.updateLimit( + self.api_client, + account=self.account.name, + domainid=self.domain.id, + max=-1, + resourcetype=i + ) + self.cleanup.append(self.account) def tearDown(self): # Clean up, terminate the created resources - for vm in self.cleanup_vm: - vm.delete(self.apiClient, expunge=True) cleanup_resources(self.apiClient, self.cleanup) return @@ -2242,7 +2239,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.append(vm_created) # Listing all the VMs for a user again for matching zone list_vms_after = VirtualMachine.list( self.userapiclient, @@ -2372,7 +2368,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.append(vm_created) # Listing all the VMs for a user again list_vms_after = VirtualMachine.list( self.userapiclient, @@ -2535,7 +2530,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.append(vm_created) # Listing all the VMs for a user again list_vms_after = VirtualMachine.list( self.userapiclient, @@ -2568,6 +2562,7 @@ class TestInstances(cloudstackTestCase): snapshot1 = VmSnapshot.create( self.userapiclient, vm_created.id, + snapshotmemory=True ) self.assertIsNotNone( snapshot1, @@ -2576,6 +2571,7 @@ class TestInstances(cloudstackTestCase): snapshot2 = VmSnapshot.create( self.userapiclient, vm_created.id, + snapshotmemory=True ) self.assertIsNotNone( snapshot2, @@ -2699,7 +2695,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.append(vm_created) # Listing all the VMs for a user again list_vms_after = VirtualMachine.list( self.userapiclient, @@ -2747,7 +2742,6 @@ class TestInstances(cloudstackTestCase): volume_created, "Volume is not created" ) - self.cleanup.append(volume_created) # Attaching all the volumes created to VM vm_created.attach_volume( self.userapiclient, @@ -2943,7 +2937,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.append(vm_created) # Listing details of current Service Offering vm_so_list = ServiceOffering.list( self.userapiclient, @@ -3074,7 +3067,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.append(vm_created) # Listing details of current Service Offering vm_so_list = ServiceOffering.list( self.userapiclient, @@ -3198,7 +3190,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.append(vm_created) # Listing all the VMs for a user again list_vms_after = VirtualMachine.list( self.userapiclient, @@ -3326,7 +3317,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.append(vm_created) # Verifying the displayname and group details for deployed VM self.assertEquals( self.services["virtual_machine"]["displayname"], @@ -3425,7 +3415,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.append(vm_created) # Listing all the VMs for a user again list_vms_after = VirtualMachine.list( self.userapiclient, @@ -3533,7 +3522,6 @@ class TestInstances(cloudstackTestCase): network, "Network creation failed" ) - self.cleanup.append(network) networks_list_size = networks_list_size + 1 # Listing the networks again @@ -3573,7 +3561,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.insert(-2, vm_created) # Listing all the VMs for a user again list_vms_after = VirtualMachine.list( self.userapiclient, @@ -3693,7 +3680,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup_vm.append(vm_created) # Listing all the VMs for a user again list_vms_after = VirtualMachine.list( self.userapiclient, @@ -3797,8 +3783,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - # self.cleanup.append(vm_created) - self.cleanup.append(network) # Listing all the VMs for a user again vm_response = VirtualMachine.list( self.userapiclient, @@ -3921,7 +3905,6 @@ class TestInstances(cloudstackTestCase): vm1, "VM1 creation failed with ip address %s and host name %s" % (vm_ip1, name1)) # self.cleanup.append(vm_created) - self.cleanup.append(network) # Listing all the VMs for a user again vm_response = VirtualMachine.list( self.userapiclient, From 94814603db6d825d23710071d4aba1c5893d3360 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Wed, 17 Dec 2014 14:11:16 +0530 Subject: [PATCH 30/39] CLOUDSTACK-8055: test_portable_ip.py - Tagging test case which can't be run on simulator Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_portable_ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/component/test_portable_ip.py b/test/integration/component/test_portable_ip.py index 6bdc61a3d9f..83ef6b51907 100644 --- a/test/integration/component/test_portable_ip.py +++ b/test/integration/component/test_portable_ip.py @@ -1288,7 +1288,7 @@ class TestPortableIpTransferAcrossNetworks(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return - @attr(tags=["advanced","swamy"], required_hardware="false") + @attr(tags=["advanced","swamy"], required_hardware="true") def test_list_portable_ip_range_non_root_admin(self): """Test list portable ip ranges with non admin root account """ From ed5bc1c7ea1885e95c7cad0045243c7f43e9f516 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Mon, 15 Dec 2014 15:54:51 +0530 Subject: [PATCH 31/39] CLOUDSTACK-8071: Fixed api key issue in test_snapshots_improvement.py Signed-off-by: SrikanteswaraRao Talluri --- .../component/test_snapshots_improvement.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/integration/component/test_snapshots_improvement.py b/test/integration/component/test_snapshots_improvement.py index 6638dddf035..1fd42392db3 100644 --- a/test/integration/component/test_snapshots_improvement.py +++ b/test/integration/component/test_snapshots_improvement.py @@ -303,7 +303,9 @@ class TestCreateSnapshot(cloudstackTestCase): def setUpClass(cls): cls.testClient = super(TestCreateSnapshot, cls).getClsTestClient() cls.api_client = cls.testClient.getApiClient() - + cls.hypervisor = cls.testClient.getHypervisorInfo() + if cls.hypervisor.lower() in ['hyperv']: + raise unittest.SkipTest("Snapshots feature is not supported on Hyper-V") cls.services = Services().services # Get Zone, Domain and templates cls.domain = get_domain(cls.api_client) @@ -345,13 +347,17 @@ class TestCreateSnapshot(cloudstackTestCase): self.services["account"], domainid=self.domain.id ) + + self.apiclient = self.testClient.getUserApiClient( + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup = [self.account, ] return def tearDown(self): try: # Clean up, terminate the created instance, volumes and snapshots - cleanup_resources(self.apiclient, self.cleanup) + cleanup_resources(self.api_client, self.cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @@ -615,13 +621,8 @@ class TestCreateSnapshot(cloudstackTestCase): self.debug("Create a template from snapshot: %s" % snapshot.name) jobs.append(self.create_Template_from_Snapshot(snapshot)) - - userapiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) - # Verify IO usage by submitting the concurrent jobs - self.testClient.submitCmdsAndWait(jobs, apiclient=userapiclient) + self.testClient.submitCmdsAndWait(jobs) self.debug("Verifying if templates are created properly or not?") templates = Template.list( From 11fa48108ffc5ef7b276f7261b29eaca406105b8 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Wed, 17 Dec 2014 10:58:43 -0800 Subject: [PATCH 32/39] CLOUDSTACK-8075: UI > Instances menu > Add Instance > Select template/ISO > "shared" tab > select a shared template, click Next button => fix error "unable to find matched template object". --- ui/scripts/instanceWizard.js | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ui/scripts/instanceWizard.js b/ui/scripts/instanceWizard.js index a067d7d7a69..984c6e3dee1 100644 --- a/ui/scripts/instanceWizard.js +++ b/ui/scripts/instanceWizard.js @@ -323,11 +323,22 @@ } } } - } - if (selectedTemplateObj == null) + } + if (selectedTemplateObj == null) { + if (sharedTemplateObjs != null && sharedTemplateObjs.length > 0) { + for (var i = 0; i < sharedTemplateObjs.length; i++) { + if (sharedTemplateObjs[i].id == args.currentData.templateid) { + selectedTemplateObj = sharedTemplateObjs[i]; + break; + } + } + } + } + if (selectedTemplateObj == null) { alert("unable to find matched template object"); - else + } else { selectedHypervisor = selectedTemplateObj.hypervisor; + } } else { //(args.currentData["select-template"] == "select-iso" selectedHypervisor = args.currentData.hypervisorid; } From a7861aa5faabc9ce9422ae2c903471fbf8239abc Mon Sep 17 00:00:00 2001 From: Prachi Damle Date: Wed, 17 Dec 2014 11:42:03 -0800 Subject: [PATCH 33/39] CLOUDSTACK-8079: If the cluster capacity threshold is reached, HA-enabled VM is not migrated on another host during HA Changes: - When there is HA we try to redeploy the affected vm using regular planners and if that fails we retry using the special planner for HA (which skips checking disable threshold) Now because of job framework the InsufficientCapacittyException gets masked and the special planners are not called. Job framework needs to be fixed to rethrow the correct exception. - Also the VM Work Job framework is not setting the DeploymentPlanner to the VmWorkJob. So the HA Planner being passed by HAMgr was not getting used. - Now the job framework sets the planner passed in by any caller of the VM Start operation, to the job --- .../src/com/cloud/vm/VirtualMachineManagerImpl.java | 3 +++ .../orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java | 4 ++-- server/src/com/cloud/storage/VolumeApiServiceImpl.java | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index 83779064f49..c87b7729b2e 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -3949,6 +3949,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac VmWorkStart workInfo = new VmWorkStart(callingUser.getId(), callingAccount.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER); workInfo.setPlan(planToDeploy); workInfo.setParams(params); + if (planner != null) { + workInfo.setDeploymentPlanner(planner.getName()); + } workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); diff --git a/engine/orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java b/engine/orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java index f3020021840..c023511a224 100644 --- a/engine/orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java +++ b/engine/orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java @@ -111,8 +111,8 @@ public class VmWorkJobDispatcher extends AdapterBase implements AsyncJobDispatch } catch(Throwable e) { s_logger.error("Unable to complete " + job + ", job origin:" + job.getRelated(), e); - RuntimeException ex = new RuntimeException("Job failed due to exception " + e.getMessage()); - _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, 0, _asyncJobMgr.marshallResultObject(ex)); + //RuntimeException ex = new RuntimeException("Job failed due to exception " + e.getMessage()); + _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, 0, _asyncJobMgr.marshallResultObject(e)); } } } diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index 0914c57132c..03b64ac6101 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -925,6 +925,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (jobResult instanceof ConcurrentOperationException) { throw (ConcurrentOperationException)jobResult; } + else if (jobResult instanceof ResourceAllocationException) { + throw (ResourceAllocationException)jobResult; + } else if (jobResult instanceof RuntimeException) { throw (RuntimeException)jobResult; } From 01ae7120ac2d8a9c673ab5d54d45f604a4a88581 Mon Sep 17 00:00:00 2001 From: Prachi Damle Date: Tue, 16 Dec 2014 16:46:28 -0800 Subject: [PATCH 34/39] CLOUDSTACK-8078: [Automation] Deletion of Affinity Groups - CloudRuntimeException: No Event Pubish can be wrapped within DB Transaction! Changes: - The event of deleteing an affinity group is published on the MessageBus so that IAM Service can listen and process the event, However the publish operation should not be handled within a DB transaction, since it may take longer and hold the DB transaction for long unnecessarily - Publish any events to MessageBus outside of the transaction Conflicts: server/src/org/apache/cloudstack/affinity/AffinityGroupServiceImpl.java --- .../cloudstack/affinity/AffinityGroupServiceImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/org/apache/cloudstack/affinity/AffinityGroupServiceImpl.java b/server/src/org/apache/cloudstack/affinity/AffinityGroupServiceImpl.java index 8e606ca0584..91835ea8b9d 100644 --- a/server/src/org/apache/cloudstack/affinity/AffinityGroupServiceImpl.java +++ b/server/src/org/apache/cloudstack/affinity/AffinityGroupServiceImpl.java @@ -298,13 +298,14 @@ public class AffinityGroupServiceImpl extends ManagerBase implements AffinityGro if (groupDomain != null) { _affinityGroupDomainMapDao.remove(groupDomain.getId()); } - // remove its related ACL permission - Pair, Long> params = new Pair, Long>(AffinityGroup.class, affinityGroupIdFinal); - _messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, params); } } }); + // remove its related ACL permission + Pair, Long> params = new Pair, Long>(AffinityGroup.class, affinityGroupIdFinal); + _messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, params); + if (s_logger.isDebugEnabled()) { s_logger.debug("Deleted affinity group id=" + affinityGroupId); } From 1c5e8ebb3113e97c6c0be9b602f9a15852f2d2fd Mon Sep 17 00:00:00 2001 From: Prachi Damle Date: Tue, 16 Dec 2014 16:26:16 -0800 Subject: [PATCH 35/39] CLOUDSTACK-8022: [Automation] Deletion of Domain with Cleanup set to "true" fails Changes: - This is a race condition between the deleteDomain thread and AccountChecker thread. DeleteDomain thread marks the domain as inactive and proceeds for cleanup, AccountChecker thread that runs at the same time cleans up any domains marked as inactive. - When the DeleteDomain thread finds that domain is already removed, it need not error out since the domain deletion has already happened --- engine/schema/src/com/cloud/domain/dao/DomainDaoImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/com/cloud/domain/dao/DomainDaoImpl.java b/engine/schema/src/com/cloud/domain/dao/DomainDaoImpl.java index e02daf16667..4c4fe86e3c8 100644 --- a/engine/schema/src/com/cloud/domain/dao/DomainDaoImpl.java +++ b/engine/schema/src/com/cloud/domain/dao/DomainDaoImpl.java @@ -154,8 +154,8 @@ public class DomainDaoImpl extends GenericDaoBase implements Dom DomainVO domain = findById(id); if (domain == null) { - s_logger.error("Unable to remove domain as domain " + id + " no longer exists"); - return false; + s_logger.info("Unable to remove domain as domain " + id + " no longer exists"); + return true; } if (domain.getParent() == null) { From 50d5d488d4f7288cd5f1c0b6f8dd8352c93eed2d Mon Sep 17 00:00:00 2001 From: Edison Su Date: Mon, 8 Dec 2014 15:08:45 -0800 Subject: [PATCH 36/39] add event for volume migration Reviewed-by: Frank --- server/src/com/cloud/storage/VolumeApiServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index 03b64ac6101..40b957446af 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -1691,6 +1691,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @DB @Override + @ActionEvent(eventType = EventTypes.EVENT_VOLUME_MIGRATE, eventDescription = "migrating volume", async = true) public Volume migrateVolume(MigrateVolumeCmd cmd) { Long volumeId = cmd.getVolumeId(); Long storagePoolId = cmd.getStoragePoolId(); From 8f2bd456600b65f376c93a64075e9f8c1c949d4f Mon Sep 17 00:00:00 2001 From: Edison Su Date: Mon, 8 Dec 2014 15:25:06 -0800 Subject: [PATCH 37/39] if local storage is in maitenance mode or removed, won't change its status in case of mgt server restart Reviewed-by:Frank --- server/src/com/cloud/storage/StorageManagerImpl.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 1feea016202..7de4ea0c844 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -583,8 +583,12 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C store = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); } - HostScope scope = new HostScope(host.getId(), host.getClusterId(), host.getDataCenterId()); - lifeCycle.attachHost(store, scope, pInfo); + pool = _storagePoolDao.findById(store.getId()); + if (pool.getStatus() != StoragePoolStatus.Maintenance && pool.getStatus() != StoragePoolStatus.Removed) { + HostScope scope = new HostScope(host.getId(), host.getClusterId(), host.getDataCenterId()); + lifeCycle.attachHost(store, scope, pInfo); + } + } catch (Exception e) { s_logger.warn("Unable to setup the local storage pool for " + host, e); throw new ConnectionException(true, "Unable to setup the local storage pool for " + host, e); From 1eb20ee09559a505a6f5cc60bbdfa601f439194f Mon Sep 17 00:00:00 2001 From: Edison Su Date: Tue, 9 Dec 2014 15:38:58 -0800 Subject: [PATCH 38/39] if concurrent snapshot happening on the same vm, throw invalid parameter exception to user. Reviewed-by: Frank --- server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 1e136825597..a3eb7b02ec6 100755 --- a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -938,7 +938,7 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager, List activeSnapshots = _snapshotDao.listByInstanceId(volume.getInstanceId(), Snapshot.State.Creating, Snapshot.State.CreatedOnPrimary, Snapshot.State.BackingUp); if (activeSnapshots.size() > 0) { - throw new CloudRuntimeException("There is other active snapshot tasks on the instance to which the volume is attached, please try again later"); + throw new InvalidParameterValueException("There is other active snapshot tasks on the instance to which the volume is attached, please try again later"); } } From 8b6e251b5d5e47a45f392d999fde0f14765c3ca1 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Thu, 18 Dec 2014 12:26:14 +0100 Subject: [PATCH 39/39] CLOUDSTACK-7184 config value for xen heartbeat timeout --- setup/db/db/schema-442to450.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/db/db/schema-442to450.sql b/setup/db/db/schema-442to450.sql index c51bd00cb37..a12c0f5941d 100644 --- a/setup/db/db/schema-442to450.sql +++ b/setup/db/db/schema-442to450.sql @@ -988,3 +988,5 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'LXC', 'default', 'Debian GNU/Linux 7(32-bit)', 183, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'LXC', 'default', 'Debian GNU/Linux 7(64-bit)', 184, utc_timestamp(), 0); +INSERT IGNORE INTO `cloud`.`configuration` (`category`, `instance`, `component`, `name`, `value`, `default`, `description`) VALUES ('Advanced', 'DEFAULT', 'ManagementServer', 'xen.heartbeat.timeout' , '180', '120', 'Timeout value to send to the xenheartbeat script for guarding the self fencing functionality'); +