# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ P1 tests for dedicated Host high availability """ #Import Local Modules from nose.plugins.attrib import attr from marvin.cloudstackTestCase import * from marvin.cloudstackAPI import * from marvin.lib.utils import * from marvin.lib.base import * from marvin.lib.common import * class Services: """ Dedicated host HA test cases """ def __init__(self): self.services = { "account": { "email": "test@test.com", "firstname": "HA", "lastname": "HA", "username": "HA", # Random characters are appended for unique # username "password": "password", }, "service_offering_with_ha": { "name": "Tiny Instance With HA Enabled", "displaytext": "Tiny Instance", "cpunumber": 1, "cpuspeed": 100, # in MHz "memory": 128, # In MBs }, "service_offering_without_ha": { "name": "Tiny Instance Without HA", "displaytext": "Tiny Instance", "cpunumber": 1, "cpuspeed": 100, # in MHz "memory": 128, # In MBs }, "virtual_machine": { "displayname": "VM", "username": "root", "password": "password", "ssh_port": 22, "hypervisor": 'XenServer', # Hypervisor type should be same as # hypervisor type of cluster "privateport": 22, "publicport": 22, "protocol": 'TCP', }, "ostype": 'CentOS 5.3 (64-bit)', "timeout": 100, } class TestHostHighAvailability(cloudstackTestCase): """ Dedicated host HA test cases """ @classmethod def setUpClass(cls): cls.testClient = super(TestHostHighAvailability, cls).getClsTestClient() cls.api_client = cls.testClient.getApiClient() cls.services = Services().services # Get Zone, Domain and templates cls.domain = get_domain(cls.api_client) cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) cls.template = get_template( cls.api_client, cls.zone.id, cls.services["ostype"] ) cls.services["virtual_machine"]["zoneid"] = cls.zone.id cls.services["virtual_machine"]["template"] = cls.template.id cls.service_offering_with_ha = ServiceOffering.create( cls.api_client, cls.services["service_offering_with_ha"], offerha=True ) cls.service_offering_without_ha = ServiceOffering.create( cls.api_client, cls.services["service_offering_without_ha"], offerha=False ) cls._cleanup = [ cls.service_offering_with_ha, cls.service_offering_without_ha, ] return @classmethod def tearDownClass(cls): try: #Cleanup resources used cleanup_resources(cls.api_client, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() self.account = Account.create( self.apiclient, self.services["account"], admin=True, domainid=self.domain.id ) self.cleanup = [self.account] return def tearDown(self): try: #Clean up, terminate the created accounts, domains etc cleanup_resources(self.apiclient, self.cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @attr(configuration="ha.tag") @attr(tags=["advanced", "advancedns", "sg", "basic", "eip", "simulator"]) def test_01_vm_deployment_with_compute_offering_with_ha_enabled(self): """ Test VM deployments (Create HA enabled Compute Service Offering and VM) """ # Steps, #1. Create a Compute service offering with the 'Offer HA' option selected. #2. Create a Guest VM with the compute service offering created above. # Validations, #1. Ensure that the offering is created and that in the UI the 'Offer HA' field is enabled (Yes) #The listServiceOffering API should list 'offerha' as true. #2. Select the newly created VM and ensure that the Compute offering field value lists the compute service offering that was selected. # Also, check that the HA Enabled field is enabled 'Yes'. #list and validate above created service offering with Ha enabled list_service_response = list_service_offering( self.apiclient, id=self.service_offering_with_ha.id ) self.assertEqual( isinstance(list_service_response, list), True, "listServiceOfferings returned invalid object in response." ) self.assertNotEqual( len(list_service_response), 0, "listServiceOfferings returned empty list." ) self.assertEqual( list_service_response[0].offerha, True, "The service offering is not HA enabled" ) #create virtual machine with the service offering with Ha enabled virtual_machine = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering_with_ha.id ) vms = VirtualMachine.list( self.apiclient, id=virtual_machine.id, listall=True ) self.assertEqual( isinstance(vms, list), True, "listVirtualMachines returned invalid object in response." ) self.assertNotEqual( len(vms), 0, "listVirtualMachines returned empty list." ) self.debug("Deployed VM on host: %s" % vms[0].hostid) self.assertEqual( vms[0].haenable, True, "VM not created with HA enable tag" ) @attr(configuration="ha.tag") @attr(tags=["advanced", "advancedns", "sg", "basic", "eip", "simulator", "multihost"]) def test_02_no_vm_creation_on_host_with_haenabled(self): """ Verify you can not create new VMs on hosts with an ha.tag """ # Steps, #1. Fresh install CS (Bonita) that supports this feature #2. Create Basic zone, pod, cluster, add 3 hosts to cluster (host1, host2, host3), secondary & primary Storage #3. When adding host3, assign the HA host tag. #4. You should already have a compute service offering with HA already create from above. If not, create one for HA. #5. Create VMs with the service offering with and without the HA tag # Validations, #Check to make sure the newly created VM is not on any HA enabled hosts #The VM should be created only on host1 or host2 and never host3 (HA enabled) #create and verify virtual machine with HA enabled service offering virtual_machine_with_ha = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering_with_ha.id ) vms = VirtualMachine.list( self.apiclient, id=virtual_machine_with_ha.id, listall=True ) self.assertEqual( isinstance(vms, list), True, "listVirtualMachines returned invalid object in response." ) self.assertNotEqual( len(vms), 0, "listVirtualMachines returned empty list." ) vm = vms[0] self.debug("Deployed VM on host: %s" % vm.hostid) #validate the virtual machine created is host Ha enabled list_hosts_response = list_hosts( self.apiclient, id=vm.hostid ) self.assertEqual( isinstance(list_hosts_response, list), True, "listHosts returned invalid object in response." ) self.assertNotEqual( len(list_hosts_response), 0, "listHosts retuned empty list in response." ) self.assertEqual( list_hosts_response[0].hahost, False, "VM created on HA enabled host." ) #create and verify virtual machine with Ha disabled service offering virtual_machine_without_ha = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering_without_ha.id ) vms = VirtualMachine.list( self.apiclient, id=virtual_machine_without_ha.id, listall=True ) self.assertEqual( isinstance(vms, list), True, "listVirtualMachines returned invalid object in response." ) self.assertNotEqual( len(vms), 0, "listVirtualMachines returned empty list." ) vm = vms[0] self.debug("Deployed VM on host: %s" % vm.hostid) #verify that the virtual machine created on the host is Ha disabled list_hosts_response = list_hosts( self.apiclient, id=vm.hostid ) self.assertEqual( isinstance(list_hosts_response, list), True, "listHosts returned invalid object in response." ) self.assertNotEqual( len(list_hosts_response), 0, "listHosts returned empty list." ) host = list_hosts_response[0] self.assertEqual( host.hahost, False, "VM migrated to HA enabled host." ) @attr(configuration="ha.tag") @attr(tags=["advanced", "advancedns", "sg", "basic", "eip", "simulator", "multihost"]) def test_03_cant_migrate_vm_to_host_with_ha_positive(self): """ Verify you can not migrate VMs to hosts with an ha.tag (positive) """ # Steps, #1. Create a Compute service offering with the 'Offer HA' option selected. #2. Create a Guest VM with the compute service offering created above. #3. Select the VM and migrate VM to another host. Choose a 'Suitable' host (i.e. host2) # Validations #The option from the 'Migrate instance to another host' dialog box' should list host3 as 'Not Suitable' for migration. #Confirm that the VM is migrated to the 'Suitable' host you selected (i.e. host2) #create and verify the virtual machine with HA enabled service offering virtual_machine_with_ha = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering_with_ha.id ) vms = VirtualMachine.list( self.apiclient, id=virtual_machine_with_ha.id, listall=True, ) self.assertEqual( isinstance(vms, list), True, "List VMs should return valid response for deployed VM" ) self.assertNotEqual( len(vms), 0, "List VMs should return valid response for deployed VM" ) vm = vms[0] self.debug("Deployed VM on host: %s" % vm.hostid) #Find out a Suitable host for VM migration list_hosts_response = list_hosts( self.apiclient, ) self.assertEqual( isinstance(list_hosts_response, list), True, "The listHosts API returned the invalid list" ) self.assertNotEqual( len(list_hosts_response), 0, "The listHosts returned nothing." ) suitableHost = None for host in list_hosts_response: if host.suitableformigration == True and host.hostid != vm.hostid: suitableHost = host break self.assertTrue(suitableHost is not None, "suitablehost should not be None") #Migration of the VM to a suitable host self.debug("Migrating VM-ID: %s to Host: %s" % (self.vm.id, suitableHost.id)) cmd = migrateVirtualMachine.migrateVirtualMachineCmd() cmd.hostid = suitableHost.id cmd.virtualmachineid = self.vm.id self.apiclient.migrateVirtualMachine(cmd) #Verify that the VM migrated to a targeted Suitable host list_vm_response = list_virtual_machines( self.apiclient, id=vm.id ) self.assertEqual( isinstance(list_vm_response, list), True, "The listVirtualMachines returned the invalid list." ) self.assertNotEqual( list_vm_response, None, "The listVirtualMachines API returned nothing." ) vm_response = list_vm_response[0] self.assertEqual( vm_response.id, vm.id, "The virtual machine id and the the virtual machine from listVirtualMachines is not matching." ) self.assertEqual( vm_response.hostid, suitableHost.id, "The VM is not migrated to targeted suitable host." ) @attr(configuration="ha.tag") @attr(tags=["advanced", "advancedns", "sg", "basic", "eip", "simulator", "multihost"]) def test_04_cant_migrate_vm_to_host_with_ha_negative(self): """ Verify you can not migrate VMs to hosts with an ha.tag (negative) """ # Steps, #1. Create a Compute service offering with the 'Offer HA' option selected. #2. Create a Guest VM with the compute service offering created above. #3. Select the VM and migrate VM to another host. Choose a 'Not Suitable' host. # Validations, #The option from the 'Migrate instance to another host' dialog box should list host3 as 'Not Suitable' for migration. #By design, The Guest VM can STILL can be migrated to host3 if the admin chooses to do so. #create and verify virtual machine with HA enabled service offering virtual_machine_with_ha = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering_with_ha.id ) vms = VirtualMachine.list( self.apiclient, id=virtual_machine_with_ha.id, listall=True ) self.assertEqual( isinstance(vms, list), True, "The listVirtualMachines returned invalid object in response." ) self.assertNotEqual( len(vms), 0, "The listVirtualMachines returned empty response." ) vm = vms[0] self.debug("Deployed VM on host: %s" % vm.hostid) #Find out Non-Suitable host for VM migration list_hosts_response = list_hosts( self.apiclient, ) self.assertEqual( isinstance(list_hosts_response, list), True, "listHosts returned invalid object in response." ) self.assertNotEqual( len(list_hosts_response), 0, "listHosts returned empty response." ) notSuitableHost = None for host in list_hosts_response: if not host.suitableformigration and host.hostid != vm.hostid: notSuitableHost = host break self.assertTrue(notSuitableHost is not None, "notsuitablehost should not be None") #Migrate VM to Non-Suitable host self.debug("Migrating VM-ID: %s to Host: %s" % (vm.id, notSuitableHost.id)) cmd = migrateVirtualMachine.migrateVirtualMachineCmd() cmd.hostid = notSuitableHost.id cmd.virtualmachineid = vm.id self.apiclient.migrateVirtualMachine(cmd) #Verify that the virtual machine got migrated to targeted Non-Suitable host list_vm_response = list_virtual_machines( self.apiclient, id=vm.id ) self.assertEqual( isinstance(list_vm_response, list), True, "listVirtualMachine returned invalid object in response." ) self.assertNotEqual( len(list_vm_response), 0, "listVirtualMachines returned empty response." ) self.assertEqual( list_vm_response[0].id, vm.id, "Virtual machine id with the virtual machine from listVirtualMachine is not matching." ) self.assertEqual( list_vm_response[0].hostid, notSuitableHost.id, "The detination host id of migrated VM is not matching." ) @attr(configuration="ha.tag") @attr(speed="slow") @attr(tags=["advanced", "advancedns", "sg", "basic", "eip", "simulator", "multihost"]) def test_05_no_vm_with_ha_gets_migrated_to_ha_host_in_live_migration(self): """ Verify that none of the VMs with HA enabled migrate to an ha tagged host during live migration """ # Steps, #1. Fresh install CS that supports this feature #2. Create Basic zone, pod, cluster, add 3 hosts to cluster (host1, host2, host3), secondary & primary Storage #3. When adding host3, assign the HA host tag. #4. Create VMs with and without the Compute Service Offering with the HA tag. #5. Note the VMs on host1 and whether any of the VMs have their 'HA enabled' flags enabled. #6. Put host1 into maintenance mode. # Validations, #1. Make sure the VMs are created on either host1 or host2 and not on host3 #2. Putting host1 into maintenance mode should trigger a live migration. Make sure the VMs are not migrated to HA enabled host3. # create and verify virtual machine with HA disabled service offering virtual_machine_with_ha = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering_with_ha.id ) vms = VirtualMachine.list( self.apiclient, id=virtual_machine_with_ha.id, listall=True ) self.assertEqual( isinstance(vms, list), True, "List VMs should return valid response for deployed VM" ) self.assertNotEqual( len(vms), 0, "List VMs should return valid response for deployed VM" ) vm_with_ha_enabled = vms[0] #Verify the virtual machine got created on non HA host list_hosts_response = list_hosts( self.apiclient, id=vm_with_ha_enabled.hostid ) self.assertEqual( isinstance(list_hosts_response, list), True, "Check list response returns a valid list" ) self.assertNotEqual( len(list_hosts_response), 0, "Check Host is available" ) self.assertEqual( list_hosts_response[0].hahost, False, "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled" ) #put the Host in maintenance mode self.debug("Enabling maintenance mode for host %s" % vm_with_ha_enabled.hostid) cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() cmd.id = vm_with_ha_enabled.hostid self.apiclient.prepareHostForMaintenance(cmd) timeout = self.services["timeout"] #verify the VM live migration happened to another running host self.debug("Waiting for VM to come up") wait_for_vm( self.apiclient, virtualmachineid=vm_with_ha_enabled.id, interval=timeout ) vms = VirtualMachine.list( self.apiclient, id=vm_with_ha_enabled.id, listall=True, ) self.assertEqual( isinstance(vms, list), True, "List VMs should return valid response for deployed VM" ) self.assertNotEqual( len(vms), 0, "List VMs should return valid response for deployed VM" ) vm_with_ha_enabled1 = vms[0] list_hosts_response = list_hosts( self.apiclient, id=vm_with_ha_enabled1.hostid ) self.assertEqual( isinstance(list_hosts_response, list), True, "Check list response returns a valid list" ) self.assertNotEqual( len(list_hosts_response), 0, "Check Host is available" ) self.assertEqual( list_hosts_response[0].hahost, False, "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled" ) self.debug("Disabling the maintenance mode for host %s" % vm_with_ha_enabled.hostid) cmd = cancelHostMaintenance.cancelHostMaintenanceCmd() cmd.id = vm_with_ha_enabled.hostid self.apiclient.cancelHostMaintenance(cmd) @attr(configuration="ha.tag") @attr(speed="slow") @attr(tags=["advanced", "advancedns", "sg", "basic", "eip", "simulator", "multihost"]) def test_06_no_vm_without_ha_gets_migrated_to_ha_host_in_live_migration(self): """ Verify that none of the VMs without HA enabled migrate to an ha tagged host during live migration """ # Steps, #1. Fresh install CS that supports this feature #2. Create Basic zone, pod, cluster, add 3 hosts to cluster (host1, host2, host3), secondary & primary Storage #3. When adding host3, assign the HA host tag. #4. Create VMs with and without the Compute Service Offering with the HA tag. #5. Note the VMs on host1 and whether any of the VMs have their 'HA enabled' flags enabled. #6. Put host1 into maintenance mode. # Validations, #1. Make sure the VMs are created on either host1 or host2 and not on host3 #2. Putting host1 into maintenance mode should trigger a live migration. Make sure the VMs are not migrated to HA enabled host3. # create and verify virtual machine with HA disabled service offering virtual_machine_without_ha = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering_without_ha.id ) vms = VirtualMachine.list( self.apiclient, id=virtual_machine_without_ha.id, listall=True ) self.assertEqual( isinstance(vms, list), True, "List VMs should return valid response for deployed VM" ) self.assertNotEqual( len(vms), 0, "List VMs should return valid response for deployed VM" ) vm_with_ha_disabled = vms[0] #Verify the virtual machine got created on non HA host list_hosts_response = list_hosts( self.apiclient, id=vm_with_ha_disabled.hostid ) self.assertEqual( isinstance(list_hosts_response, list), True, "Check list response returns a valid list" ) self.assertNotEqual( len(list_hosts_response), 0, "Check Host is available" ) self.assertEqual( list_hosts_response[0].hahost, False, "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled" ) #put the Host in maintenance mode self.debug("Enabling maintenance mode for host %s" % vm_with_ha_disabled.hostid) cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() cmd.id = vm_with_ha_disabled.hostid self.apiclient.prepareHostForMaintenance(cmd) timeout = self.services["timeout"] #verify the VM live migration happened to another running host self.debug("Waiting for VM to come up") wait_for_vm( self.apiclient, virtualmachineid=vm_with_ha_disabled.id, interval=timeout ) vms = VirtualMachine.list( self.apiclient, id=vm_with_ha_disabled.id, listall=True ) self.assertEqual( isinstance(vms, list), True, "List VMs should return valid response for deployed VM" ) self.assertNotEqual( len(vms), 0, "List VMs should return valid response for deployed VM" ) list_hosts_response = list_hosts( self.apiclient, id=vms[0].hostid ) self.assertEqual( isinstance(list_hosts_response, list), True, "Check list response returns a valid list" ) self.assertNotEqual( len(list_hosts_response), 0, "Check Host is available" ) self.assertEqual( list_hosts_response[0].hahost, False, "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled" ) self.debug("Disabling the maintenance mode for host %s" % vm_with_ha_disabled.hostid) cmd = cancelHostMaintenance.cancelHostMaintenanceCmd() cmd.id = vm_with_ha_disabled.hostid self.apiclient.cancelHostMaintenance(cmd)