# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ Test cases for VM/Volume snapshot Test Path """ from nose.plugins.attrib import attr from marvin.cloudstackTestCase import cloudstackTestCase import unittest from marvin.lib.utils import (cleanup_resources, is_snapshot_on_nfs, validateList) from marvin.lib.base import (Account, Cluster, StoragePool, DiskOffering, ServiceOffering, Host, Configurations, Template, VirtualMachine, Snapshot, SnapshotPolicy, Volume ) from marvin.lib.common import (get_domain, get_zone, get_template, list_volumes, list_snapshots, list_virtual_machines, createChecksum, compareChecksum ) from marvin.sshClient import SshClient import time from marvin.codes import ( CLUSTERTAG1, CLUSTERTAG2, PASS, BACKED_UP, UP) from threading import Thread def checkIntegrityOfSnapshot( self, snapshotsToRestore, checksumToCompare, disk_type="root"): if disk_type == "root": # Create template from snapshot template_from_snapshot = Template.create_from_snapshot( self.apiclient, snapshotsToRestore, self.testdata["template_2"]) self.assertNotEqual( template_from_snapshot, None, "Check if result exists in list item call" ) time.sleep(60) # Deploy VM vm_from_temp = VirtualMachine.create( self.apiclient, self.testdata["small"], templateid=template_from_snapshot.id, accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering.id, zoneid=self.zone.id, mode=self.zone.networktype ) self.assertNotEqual( vm_from_temp, None, "Check if result exists in list item call" ) time.sleep(60) # Verify contents of ROOT disk match with snapshot compareChecksum( self.apiclient, service=self.testdata, original_checksum=checksumToCompare, disk_type="rootdiskdevice", virt_machine=vm_from_temp ) vm_from_temp.delete(self.apiclient) template_from_snapshot.delete(self.apiclient) else: volumeFormSnap = Volume.create_from_snapshot( self.apiclient, snapshotsToRestore.id, self.testdata["volume"], account=self.account.name, domainid=self.account.domainid, zoneid=self.zone.id ) temp_vm = VirtualMachine.create( self.apiclient, self.testdata["small"], templateid=self.template.id, accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering.id, zoneid=self.zone.id, mode=self.zone.networktype ) temp_vm.attach_volume( self.apiclient, volumeFormSnap ) temp_vm.reboot(self.apiclient) compareChecksum( self.apiclient, service=self.testdata, original_checksum=checksumToCompare, disk_type="datadiskdevice_1", virt_machine=temp_vm ) temp_vm.delete(self.apiclient) volumeFormSnap.delete(self.apiclient) return def GetDestinationHost(self, hostid, currentHosts, hostList ): """ Get destination host in same cluster to migrate vm to """ destinationHost = None clusterid = None for host in self.Hosts: if host.id == hostid: clusterid = host.clusterid break for host in hostList: if host.clusterid == clusterid: destinationHost = host break return destinationHost def MigrateRootVolume(self, vm, destinationHost, expectexception=False): """ Migrate given volume to type of storage pool mentioned in migrateto: Inputs: 1. volume: Volume to be migrated 2. migrate_to: Scope of desired Storage pool to which volume is to be migrated 3. expectexception: If exception is expected while migration """ if expectexception: with self.assertRaises(Exception): VirtualMachine.migrate( vm, self.apiclient, hostid=destinationHost.id, ) else: VirtualMachine.migrate( vm, self.apiclient, hostid=destinationHost.id, ) migrated_vm_response = list_virtual_machines( self.apiclient, id=vm.id ) self.assertEqual( isinstance(migrated_vm_response, list), True, "Check list virtual machines response for valid list" ) self.assertNotEqual( migrated_vm_response, None, "Check if virtual machine exists in ListVirtualMachines" ) migrated_vm = migrated_vm_response[0] vm_list = VirtualMachine.list( self.apiclient, id=migrated_vm.id ) self.assertEqual( vm_list[0].hostid, destinationHost.id, "Check volume is on migrated pool" ) return class TestSnapshotsHardning(cloudstackTestCase): @classmethod def setUpClass(cls): testClient = super(TestSnapshotsHardning, cls).getClsTestClient() cls.apiclient = testClient.getApiClient() cls.testdata = testClient.getParsedTestDataConfig() cls.hypervisor = cls.testClient.getHypervisorInfo() # Get Zone, Domain and templates cls.domain = get_domain(cls.apiclient) cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) cls.template = get_template( cls.apiclient, cls.zone.id, cls.testdata["ostype"]) cls._cleanup = [] cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ try: # Create an account cls.account = Account.create( cls.apiclient, cls.testdata["account"], domainid=cls.domain.id ) # Create user api client of the account cls.userapiclient = testClient.getUserApiClient( UserName=cls.account.name, DomainName=cls.account.domain ) # Create Service offering cls.service_offering = ServiceOffering.create( cls.apiclient, cls.testdata["service_offering"], ) cls._cleanup.append(cls.service_offering) cls.service_offering_ha = ServiceOffering.create( cls.apiclient, cls.testdata["service_offering"], offerha=True ) cls.disk_offering = DiskOffering.create( cls.apiclient, cls.testdata["disk_offering"], ) cls._cleanup.append(cls.disk_offering) cls.vm = VirtualMachine.create( cls.apiclient, cls.testdata["small"], templateid=cls.template.id, accountid=cls.account.name, domainid=cls.account.domainid, serviceofferingid=cls.service_offering.id, zoneid=cls.zone.id, diskofferingid=cls.disk_offering.id, mode=cls.zone.networktype ) cls._cleanup.append(cls.vm) cls.root_volume = list_volumes( cls.userapiclient, virtualmachineid=cls.vm.id, type='ROOT', listall=True ) cls.data_volume = list_volumes( cls.userapiclient, virtualmachineid=cls.vm.id, type='DATA', listall=True ) cls.vm_ha = VirtualMachine.create( cls.apiclient, cls.testdata["small"], templateid=cls.template.id, accountid=cls.account.name, domainid=cls.account.domainid, serviceofferingid=cls.service_offering_ha.id, zoneid=cls.zone.id, diskofferingid=cls.disk_offering.id, mode=cls.zone.networktype ) cls._cleanup.append(cls.vm_ha) cls._cleanup.append(cls.account) cls.root_volume_ha = list_volumes( cls.userapiclient, virtualmachineid=cls.vm_ha.id, type='ROOT', listall=True ) cls.clusterList = Cluster.list(cls.apiclient) cls.Hosts = Host.list(cls.apiclient) cls.exceptionList = [] configs = Configurations.list( cls.apiclient, name="snapshot.delta.max") cls.delta_max = configs[0].value try: cls.pools = StoragePool.list(cls.apiclient, zoneid=cls.zone.id) except Exception as e: raise unittest.SkipTest(e) except Exception as e: cls.tearDownClass() raise e return @classmethod def tearDownClass(cls): try: cleanup_resources(cls.apiclient, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() self.cleanup = [] def tearDown(self): try: cleanup_resources(self.apiclient, self.cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @classmethod def RestartServer(cls): """Restart management server""" sshClient = SshClient( cls.mgtSvrDetails["mgtSvrIp"], 22, cls.mgtSvrDetails["user"], cls.mgtSvrDetails["passwd"] ) command = "service cloudstack-management restart" sshClient.execute(command) return def EnableMaintenance(self, hostid): Host.enableMaintenance(self.apiclient, id=hostid) return def StartVM(self, vm): vm.start(self.apiclient) return def RebootVM(self, vm): vm.reboot(self.apiclient) return def StopVM(self, vm): vm.stop(self.apiclient) return def ClearSnapshots(self, snapshots): if snapshots: for snap in snapshots: snap.delete(self.apiclient) return def CreateSnapshot(self, root_volume, is_recurring): """Create Snapshot""" try: if is_recurring: recurring_snapshot = SnapshotPolicy.create( self.apiclient, root_volume.id, self.testdata["recurring_snapshot"] ) self.rec_policy_pool.append(recurring_snapshot) else: root_vol_snap = Snapshot.create( self.apiclient, root_volume.id) self.snapshot_pool.append(root_vol_snap) except Exception as e: self.exceptionList = [] self.exceptionList.append(e) return def CreateDeltaSnapshot(self, volume): for i in range(int(self.delta_max)): Snapshot.create( self.apiclient, volume.id) return @attr(tags=["advanced", "basic"], required_hardware="true") def test_01_snapshot_hardning_kvm(self): """snapshot hardning 1. Take VM snapshot then migrate the VM to another host and again take volume snapshot and check its integrity 2. Verify that snapshot gets created successfully while VM is getting Migrated to another host 3. Verify that snapshot should succeed after vm's are HA-ed to different host and also check its integrity 4. Take ROOT volume snapshot and when snapshot is in progress bring the host down then once the VM is HA-ed to different host take snapshot of root volume and Check the integrity of this snapshot 5. Stop the VM, initiate VM snapshot and while snapshot is still in progress start the VM and check the integrity of the snapshot 6. Initiate ROOT volume snapshot and while snapshot is in pregress Stop the VM Verify that the VM stops successfully and check integrity of snapshot 7. Initiate ROOT volume snapshot and while snapshot is in pregress Reboot the VM Verify that the VM reboot successfully and check integrity of snapshot 8. Initiate ROOT volume snapshot and while snapshot is in progress create snapshot of the same volume and check integrity of both the snapshots 9. Initiate snapshot of DATA volume and while snapshot is in progress detach the volume verify that volume gets detached successfully also check integrity of snapshot 10. Initiate snapshot of a detached volume and while snapshot is in progress attach the volume to A VM verify that volume gets attached successfully also check integrity of snapshot """ if self.hypervisor.lower() != "kvm": self.skipTest("Skip test for Hypervisor other than KVM") # Step 1 root_volume = list_volumes( self.userapiclient, virtualmachineid=self.vm.id, type='ROOT', listall=True ) checksum_root = createChecksum( service=self.testdata, virtual_machine=self.vm, disk=root_volume[0], disk_type="rootdiskdevice") Snapshot.create( self.apiclient, root_volume[0].id) snapshots = list_snapshots( self.apiclient, volumeid=root_volume[0].id, listall=True) destinationHost = Host.listForMigration( self.apiclient, virtualmachineid=self.vm.id) sameClusHosts = Host.list(self.apiclient, virtualmachineid=self.vm.id) current_host = self.vm.hostid hostToMigarte = GetDestinationHost( self, current_host, sameClusHosts, destinationHost) MigrateRootVolume(self, self.vm, hostToMigarte) Snapshot.create( self.apiclient, root_volume[0].id) snapshots = list_snapshots( self.apiclient, volumeid=root_volume[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root) # Step 2 try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.root_volume[0], False)) destinationHost = Host.listForMigration( self.apiclient, virtualmachineid=self.vm.id) migrate_rootvolume_thread = Thread(target=MigrateRootVolume, args=(self, self.vm, destinationHost[0])) create_snapshot_thread.start() migrate_rootvolume_thread.start() create_snapshot_thread.join() migrate_rootvolume_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root) # Step 3 vm_host_id = self.vm_ha.hostid checksum_root_ha = createChecksum( service=self.testdata, virtual_machine=self.vm_ha, disk=self.root_volume_ha[0], disk_type="rootdiskdevice") self.CreateSnapshot( self.root_volume_ha[0], False) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume_ha[0].id, listall=True) Host.enableMaintenance(self.apiclient, id=vm_host_id) time.sleep(180) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume_ha[0].id, listall=True) Host.cancelMaintenance(self.apiclient, id=vm_host_id) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root_ha) # Step 4 # Scenario to be tested try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.root_volume_ha[0], False)) host_enable_maint_thread = Thread( target=Host.enableMaintenance, args=( self.apiclient, vm_host_id)) create_snapshot_thread.start() host_enable_maint_thread.start() create_snapshot_thread.join() host_enable_maint_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) self.CreateSnapshot(self.root_volume_ha[0], False) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume_ha[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root_ha) Host.cancelMaintenance(self.apiclient, vm_host_id) # Step 5 self.vm.stop(self.apiclient) time.sleep(90) try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.root_volume[0], False)) start_vm_thread = Thread(target=self.StartVM, args=([self.vm])) create_snapshot_thread.start() start_vm_thread.start() create_snapshot_thread.join() start_vm_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) state = self.dbclient.execute( "select state from vm_instance where name='%s'" % self.vm.name)[0][0] self.assertEqual( state, "Running", "Check if vm has started properly") snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root) # Step 6 try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.root_volume[0], False)) stop_vm_thread = Thread(target=self.StopVM, args=([self.vm])) create_snapshot_thread.start() stop_vm_thread.start() create_snapshot_thread.join() stop_vm_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) state = self.dbclient.execute( "select state from vm_instance where name='%s'" % self.vm.name)[0][0] self.assertEqual( state, "Stopped", "Check if vm has started properly") self.vm.start(self.apiclient) time.sleep(180) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root) # Step 7 try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.root_volume[0], False)) reboot_vm_thread = Thread(target=self.RebootVM, args=([self.vm])) create_snapshot_thread.start() reboot_vm_thread.start() create_snapshot_thread.join() reboot_vm_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) state = self.dbclient.execute( "select state from vm_instance where name='%s'" % self.vm.name)[0][0] self.assertEqual( state, "Running", "Check if vm has started properly") time.sleep(180) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root) # Step 8 pending(actual 9) # Step 9 checksum_data = createChecksum( service=self.testdata, virtual_machine=self.vm, disk=self.data_volume[0], disk_type="datadiskdevice_1") try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.data_volume[0], False)) detach_vm_thread = Thread( target=self.vm.detach_volume, args=( self.apiclient, self.data_volume[0])) create_snapshot_thread.start() detach_vm_thread.start() create_snapshot_thread.join() detach_vm_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) self.vm.reboot(self.apiclient) snapshots = list_snapshots( self.apiclient, volumeid=self.data_volume[0].id, listall=True) data_volume_list = list_volumes( self.apiclient, virtualmachineid=self.vm.id, type='DATA', listall=True ) self.assertEqual( data_volume_list, None, "check if volume is detached" ) checkIntegrityOfSnapshot( self, snapshots[0], checksum_data, disk_type="data") # Step 10 try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.data_volume[0], False)) attach_vm_thread = Thread( target=self.vm.attach_volume, args=( self.apiclient, self.data_volume[0])) create_snapshot_thread.start() attach_vm_thread.start() create_snapshot_thread.join() attach_vm_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) self.vm.reboot(self.apiclient) snapshots = list_snapshots( self.apiclient, volumeid=self.data_volume[0].id, listall=True) data_volume_list = list_volumes( self.apiclient, virtualmachineid=self.vm.id, type='DATA', listall=True ) self.assertNotEqual( data_volume_list, [], "check if volume is detached" ) checkIntegrityOfSnapshot( self, snapshots[0], checksum_root, disk_type="data") @attr(tags=["advanced", "basic"], required_hardware="true") def test_02_snapshot_hardning_xenserver(self): """snapshot hardning 1. Take VM snapshot then migrate the VM to another host and again take volume snapshot and check its intigrity 2. Verify that snapshot gets created successfully while VM is getting Migrated to another host 3. Verify that snapshot should succeed after vm's are HA-ed to different host and also check its integrity 4. Take ROOT volume snapshot and when snapshot is in progress bring the host down then once the VM is HA-ed to different host take snapshot of root volume and Check the integrity of this snapshot 5. Stop the VM, initiate VM snapshot and while snapshot is still in progress start the VM and check the integrity of the snapshot 6. Initiate ROOT volume snapshot and while snapshot is in pregress Stop the VM Verify that the VM stops successfully and check integrity of snapshot 7. Initiate ROOT volume snapshot and while snapshot is in pregress Reboot the VM Verify that the VM reboot successfully and check integrity of snapshot 8. Initiate ROOT volume snapshot and while snapshot is in pregress create snapshot of the same volume and check integrity of both the snapshots 9. Initiate snapshot of DATA volume and while snapshot is in progress detach the volume verify that volume gets detached successfully also check integrity of snapshot 10. Initiate snapshot of a detached volume and while snapshot is in progress attach the volume to A VM verify that volume gets attached successfully also check integrity of snapshot """ if self.hypervisor != "xenserver": self.skipTest("Skip test for server other than XenServer") # Step 1 root_volume = list_volumes( self.userapiclient, virtualmachineid=self.vm.id, type='ROOT', listall=True ) checksum_root = createChecksum( service=self.testdata, virtual_machine=self.vm, disk=root_volume[0], disk_type="rootdiskdevice") Snapshot.create( self.apiclient, root_volume[0].id) snapshots = list_snapshots( self.apiclient, volumeid=root_volume[0].id, listall=True) destinationHost = Host.listForMigration( self.apiclient, virtualmachineid=self.vm.id) sameClusHosts = Host.list(self.apiclient, virtualmachineid=self.vm.id) current_host = self.vm.hostid hostToMigarte = GetDestinationHost( self, current_host, sameClusHosts, destinationHost) MigrateRootVolume(self, self.vm, hostToMigarte) self.CreateDeltaSnapshot(root_volume[0]) Snapshot.create( self.apiclient, root_volume[0].id) snapshots = list_snapshots( self.apiclient, volumeid=root_volume[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root) self.CreateDeltaSnapshot(root_volume[0]) # Step 2 try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.root_volume[0], False)) destinationHost = Host.listForMigration( self.apiclient, virtualmachineid=self.vm.id) migrate_rootvolume_thread = Thread(target=MigrateRootVolume, args=(self, self.vm, destinationHost[0])) create_snapshot_thread.start() migrate_rootvolume_thread.start() create_snapshot_thread.join() migrate_rootvolume_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root) self.CreateDeltaSnapshot(root_volume[0]) # Step 3 vm_host_id = self.vm_ha.hostid checksum_root_ha = createChecksum( service=self.testdata, virtual_machine=self.vm_ha, disk=self.root_volume_ha[0], disk_type="rootdiskdevice") self.CreateSnapshot( self.root_volume_ha[0], False) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume_ha[0].id, listall=True) self.CreateDeltaSnapshot(self.root_volume_ha[0]) Host.enableMaintenance(self.apiclient, id=vm_host_id) time.sleep(180) self.CreateSnapshot(self.root_volume[0], False) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume_ha[0].id, listall=True) Host.cancelMaintenance(self.apiclient, id=vm_host_id) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root_ha) self.CreateDeltaSnapshot(self.root_volume_ha[0]) # Step 4 # Scenario to be tested try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.root_volume_ha[0], False)) host_enable_maint_thread = Thread( target=Host.enableMaintenance, args=( self.apiclient, vm_host_id)) create_snapshot_thread.start() host_enable_maint_thread.start() create_snapshot_thread.join() host_enable_maint_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) self.CreateDeltaSnapshot(self.root_volume_ha[0]) self.CreateSnapshot(self.root_volume_ha[0], False) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume_ha[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root_ha) Host.cancelMaintenance(self.apiclient, vm_host_id) # Step 5 self.vm.stop(self.apiclient) time.sleep(90) try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.root_volume[0], False)) start_vm_thread = Thread(target=self.StartVM, args=([self.vm])) create_snapshot_thread.start() start_vm_thread.start() create_snapshot_thread.join() start_vm_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) state = self.dbclient.execute( "select state from vm_instance where name='%s'" % self.vm.name)[0][0] self.assertEqual( state, "Running", "Check if vm has started properly") snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root) self.CreateDeltaSnapshot(root_volume[0]) # Step 6 try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.root_volume[0], False)) stop_vm_thread = Thread(target=self.StopVM, args=([self.vm])) create_snapshot_thread.start() stop_vm_thread.start() create_snapshot_thread.join() stop_vm_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) state = self.dbclient.execute( "select state from vm_instance where name='%s'" % self.vm.name)[0][0] self.assertEqual( state, "Stopped", "Check if vm has started properly") self.vm.start(self.apiclient) time.sleep(180) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root) self.CreateDeltaSnapshot(root_volume[0]) # Step 7 try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.root_volume[0], False)) reboot_vm_thread = Thread(target=self.RebootVM, args=([self.vm])) create_snapshot_thread.start() reboot_vm_thread.start() create_snapshot_thread.join() reboot_vm_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) state = self.dbclient.execute( "select state from vm_instance where name='%s'" % self.vm.name)[0][0] self.assertEqual( state, "Running", "Check if vm has started properly") time.sleep(180) snapshots = list_snapshots( self.apiclient, volumeid=self.root_volume[0].id, listall=True) checkIntegrityOfSnapshot(self, snapshots[0], checksum_root) self.CreateDeltaSnapshot(root_volume[0]) # Step 8 pending(actual 9) # Step 9 checksum_data = createChecksum( service=self.testdata, virtual_machine=self.vm, disk=self.data_volume[0], disk_type="datadiskdevice_1") try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.data_volume[0], False)) detach_vm_thread = Thread( target=self.vm.detach_volume, args=( self.apiclient, self.data_volume[0])) create_snapshot_thread.start() detach_vm_thread.start() create_snapshot_thread.join() detach_vm_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) self.vm.reboot(self.apiclient) snapshots = list_snapshots( self.apiclient, volumeid=self.data_volume[0].id, listall=True) data_volume_list = list_volumes( self.apiclient, virtualmachineid=self.vm.id, type='DATA', listall=True ) self.assertEqual( data_volume_list, None, "check if volume is detached" ) checkIntegrityOfSnapshot( self, snapshots[0], checksum_data, disk_type="data") self.CreateDeltaSnapshot(self.data_volume[0]) # Step 10 try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self.data_volume[0], False)) attach_volume_thread = Thread( target=self.vm.attach_volume, args=( self.apiclient, self.data_volume[0])) create_snapshot_thread.start() attach_volume_thread.start() create_snapshot_thread.join() attach_volume_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) self.vm.reboot(self.apiclient) snapshots = list_snapshots( self.apiclient, volumeid=self.data_volume[0].id, listall=True) data_volume_list = list_volumes( self.apiclient, virtualmachineid=self.vm.id, type='DATA', listall=True ) self.assertNotEqual( data_volume_list, [], "check if volume is detached" ) checkIntegrityOfSnapshot( self, snapshots[0], checksum_root, disk_type="data") self.CreateDeltaSnapshot(self.data_volume[0]) return def test_03_snapshot_hardning_configuration(self): """snapshot hardning 1. Verify the snapshot failuar for smaller value of backup.snapshot.wait then snapshot success for larger value of backup.snapshot.wait and check the integrity of the snapshot 2. """ # Step 1 if not self.testdata["configurableData"][ "restartManagementServerThroughTestCase"]: self.skipTest( "Skip test if restartManagementServerThroughTestCase\ is not provided") configs = Configurations.list( self.apiclient, name="backup.snapshot.wait") orig_backup = configs[0].value Configurations.update(self.apiclient, name="backup.snapshot.wait", value="10" ) # Restart management server self.RestartServer() time.sleep(120) checksum_root = createChecksum( service=self.testdata, virtual_machine=self.vm, disk=self.root_volume, disk_type="rootdiskdevice") with self.assertRaises(Exception): Snapshot.create( self.apiclient, self.root_volume[0].id) Configurations.update(self.apiclient, name="backup.snapshot.wait", value=orig_backup ) # Restart management server self.RestartServer() time.sleep(120) configs = Configurations.list( self.apiclient, name="backup.snapshot.wait") orig_backup = configs[0].value snapshot_2 = Snapshot.create( self.apiclient, self.root_volume[0].id) time.sleep(360) self.assertEqual( self.dbclient.execute( "select status from snapshots where id='%s'" % snapshot_2.id)[0][0], "BackedUp" ) checkIntegrityOfSnapshot(self, snapshot_2, checksum_root) return class TestHardening(cloudstackTestCase): @classmethod def setUpClass(cls): testClient = super(TestHardening, cls).getClsTestClient() cls.apiclient = testClient.getApiClient() cls.testdata = testClient.getParsedTestDataConfig() cls.hypervisor = cls.testClient.getHypervisorInfo() # Get Zone, Domain and templates cls.domain = get_domain(cls.apiclient) cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) cls.template = get_template( cls.apiclient, cls.zone.id, cls.testdata["ostype"]) cls._cleanup = [] configs = Configurations.list( cls.apiclient, name="snapshot.delta.max") cls.delta_max = configs[0].value clusterid_tag_mapping = {} cwps_no = 0 cls.unsupportedHypervisor = False if cls.hypervisor.lower() not in [ "vmware", "kvm", "xenserver", "hyper-v"]: cls.unsupportedHypervisor = True return try: cls.pools = StoragePool.list( cls.apiclient, zoneid=cls.zone.id, scope="CLUSTER") for storagePool in cls.pools: if storagePool.state.lower() == UP: if storagePool.clusterid not in clusterid_tag_mapping: cwps_no += 1 StoragePool.update( cls.apiclient, id=storagePool.id, tags=['cwps' + repr(cwps_no)]) clusterid_tag_mapping[ storagePool.clusterid] = [cwps_no] else: cwps_no = clusterid_tag_mapping[ storagePool.clusterid][0] StoragePool.update( cls.apiclient, id=storagePool.id, tags=['cwps' + repr(cwps_no)]) clusterid_tag_mapping[ storagePool.clusterid].append(cwps_no) cls.pools = StoragePool.list( cls.apiclient, zoneid=cls.zone.id, scope="CLUSTER") # Check clusterid count is 2 # Check each clusterid has two Storage Pool Tags # which indicate two Storage Pools exist. assert (len(clusterid_tag_mapping)) >= 2 and\ (len(tags) for tags in clusterid_tag_mapping.values( )) >= 2, "There must be at least two Clusters and\ each must have at least two cluster wide storage pools in\ Up state in the setup" except Exception as e: raise unittest.SkipTest(e) try: # Create an account cls.account = Account.create( cls.apiclient, cls.testdata["account"], domainid=cls.domain.id ) cls._cleanup.append(cls.account) # Create user api client of the account cls.userapiclient = testClient.getUserApiClient( UserName=cls.account.name, DomainName=cls.account.domain ) # Create Service offering cls.service_offering_cluster1 = ServiceOffering.create( cls.apiclient, cls.testdata["service_offering"], tags=CLUSTERTAG1 ) cls._cleanup.append(cls.service_offering_cluster1) cls.service_offering_cluster2 = ServiceOffering.create( cls.apiclient, cls.testdata["service_offering"], tags=CLUSTERTAG2 ) cls._cleanup.append(cls.service_offering_cluster1) cls.service_offering = ServiceOffering.create( cls.apiclient, cls.testdata["service_offering"] ) cls._cleanup.append(cls.service_offering) # Create Disk offering cls.disk_offering_cluster1 = DiskOffering.create( cls.apiclient, cls.testdata["disk_offering"], tags=CLUSTERTAG1 ) cls._cleanup.append(cls.disk_offering_cluster1) # Create VM on CWPS cls.vm = VirtualMachine.create( cls.apiclient, cls.testdata["small"], templateid=cls.template.id, accountid=cls.account.name, domainid=cls.account.domainid, serviceofferingid=cls.service_offering_cluster1.id, zoneid=cls.zone.id, mode=cls.zone.networktype ) except Exception as e: cls.tearDownClass() raise e return @classmethod def tearDownClass(cls): try: cleanup_resources(cls.apiclient, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() self.cleanup = [] if self.unsupportedHypervisor: self.skipTest("VM migration is not supported on %s" % self.hypervisor) def tearDown(self): try: for storagePool in self.pools: StoragePool.update(self.apiclient, id=storagePool.id, tags="") cleanup_resources(self.apiclient, self.cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return def GetStoragePoolTag(self, rootVolume): storagePoolTag = None # Storage Pool Id to migrate to for storagePool in self.pools: if storagePool.id == rootVolume.storageid: storagePoolTag = storagePool.tags return storagePoolTag def GetDestinationPool( self, currentStoragePoolTag, poolsToavoid, migrateToDiffCluster=False): destinationPool = None # Storage Pool Id to migrate to if migrateToDiffCluster: for storagePool in self.pools: if storagePool.tags != currentStoragePoolTag: destinationPool = storagePool break else: for storagePool in self.pools: if storagePool.name not in poolsToavoid: if storagePool.tags == currentStoragePoolTag: destinationPool = storagePool break return destinationPool def CreateSnapshot(self, root_volume, is_recurring): """Create Snapshot""" try: if is_recurring: recurring_snapshot = SnapshotPolicy.create( self.apiclient, root_volume.id, self.testdata["recurring_snapshot"] ) self.rec_policy_pool.append(recurring_snapshot) else: root_vol_snap = Snapshot.create( self.apiclient, root_volume.id) self.snapshot_pool.append(root_vol_snap) except Exception as e: self.exceptionList = [] self.exceptionList.append(e) return def CreateDeltaSnapshot(self, volume): for i in range(int(self.delta_max)): Snapshot.create( self.apiclient, volume.id) return def GetUpdatedRootVolume(self): """ Return Updated Root Volume. The storage pool of ROOT Volume changes after migration. """ # Get ROOT Volume root_volumes_list = Volume.list( self.apiclient, virtualmachineid=self.vm.id, type='ROOT', listall=True ) status = validateList(root_volumes_list) self.assertEqual(status[0], PASS, "Check list of ROOT Volumes.") rootVolume = root_volumes_list[0] return rootVolume def LiveMigrateVolume(self, volume, destinationPool, islive=False, expectexception=False ): """ Migrate given volume to type of storage pool mentioned in migrateto: Inputs: 1. volume: Volume to be migrated 2. migrate_to: Scope of desired Storage pool to which volume is to be migrated 3. expectexception: If exception is expected while migration """ if expectexception: with self.assertRaises(Exception): Volume.migrate( self.apiclient, volumeid=volume.id, storageid=destinationPool.id, livemigrate=islive ) else: Volume.migrate( self.apiclient, volumeid=volume.id, storageid=destinationPool.id, livemigrate=islive ) migrated_volume_response = list_volumes( self.apiclient, id=volume.id ) self.assertEqual( isinstance(migrated_volume_response, list), True, "Check list volumes response for valid list" ) self.assertNotEqual( migrated_volume_response, None, "Check if volume exists in ListVolumes" ) migrated_volume = migrated_volume_response[0] self.assertEqual( str(migrated_volume.state).lower(), 'ready', "Check migrated volume is in Ready state" ) self.assertEqual( migrated_volume.storage, destinationPool.name, "Check volume is on migrated pool" ) return @attr(tags=["basic", "advanced"], required_hardware="true") def test_06_hardening(self): """ Hardening 1. Attach Volume when snapshot on this volume is still in progress to a VM in different cluster. 2. Volume Snapshot after Vms have migrated to a different storage pool in same cluster. 3. Volume Snapshot after Vms have migrated to a different cluster. 4. Volume Snapshot after Vm has live migrated to a different storage with in the same cluster. 5. Volume Snapshot after Vm has live migrated to a different cluster. 6. Volume migration when snapshot is in progress. 7. Storage live migration when snapshot is in progress. """ # Get ROOT Volume root_volume = self.GetUpdatedRootVolume() checksum_root = createChecksum( service=self.testdata, virtual_machine=self.vm, disk=root_volume, disk_type="rootdiskdevice") data_volume_created = Volume.create( self.apiclient, self.testdata["volume"], zoneid=self.zone.id, account=self.account.name, domainid=self.account.domainid, diskofferingid=self.disk_offering_cluster1.id ) self.vm.attach_volume( self.apiclient, data_volume_created ) self.vm.reboot(self.apiclient) data_volumes_list = Volume.list( self.apiclient, id=data_volume_created.id ) data_volume = data_volumes_list[0] checksum_data = createChecksum( service=self.testdata, virtual_machine=self.vm, disk=data_volume, disk_type="datadiskdevice_1") # Detach DATA Volume self.vm.detach_volume( self.apiclient, data_volume ) self.vm.reboot(self.apiclient) current_storagepool_tag = self.GetStoragePoolTag(root_volume) # Create VM on CWPS vm_in_cluster2 = VirtualMachine.create( self.apiclient, self.testdata["small"], templateid=self.template.id, accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering_cluster2.id, zoneid=self.zone.id, mode=self.zone.networktype ) # Step 1 self.CreateSnapshot(data_volume, False) vm_in_cluster2.attach_volume(self.apiclient, data_volume) vm_in_cluster2.reboot(self.apiclient) snapshots_list = list_snapshots( self.apiclient, volumeid=data_volume.id, listall=True) data_volume_list = list_volumes( self.apiclient, virtualmachineid=vm_in_cluster2.id, type='DATA', listall=True ) self.assertNotEqual( data_volume_list, [], "check if volume is detached" ) self.CreateDeltaSnapshot(data_volume) self.CreateSnapshot(data_volume, False) snapshots_list = list_snapshots( self.apiclient, volumeid=data_volume_list[0].id, listall=True) status = validateList(snapshots_list) self.assertEqual(status[0], PASS, "Snapshots List Validation Failed") # Verify Snapshot state self.assertEqual( snapshots_list[0].state.lower() in [ BACKED_UP, ], True, "Snapshot state is not as expected. It is %s" % snapshots_list[0].state ) checkIntegrityOfSnapshot( self, snapshots_list[0], checksum_data, disk_type="data") # Detach DATA Volume vm_in_cluster2.detach_volume( self.apiclient, data_volume_list[0] ) vm_in_cluster2.reboot(self.apiclient) # Step 2 self.CreateSnapshot(self, root_volume, False) snapshots_list = Snapshot.list( self.userapiclient, volumeid=root_volume.id) status = validateList(snapshots_list) self.assertEqual(status[0], PASS, "Snapshots List Validation Failed") # Verify Snapshot state self.assertEqual( snapshots_list[0].state.lower() in [ BACKED_UP, ], True, "Snapshot state is not as expected. It is %s" % snapshots_list[0].state ) self.vm.stop(self.apiclient) # Migration destination_storage_pool = self.GetDestinationPool( self, current_storagepool_tag, root_volume.storage, migrateToDiffCluster=False) MigrateRootVolume(self, self.vm, destination_storage_pool) self.vm.start(self.apiclient) self.CreateSnapshot(self, root_volume, False) new_snapshots_list = list_snapshots( self.userapiclient, volumeid=root_volume.id) status = validateList(new_snapshots_list) self.assertEqual(status[0], PASS, "Snapshots List Validation Failed") # Verify Snapshot state self.assertEqual( new_snapshots_list[0].state.lower() in [ BACKED_UP, ], True, "Snapshot state is not as expected. It is %s" % new_snapshots_list[0].state ) checkIntegrityOfSnapshot( self, new_snapshots_list[0], checksum_root, disk_type="root") # Step 3 self.vm.stop(self.apiclient) # Get Updated ROOT Volume root_volume = self.GetUpdatedRootVolume(self) current_storagepool_tag = self.GetStoragePoolTag(self, root_volume) destination_storage_pool = self.GetDestinationPool( self, current_storagepool_tag, [], migrateToDiffCluster=True) # Migration MigrateRootVolume(self, self.vm, destination_storage_pool) self.vm.start(self.apiclient) self.CreateSnapshot(self, root_volume, False) snapshots_list = list_snapshots( self.userapiclient, volumeid=root_volume.id) checkIntegrityOfSnapshot( self, snapshots_list[0], checksum_root, disk_type="root") # Get Updated ROOT Volume root_volume = self.GetUpdatedRootVolume(self) current_storagepool_tag = self.GetStoragePoolTag(self, root_volume) # Step 4 # Migration destination_storage_pool = self.GetDestinationPool( self, current_storagepool_tag, root_volume.storage, migrateToDiffCluster=False) # Migrate self.LiveMigrateVolume( self, root_volume, destination_storage_pool, islive=True) self.CreateSnapshot(self, root_volume, False) snapshots_list = list_snapshots( self.userapiclient, volumeid=root_volume.id) checkIntegrityOfSnapshot( self, snapshots_list[0], checksum_root, disk_type="root") # Step 5 # Live Migration # Get Updated ROOT Volume root_volume = self.GetUpdatedRootVolume(self) current_storagepool_tag = self.GetStoragePoolTag(self, root_volume) destination_storage_pool = self.GetDestinationPool( self, current_storagepool_tag, [], migrateToDiffCluster=True) # Migrate self.LiveMigrateVolume( self, root_volume, destination_storage_pool, islive=True) current_storagepool_tag = self.GetStoragePoolTag(self, data_volume) destination_storage_pool = self.GetDestinationPool( self, current_storagepool_tag, [], migrateToDiffCluster=True) # Migrate self.LiveMigrateVolume( self, data_volume, destination_storage_pool, islive=True) self.CreateSnapshot(self, root_volume, False) snapshots_list = list_snapshots( self.userapiclient, volumeid=root_volume.id) checkIntegrityOfSnapshot( self, snapshots_list[0], checksum_root, disk_type="root") self.vm.stop(self.apiclient) # Get Updated ROOT Volume root_volume = self.GetUpdatedRootVolume(self) current_storagepool_tag = self.GetStoragePoolTag(self, root_volume) # Step 6 try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self, root_volume, False)) destination_storage_pool = self.GetDestinationPool( self, current_storagepool_tag, root_volume.storage, migrateToDiffCluster=False) migrate_volume_thread = Thread(target=MigrateRootVolume, args=(self, self.vm, destination_storage_pool)) create_snapshot_thread.start() migrate_volume_thread.start() create_snapshot_thread.join() migrate_volume_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) snapshots_list = list_snapshots( self.userapiclient, volumeid=root_volume.id) self.assertTrue( is_snapshot_on_nfs( self.apiclient, self.dbclient, self.config, self.zone.id, snapshots_list[0].id), "Snapshot is not on Secondary Storage.") checkIntegrityOfSnapshot(self, snapshots_list[0], checksum_root) # Step 7 self.vm.start(self.apiclient) # Get Updated ROOT Volume root_volume = self.GetUpdatedRootVolume(self) current_storagepool_tag = self.GetStoragePoolTag(self, root_volume) # live try: create_snapshot_thread = Thread( target=self.CreateSnapshot, args=( self, root_volume, False)) destination_storage_pool = self.GetDestinationPool( self, current_storagepool_tag, root_volume.storage, migrateToDiffCluster=False) live_migrate_volume_thread = Thread(target=self.LiveMigrateVolume, args=(self, root_volume, destination_storage_pool, True)) create_snapshot_thread.start() live_migrate_volume_thread.start() create_snapshot_thread.join() live_migrate_volume_thread.join() except Exception as e: raise Exception( "Warning: Exception unable to start thread : %s" % e) snapshots_list = list_snapshots( self.userapiclient, volumeid=root_volume.id) checkIntegrityOfSnapshot( self, snapshots_list[0], checksum_root, disk_type="root") data_volume_created.delete(self.apiclient) return