cloudstack/test/integration/testpaths/testpath_volume_cuncurrent_snapshots.py
John Bampton def7ce655d
Fix spelling (#6898)
Co-authored-by: davidjumani <dj.davidjumani1994@gmail.com>
2022-12-13 14:58:14 +01:00

861 lines
27 KiB
Python

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
""" Test cases for VM/Volume snapshot Test Path
Snapshot Test cases related to handling concurrent snapshots at host level.
"""
from nose.plugins.attrib import attr
from marvin.cloudstackTestCase import cloudstackTestCase
import unittest
from marvin.lib.utils import (cleanup_resources, is_snapshot_on_nfs,
validateList)
from marvin.lib.base import (Account,
StoragePool,
Host,
ServiceOffering,
VirtualMachine,
Configurations,
Snapshot,
SnapshotPolicy,
)
from marvin.lib.common import (get_domain,
list_snapshot_policy,
get_zone,
get_template,
list_volumes,
list_snapshots,
list_virtual_machines,
createChecksum,
)
from marvin.sshClient import SshClient
import time
from threading import Thread
from marvin.codes import PASS
def MigrateRootVolume(self,
vm,
destinationHost,
expectexception=False):
""" Migrate given volume to type of storage pool mentioned in migrateto:
Inputs:
1. vm: VM to be migrated
is to be migrated
2. expectexception: If exception is expected while migration
3. destinationHost: Destination host where the VM\
should get migrated
"""
if expectexception:
with self.assertRaises(Exception):
VirtualMachine.migrate(
vm,
self.apiclient,
hostid=destinationHost.id,
)
else:
VirtualMachine.migrate(
vm,
self.apiclient,
hostid=destinationHost.id,
)
migrated_vm_response = list_virtual_machines(
self.apiclient,
id=vm.id
)
self.assertEqual(
isinstance(migrated_vm_response, list),
True,
"Check list virtual machines response for valid list"
)
self.assertNotEqual(
migrated_vm_response,
None,
"Check if virtual machine exists in ListVirtualMachines"
)
migrated_vm = migrated_vm_response[0]
vm_list = VirtualMachine.list(
self.apiclient,
id=migrated_vm.id
)
self.assertEqual(
vm_list[0].hostid,
destinationHost.id,
"Check volume is on migrated pool"
)
return
def CreateSnapshot(self, root_volume, is_recurring):
"""Create Snapshot"""
if is_recurring:
self.testdata["recurring_snapshot"]["intervaltype"] = 'HOURLY'
self.testdata["recurring_snapshot"]["schedule"] = 1
recurring_snapshot = SnapshotPolicy.create(
self.apiclient,
root_volume.id,
self.testdata["recurring_snapshot"]
)
self.rec_policy_pool.append(recurring_snapshot)
else:
root_vol_snapshot = Snapshot.create(
self.apiclient,
root_volume.id)
self.snapshot_pool.append(root_vol_snapshot)
return
class TestConcurrentSnapshots(cloudstackTestCase):
@classmethod
def setUpClass(cls):
testClient = super(TestConcurrentSnapshots, cls).getClsTestClient()
cls.apiclient = testClient.getApiClient()
cls.testdata = testClient.getParsedTestDataConfig()
cls.hypervisor = cls.testClient.getHypervisorInfo()
# Get Zone, Domain and templates
cls.domain = get_domain(cls.apiclient)
cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
cls.template = get_template(
cls.apiclient,
cls.zone.id,
cls.testdata["ostype"])
cls._cleanup = []
cls.vm_pool = []
cls.snapshotSupported = True
if cls.hypervisor.lower() in ["hyperv", "lxc"]:
cls.snapshotSupported = False
return
# Set sleep time as per Snapshot Recurring Policy - HOURLY
cls.sleep_time_for_hourly_policy = 60 * 60 * 1
cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__
try:
# Create an account
cls.account = Account.create(
cls.apiclient,
cls.testdata["account"],
domainid=cls.domain.id
)
cls._cleanup.append(cls.account)
# Create user api client of the account
cls.userapiclient = testClient.getUserApiClient(
UserName=cls.account.name,
DomainName=cls.account.domain
)
# Create Service offering
cls.service_offering = ServiceOffering.create(
cls.apiclient,
cls.testdata["service_offering"],
)
cls._cleanup.append(cls.service_offering)
for i in range(4):
cls.vm = VirtualMachine.create(
cls.apiclient,
cls.testdata["small"],
templateid=cls.template.id,
accountid=cls.account.name,
domainid=cls.account.domainid,
serviceofferingid=cls.service_offering.id,
zoneid=cls.zone.id,
mode=cls.zone.networktype
)
cls.vm_pool.append(cls.vm)
cls._cleanup.append(cls.vm)
cls.checksum_pool = []
cls.root_pool = []
cls.snapshot_pool = []
cls.rec_policy_pool = []
for vm in cls.vm_pool:
root_volumes = list_volumes(
cls.apiclient,
virtualmachineid=vm.id,
type='ROOT',
listall=True
)
checksum_root = createChecksum(
cls.testdata,
vm,
root_volumes[0],
"rootdiskdevice")
cls.checksum_pool.append(checksum_root)
cls.root_pool.append(root_volumes[0])
try:
cls.pools = StoragePool.list(cls.apiclient, zoneid=cls.zone.id)
except Exception as e:
raise unittest.SkipTest(e)
except Exception as e:
cls.tearDownClass()
raise e
return
@classmethod
def tearDownClass(cls):
try:
for vm in cls.vm_pool:
vm.delete(cls.apiclient)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
def setUp(self):
self.apiclient = self.testClient.getApiClient()
self.dbclient = self.testClient.getDbConnection()
self.cleanup = []
if not self.snapshotSupported:
self.skipTest("Snapshot is not supported on %s" % self.hypervisor)
def tearDown(self):
try:
cleanup_resources(self.apiclient, self.cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
@classmethod
def RestartServer(cls):
"""Restart management server"""
sshClient = SshClient(
cls.mgtSvrDetails["mgtSvrIp"],
22,
cls.mgtSvrDetails["user"],
cls.mgtSvrDetails["passwd"]
)
command = "service cloudstack-management restart"
sshClient.execute(command)
return
@classmethod
def StopVM(cls, vms):
for vm in vms:
vm.stop(cls.apiclient)
return
@attr(tags=["advanced", "basic"], required_hardware="true")
def test_01_concurrent_snapshots(self):
"""Concurrent Snapshots
1. Create snapshot on 2 new VMs in parallel and check
1. all snapshot jobs are running
2. listSnapshots should list all the snapshots
3. Verify secondary_storage NFS share
contains the required volume under
/secondary/snapshots/$accountid/$volumeid/$snapshot_uuid.
4. Verify backup_snap_id was non null in "snapshots"table
2. Perform step 1 for all the 4 VM's.
3. Verify that VM gets migrated when snapshot
is in progress for the VM.
4. Verify that snapshots get created when
VM's are stopped in between snapshot creation.
5. Perform live Migration then stop all the
VM's after that verify that snapshot creation success .
6. Verify success of snapshots creation in case:
Stop the running VM while performing
concurrent snapshot on volumes
7. Verify success of snapshots creation in case:
Start Migration of VM's and then Stop the running VM then
performing concurrent snapshot on volumes
"""
# Step 1
try:
create_snapshot_thread_1 = Thread(
target=CreateSnapshot,
args=(
self,
self.root_pool[0],
False))
create_snapshot_thread_2 = Thread(
target=CreateSnapshot,
args=(
self,
self.root_pool[1],
False))
create_snapshot_thread_1.start()
create_snapshot_thread_2.start()
create_snapshot_thread_1.join()
create_snapshot_thread_2.join()
except:
self.debug("Error: unable to start thread")
snapshots = list_snapshots(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
for snapshot in self.snapshot_pool:
self.assertTrue(snapshot.id in any(
s.id) for s in snapshots)
for snapshot in self.snapshot_pool:
self.assertTrue(
is_snapshot_on_nfs(
self.apiclient,
self.dbclient,
self.config,
self.zone.id,
snapshot.id))
for snapshot in self.snapshot_pool:
snapshot.delete(self.apiclient)
self.snapshot_pool = []
# Step 2
thread_pool = []
for i in range(4):
try:
create_snapshot_thread_1 = Thread(
target=CreateSnapshot,
args=(
self,
self.root_pool[i],
False))
thread_pool.append(create_snapshot_thread_1)
except Exception as e:
raise Exception(
"Warning: Exception unable to start thread : %s" %
e)
for thread in thread_pool:
thread.start()
for thread in thread_pool:
thread.join()
snapshots = list_snapshots(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
for snapshot in self.snapshot_pool:
self.assertTrue(snapshot.id in any(
s.id) for s in snapshots)
for snapshot in self.snapshot_pool:
self.assertTrue(
is_snapshot_on_nfs(
self.apiclient,
self.dbclient,
self.config,
self.zone.id,
snapshot.id))
for snapshot in self.snapshot_pool:
snapshot.delete(self.apiclient)
self.snapshot_pool = []
# Step 3
# Recurring snapshot
try:
create_snapshot_thread_1 = Thread(
target=CreateSnapshot,
args=(
self,
self.root_pool[0],
True))
create_snapshot_thread_2 = Thread(
target=CreateSnapshot,
args=(
self,
self.root_pool[1],
True))
create_snapshot_thread_1.start()
create_snapshot_thread_2.start()
create_snapshot_thread_1.join()
create_snapshot_thread_2.join()
except:
self.debug("Error: unable to start thread")
for rec_snap in self.rec_policy_pool:
list_snapshots_policy_1 = list_snapshot_policy(
self.apiclient,
id=rec_snap.id,
)
list_validation = validateList(list_snapshots_policy_1)
self.assertEqual(
list_validation[0],
PASS,
"snapshot list validation failed due to %s" %
list_validation[2])
time.sleep(self.sleep_time_for_hourly_policy)
snapshots = list_snapshots(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
for snapshot in self.snapshot_pool:
self.assertTrue(snapshot.id in any(
s.id) for s in snapshots)
for snapshot in self.snapshot_pool:
self.assertTrue(
is_snapshot_on_nfs(
self.apiclient,
self.dbclient,
self.config,
self.zone.id,
snapshot.id))
for snapshot in self.snapshot_pool:
snapshot.delete(self.apiclient)
self.snapshot_pool = []
for rec_snap_pol in self.rec_policy_pool:
rec_snap_pol.delete(self.apiclient)
self.rec_policy_pool = []
# Step 4
thread_pool = []
for i in range(4):
try:
create_snapshot_thread_1 = Thread(
target=CreateSnapshot,
args=(
self,
self.root_pool[i],
True))
thread_pool.append(create_snapshot_thread_1)
except Exception as e:
raise Exception(
"Warning: Exception unable to start thread : %s" %
e)
for thread in thread_pool:
thread.start()
for thread in thread_pool:
thread.join()
for rec_snap in self.rec_policy_pool:
list_snapshots_policy_1 = list_snapshot_policy(
self.apiclient,
id=rec_snap.id,
)
list_validation_1 = validateList(list_snapshots_policy_1)
self.assertEqual(
list_validation[0],
PASS,
"snapshot list validation failed due to %s" %
list_validation_1[2])
time.sleep(self.sleep_time_for_hourly_policy)
for snapshot in self.snapshot_pool:
self.assertTrue(snapshot.id in any(
s.id) for s in snapshots)
for snapshot in self.snapshot_pool:
self.assertTrue(
is_snapshot_on_nfs(
self.apiclient,
self.dbclient,
self.config,
self.zone.id,
snapshot.id))
for snapshot in self.snapshot_pool:
snapshot.delete(self.apiclient)
self.snapshot_pool = []
for rec_snap_pol in self.rec_policy_pool:
rec_snap_pol.delete(self.apiclient)
self.rec_policy_pool = []
# Step 5
try:
thread_pool = []
for i in range(4):
create_snapshot_thread_1 = Thread(
target=CreateSnapshot,
args=(
self,
self.root_pool[i],
False))
thread_pool.append(create_snapshot_thread_1)
destinationHost = Host.listForMigration(
self.apiclient,
virtualmachineid=self.vm_pool[3].id)
migrate_volume_thread_1 = Thread(target=MigrateRootVolume,
args=(self,
self.vm_pool[3],
destinationHost[0]))
thread_pool.append(migrate_volume_thread_1)
for thread in thread_pool:
thread.start()
for thread in thread_pool:
thread.join()
except Exception as e:
raise Exception(
"Warning: Exception unable to start thread : %s" %
e)
snapshots = list_snapshots(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
for snapshot in self.snapshot_pool:
self.assertTrue(snapshot.id in any(
s.id) for s in snapshots)
for snapshot in self.snapshot_pool:
self.assertTrue(
is_snapshot_on_nfs(
self.apiclient,
self.dbclient,
self.config,
self.zone.id,
snapshot.id))
for snapshot in self.snapshot_pool:
snapshot.delete(self.apiclient)
self.snapshot_pool = []
# Step 6
try:
thread_pool = []
for i in range(4):
create_snapshot_thread_1 = Thread(
target=CreateSnapshot,
args=(
self,
self.root_pool[i],
False))
thread_pool.append(create_snapshot_thread_1)
stop_vm_thread_1 = Thread(target=self.StopVM,
args=(self.vm_pool,
))
thread_pool.append(stop_vm_thread_1)
for thread in thread_pool:
thread.start()
for thread in thread_pool:
thread.join()
except Exception as e:
raise Exception(
"Warning: Exception unable to start thread : %s" %
e)
snapshots = list_snapshots(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
for snapshot in self.snapshot_pool:
self.assertTrue(snapshot.id in any(
s.id) for s in snapshots)
for snapshot in self.snapshot_pool:
self.assertTrue(
is_snapshot_on_nfs(
self.apiclient,
self.dbclient,
self.config,
self.zone.id,
snapshot.id))
for snapshot in self.snapshot_pool:
snapshot.delete(self.apiclient)
self.snapshot_pool = []
# Step 7
thread_pool = []
try:
for i in range(2):
destinationHost = Host.listForMigration(
self.apiclient,
virtualmachineid=self.vm_pool[i].id)
migrate_volume_thread_1 = Thread(target=MigrateRootVolume,
args=(self,
self.vm_pool[i],
destinationHost[0]))
thread_pool.append(migrate_volume_thread_1)
for thread in thread_pool:
thread.start()
for thread in thread_pool:
thread.join()
for vm in self.vm_pool:
if vm.state != "Stopped":
vm.stop(self.apiclient)
thread_pool = []
for vm in self.vm_pool[:2]:
create_snapshot_thread_1 = Thread(
target=CreateSnapshot,
args=(
self,
self.root_pool[0],
False))
thread_pool.append(create_snapshot_thread_1)
for thread in thread_pool:
thread.start()
for thread in thread_pool:
thread.join()
except:
self.debug("Error: unable to start thread")
snapshots = list_snapshots(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
for snapshot in self.snapshot_pool:
self.assertTrue(snapshot.id in any(
s.id) for s in snapshots)
for snapshot in self.snapshot_pool:
self.assertTrue(
is_snapshot_on_nfs(
self.apiclient,
self.dbclient,
self.config,
self.zone.id,
snapshot.id))
for snapshot in self.snapshot_pool:
snapshot.delete(self.apiclient)
self.snapshot_pool = []
@attr(tags=["advanced", "basic"])
def test_02_concurrent_snapshots_configuration(self):
"""Concurrent Snapshots
1. Verify that CreateSnapshot command fails when it
takes more time than job.expire.minute
2. Verify that snapshot creation fails if CreateSnapshot command
takes more time than concurrent.snapshots.threshold.perhost
3. Check the event generation when snapshot creation
fails if CreateSnapshot takes more time than
concurrent.snapshots.threshold.perhost
"""
# Step 1
if not self.testdata["configurableData"][
"restartManagementServerThroughTestCase"]:
self.skipTest(
"Skip test if restartManagementServerThroughTestCase \
is not provided")
configs = Configurations.list(
self.apiclient,
name="job.expire.minutes")
orig_expire = configs[0].value
Configurations.update(self.apiclient,
name="concurrent.snapshots.threshold.perhost",
value="2"
)
Configurations.update(self.apiclient,
name="job.expire.minutes",
value="1"
)
# Restart management server
self.RestartServer()
time.sleep(120)
try:
thread_pool = []
for i in range(4):
create_snapshot_thread_1 = Thread(
target=CreateSnapshot,
args=(
self,
self.root_pool[i],
False))
thread_pool.append(create_snapshot_thread_1)
for thread in thread_pool:
thread.start()
for thread in thread_pool:
thread.join()
except Exception as e:
raise Exception(
"Warning: Exception unable to start thread : %s" %
e)
Configurations.update(self.apiclient,
name="job.expire.minutes",
value=orig_expire
)
# Restart management server
self.RestartServer()
time.sleep(120)
# Step 2
Configurations.update(self.apiclient,
name="concurrent.snapshots.threshold.perhost",
value="3"
)
Configurations.update(self.apiclient,
name="job.expire.minutes",
value="1"
)
# Restart management server
self.RestartServer()
time.sleep(120)
configs = Configurations.list(
self.apiclient,
name="job.expire.minutes")
with self.assertRaises(Exception):
CreateSnapshot(self, self.root_pool[0], False)
Configurations.update(self.apiclient,
name="job.expire.minutes",
value=orig_expire
)
# Restart management server
self.RestartServer()
time.sleep(120)
# Step 3
configs = Configurations.list(
self.apiclient,
name="job.cancel.threshold.minutes")
orig_cancel = configs[0].value
Configurations.update(self.apiclient,
name="concurrent.snapshots.threshold.perhost",
value="3"
)
Configurations.update(self.apiclient,
name="job.cancel.threshold.minutes",
value="1"
)
self.RestartServer()
time.sleep(120)
configs = Configurations.list(
self.apiclient,
name="job.expire.minutes")
with self.assertRaises(Exception):
CreateSnapshot(self, self.root_pool[0], False)
Configurations.update(self.apiclient,
name="job.cancel.threshold.minutes",
value=orig_cancel
)
self.RestartServer()
time.sleep(120)
# Step 4
Configurations.update(self.apiclient,
name="concurrent.snapshots.threshold.perhost",
value="3"
)
self.RestartServer()
time.sleep(120)
Configurations.update(self.apiclient,
name="job.cancel.threshold.minutes",
value="1"
)
self.RestartServer()
time.sleep(120)
with self.assertRaises(Exception):
CreateSnapshot(self, self.root_pool[0], True)
return