From 5f4439007c22798b80186b13cd1dbb2d29492224 Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Tue, 22 Dec 2015 23:51:31 +0530 Subject: [PATCH] Added a test to test update router in sequence --- .../component/maint/test_redundant_router.py | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/test/integration/component/maint/test_redundant_router.py b/test/integration/component/maint/test_redundant_router.py index 98b0bae1ba0..cad1d341d55 100644 --- a/test/integration/component/maint/test_redundant_router.py +++ b/test/integration/component/maint/test_redundant_router.py @@ -29,6 +29,7 @@ from marvin.lib.common import (get_domain, get_zone, get_process_status) import time +import multiprocessing # Import Local Modules from marvin.cloudstackTestCase import cloudstackTestCase @@ -872,9 +873,16 @@ class TestRvRRedundancy(cloudstackTestCase): cls.testdata["nw_off_isolated_RVR"], conservemode=True ) + cls.network_offering_for_update=NetworkOffering.create( + cls.api_client, + cls.testdata["nw_off_isolated_RVR"], + conservemode=True + ) + cls._cleanup.append(cls.network_offering_for_update) cls._cleanup.append(cls.network_offering) # Enable Network offering cls.network_offering.update(cls.api_client, state='Enabled') + cls.network_offering_for_update.update(cls.api_client, state='Enabled') return @classmethod @@ -1511,3 +1519,149 @@ class TestRvRRedundancy(cloudstackTestCase): "Redundant state of the router should be BACKUP but is %s" % routers[0].redundantstate) return + + def updateNetwork(self, conn): + try: + self.network.update( + self.api_client, + networkofferingid=self.network_offering_for_update.id, + updateinsequence=True, + forced=True, + changecidr=False + ) + except Exception as e: + conn.send("Failed to update network: %s due to %s"%(self.network.name, e)) + conn.send("update Network Complete") + return + + + + def get_master_and_backupRouter(self): + retry = 4 + master_router = backup_router=None + while retry > 0: + routers = Router.list( + self.apiclient, + networkid=self.network.id, + listall=True + ) + retry = retry-1 + if not (routers[0].redundantstate == 'MASTER' or routers[1].redundantstate == 'MASTER'): + continue; + if routers[0].redundantstate == 'MASTER': + master_router = routers[0] + backup_router = routers[1] + break + else: + master_router = routers[1] + backup_router = routers[0] + break + return master_router, backup_router + + + def chek_for_new_backupRouter(self,old_backup_router): + master_router, backup_router = self.get_master_and_backupRouter() + retry = 4 + self.info("Checking if new router is getting created.") + self.info("old_backup_router:"+old_backup_router.name+" new_backup_router:"+backup_router.name) + while old_backup_router.name == backup_router.name: + self.debug("waiting for new router old router:"+backup_router.name) + retry = retry-1 + if retry == 0: + break; + time.sleep(self.testdata["sleep"]) + master_router, backup_router = self.get_master_and_backupRouter() + if retry == 0: + self.fail("New router creation taking too long, timed out") + + def wait_untill_router_stabilises(self): + retry=4 + while retry > 0: + routers = Router.list( + self.apiclient, + networkid=self.network.id, + listall=True + ) + retry = retry-1 + self.info("waiting untill state of the routers is stable") + if routers[0].redundantstate != 'UNKNOWN' and routers[1].redundantstate != 'UNKNOWN': + return + elif retry==0: + self.fail("timedout while waiting for routers to stabilise") + return + time.sleep(self.testdata["sleep"]) + + @attr(tags=["bharat"]) + def test_06_updateVRs_in_sequence(self): + """Test update network and check if VRs are updated in sequence + """ + + # Steps to validate + # update network to a new offering + # check if the master router is running while backup is starting. + # check if the backup is running while master is starting. + # check if both the routers are running after the update is complete. + + #clean up the network to make sure it is in proper state. + self.network.restart(self.apiclient,cleanup=True) + time.sleep(self.testdata["sleep"]) + self.wait_untill_router_stabilises() + old_master_router, old_backup_router = self.get_master_and_backupRouter() + self.info("old_master_router:"+old_master_router.name+" old_backup_router"+old_backup_router.name) + #chek if the network is in correct state + self.assertEqual(old_master_router.state, "Running", "The master router is not running, network is not in a correct state to start the test") + self.assertEqual(old_backup_router.state, "Running", "The backup router is not running, network is not in a correct state to start the test") + + worker, monitor = multiprocessing.Pipe() + worker_process = multiprocessing.Process(target=self.updateNetwork, args=(worker,)) + worker_process.start() + if not worker_process.is_alive(): + message = monitor.recv() + if "Complete" not in message: + self.fail(message) + + self.info("Network update Started, the old backup router will get destroyed and a new router will be created") + + self.chek_for_new_backupRouter(old_backup_router) + master_router, new_backup_router=self.get_master_and_backupRouter() + #the state of the master router should be running. while backup is being updated + self.assertEqual(master_router.state, "Running", "State of the master router is not running") + self.assertEqual(master_router.redundantstate, 'MASTER', "Redundant state of the master router should be MASTER, but it is %s"%master_router.redundantstate) + self.info("Old backup router:"+old_backup_router.name+" is destroyed and new router:"+new_backup_router.name+" got created") + + #wait for the new backup to become master. + retry = 4 + while new_backup_router.name != master_router.name: + retry = retry-1 + if retry == 0: + break + time.sleep(self.testdata["sleep"]) + self.info("wating for backup router to become master router name:"+new_backup_router.name) + master_router, backup_router = self.get_master_and_backupRouter() + if retry == 0: + self.fail("timed out while waiting for new backup router to change state to MASTER.") + + #new backup router has become master. + self.info("newly created router:"+new_backup_router.name+" has changed state to Master") + self.info("old master router:"+old_master_router.name+"is destroyed") + #old master will get destroyed and a new backup will be created. + #wait until new backup changes state from unknown to backup + master_router, backup_router = self.get_master_and_backupRouter() + retry = 4 + while backup_router.redundantstate != 'BACKUP': + retry = retry-1 + self.info("waiting for router:"+backup_router.name+" to change state to Backup") + if retry == 0: + break + time.sleep(self.testdata["sleep"]) + master_router, backup_router = self.get_master_and_backupRouter() + self.assertEqual(master_router.state, "Running", "State of the master router is not running") + self.assertEqual(master_router.redundantstate, 'MASTER', "Redundant state of the master router should be MASTER, but it is %s"%master_router.redundantstate) + if retry == 0: + self.fail("timed out while waiting for new backup rotuer to change state to MASTER.") + + #the network update is complete.finally both the router should be running. + new_master_router, new_backup_router=self.get_master_and_backupRouter() + self.assertEqual(new_master_router.state, "Running", "State of the master router:"+new_master_router.name+" is not running") + self.assertEqual(new_backup_router.state, "Running", "State of the backup router:"+new_backup_router.name+" is not running") + worker_process.join()