# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from marvin.cloudstackTestCase import * from marvin.cloudstackAPI import * from marvin.lib.utils import * from marvin.lib.base import * from marvin.lib.common import * from nose.plugins.attrib import attr from ipmisim.ipmisim import IpmiServerContext, IpmiServer, ThreadedIpmiServer import random import socket import _thread class TestOutOfBandManagement(cloudstackTestCase): """ Test cases for out of band management """ @classmethod def setUpClass(cls): testClient = super(TestOutOfBandManagement, cls).getClsTestClient() cls.apiclient = testClient.getApiClient() cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ cls.services = testClient.getParsedTestDataConfig() cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) cls.host = None cls.cleanup = [] cls.hypervisor = cls.testClient.getHypervisorInfo() if (cls.hypervisor.lower() != "simulator"): cls.skipIfMSIsUnsupported(cls) # use random port for ipmisim s = socket.socket() s.bind(('', 0)) cls.serverPort = s.getsockname()[1] s.close() def startIpmiServer(tname, server): try: server.serve_forever() except Exception: pass IpmiServerContext('reset') ThreadedIpmiServer.allow_reuse_address = True server = ThreadedIpmiServer(('0.0.0.0', cls.serverPort), IpmiServer) _thread.start_new_thread(startIpmiServer, ("ipmi-server", server,)) cls.server = server @classmethod def tearDownClass(cls): try: cleanup_resources(cls.apiclient, cls.cleanup) cls.server.shutdown() cls.server.server_close() IpmiServerContext('reset') except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() self.mgtSvrDetails = self.config.__dict__["mgtSvr"][0].__dict__ self.fakeMsId = random.randint(10000, 99999) * random.randint(10, 20) self.cleanup = [] def tearDown(self): try: self.dbclient.execute("delete from oobm where port=%d" % self.getIpmiServerPort()) self.dbclient.execute("delete from mshost_peer where peer_runid=%s" % self.getFakeMsRunId()) self.dbclient.execute("delete from mshost where runid=%s" % self.getFakeMsRunId()) self.dbclient.execute("delete from cluster_details where name='outOfBandManagementEnabled'") self.dbclient.execute("delete from data_center_details where name='outOfBandManagementEnabled'") cleanup_resources(self.apiclient, self.cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) def skipIfMSIsUnsupported(self) : os_details = SshClient(self.mgtSvrDetails["mgtSvrIp"], 22, self.mgtSvrDetails["user"], self.mgtSvrDetails["passwd"]).execute \ ("/usr/share/cloudstack-common/scripts/vm/hypervisor/versions.sh | cut -d '=' -f2") os = os_details[0].lower() if 'ubuntu' in os or 'debian' in os : return # RHEL < 8 works fine os_ver = os_details[1].split('.')[0] if float(os_ver) < 8: return self.skipTest(self, reason="Skipping since RHEL8 / SUSE have known IPMI issues") def getFakeMsId(self): return self.fakeMsId def getFakeMsRunId(self): return self.fakeMsId * 1000 def getHost(self, hostId=None): if self.host and hostId is None: return self.host response = list_hosts( self.apiclient, zoneid=self.zone.id, type='Routing', id=hostId ) if len(response) > 0: self.host = response[0] return self.host raise self.skipTest("No hosts found, skipping out-of-band management test") def getIpmiServerIp(self): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect((self.mgtSvrDetails["mgtSvrIp"], self.mgtSvrDetails["port"])) return s.getsockname()[0] def getIpmiServerPort(self): return self.serverPort def getOobmConfigCmd(self): cmd = configureOutOfBandManagement.configureOutOfBandManagementCmd() cmd.driver = 'ipmitool' # The default available driver cmd.address = self.getIpmiServerIp() cmd.port = self.getIpmiServerPort() cmd.username = 'admin' cmd.password = 'password' cmd.hostid = self.getHost().id return cmd def getOobmEnableCmd(self): cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd() cmd.hostid = self.getHost().id return cmd def getOobmDisableCmd(self): cmd = disableOutOfBandManagementForHost.disableOutOfBandManagementForHostCmd() cmd.hostid = self.getHost().id return cmd def getOobmIssueActionCmd(self): cmd = issueOutOfBandManagementPowerAction.issueOutOfBandManagementPowerActionCmd() cmd.hostid = self.getHost().id cmd.action = 'STATUS' return cmd def issuePowerActionCmd(self, action, timeout=None): cmd = self.getOobmIssueActionCmd() cmd.action = action if timeout: cmd.timeout = timeout tries = 3 while tries > 0: tries -= 1 try: return self.apiclient.issueOutOfBandManagementPowerAction(cmd) except Exception as e: if tries <= 0: if "packet session id 0x0 does not match active session" in str(e): raise self.skipTest("Known ipmitool issue hit, skipping test") raise e def configureAndEnableOobm(self, power_state=None): """ Setup ipmisim and enable out-of-band management for host """ self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd()) response = self.apiclient.enableOutOfBandManagementForHost(self.getOobmEnableCmd()) self.assertEqual(response.enabled, True) if power_state: bmc = IpmiServerContext().bmc bmc.powerstate = power_state def checkSyncToState(self, state, interval): def checkForStateSync(expectedState): response = self.getHost(hostId=self.getHost().id).outofbandmanagement return response.powerstate == expectedState, None sync_interval = 1 + int(interval)/1000 res, _ = wait_until(sync_interval, 20, checkForStateSync, state) if not res: self.fail("Failed to get host.powerstate synced to expected state:" + state) response = self.getHost(hostId=self.getHost().id).outofbandmanagement self.assertEqual(response.powerstate, state) @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_configure_invalid_driver(self): """ Tests out-of-band management configuration with invalid driver """ cmd = self.getOobmConfigCmd() cmd.driver = 'randomDriverThatDoesNotExist' try: response = self.apiclient.configureOutOfBandManagement(cmd) except Exception: pass else: self.fail("Expected an exception to be thrown, failing") @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_configure_default_driver(self): """ Tests out-of-band management configuration with valid data """ cmd = self.getOobmConfigCmd() response = self.apiclient.configureOutOfBandManagement(cmd) self.assertEqual(response.hostid, cmd.hostid) self.assertEqual(response.driver, cmd.driver) self.assertEqual(response.address, cmd.address) self.assertEqual(response.port, str(cmd.port)) self.assertEqual(response.username, cmd.username) @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_enable_feature_invalid(self): """ Tests out-of-band management host enable feature with invalid options """ cmd = self.getOobmEnableCmd() cmd.hostid = -1 try: response = self.apiclient.enableOutOfBandManagementForHost(cmd) except Exception: pass else: self.fail("Expected an exception to be thrown, failing") try: cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd() response = self.apiclient.enableOutOfBandManagementForCluster(cmd) except Exception: pass else: self.fail("Expected an exception to be thrown, failing") try: cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd() response = self.apiclient.enableOutOfBandManagementForZone(cmd) except Exception: pass else: self.fail("Expected an exception to be thrown, failing") @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_disable_feature_invalid(self): """ Tests out-of-band management host disable feature with invalid options """ cmd = self.getOobmDisableCmd() cmd.hostid = -1 try: response = self.apiclient.disableOutOfBandManagementForHost(cmd) except Exception: pass else: self.fail("Expected an exception to be thrown, failing") try: cmd = disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd() response = self.apiclient.disableOutOfBandManagementForCluster(cmd) except Exception: pass else: self.fail("Expected an exception to be thrown, failing") try: cmd = disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd() response = self.apiclient.disableOutOfBandManagementForZone(cmd) except Exception: pass else: self.fail("Expected an exception to be thrown, failing") @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_enable_feature_valid(self): """ Tests out-of-band management host enable feature with valid options """ self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd()) cmd = self.getOobmEnableCmd() response = self.apiclient.enableOutOfBandManagementForHost(cmd) self.assertEqual(response.hostid, cmd.hostid) self.assertEqual(response.enabled, True) @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_disable_feature_valid(self): """ Tests out-of-band management host disable feature with valid options """ self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd()) cmd = self.getOobmDisableCmd() response = self.apiclient.disableOutOfBandManagementForHost(cmd) self.assertEqual(response.hostid, cmd.hostid) self.assertEqual(response.enabled, False) response = self.getHost(hostId=cmd.hostid).outofbandmanagement self.assertEqual(response.powerstate, 'Disabled') @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_enabledisable_across_clusterzones(self): """ Tests out-of-band management enable/disable feature at cluster and zone level sequentially Zone > Cluster > Host """ self.configureAndEnableOobm() bmc = IpmiServerContext().bmc bmc.powerstate = 'off' host = self.getHost() # Disable at zone level cmd = disableOutOfBandManagementForZone.disableOutOfBandManagementForZoneCmd() cmd.zoneid = host.zoneid response = self.apiclient.disableOutOfBandManagementForZone(cmd) # Disable at cluster level cmd = disableOutOfBandManagementForCluster.disableOutOfBandManagementForClusterCmd() cmd.clusterid = host.clusterid response = self.apiclient.disableOutOfBandManagementForCluster(cmd) # Disable at host level cmd = disableOutOfBandManagementForHost.disableOutOfBandManagementForHostCmd() cmd.hostid = host.id response = self.apiclient.disableOutOfBandManagementForHost(cmd) try: self.issuePowerActionCmd('STATUS') except Exception: pass else: self.fail("Expected an exception to be thrown, failing") # Enable at zone level cmd = enableOutOfBandManagementForZone.enableOutOfBandManagementForZoneCmd() cmd.zoneid = host.zoneid response = self.apiclient.enableOutOfBandManagementForZone(cmd) try: self.issuePowerActionCmd('STATUS') except Exception: pass else: self.fail("Expected an exception to be thrown, failing") # Check background thread syncs state to Disabled response = self.getHost(hostId=host.id).outofbandmanagement self.assertEqual(response.powerstate, 'Disabled') self.dbclient.execute("update oobm set power_state='On' where port=%d" % self.getIpmiServerPort()) self.checkSyncToState('Disabled', 2) # Enable at cluster level cmd = enableOutOfBandManagementForCluster.enableOutOfBandManagementForClusterCmd() cmd.clusterid = host.clusterid response = self.apiclient.enableOutOfBandManagementForCluster(cmd) try: self.issuePowerActionCmd('STATUS') except Exception: pass else: self.fail("Expected an exception to be thrown, failing") # Enable at host level cmd = enableOutOfBandManagementForHost.enableOutOfBandManagementForHostCmd() cmd.hostid = host.id response = self.apiclient.enableOutOfBandManagementForHost(cmd) response = self.issuePowerActionCmd('STATUS') self.assertEqual(response.powerstate, 'Off') def assertIssueCommandState(self, command, expected): """ Asserts power action result for a given power command """ if command != 'STATUS': self.issuePowerActionCmd(command) response = self.issuePowerActionCmd('STATUS') self.assertEqual(response.powerstate, expected) @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_issue_power_status(self): """ Tests out-of-band management issue power action """ self.configureAndEnableOobm(power_state='on') self.assertIssueCommandState('STATUS', 'On') @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_issue_power_on(self): """ Tests out-of-band management issue power on action """ self.configureAndEnableOobm() self.assertIssueCommandState('ON', 'On') @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_issue_power_off(self): """ Tests out-of-band management issue power off action """ self.configureAndEnableOobm() self.assertIssueCommandState('OFF', 'Off') @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_issue_power_cycle(self): """ Tests out-of-band management issue power cycle action """ self.configureAndEnableOobm() self.assertIssueCommandState('CYCLE', 'On') @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_issue_power_reset(self): """ Tests out-of-band management issue power reset action """ self.configureAndEnableOobm() self.assertIssueCommandState('RESET', 'On') @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_issue_power_soft(self): """ Tests out-of-band management issue power soft action """ self.configureAndEnableOobm() self.assertIssueCommandState('SOFT', 'Off') @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_background_powerstate_sync(self): """ Tests out-of-band management background powerstate sync """ self.debug("Testing oobm background sync") self.configureAndEnableOobm() bmc = IpmiServerContext().bmc bmc.powerstate = 'on' self.checkSyncToState('On', 2) bmc.powerstate = 'off' self.checkSyncToState('Off', 2) # Check for unknown state (ipmi server not reachable) cmd = self.getOobmConfigCmd() cmd.port = 1 response = self.apiclient.configureOutOfBandManagement(cmd) self.checkSyncToState('Unknown', 2) @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_multiple_mgmt_server_ownership(self): """ Tests out-of-band management ownership expiry across multi-mgmt server """ self.configureAndEnableOobm() cloudstackVersion = Configurations.listCapabilities(self.apiclient).cloudstackversion currentMsHosts = [] mshosts = self.dbclient.execute("select msid from mshost where version='%s' and removed is NULL and state='Up'" % (cloudstackVersion)) if len(mshosts) > 0: currentMsHosts = [row[0] for row in mshosts] # Inject fake ms host self.dbclient.execute("insert into mshost (msid,runid,name,state,version,service_ip,service_port,last_update) values (%s,%s,'oobm-marvin-fakebox', 'Down', '%s', '127.0.0.1', '22', NOW())" % (self.getFakeMsId(), self.getFakeMsRunId(), cloudstackVersion)) # Pass ownership to the fake ms id self.dbclient.execute("update oobm set mgmt_server_id=%d where port=%d" % (self.getFakeMsId(), self.getIpmiServerPort())) self.debug("Testing oobm background sync") pingInterval = float(list_configurations( self.apiclient, name='ping.interval' )[0].value) pingTimeout = float(list_configurations( self.apiclient, name='ping.timeout' )[0].value) def removeFakeMgmtServer(fakeMsRunId): rows = self.dbclient.execute("select * from mshost_peer where peer_runid=%s" % fakeMsRunId) if len(rows) > 0: self.debug("Mgmt server is now trying to contact the fake mgmt server") self.dbclient.execute("update mshost set removed=now() where runid=%s" % fakeMsRunId) self.dbclient.execute("update mshost_peer set peer_state='Down' where peer_runid=%s" % fakeMsRunId) return True, None return False, None def checkOobmOwnershipExpiry(serverPort, fakeMsId): rows = self.dbclient.execute("select mgmt_server_id from oobm where port=%d" % (serverPort)) if len(rows) > 0 and rows[0][0] != fakeMsId: self.debug("Out-of-band management ownership expired as node was detected to be gone") return True, None return False, None retry_interval = 1 + (pingInterval * pingTimeout / 10) res, _ = wait_until(retry_interval, 10, removeFakeMgmtServer, self.getFakeMsRunId()) if not res: self.fail("Management server failed to turn down or remove fake mgmt server") res, _ = wait_until(retry_interval, 100, checkOobmOwnershipExpiry, self.getIpmiServerPort(), self.getFakeMsId()) if not res: self.fail("Management server failed to expire ownership of fenced peer") self.debug("Testing oobm background sync should claim new ownership") bmc = IpmiServerContext().bmc bmc.powerstate = 'on' self.checkSyncToState('On', 2) result = self.dbclient.execute("select mgmt_server_id from oobm where port=%d" % (self.getIpmiServerPort())) newOwnerId = result[0][0] self.assertTrue(newOwnerId in currentMsHosts) @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_oobm_zchange_password(self): """ Tests out-of-band management change password feature """ self.configureAndEnableOobm() self.debug("Testing oobm change password") alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True) alertCount = 0 if alerts: alertCount = len(alerts) cmd = changeOutOfBandManagementPassword.changeOutOfBandManagementPasswordCmd() cmd.hostid = self.getHost().id cmd.password = "Password12345" try: response = self.apiclient.changeOutOfBandManagementPassword(cmd) self.assertEqual(response.status, True) except Exception as e: if "packet session id 0x0 does not match active session" in str(e): raise self.skipTest("Known ipmitool issue hit, skipping test") raise e bmc = IpmiServerContext().bmc bmc.powerstate = 'on' response = self.issuePowerActionCmd('STATUS') self.assertEqual(response.status, True) self.assertEqual(response.powerstate, 'On') # Reset configuration, resets password self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd()) self.assertEqual(response.status, True) try: response = self.issuePowerActionCmd('STATUS') except Exception: pass else: self.fail("Expected an exception to be thrown, failing") alerts = Alert.list(self.apiclient, keyword="auth-error", listall=True) self.assertTrue((len(alerts) - alertCount) >= 0)