mirror of
https://github.com/apache/cloudstack.git
synced 2025-10-26 08:42:29 +01:00
778 lines
29 KiB
Python
778 lines
29 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import marvin
|
|
from marvin.cloudstackTestCase import *
|
|
from marvin.cloudstackAPI import *
|
|
from marvin.lib.utils import *
|
|
from marvin.lib.base import *
|
|
from marvin.lib.common import *
|
|
from nose.plugins.attrib import attr
|
|
|
|
from ipmisim.ipmisim import IpmiServerContext, IpmiServer, ThreadedIpmiServer
|
|
|
|
import random
|
|
import socket
|
|
import sys
|
|
import _thread
|
|
import time
|
|
|
|
|
|
class TestHostHA(cloudstackTestCase):
|
|
""" Test host-ha business logic using Simulator
|
|
"""
|
|
|
|
def setUp(self):
|
|
self.apiclient = self.testClient.getApiClient()
|
|
self.hypervisor = self.testClient.getHypervisorInfo()
|
|
self.dbclient = self.testClient.getDbConnection()
|
|
self.services = self.testClient.getParsedTestDataConfig()
|
|
self.mgtSvrDetails = self.config.__dict__["mgtSvr"][0].__dict__
|
|
self.fakeMsId = random.randint(10000, 99999) * random.randint(10, 20)
|
|
self.host = None
|
|
|
|
# Cleanup any existing configs
|
|
self.dbclient.execute("delete from ha_config where resource_type='Host'")
|
|
|
|
# use random port for ipmisim
|
|
s = socket.socket()
|
|
s.bind(('', 0))
|
|
self.serverPort = s.getsockname()[1]
|
|
s.close()
|
|
|
|
# Get a host to run tests against
|
|
self.host = self.getHost()
|
|
|
|
self.cleanup = []
|
|
|
|
|
|
def tearDown(self):
|
|
try:
|
|
host = self.getHost()
|
|
self.configureAndDisableHostHa(host.id)
|
|
self.host = None
|
|
self.dbclient.execute("delete from mshost_peer where peer_runid=%s" % self.getFakeMsRunId())
|
|
self.dbclient.execute("delete from mshost where runid=%s" % self.getFakeMsRunId())
|
|
self.dbclient.execute("delete from cluster_details where name='resourceHAEnabled'")
|
|
self.dbclient.execute("delete from data_center_details where name='resourceHAEnabled'")
|
|
self.dbclient.execute("delete from ha_config where resource_type='Host'")
|
|
self.dbclient.execute("update host set resource_state='Enabled' where type='Routing' and resource_state='Maintenance'")
|
|
except Exception as e:
|
|
raise Exception("Warning: Exception during db cleanup : %s" % e)
|
|
super(TestHostHA,self).tearDown()
|
|
|
|
|
|
def getFakeMsId(self):
|
|
return self.fakeMsId
|
|
|
|
|
|
def getFakeMsRunId(self):
|
|
return self.fakeMsId * 1000
|
|
|
|
|
|
def getHost(self, hostId=None):
|
|
if self.host and hostId is None:
|
|
return self.host
|
|
|
|
response = list_hosts(
|
|
self.apiclient,
|
|
type='Routing',
|
|
hypervisor='Simulator',
|
|
resourcestate='Enabled',
|
|
id=hostId
|
|
)
|
|
|
|
if response and len(response) > 0:
|
|
random.shuffle(response)
|
|
self.host = response[0]
|
|
return self.host
|
|
raise self.skipTest("No suitable hosts found, skipping host-ha test")
|
|
|
|
|
|
def getHostHaConfigCmd(self, provider='simulatorhaprovider'):
|
|
cmd = configureHAForHost.configureHAForHostCmd()
|
|
cmd.provider = provider
|
|
cmd.hostid = self.getHost().id
|
|
return cmd
|
|
|
|
|
|
def getHostHaEnableCmd(self):
|
|
cmd = enableHAForHost.enableHAForHostCmd()
|
|
cmd.hostid = self.getHost().id
|
|
return cmd
|
|
|
|
|
|
def getHostHaDisableCmd(self):
|
|
cmd = disableHAForHost.disableHAForHostCmd()
|
|
cmd.hostid = self.getHost().id
|
|
return cmd
|
|
|
|
|
|
def getListHostHAResources(self):
|
|
cmd = listHostHAResources.listHostHAResourcesCmd()
|
|
cmd.hostid = self.getHost().id
|
|
return cmd
|
|
|
|
|
|
def configureAndEnableHostHa(self, initialize=True):
|
|
self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
|
|
response = self.apiclient.enableHAForHost(self.getHostHaEnableCmd())
|
|
self.assertEqual(response.haenable, True)
|
|
if initialize:
|
|
self.configureSimulatorHAProviderState(True, True, True, False)
|
|
|
|
|
|
def configureAndDisableHostHa(self, hostId):
|
|
self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
|
|
cmd = self.getHostHaDisableCmd()
|
|
cmd.hostid = hostId
|
|
response = self.apiclient.disableHAForHost(cmd)
|
|
self.assertEqual(response.hostid, cmd.hostid)
|
|
self.assertEqual(response.haenable, False)
|
|
|
|
|
|
def enableHostHa(self, hostId):
|
|
cmd = self.getHostHaEnableCmd()
|
|
cmd.hostid = hostId
|
|
response = self.apiclient.enableHAForHost(cmd)
|
|
self.assertEqual(response.hostid, cmd.hostid)
|
|
self.assertEqual(response.haenable, True)
|
|
|
|
|
|
def configureSimulatorHAProviderState(self, health, activity, recover, fence):
|
|
cmd = configureSimulatorHAProviderState.configureSimulatorHAProviderStateCmd()
|
|
cmd.hostid = self.getHost().id
|
|
cmd.health = health
|
|
cmd.activity = activity
|
|
cmd.recover = recover
|
|
cmd.fence = fence
|
|
response = self.apiclient.configureSimulatorHAProviderState(cmd)
|
|
self.assertEqual(response.success, True)
|
|
|
|
|
|
def getSimulatorHAStateTransitions(self, hostId):
|
|
cmd = listSimulatorHAStateTransitions.listSimulatorHAStateTransitionsCmd()
|
|
cmd.hostid = hostId
|
|
return self.apiclient.listSimulatorHAStateTransitions(cmd)
|
|
|
|
|
|
def checkSyncToState(self, state, interval=5000):
|
|
def checkForStateSync(expectedState):
|
|
response = self.getHost(hostId=self.getHost().id).hostha
|
|
print(("checkForStateSync:: response=%s, expected=%s" % (response, expectedState)))
|
|
return response.hastate == expectedState, None
|
|
|
|
sync_interval = 1 + int(interval) / 1000
|
|
res, _ = wait_until(sync_interval, 100, checkForStateSync, state)
|
|
if not res:
|
|
self.fail("Failed to get host.hastate synced to expected state:" + state)
|
|
response = self.getHost(hostId=self.getHost().id).hostha
|
|
self.assertEqual(response.hastate, state)
|
|
|
|
|
|
def getNonConfiguredHaHost(self):
|
|
response = list_hosts(
|
|
self.apiclient,
|
|
type='Routing'
|
|
)
|
|
for host in response:
|
|
if host.haprovider is None:
|
|
return host
|
|
else:
|
|
return None
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_hostha_enable_feature_without_setting_provider(self):
|
|
"""
|
|
Tests Enable HA without setting the provider, Exception is thrown
|
|
"""
|
|
host = self.getNonConfiguredHaHost()
|
|
|
|
if host is None:
|
|
cloudstackTestCase.skipTest(self, "There is no non configured hosts. Skipping test.")
|
|
|
|
cmd = self.getHostHaEnableCmd()
|
|
cmd.hostid = host.id
|
|
|
|
try:
|
|
response = self.apiclient.enableHAForHost(cmd)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self.fail("Expected an exception to be thrown, failing")
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_ha_list_providers(self):
|
|
"""
|
|
Tests default ha providers list
|
|
"""
|
|
cmd = listHostHAProviders.listHostHAProvidersCmd()
|
|
|
|
cmd.hypervisor = 'Simulator'
|
|
response = self.apiclient.listHostHAProviders(cmd)[0]
|
|
self.assertEqual(response.haprovider, 'SimulatorHAProvider')
|
|
|
|
cmd.hypervisor = 'KVM'
|
|
response = self.apiclient.listHostHAProviders(cmd)[0]
|
|
self.assertEqual(response.haprovider, 'KVMHAProvider')
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_hostha_configure_invalid_provider(self):
|
|
"""
|
|
Tests host-ha configuration with invalid driver
|
|
"""
|
|
cmd = self.getHostHaConfigCmd()
|
|
cmd.provider = 'randomDriverThatDoesNotExist'
|
|
try:
|
|
response = self.apiclient.configureHAForHost(cmd)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self.fail("Expected an exception to be thrown, failing")
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_hostha_configure_default_driver(self):
|
|
"""
|
|
Tests host-ha configuration with valid data
|
|
"""
|
|
cmd = self.getHostHaConfigCmd()
|
|
response = self.apiclient.configureHAForHost(cmd)
|
|
self.assertEqual(response.hostid, cmd.hostid)
|
|
self.assertEqual(response.haprovider, cmd.provider.lower())
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_ha_enable_feature_invalid(self):
|
|
"""
|
|
Tests ha feature enable command with invalid options
|
|
"""
|
|
cmd = self.getHostHaEnableCmd()
|
|
cmd.hostid = -1
|
|
try:
|
|
response = self.apiclient.enableHAForHost(cmd)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self.fail("Expected an exception to be thrown, failing")
|
|
try:
|
|
cmd = enableHAForCluster.enableHAForClusterCmd()
|
|
response = self.apiclient.enableHAForCluster(cmd)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self.fail("Expected an exception to be thrown, failing")
|
|
try:
|
|
cmd = enableHAForZone.enableHAForZoneCmd()
|
|
response = self.apiclient.enableHAForZone(cmd)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self.fail("Expected an exception to be thrown, failing")
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_ha_disable_feature_invalid(self):
|
|
"""
|
|
Tests ha feature disable command with invalid options
|
|
"""
|
|
cmd = self.getHostHaDisableCmd()
|
|
cmd.hostid = -1
|
|
try:
|
|
response = self.apiclient.disableHAForHost(cmd)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self.fail("Expected an exception to be thrown, failing")
|
|
try:
|
|
cmd = disableHAForCluster.disableHAForClusterCmd()
|
|
response = self.apiclient.disableHAForCluster(cmd)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self.fail("Expected an exception to be thrown, failing")
|
|
|
|
try:
|
|
cmd = disableHAForZone.disableHAForZoneCmd()
|
|
response = self.apiclient.disableHAForZone(cmd)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self.fail("Expected an exception to be thrown, failing")
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_hostha_enable_feature_valid(self):
|
|
"""
|
|
Tests host-ha enable feature with valid options
|
|
"""
|
|
self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
|
|
cmd = self.getHostHaEnableCmd()
|
|
response = self.apiclient.enableHAForHost(cmd)
|
|
self.assertEqual(response.hostid, cmd.hostid)
|
|
self.assertEqual(response.haenable, True)
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_hostha_disable_feature_valid(self):
|
|
"""
|
|
Tests host-ha disable feature with valid options
|
|
"""
|
|
self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
|
|
cmd = self.getHostHaDisableCmd()
|
|
response = self.apiclient.disableHAForHost(cmd)
|
|
self.assertEqual(response.hostid, cmd.hostid)
|
|
self.assertEqual(response.haenable, False)
|
|
|
|
response = self.getHost(hostId=cmd.hostid).hostha
|
|
self.assertEqual(response.hastate, 'Disabled')
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_ha_configure_enabledisable_across_clusterzones(self):
|
|
"""
|
|
Tests ha enable/disable feature at cluster and zone level
|
|
Zone > Cluster > Host
|
|
"""
|
|
host = self.getHost()
|
|
self.configureAndDisableHostHa(host.id)
|
|
self.configureAndEnableHostHa()
|
|
|
|
self.checkSyncToState('Available')
|
|
response = self.getHost(hostId=host.id).hostha
|
|
self.assertTrue(response.hastate == 'Available')
|
|
|
|
# Disable at host level
|
|
cmd = disableHAForHost.disableHAForHostCmd()
|
|
cmd.hostid = host.id
|
|
response = self.apiclient.disableHAForHost(cmd)
|
|
|
|
# Disable at cluster level
|
|
cmd = disableHAForCluster.disableHAForClusterCmd()
|
|
cmd.clusterid = host.clusterid
|
|
response = self.apiclient.disableHAForCluster(cmd)
|
|
|
|
# Disable at zone level
|
|
cmd = disableHAForZone.disableHAForZoneCmd()
|
|
cmd.zoneid = host.zoneid
|
|
response = self.apiclient.disableHAForZone(cmd)
|
|
|
|
# HA state check
|
|
response = self.getHost(hostId=host.id).hostha
|
|
self.assertTrue(response.hastate == 'Disabled')
|
|
|
|
# Check ha-state check and sync
|
|
self.dbclient.execute("update ha_config set ha_state='Available' where enabled='1' and resource_type='Host'")
|
|
self.checkSyncToState('Disabled')
|
|
|
|
# Enable at zone level
|
|
cmd = enableHAForZone.enableHAForZoneCmd()
|
|
cmd.zoneid = host.zoneid
|
|
response = self.apiclient.enableHAForZone(cmd)
|
|
|
|
# Enable at cluster level
|
|
cmd = enableHAForCluster.enableHAForClusterCmd()
|
|
cmd.clusterid = host.clusterid
|
|
response = self.apiclient.enableHAForCluster(cmd)
|
|
|
|
# Enable at host level
|
|
cmd = enableHAForHost.enableHAForHostCmd()
|
|
cmd.hostid = host.id
|
|
response = self.apiclient.enableHAForHost(cmd)
|
|
|
|
# Check state sync
|
|
self.checkSyncToState('Available')
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_ha_multiple_mgmt_server_ownership(self):
|
|
"""
|
|
Tests ha resource ownership expiry across multi-mgmt server
|
|
"""
|
|
host = self.getHost()
|
|
self.configureAndDisableHostHa(host.id)
|
|
self.configureSimulatorHAProviderState(True, True, True, False)
|
|
self.configureAndEnableHostHa(False)
|
|
|
|
cloudstackVersion = Configurations.listCapabilities(self.apiclient).cloudstackversion
|
|
|
|
currentMsHosts = []
|
|
mshosts = self.dbclient.execute(
|
|
"select msid from mshost where version='%s' and removed is NULL and state='Up'" % (cloudstackVersion))
|
|
if len(mshosts) > 0:
|
|
currentMsHosts = [row[0] for row in mshosts]
|
|
|
|
# Inject fake ms host
|
|
self.dbclient.execute(
|
|
"insert into mshost (msid,runid,name,state,version,service_ip,service_port,last_update) values (%s,%s,'ha-marvin-fakebox', 'Down', '%s', '127.0.0.1', '22', NOW())" % (
|
|
self.getFakeMsId(), self.getFakeMsRunId(), cloudstackVersion))
|
|
|
|
# Pass ownership to the fake ms id
|
|
self.dbclient.execute(
|
|
"update ha_config set mgmt_server_id=%d where resource_type='Host' and enabled=1 and provider='simulatorhaprovider'" % self.getFakeMsId())
|
|
|
|
pingInterval = float(list_configurations(
|
|
self.apiclient,
|
|
name='ping.interval'
|
|
)[0].value)
|
|
|
|
pingTimeout = float(list_configurations(
|
|
self.apiclient,
|
|
name='ping.timeout'
|
|
)[0].value)
|
|
|
|
def removeFakeMgmtServer(fakeMsRunId):
|
|
rows = self.dbclient.execute("select * from mshost_peer where peer_runid=%s" % fakeMsRunId)
|
|
if len(rows) > 0:
|
|
self.debug("Mgmt server is now trying to contact the fake mgmt server")
|
|
self.dbclient.execute("update mshost set removed=now() where runid=%s" % fakeMsRunId)
|
|
self.dbclient.execute("update mshost_peer set peer_state='Down' where peer_runid=%s" % fakeMsRunId)
|
|
return True, None
|
|
return False, None
|
|
|
|
def checkHaOwnershipExpiry(fakeMsId):
|
|
rows = self.dbclient.execute(
|
|
"select mgmt_server_id from ha_config where resource_type='Host' and enabled=1 and provider='simulatorhaprovider'")
|
|
if len(rows) > 0 and rows[0][0] != fakeMsId:
|
|
self.debug("HA resource ownership expired as node was detected to be gone")
|
|
return True, None
|
|
return False, None
|
|
|
|
retry_interval = 1 + (pingInterval * pingTimeout / 10)
|
|
|
|
res, _ = wait_until(retry_interval, 100, removeFakeMgmtServer, self.getFakeMsRunId())
|
|
if not res:
|
|
self.fail("Management server failed to turn down or remove fake mgmt server")
|
|
|
|
res, _ = wait_until(retry_interval, 100, checkHaOwnershipExpiry, self.getFakeMsId())
|
|
if not res:
|
|
self.fail("Management server failed to expire ownership of fenced peer")
|
|
|
|
self.debug("Testing ha background sync should claim new ownership")
|
|
self.checkSyncToState('Available')
|
|
|
|
result = self.dbclient.execute(
|
|
"select mgmt_server_id from ha_config where resource_type='Host' and enabled=1 and provider='simulatorhaprovider'")
|
|
newOwnerId = result[0][0]
|
|
self.assertTrue(newOwnerId in currentMsHosts)
|
|
|
|
|
|
def checkFSMTransition(self, transition, event, haState, prevHaState, hasActiviyCounter, hasRecoveryCounter):
|
|
print(("checkFSMTransition:: transition=%s, event=%s, state=%s" % (transition, event, haState)))
|
|
self.assertEqual(transition.event, event)
|
|
self.assertEqual(transition.hastate, haState)
|
|
self.assertEqual(transition.prevhastate, prevHaState)
|
|
|
|
if hasActiviyCounter is None:
|
|
pass
|
|
elif hasActiviyCounter:
|
|
self.assertTrue(transition.activitycounter > 0)
|
|
else:
|
|
self.assertEqual(transition.activitycounter, 0)
|
|
|
|
if hasRecoveryCounter is None:
|
|
pass
|
|
elif hasRecoveryCounter:
|
|
self.assertTrue(transition.recoverycounter > 0)
|
|
else:
|
|
self.assertEqual(transition.recoverycounter, 0)
|
|
|
|
|
|
def findFSMTransitionToState(self, state, host):
|
|
transitions = self.getSimulatorHAStateTransitions(host.id)
|
|
if not transitions:
|
|
print("findFSMTransition:: no transitions returned")
|
|
return False, (None, None, None)
|
|
|
|
previousTransition = None
|
|
stateTransition = None
|
|
nextTransition = None
|
|
for transition in transitions:
|
|
if stateTransition:
|
|
nextTransition = transition
|
|
break
|
|
if transition.hastate == state:
|
|
stateTransition = transition
|
|
if not stateTransition:
|
|
previousTransition = transition
|
|
|
|
print(("findFSMTransition:: prev=%s, cur=%s, next=%s, find state=%s" % (previousTransition, stateTransition, nextTransition, state)))
|
|
if stateTransition:
|
|
return True, (previousTransition, stateTransition, nextTransition,)
|
|
return False, (previousTransition, stateTransition, nextTransition,)
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_ha_verify_fsm_available(self):
|
|
"""
|
|
Tests ha FSM transitions for valid healthy host
|
|
Simulates health check passing
|
|
"""
|
|
|
|
host = self.getHost()
|
|
self.configureAndDisableHostHa(host.id)
|
|
self.configureSimulatorHAProviderState(True, True, True, False)
|
|
self.configureAndEnableHostHa(False)
|
|
|
|
res, (_, T, _) = wait_until(3, 20, self.findFSMTransitionToState, 'available', host)
|
|
if not res:
|
|
self.fail("FSM did not transition to available state")
|
|
|
|
self.checkFSMTransition(T, 'enabled', 'available', 'disabled', False, False)
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_ha_verify_fsm_degraded(self):
|
|
"""
|
|
Tests ha FSM transitions leading to degraded state
|
|
Simulates health check failures with activity checks passing
|
|
FSM transitions should happen indefinitely between:
|
|
Available->Suspect<->Checking->Degraded->Available
|
|
"""
|
|
host = self.getHost()
|
|
self.configureAndDisableHostHa(host.id)
|
|
self.configureSimulatorHAProviderState(False, True, True, False)
|
|
self.configureAndEnableHostHa(False)
|
|
|
|
# Initial health check failure
|
|
res, (_, T, _) = wait_until(3, 50, self.findFSMTransitionToState, 'suspect', host)
|
|
if not res:
|
|
self.fail("FSM did not transition to suspect state")
|
|
|
|
self.checkFSMTransition(T, 'healthcheckfailed', 'suspect', 'available', False, False)
|
|
|
|
# Check transition to Degraded
|
|
res, (prevT, T, _) = wait_until(3, 100, self.findFSMTransitionToState, 'degraded', host)
|
|
if not res:
|
|
self.fail("FSM did not transition to degraded state")
|
|
|
|
if prevT:
|
|
self.checkFSMTransition(prevT, 'performactivitycheck', 'checking', 'suspect', True, False)
|
|
self.checkFSMTransition(T, 'activitycheckfailureunderthresholdratio', 'degraded', 'checking', True, False)
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_ha_verify_fsm_recovering(self):
|
|
"""
|
|
Tests ha FSM transitions leading to recovering
|
|
Simulates both health and activity check failures
|
|
FSM transitions should happen indefinitely between:
|
|
Available->Suspect<->Checking->Recovering->Recovered<-retry-loop->->Fencing
|
|
"""
|
|
host = self.getHost()
|
|
self.configureAndDisableHostHa(host.id)
|
|
self.configureSimulatorHAProviderState(False, False, True, False)
|
|
self.configureAndEnableHostHa(False)
|
|
|
|
# Initial health check failure
|
|
res, (_, T, _) = wait_until(3, 50, self.findFSMTransitionToState, 'suspect', host)
|
|
if not res:
|
|
self.fail("FSM did not transition to suspect state")
|
|
|
|
self.checkFSMTransition(T, 'healthcheckfailed', 'suspect', 'available', False, False)
|
|
|
|
# Check transition to recovering
|
|
res, (prevT, T, _) = wait_until(3, 100, self.findFSMTransitionToState, 'recovering', host)
|
|
if not res:
|
|
self.fail("FSM did not transition to recovering state")
|
|
|
|
if prevT:
|
|
self.checkFSMTransition(prevT, 'performactivitycheck', 'checking', 'suspect', True, False)
|
|
self.checkFSMTransition(T, 'activitycheckfailureoverthresholdratio', 'recovering', 'checking', True, False)
|
|
|
|
# Check transition to fencing due to recovery attempts exceeded
|
|
res, (_, T, _) = wait_until(3, 100, self.findFSMTransitionToState, 'fencing', host)
|
|
if not res:
|
|
self.fail("FSM did not transition to fencing state")
|
|
|
|
self.checkFSMTransition(T, 'recoveryoperationthresholdexceeded', 'fencing', 'recovering', None, True)
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_ha_verify_fsm_fenced(self):
|
|
"""
|
|
Tests ha FSM transitions for failures leading to fenced state
|
|
FSM transitions should happen indefinitely between:
|
|
Available->Suspect<->Checking->Recovering<-fail recovery->->Fencing->Fenced
|
|
"""
|
|
host = self.getHost()
|
|
self.configureAndDisableHostHa(host.id)
|
|
self.configureSimulatorHAProviderState(False, False, False, True)
|
|
self.configureAndEnableHostHa(False)
|
|
|
|
# Check for transition to fenced
|
|
res, (prevT, T, _) = wait_until(3, 100, self.findFSMTransitionToState, 'fenced', host)
|
|
if not res:
|
|
self.fail("FSM did not transition to fenced state")
|
|
|
|
self.checkFSMTransition(prevT, 'recoveryoperationthresholdexceeded', 'fencing', 'recovering', False, True)
|
|
self.checkFSMTransition(T, 'fenced', 'fenced', 'fencing', False, False)
|
|
|
|
# Simulate manual recovery of host and cancel maintenance mode
|
|
self.configureSimulatorHAProviderState(True, True, True, False)
|
|
cancelCmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
|
|
cancelCmd.id = host.id
|
|
self.apiclient.cancelHostMaintenance(cancelCmd)
|
|
|
|
# Check for transition to available after manual recovery
|
|
res, (prevT, T, _) = wait_until(3, 50, self.findFSMTransitionToState, 'available', host)
|
|
if not res:
|
|
self.fail("FSM did not transition to available state")
|
|
|
|
self.checkFSMTransition(T, 'eligible', 'available', 'ineligible', False, False)
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_configure_ha_provider_invalid(self):
|
|
"""
|
|
Tests configure HA Provider with invalid provider options
|
|
"""
|
|
|
|
# Enable ha for host
|
|
self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
|
|
cmd = self.getHostHaEnableCmd()
|
|
response = self.apiclient.enableHAForHost(cmd)
|
|
self.assertEqual(response.hostid, cmd.hostid)
|
|
self.assertEqual(response.haenable, True)
|
|
|
|
host = self.getHost(response.hostid)
|
|
|
|
# Setup wrong configuration for the host
|
|
conf_ha_cmd = configureHAForHost.configureHAForHostCmd()
|
|
if host.hypervisor.lower() in "simulator":
|
|
conf_ha_cmd.provider = "kvmhaprovider"
|
|
if host.hypervisor.lower() in "kvm":
|
|
conf_ha_cmd.provider = "simulatorhaprovider"
|
|
|
|
conf_ha_cmd.hostid = cmd.hostid
|
|
|
|
# Call the configure HA provider API with not supported provider for HA
|
|
try:
|
|
self.apiclient.configureHAForHost(conf_ha_cmd)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self.fail("Expected an exception to be thrown, failing")
|
|
|
|
|
|
@attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
|
|
def test_configure_ha_provider_valid(self):
|
|
"""
|
|
Tests configure HA Provider with valid provider options
|
|
"""
|
|
|
|
# Enable ha for host
|
|
self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
|
|
cmd = self.getHostHaEnableCmd()
|
|
response = self.apiclient.enableHAForHost(cmd)
|
|
self.assertEqual(response.hostid, cmd.hostid)
|
|
self.assertEqual(response.haenable, True)
|
|
|
|
host = self.getHost(response.hostid)
|
|
|
|
|
|
# Setup wrong configuration for the host
|
|
conf_ha_cmd = configureHAForHost.configureHAForHostCmd()
|
|
if host.hypervisor.lower() in "kvm":
|
|
conf_ha_cmd.provider = "kvmhaprovider"
|
|
if host.hypervisor.lower() in "simulator":
|
|
conf_ha_cmd.provider = "simulatorhaprovider"
|
|
|
|
conf_ha_cmd.hostid = cmd.hostid
|
|
|
|
# Call the configure HA provider API with not supported provider for HA
|
|
response = self.apiclient.configureHAForHost(conf_ha_cmd)
|
|
|
|
# Check the response contains the set provider and hostID
|
|
self.assertEqual(response.haprovider, conf_ha_cmd.provider)
|
|
self.assertEqual(response.hostid, conf_ha_cmd.hostid)
|
|
|
|
|
|
def getHaProvider(self, host):
|
|
cmd = listHostHAProviders.listHostHAProvidersCmd()
|
|
cmd.hypervisor = host.hypervisor
|
|
response = self.apiclient.listHostHAProviders(cmd)
|
|
return response[0].haprovider
|
|
|
|
|
|
def configureHaProvider(self):
|
|
cmd = self.getHostHaConfigCmd(self.getHaProvider(self.getHost()))
|
|
return self.apiclient.configureHAForHost(cmd)
|
|
|
|
|
|
@attr(tags=["advanced",
|
|
"advancedns",
|
|
"smoke",
|
|
"basic",
|
|
"sg"],
|
|
required_hardware="false")
|
|
def test_list_ha_for_host(self):
|
|
"""
|
|
Test that verifies the listHAForHost API
|
|
"""
|
|
self.configureHaProvider()
|
|
db_count = self.dbclient.execute("SELECT count(*) FROM cloud.ha_config")
|
|
|
|
cmd = self.getListHostHAResources()
|
|
del cmd.hostid
|
|
response = self.apiclient.listHostHAResources(cmd)
|
|
|
|
self.assertEqual(db_count[0][0], len(response))
|
|
|
|
|
|
@attr(tags=["advanced",
|
|
"advancedns",
|
|
"smoke",
|
|
"basic",
|
|
"sg"],
|
|
required_hardware="false")
|
|
def test_list_ha_for_host_valid(self):
|
|
"""
|
|
Valid test for listing a specific host HA resources
|
|
"""
|
|
|
|
self.configureHaProvider()
|
|
cmd = self.getListHostHAResources()
|
|
response = self.apiclient.listHostHAResources(cmd)
|
|
self.assertEqual(response[0].hostid, cmd.hostid)
|
|
|
|
|
|
@attr(tags=["advanced",
|
|
"advancedns",
|
|
"smoke",
|
|
"basic",
|
|
"sg"],
|
|
required_hardware="false")
|
|
def test_list_ha_for_host_invalid(self):
|
|
"""
|
|
Test that listHostHAResources is returning exception when called with invalid data
|
|
"""
|
|
|
|
self.configureHaProvider()
|
|
cmd = self.getListHostHAResources()
|
|
cmd.hostid = "someinvalidvalue"
|
|
|
|
try:
|
|
response = self.apiclient.listHostHAResources(cmd)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
self.fail("Expected an exception to be thrown, failing")
|