cloudstack/test/integration/smoke/test_hostha_simulator.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import marvin
from marvin.cloudstackTestCase import *
from marvin.cloudstackAPI import *
from marvin.lib.utils import *
from marvin.lib.base import *
from marvin.lib.common import *
from nose.plugins.attrib import attr

from ipmisim.ipmisim import IpmiServerContext, IpmiServer, ThreadedIpmiServer

import random
import socket
import sys
import _thread
import time


class TestHostHA(cloudstackTestCase):
    """ Test host-ha business logic using Simulator
    """

    def setUp(self):
        self.apiclient = self.testClient.getApiClient()
        self.hypervisor = self.testClient.getHypervisorInfo()
        self.dbclient = self.testClient.getDbConnection()
        self.services = self.testClient.getParsedTestDataConfig()
        self.mgtSvrDetails = self.config.__dict__["mgtSvr"][0].__dict__
        self.fakeMsId = random.randint(10000, 99999) * random.randint(10, 20)
        self.host = None

        # Cleanup any existing configs
        self.dbclient.execute("delete from ha_config where resource_type='Host'")

        # use random port for ipmisim
        s = socket.socket()
        s.bind(('', 0))
        self.serverPort = s.getsockname()[1]
        s.close()

        # Get a host to run tests against
        self.host = self.getHost()

        self.cleanup = []


    def tearDown(self):
        try:
            host = self.getHost()
            self.configureAndDisableHostHa(host.id)
            self.host = None
            self.dbclient.execute("delete from mshost_peer where peer_runid=%s" % self.getFakeMsRunId())
            self.dbclient.execute("delete from mshost where runid=%s" % self.getFakeMsRunId())
            self.dbclient.execute("delete from cluster_details where name='resourceHAEnabled'")
            self.dbclient.execute("delete from data_center_details where name='resourceHAEnabled'")
            self.dbclient.execute("delete from ha_config where resource_type='Host'")
            self.dbclient.execute("update host set resource_state='Enabled' where type='Routing' and resource_state='Maintenance'")
        except Exception as e:
            raise Exception("Warning: Exception during db cleanup : %s" % e)
        super(TestHostHA,self).tearDown()


    def getFakeMsId(self):
        return self.fakeMsId


    def getFakeMsRunId(self):
        return self.fakeMsId * 1000


    def getHost(self, hostId=None):
        if self.host and hostId is None:
            return self.host

        response = list_hosts(
            self.apiclient,
            type='Routing',
            hypervisor='Simulator',
            resourcestate='Enabled',
            id=hostId
        )

        if response and len(response) > 0:
            random.shuffle(response)
            self.host = response[0]
            return self.host
        raise self.skipTest("No suitable hosts found, skipping host-ha test")


    def getHostHaConfigCmd(self, provider='simulatorhaprovider'):
        cmd = configureHAForHost.configureHAForHostCmd()
        cmd.provider = provider
        cmd.hostid = self.getHost().id
        return cmd


    def getHostHaEnableCmd(self):
        cmd = enableHAForHost.enableHAForHostCmd()
        cmd.hostid = self.getHost().id
        return cmd


    def getHostHaDisableCmd(self):
        cmd = disableHAForHost.disableHAForHostCmd()
        cmd.hostid = self.getHost().id
        return cmd


    def getListHostHAResources(self):
        cmd = listHostHAResources.listHostHAResourcesCmd()
        cmd.hostid = self.getHost().id
        return cmd


    def configureAndEnableHostHa(self, initialize=True):
        self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
        response = self.apiclient.enableHAForHost(self.getHostHaEnableCmd())
        self.assertEqual(response.haenable, True)
        if initialize:
            self.configureSimulatorHAProviderState(True, True, True, False)


    def configureAndDisableHostHa(self, hostId):
        self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
        cmd = self.getHostHaDisableCmd()
        cmd.hostid = hostId
        response = self.apiclient.disableHAForHost(cmd)
        self.assertEqual(response.hostid, cmd.hostid)
        self.assertEqual(response.haenable, False)


    def enableHostHa(self, hostId):
        cmd = self.getHostHaEnableCmd()
        cmd.hostid = hostId
        response = self.apiclient.enableHAForHost(cmd)
        self.assertEqual(response.hostid, cmd.hostid)
        self.assertEqual(response.haenable, True)


    def configureSimulatorHAProviderState(self, health, activity, recover, fence):
        cmd = configureSimulatorHAProviderState.configureSimulatorHAProviderStateCmd()
        cmd.hostid = self.getHost().id
        cmd.health = health
        cmd.activity = activity
        cmd.recover = recover
        cmd.fence = fence
        response = self.apiclient.configureSimulatorHAProviderState(cmd)
        self.assertEqual(response.success, True)


    def getSimulatorHAStateTransitions(self, hostId):
        cmd = listSimulatorHAStateTransitions.listSimulatorHAStateTransitionsCmd()
        cmd.hostid = hostId
        return self.apiclient.listSimulatorHAStateTransitions(cmd)


    def checkSyncToState(self, state, interval=5000):
        def checkForStateSync(expectedState):
            response = self.getHost(hostId=self.getHost().id).hostha
            print(("checkForStateSync:: response=%s, expected=%s" % (response, expectedState)))
            return response.hastate == expectedState, None

        sync_interval = 1 + int(interval) / 1000
        res, _ = wait_until(sync_interval, 100, checkForStateSync, state)
        if not res:
            self.fail("Failed to get host.hastate synced to expected state:" + state)
        response = self.getHost(hostId=self.getHost().id).hostha
        self.assertEqual(response.hastate, state)


    def getNonConfiguredHaHost(self):
        response = list_hosts(
            self.apiclient,
            type='Routing'
        )
        for host in response:
            if host.haprovider is None:
                return host
            else:
                return None


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_hostha_enable_feature_without_setting_provider(self):
        """
            Tests Enable HA without setting the provider, Exception is thrown
        """
        host = self.getNonConfiguredHaHost()

        if host is None:
            cloudstackTestCase.skipTest(self, "There is no non configured hosts. Skipping test.")

        cmd = self.getHostHaEnableCmd()
        cmd.hostid = host.id

        try:
            response = self.apiclient.enableHAForHost(cmd)
        except Exception:
            pass
        else:
            self.fail("Expected an exception to be thrown, failing")


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_ha_list_providers(self):
        """
            Tests default ha providers list
        """
        cmd = listHostHAProviders.listHostHAProvidersCmd()

        cmd.hypervisor = 'Simulator'
        response = self.apiclient.listHostHAProviders(cmd)[0]
        self.assertEqual(response.haprovider, 'SimulatorHAProvider')

        cmd.hypervisor = 'KVM'
        response = self.apiclient.listHostHAProviders(cmd)[0]
        self.assertEqual(response.haprovider, 'KVMHAProvider')


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_hostha_configure_invalid_provider(self):
        """
            Tests host-ha configuration with invalid driver
        """
        cmd = self.getHostHaConfigCmd()
        cmd.provider = 'randomDriverThatDoesNotExist'
        try:
            response = self.apiclient.configureHAForHost(cmd)
        except Exception:
            pass
        else:
            self.fail("Expected an exception to be thrown, failing")


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_hostha_configure_default_driver(self):
        """
            Tests host-ha configuration with valid data
        """
        cmd = self.getHostHaConfigCmd()
        response = self.apiclient.configureHAForHost(cmd)
        self.assertEqual(response.hostid, cmd.hostid)
        self.assertEqual(response.haprovider, cmd.provider.lower())


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_ha_enable_feature_invalid(self):
        """
            Tests ha feature enable command with invalid options
        """
        cmd = self.getHostHaEnableCmd()
        cmd.hostid = -1
        try:
            response = self.apiclient.enableHAForHost(cmd)
        except Exception:
            pass
        else:
            self.fail("Expected an exception to be thrown, failing")
        try:
            cmd = enableHAForCluster.enableHAForClusterCmd()
            response = self.apiclient.enableHAForCluster(cmd)
        except Exception:
            pass
        else:
            self.fail("Expected an exception to be thrown, failing")
        try:
            cmd = enableHAForZone.enableHAForZoneCmd()
            response = self.apiclient.enableHAForZone(cmd)
        except Exception:
            pass
        else:
            self.fail("Expected an exception to be thrown, failing")


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_ha_disable_feature_invalid(self):
        """
            Tests ha feature disable command with invalid options
        """
        cmd = self.getHostHaDisableCmd()
        cmd.hostid = -1
        try:
            response = self.apiclient.disableHAForHost(cmd)
        except Exception:
            pass
        else:
            self.fail("Expected an exception to be thrown, failing")
        try:
            cmd = disableHAForCluster.disableHAForClusterCmd()
            response = self.apiclient.disableHAForCluster(cmd)
        except Exception:
            pass
        else:
            self.fail("Expected an exception to be thrown, failing")

        try:
            cmd = disableHAForZone.disableHAForZoneCmd()
            response = self.apiclient.disableHAForZone(cmd)
        except Exception:
            pass
        else:
            self.fail("Expected an exception to be thrown, failing")


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_hostha_enable_feature_valid(self):
        """
            Tests host-ha enable feature with valid options
        """
        self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
        cmd = self.getHostHaEnableCmd()
        response = self.apiclient.enableHAForHost(cmd)
        self.assertEqual(response.hostid, cmd.hostid)
        self.assertEqual(response.haenable, True)


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_hostha_disable_feature_valid(self):
        """
            Tests host-ha disable feature with valid options
        """
        self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
        cmd = self.getHostHaDisableCmd()
        response = self.apiclient.disableHAForHost(cmd)
        self.assertEqual(response.hostid, cmd.hostid)
        self.assertEqual(response.haenable, False)

        response = self.getHost(hostId=cmd.hostid).hostha
        self.assertEqual(response.hastate, 'Disabled')


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_ha_configure_enabledisable_across_clusterzones(self):
        """
            Tests ha enable/disable feature at cluster and zone level
            Zone > Cluster > Host
        """
        host = self.getHost()
        self.configureAndDisableHostHa(host.id)
        self.configureAndEnableHostHa()

        self.checkSyncToState('Available')
        response = self.getHost(hostId=host.id).hostha
        self.assertTrue(response.hastate == 'Available')

        # Disable at host level
        cmd = disableHAForHost.disableHAForHostCmd()
        cmd.hostid = host.id
        response = self.apiclient.disableHAForHost(cmd)

        # Disable at cluster level
        cmd = disableHAForCluster.disableHAForClusterCmd()
        cmd.clusterid = host.clusterid
        response = self.apiclient.disableHAForCluster(cmd)

        # Disable at zone level
        cmd = disableHAForZone.disableHAForZoneCmd()
        cmd.zoneid = host.zoneid
        response = self.apiclient.disableHAForZone(cmd)

        # HA state check
        response = self.getHost(hostId=host.id).hostha
        self.assertTrue(response.hastate == 'Disabled')

        # Check ha-state check and sync
        self.dbclient.execute("update ha_config set ha_state='Available' where enabled='1' and resource_type='Host'")
        self.checkSyncToState('Disabled')

        # Enable at zone level
        cmd = enableHAForZone.enableHAForZoneCmd()
        cmd.zoneid = host.zoneid
        response = self.apiclient.enableHAForZone(cmd)

        # Enable at cluster level
        cmd = enableHAForCluster.enableHAForClusterCmd()
        cmd.clusterid = host.clusterid
        response = self.apiclient.enableHAForCluster(cmd)

        # Enable at host level
        cmd = enableHAForHost.enableHAForHostCmd()
        cmd.hostid = host.id
        response = self.apiclient.enableHAForHost(cmd)

        # Check state sync
        self.checkSyncToState('Available')


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_ha_multiple_mgmt_server_ownership(self):
        """
            Tests ha resource ownership expiry across multi-mgmt server
        """
        host = self.getHost()
        self.configureAndDisableHostHa(host.id)
        self.configureSimulatorHAProviderState(True, True, True, False)
        self.configureAndEnableHostHa(False)

        cloudstackVersion = Configurations.listCapabilities(self.apiclient).cloudstackversion

        currentMsHosts = []
        mshosts = self.dbclient.execute(
            "select msid from mshost where version='%s' and removed is NULL and state='Up'" % (cloudstackVersion))
        if len(mshosts) > 0:
            currentMsHosts = [row[0] for row in mshosts]

        # Inject fake ms host
        self.dbclient.execute(
            "insert into mshost (msid,runid,name,state,version,service_ip,service_port,last_update) values (%s,%s,'ha-marvin-fakebox', 'Down', '%s', '127.0.0.1', '22', NOW())" % (
                self.getFakeMsId(), self.getFakeMsRunId(), cloudstackVersion))

        # Pass ownership to the fake ms id
        self.dbclient.execute(
            "update ha_config set mgmt_server_id=%d where resource_type='Host' and enabled=1 and provider='simulatorhaprovider'" % self.getFakeMsId())

        pingInterval = float(list_configurations(
            self.apiclient,
            name='ping.interval'
        )[0].value)

        pingTimeout = float(list_configurations(
            self.apiclient,
            name='ping.timeout'
        )[0].value)

        def removeFakeMgmtServer(fakeMsRunId):
            rows = self.dbclient.execute("select * from mshost_peer where peer_runid=%s" % fakeMsRunId)
            if len(rows) > 0:
                self.debug("Mgmt server is now trying to contact the fake mgmt server")
                self.dbclient.execute("update mshost set removed=now() where runid=%s" % fakeMsRunId)
                self.dbclient.execute("update mshost_peer set peer_state='Down' where peer_runid=%s" % fakeMsRunId)
                return True, None
            return False, None

        def checkHaOwnershipExpiry(fakeMsId):
            rows = self.dbclient.execute(
                "select mgmt_server_id from ha_config where resource_type='Host' and enabled=1 and provider='simulatorhaprovider'")
            if len(rows) > 0 and rows[0][0] != fakeMsId:
                self.debug("HA resource ownership expired as node was detected to be gone")
                return True, None
            return False, None

        retry_interval = 1 + (pingInterval * pingTimeout / 10)

        res, _ = wait_until(retry_interval, 20, removeFakeMgmtServer, self.getFakeMsRunId())
        if not res:
            self.fail("Management server failed to turn down or remove fake mgmt server")

        res, _ = wait_until(retry_interval, 100, checkHaOwnershipExpiry, self.getFakeMsId())
        if not res:
            self.fail("Management server failed to expire ownership of fenced peer")

        self.debug("Testing ha background sync should claim new ownership")
        self.checkSyncToState('Available')

        result = self.dbclient.execute(
            "select mgmt_server_id from ha_config where resource_type='Host' and enabled=1 and provider='simulatorhaprovider'")
        newOwnerId = result[0][0]
        self.assertTrue(newOwnerId in currentMsHosts)


    def checkFSMTransition(self, transition, event, haState, prevHaState, hasActiviyCounter, hasRecoveryCounter):
        print(("checkFSMTransition:: transition=%s, event=%s, state=%s" % (transition, event, haState)))
        self.assertEqual(transition.event, event)
        self.assertEqual(transition.hastate, haState)
        self.assertEqual(transition.prevhastate, prevHaState)

        if hasActiviyCounter is None:
            pass
        elif hasActiviyCounter:
            self.assertTrue(transition.activitycounter > 0)
        else:
            self.assertEqual(transition.activitycounter, 0)

        if hasRecoveryCounter is None:
            pass
        elif hasRecoveryCounter:
            self.assertTrue(transition.recoverycounter > 0)
        else:
            self.assertEqual(transition.recoverycounter, 0)


    def findFSMTransitionToState(self, state, host):
        transitions = self.getSimulatorHAStateTransitions(host.id)
        if not transitions:
            print("findFSMTransition:: no transitions returned")
            return False, (None, None, None)

        previousTransition = None
        stateTransition = None
        nextTransition = None
        for transition in transitions:
            if stateTransition:
                nextTransition = transition
                break
            if transition.hastate == state:
                stateTransition = transition
            if not stateTransition:
                previousTransition = transition

        print(("findFSMTransition:: prev=%s, cur=%s, next=%s, find state=%s" % (previousTransition, stateTransition, nextTransition, state)))
        if stateTransition:
            return True, (previousTransition, stateTransition, nextTransition,)
        return False, (previousTransition, stateTransition, nextTransition,)


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_ha_verify_fsm_available(self):
        """
            Tests ha FSM transitions for valid healthy host
            Simulates health check passing
        """

        host = self.getHost()
        self.configureAndDisableHostHa(host.id)
        self.configureSimulatorHAProviderState(True, True, True, False)
        self.configureAndEnableHostHa(False)

        res, (_, T, _) = wait_until(3, 20, self.findFSMTransitionToState, 'available', host)
        if not res:
            self.fail("FSM did not transition to available state")

        self.checkFSMTransition(T, 'enabled', 'available', 'disabled', False, False)


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_ha_verify_fsm_degraded(self):
        """
            Tests ha FSM transitions leading to degraded state
            Simulates health check failures with activity checks passing
            FSM transitions should happen indefinitely between:
            Available->Suspect<->Checking->Degraded->Available
        """
        host = self.getHost()
        self.configureAndDisableHostHa(host.id)
        self.configureSimulatorHAProviderState(False, True, True, False)
        self.configureAndEnableHostHa(False)

        # Initial health check failure
        res, (_, T, _) = wait_until(3, 50, self.findFSMTransitionToState, 'suspect', host)
        if not res:
            self.fail("FSM did not transition to suspect state")

        self.checkFSMTransition(T, 'healthcheckfailed', 'suspect', 'available', False, False)

        # Check transition to Degraded
        res, (prevT, T, _) = wait_until(3, 100, self.findFSMTransitionToState, 'degraded', host)
        if not res:
            self.fail("FSM did not transition to degraded state")

        if prevT:
            self.checkFSMTransition(prevT, 'performactivitycheck', 'checking', 'suspect', True, False)
        self.checkFSMTransition(T, 'activitycheckfailureunderthresholdratio', 'degraded', 'checking', True, False)


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_ha_verify_fsm_recovering(self):
        """
            Tests ha FSM transitions leading to recovering
            Simulates both health and activity check failures
            FSM transitions should happen indefinitely between:
            Available->Suspect<->Checking->Recovering->Recovered<-retry-loop->->Fencing
        """
        host = self.getHost()
        self.configureAndDisableHostHa(host.id)
        self.configureSimulatorHAProviderState(False, False, True, False)
        self.configureAndEnableHostHa(False)

        # Initial health check failure
        res, (_, T, _) = wait_until(3, 50, self.findFSMTransitionToState, 'suspect', host)
        if not res:
            self.fail("FSM did not transition to suspect state")

        self.checkFSMTransition(T, 'healthcheckfailed', 'suspect', 'available', False, False)

        # Check transition to recovering
        res, (prevT, T, _) = wait_until(3, 100, self.findFSMTransitionToState, 'recovering', host)
        if not res:
            self.fail("FSM did not transition to recovering state")

        if prevT:
            self.checkFSMTransition(prevT, 'performactivitycheck', 'checking', 'suspect', True, False)
        self.checkFSMTransition(T, 'activitycheckfailureoverthresholdratio', 'recovering', 'checking', True, False)

        # Check transition to fencing due to recovery attempts exceeded
        res, (_, T, _) = wait_until(3, 100, self.findFSMTransitionToState, 'fencing', host)
        if not res:
            self.fail("FSM did not transition to fencing state")

        self.checkFSMTransition(T, 'recoveryoperationthresholdexceeded', 'fencing', 'recovering', None, True)


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_ha_verify_fsm_fenced(self):
        """
            Tests ha FSM transitions for failures leading to fenced state
            FSM transitions should happen indefinitely between:
            Available->Suspect<->Checking->Recovering<-fail recovery->->Fencing->Fenced
        """
        host = self.getHost()
        self.configureAndDisableHostHa(host.id)
        self.configureSimulatorHAProviderState(False, False, False, True)
        self.configureAndEnableHostHa(False)

        # Check for transition to fenced
        res, (prevT, T, _) = wait_until(3, 100, self.findFSMTransitionToState, 'fenced', host)
        if not res:
            self.fail("FSM did not transition to fenced state")

        self.checkFSMTransition(prevT, 'recoveryoperationthresholdexceeded', 'fencing', 'recovering', False, True)
        self.checkFSMTransition(T, 'fenced', 'fenced', 'fencing', False, False)

        # Simulate manual recovery of host and cancel maintenance mode
        self.configureSimulatorHAProviderState(True, True, True, False)
        cancelCmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cancelCmd.id = host.id
        self.apiclient.cancelHostMaintenance(cancelCmd)

        # Check for transition to available after manual recovery
        res, (prevT, T, _) = wait_until(3, 50, self.findFSMTransitionToState, 'available', host)
        if not res:
            self.fail("FSM did not transition to available state")

        self.checkFSMTransition(T, 'eligible', 'available', 'ineligible', False, False)


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_configure_ha_provider_invalid(self):
        """
            Tests configure HA Provider with invalid provider options
        """

        # Enable ha for host
        self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
        cmd = self.getHostHaEnableCmd()
        response = self.apiclient.enableHAForHost(cmd)
        self.assertEqual(response.hostid, cmd.hostid)
        self.assertEqual(response.haenable, True)

        host = self.getHost(response.hostid)

        # Setup wrong configuration for the host
        conf_ha_cmd = configureHAForHost.configureHAForHostCmd()
        if host.hypervisor.lower() in "simulator":
            conf_ha_cmd.provider = "kvmhaprovider"
        if host.hypervisor.lower() in "kvm":
            conf_ha_cmd.provider = "simulatorhaprovider"

        conf_ha_cmd.hostid = cmd.hostid

        # Call the configure HA provider API with not supported provider for HA
        try:
            self.apiclient.configureHAForHost(conf_ha_cmd)
        except Exception:
            pass
        else:
            self.fail("Expected an exception to be thrown, failing")


    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
    def test_configure_ha_provider_valid(self):
        """
            Tests configure HA Provider with valid provider options
        """

        # Enable ha for host
        self.apiclient.configureHAForHost(self.getHostHaConfigCmd())
        cmd = self.getHostHaEnableCmd()
        response = self.apiclient.enableHAForHost(cmd)
        self.assertEqual(response.hostid, cmd.hostid)
        self.assertEqual(response.haenable, True)

        host = self.getHost(response.hostid)


        # Setup wrong configuration for the host
        conf_ha_cmd = configureHAForHost.configureHAForHostCmd()
        if host.hypervisor.lower() in "kvm":
            conf_ha_cmd.provider = "kvmhaprovider"
        if host.hypervisor.lower() in "simulator":
            conf_ha_cmd.provider = "simulatorhaprovider"

        conf_ha_cmd.hostid = cmd.hostid

        # Call the configure HA provider API with not supported provider for HA
        response = self.apiclient.configureHAForHost(conf_ha_cmd)

        # Check the response contains the set provider and hostID
        self.assertEqual(response.haprovider, conf_ha_cmd.provider)
        self.assertEqual(response.hostid, conf_ha_cmd.hostid)


    def getHaProvider(self, host):
        cmd = listHostHAProviders.listHostHAProvidersCmd()
        cmd.hypervisor = host.hypervisor
        response = self.apiclient.listHostHAProviders(cmd)
        return response[0].haprovider


    def configureHaProvider(self):
        cmd = self.getHostHaConfigCmd(self.getHaProvider(self.getHost()))
        return self.apiclient.configureHAForHost(cmd)


    @attr(tags=["advanced",
                "advancedns",
                "smoke",
                "basic",
                "sg"],
          required_hardware="false")
    def test_list_ha_for_host(self):
        """
            Test that verifies the listHAForHost API
        """
        self.configureHaProvider()
        db_count = self.dbclient.execute("SELECT count(*) FROM cloud.ha_config")

        cmd = self.getListHostHAResources()
        del cmd.hostid
        response = self.apiclient.listHostHAResources(cmd)

        self.assertEqual(db_count[0][0], len(response))


    @attr(tags=["advanced",
                "advancedns",
                "smoke",
                "basic",
                "sg"],
          required_hardware="false")
    def test_list_ha_for_host_valid(self):
        """
            Valid test for listing a specific host HA resources
        """

        self.configureHaProvider()
        cmd = self.getListHostHAResources()
        response = self.apiclient.listHostHAResources(cmd)
        self.assertEqual(response[0].hostid, cmd.hostid)


    @attr(tags=["advanced",
                "advancedns",
                "smoke",
                "basic",
                "sg"],
          required_hardware="false")
    def test_list_ha_for_host_invalid(self):
        """
            Test that listHostHAResources is returning exception when called with invalid data
        """

        self.configureHaProvider()
        cmd = self.getListHostHAResources()
        cmd.hostid = "someinvalidvalue"

        try:
            response = self.apiclient.listHostHAResources(cmd)
        except Exception:
            pass
        else:
            self.fail("Expected an exception to be thrown, failing")