diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index ae47b1d76ed..728f5a2b180 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -32,7 +32,7 @@ import java.util.List; */ public interface HighAvailabilityManager extends Manager { - public ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", + ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", "Force High-Availability to happen even if the VM says no.", true, Cluster); ConfigKey HAWorkers = new ConfigKey<>("Advanced", Integer.class, "ha.workers", "5", @@ -112,7 +112,7 @@ public interface HighAvailabilityManager extends Manager { void cancelDestroy(VMInstanceVO vm, Long hostId); - void scheduleDestroy(VMInstanceVO vm, long hostId); + boolean scheduleDestroy(VMInstanceVO vm, long hostId); /** * Schedule restarts for all vms running on the host. @@ -143,7 +143,7 @@ public interface HighAvailabilityManager extends Manager { * @param host host the virtual machine is on. * @param type which type of stop is requested. */ - void scheduleStop(VMInstanceVO vm, long hostId, WorkType type); + boolean scheduleStop(VMInstanceVO vm, long hostId, WorkType type); void cancelScheduledMigrations(HostVO host); diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index d8fc99a0934..20435f48b52 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.ha; +import static org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone; + import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -121,6 +123,16 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur "Total number of attempts for trying migration of a VM.", true, ConfigKey.Scope.Global); + public static ConfigKey VmHaEnabled = new ConfigKey<>("Advanced", Boolean.class, "vm.ha.enabled", "true", + "Enable/Disable VM High Availability manager, it is enabled by default." + + " When enabled, the VM HA WorkItems (for VM Stop, Restart, Migration, Destroy) can be created and the scheduled items are executed; and" + + " When disabled, new VM HA WorkItems are not allowed and the scheduled items are retried until max retries configured at 'vm.ha.migration.max.retries'" + + " (executed in case HA is re-enabled during retry attempts), and then purged after 'time.between.failures' by the cleanup thread that runs" + + " regularly at 'time.between.cleanup'", true, Zone); + + protected static ConfigKey VmHaAlertsEnabled = new ConfigKey<>("Advanced", Boolean.class, "vm.ha.alerts.enabled", "true", + "Enable/Disable alerts for the VM HA operations, it is enabled by default.", true, Zone); + WorkerThread[] _workers; boolean _stopped; long _timeToSleep; @@ -185,7 +197,6 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur _haPlanners = haPlanners; } - @Inject AgentManager _agentMgr; @Inject @@ -231,6 +242,15 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur return Status.Alert; } + if (!VmHaEnabled.valueIn(host.getDataCenterId())) { + String message = String.format("Unable to investigate the host %s (%d), VM high availability manager is disabled.", host.getName(), hostId); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendHostAlert(host, message); + return Status.Alert; + } + Status hostState = null; for (Investigator investigator : investigators) { hostState = investigator.isAgentAlive(host); @@ -260,6 +280,15 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur return; } + if (!VmHaEnabled.valueIn(host.getDataCenterId())) { + String message = String.format("Unable to schedule restart for VMs on host %s (%d), VM high availability manager is disabled.", host.getName(), host.getId()); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendHostAlert(host, message); + return; + } + logger.warn("Scheduling restart for VMs on host " + host.getId() + "-" + host.getName()); final List vms = _instanceDao.listByHostId(host.getId()); @@ -314,12 +343,21 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur } @Override - public void scheduleStop(VMInstanceVO vm, long hostId, WorkType type) { + public boolean scheduleStop(VMInstanceVO vm, long hostId, WorkType type) { assert (type == WorkType.CheckStop || type == WorkType.ForceStop || type == WorkType.Stop); if (_haDao.hasBeenScheduled(vm.getId(), type)) { logger.info("There's already a job scheduled to stop " + vm); - return; + return false; + } + + if (!VmHaEnabled.valueIn(vm.getDataCenterId())) { + String message = String.format("Unable to schedule stop for the VM %s (%d) on host %d, VM high availability manager is disabled.", vm.getName(), vm.getId(), hostId); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendVMAlert(vm, message); + return false; } HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), type, Step.Scheduled, hostId, vm.getState(), 0, vm.getUpdated()); @@ -328,6 +366,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur logger.debug("Scheduled " + work); } wakeupWorkers(); + return true; } protected void wakeupWorkers() { @@ -339,17 +378,37 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur @Override public boolean scheduleMigration(final VMInstanceVO vm) { - if (vm.getHostId() != null) { - final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.Migration, Step.Scheduled, vm.getHostId(), vm.getState(), 0, vm.getUpdated()); - _haDao.persist(work); - logger.info("Scheduled migration work of VM " + vm.getUuid() + " from host " + _hostDao.findById(vm.getHostId()) + " with HAWork " + work); - wakeupWorkers(); + if (vm.getHostId() == null) { + return false; } + + if (!VmHaEnabled.valueIn(vm.getDataCenterId())) { + String message = String.format("Unable to schedule migration for the VM %s (%d) on host %d, VM high availability manager is disabled.", vm.getName(), vm.getId(), vm.getHostId()); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendVMAlert(vm, message); + return false; + } + + final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.Migration, Step.Scheduled, vm.getHostId(), vm.getState(), 0, vm.getUpdated()); + _haDao.persist(work); + logger.info("Scheduled migration work of VM " + vm.getUuid() + " from host " + _hostDao.findById(vm.getHostId()) + " with HAWork " + work); + wakeupWorkers(); return true; } @Override public void scheduleRestart(VMInstanceVO vm, boolean investigate) { + if (!VmHaEnabled.valueIn(vm.getDataCenterId())) { + String message = String.format("Unable to schedule restart for the VM %s (%d), VM high availability manager is disabled.", vm.getName(), vm.getId()); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendVMAlert(vm, message); + return; + } + logger.debug("HA schedule restart"); Long hostId = vm.getHostId(); if (hostId == null) { @@ -440,7 +499,6 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur } wakeupWorkers(); - } private void startVm(VirtualMachine vm, Map params, @@ -737,13 +795,23 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur } @Override - public void scheduleDestroy(VMInstanceVO vm, long hostId) { + public boolean scheduleDestroy(VMInstanceVO vm, long hostId) { + if (!VmHaEnabled.valueIn(vm.getDataCenterId())) { + String message = String.format("Unable to schedule destroy for the VM %s (%d) on host %d, VM high availability manager is disabled.", vm.getName(), vm.getId(), hostId); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendVMAlert(vm, message); + return false; + } + final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.Destroy, Step.Scheduled, hostId, vm.getState(), 0, vm.getUpdated()); _haDao.persist(work); if (logger.isDebugEnabled()) { logger.debug("Scheduled " + work.toString()); } wakeupWorkers(); + return true; } @Override @@ -892,7 +960,17 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur private void processWork(final HaWorkVO work) { final WorkType wt = work.getWorkType(); + final VMInstanceVO vm = _instanceDao.findById(work.getInstanceId()); try { + if (vm != null && !VmHaEnabled.valueIn(vm.getDataCenterId())) { + if (logger.isDebugEnabled()) { + logger.debug(String.format("VM high availability manager is disabled, rescheduling the HA work %s, for the VM %s (id) to retry later in case VM high availability manager is enabled on retry attempt", work, vm.getName(), vm.getId())); + } + long nextTime = getRescheduleTime(wt); + rescheduleWork(work, nextTime); + return; + } + Long nextTime = null; if (wt == WorkType.Migration) { nextTime = migrate(work); @@ -921,9 +999,10 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur // if restart failed in the middle due to exception, VM state may has been changed // recapture into the HA worker so that it can really continue in it next turn - VMInstanceVO vm = _instanceDao.findById(work.getInstanceId()); - work.setUpdateTime(vm.getUpdated()); - work.setPreviousState(vm.getState()); + if (vm != null) { + work.setUpdateTime(vm.getUpdated()); + work.setPreviousState(vm.getState()); + } } finally { if (!Step.Done.equals(work.getStep())) { if (work.getTimesTried() >= _maxRetries) { @@ -1128,11 +1207,33 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur public ConfigKey[] getConfigKeys() { return new ConfigKey[] {TimeBetweenCleanup, MigrationMaxRetries, TimeToSleep, TimeBetweenFailures, StopRetryInterval, RestartRetryInterval, MigrateRetryInterval, InvestigateRetryInterval, - HAWorkers, ForceHA, KvmHAFenceHostIfHeartbeatFailsOnStorage}; + HAWorkers, ForceHA, VmHaEnabled, VmHaAlertsEnabled, KvmHAFenceHostIfHeartbeatFailsOnStorage}; } @Override public int expungeWorkItemsByVmList(List vmIds, Long batchSize) { return _haDao.expungeByVmList(vmIds, batchSize); } + + private void sendVMAlert(VMInstanceVO vm, String message) { + if (vm == null || !VmHaAlertsEnabled.valueIn(vm.getDataCenterId())) { + return; + } + AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM; + if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) { + alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER; + } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { + alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY; + } else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) { + alertType = AlertManager.AlertType.ALERT_TYPE_SSVM; + } + _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), message, message); + } + + private void sendHostAlert(HostVO host, String message) { + if (host == null || !VmHaAlertsEnabled.valueIn(host.getDataCenterId())) { + return; + } + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), message, message); + } } diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 22837389620..94dc9169823 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -41,6 +41,7 @@ import com.cloud.alert.AlertManager; import com.cloud.cpu.CPU; import com.cloud.exception.StorageConflictException; import com.cloud.exception.StorageUnavailableException; +import com.cloud.ha.HighAvailabilityManagerImpl; import com.cloud.host.HostTagVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; @@ -1363,6 +1364,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, throw new CloudRuntimeException("Cannot perform maintain when resource state is " + hostState + ", hostId = " + hostId); } + final List vms = _vmDao.listByHostId(hostId); + if (CollectionUtils.isNotEmpty(vms) && !HighAvailabilityManagerImpl.VmHaEnabled.valueIn(host.getDataCenterId())) { + throw new CloudRuntimeException(String.format("Cannot perform maintain for the host %s (%d) as there are running VMs on it and VM high availability manager is disabled", host.getName(), hostId)); + } + final MaintainAnswer answer = (MaintainAnswer)_agentMgr.easySend(hostId, new MaintainCommand()); if (answer == null || !answer.getResult()) { logger.warn("Unable to send MaintainCommand to host: " + hostId); @@ -1382,8 +1388,6 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, /* TODO: move below to listener */ if (host.getType() == Host.Type.Routing) { - - final List vms = _vmDao.listByHostId(hostId); if (vms.size() == 0) { return true; } @@ -2841,7 +2845,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, logger.debug("Cannot transmit host " + host.getId() + " to Disabled state", e); } for (final VMInstanceVO vm : vms) { - if ((! HighAvailabilityManager.ForceHA.value() && !vm.isHaEnabled()) || vm.getState() == State.Stopping) { + if ((!HighAvailabilityManager.ForceHA.value() && !vm.isHaEnabled()) || vm.getState() == State.Stopping) { logger.debug(String.format("Stopping %s as a part of hostDelete for %s",vm, host)); try { _haMgr.scheduleStop(vm, host.getId(), WorkType.Stop); diff --git a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java index 74897967a2f..27b6e1b1b4c 100644 --- a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java +++ b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.ha; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -32,6 +33,7 @@ import javax.inject.Inject; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContext; import org.apache.logging.log4j.LogManager; @@ -174,10 +176,15 @@ public class HighAvailabilityManagerImplTest { public void scheduleRestartForVmsOnHost() { Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing); Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); Mockito.lenient().when(_instanceDao.listByHostId(42l)).thenReturn(Arrays.asList(Mockito.mock(VMInstanceVO.class))); Mockito.when(_podDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(HostPodVO.class)); Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(DataCenterVO.class)); + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); } @@ -189,11 +196,25 @@ public class HighAvailabilityManagerImplTest { highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); } + @Test + public void scheduleRestartForVmsOnHostHADisabled() { + Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing); + Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); + } + @Test public void scheduleRestartForVmsOnHostNonEmptyVMList() { Mockito.when(hostVO.getId()).thenReturn(1l); Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing); Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.XenServer); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); List vms = new ArrayList(); VMInstanceVO vm1 = Mockito.mock(VMInstanceVO.class); Mockito.lenient().when(vm1.getHostId()).thenReturn(1l); @@ -206,6 +227,7 @@ public class HighAvailabilityManagerImplTest { //Mockito.when(vm2.getInstanceName()).thenReturn("r-2-VM"); Mockito.when(vm2.getType()).thenReturn(VirtualMachine.Type.DomainRouter); Mockito.when(vm2.isHaEnabled()).thenReturn(true); + Mockito.when(vm2.getDataCenterId()).thenReturn(1L); vms.add(vm2); Mockito.when(_instanceDao.listByHostId(Mockito.anyLong())).thenReturn(vms); Mockito.when(_instanceDao.findByUuid(vm1.getUuid())).thenReturn(vm1); @@ -216,12 +238,125 @@ public class HighAvailabilityManagerImplTest { Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class)); Mockito.when(_serviceOfferingDao.findById(vm1.getServiceOfferingId())).thenReturn(Mockito.mock(ServiceOfferingVO.class)); + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); } + @Test + public void scheduleRestartHADisabled() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + highAvailabilityManager.scheduleRestart(vm, true); + } + + @Test + public void scheduleRestartHostNotSupported() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(vm.getHostId()).thenReturn(1L); + Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.VMware); + + highAvailabilityManager.scheduleRestart(vm, true); + } + + @Test + public void scheduleStop() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(_haDao.hasBeenScheduled(vm.getId(), WorkType.Stop)).thenReturn(false); + Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class)); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + + assertTrue(highAvailabilityManager.scheduleStop(vm, 1L, WorkType.Stop)); + } + + @Test + public void scheduleStopHADisabled() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(_haDao.hasBeenScheduled(vm.getId(), WorkType.Stop)).thenReturn(false); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + assertFalse(highAvailabilityManager.scheduleStop(vm, 1L, WorkType.Stop)); + } + + @Test + public void scheduleMigration() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(vm.getHostId()).thenReturn(1L); + Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class)); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + + assertTrue(highAvailabilityManager.scheduleMigration(vm)); + } + + @Test + public void scheduleMigrationHADisabled() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getHostId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + assertFalse(highAvailabilityManager.scheduleMigration(vm)); + } + + @Test + public void scheduleDestroy() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class)); + + assertTrue(highAvailabilityManager.scheduleDestroy(vm, 1L)); + } + + @Test + public void scheduleDestroyHADisabled() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + assertFalse(highAvailabilityManager.scheduleDestroy(vm, 1L)); + } + @Test public void investigateHostStatusSuccess() { Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); // Set the list of investigators, CheckOnAgentInvestigator suffices for now Investigator investigator = Mockito.mock(CheckOnAgentInvestigator.class); List investigators = new ArrayList(); @@ -230,12 +365,17 @@ public class HighAvailabilityManagerImplTest { // Mock isAgentAlive to return host status as Down Mockito.when(investigator.isAgentAlive(hostVO)).thenReturn(Status.Down); + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + assertTrue(highAvailabilityManager.investigate(1l) == Status.Down); } @Test public void investigateHostStatusFailure() { Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); // Set the list of investigators, CheckOnAgentInvestigator suffices for now // Also no need to mock isAgentAlive() as actual implementation returns null Investigator investigator = Mockito.mock(CheckOnAgentInvestigator.class); @@ -243,9 +383,25 @@ public class HighAvailabilityManagerImplTest { investigators.add(investigator); highAvailabilityManager.setInvestigators(investigators); + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + assertNull(highAvailabilityManager.investigate(1l)); } + @Test + public void investigateHostStatusHADisabled() { + Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + assertTrue(highAvailabilityManager.investigate(1L) == Status.Alert); + } + private void processWorkWithRetryCount(int count, Step expectedStep) { assertNotNull(processWorkMethod); HaWorkVO work = new HaWorkVO(1l, VirtualMachine.Type.User, WorkType.Migration, Step.Scheduled, 1l, VirtualMachine.State.Running, count, 12345678l);