apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-8628: Send an alert when fencing a KVM host failed

Browse Source 
Also change the logging a bit so that you get useful logs when
not running on DEBUG level

Signed-off-by: Wido den Hollander <wido@widodh.nl>

This closes #580
This commit is contained in:
Wido den Hollander 2015-07-16 16:15:20 +02:00
parent d1f76a2a84
commit fbe3b04a43
2 changed files with 38 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
server/src/com/cloud/ha/KVMFencer.java
View File

@ -26,6 +26,7 @@ import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
import com.cloud.alert.AlertManager;
import com.cloud.agent.api.FenceAnswer;
import com.cloud.agent.api.FenceCommand;
import com.cloud.exception.AgentUnavailableException;
@ -48,6 +49,8 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
@Inject
AgentManager _agentMgr;
@Inject
AlertManager _alertMgr;
@Inject
ResourceManager _resourceMgr;
@Override
@ -75,18 +78,22 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
@Override
public Boolean fenceOff(VirtualMachine vm, Host host) {
if (host.getHypervisorType() != HypervisorType.KVM && host.getHypervisorType() != HypervisorType.LXC) {
s_logger.debug("Don't know how to fence non kvm hosts " + host.getHypervisorType());
s_logger.warn("Don't know how to fence non kvm hosts " + host.getHypervisorType());
return null;
}
List<HostVO> hosts = _resourceMgr.listAllHostsInCluster(host.getClusterId());
FenceCommand fence = new FenceCommand(vm, host);
int i = 0;
for (HostVO h : hosts) {
if (h.getHypervisorType() == HypervisorType.KVM || h.getHypervisorType() == HypervisorType.LXC) {
if (h.getStatus() != Status.Up) {
continue;
}
i++;
if (h.getId() == host.getId()) {
continue;
}
@ -94,14 +101,10 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
try {
answer = (FenceAnswer)_agentMgr.send(h.getId(), fence);
} catch (AgentUnavailableException e) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
}
s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable");
continue;
} catch (OperationTimedoutException e) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
}
s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable");
continue;
}
if (answer != null && answer.getResult()) {
@ -110,9 +113,12 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
}
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Unable to fence off " + vm.toString() + " on " + host.toString());
}
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(),
"Unable to fence off host: " + host.getId(),
"Fencing off host " + host.getId() + " did not succeed after asking " + i + " hosts. " +
"Check Agent logs for more information.");
s_logger.error("Unable to fence off " + vm.toString() + " on " + host.toString());
return false;
}

22
server/test/com/cloud/ha/KVMFencerTest.java
View File

@ -31,6 +31,7 @@ import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import com.cloud.agent.AgentManager;
import com.cloud.alert.AlertManager;
import com.cloud.agent.api.FenceAnswer;
import com.cloud.agent.api.FenceCommand;
import com.cloud.exception.AgentUnavailableException;
@ -50,6 +51,8 @@ public class KVMFencerTest {
@Mock
AgentManager agentManager;
@Mock
AlertManager alertMgr;
@Mock
ResourceManager resourceManager;
KVMFencer fencer;
@ -58,6 +61,7 @@ public class KVMFencerTest {
public void setup() {
fencer = new KVMFencer();
fencer._agentMgr = agentManager;
fencer._alertMgr = alertMgr;
fencer._hostDao = hostDao;
fencer._resourceMgr = resourceManager;
}
@ -67,6 +71,8 @@ public class KVMFencerTest {
HostVO host = Mockito.mock(HostVO.class);
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getDataCenterId()).thenReturn(1l);
Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
Mockito.when(host.getId()).thenReturn(1l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@ -80,6 +86,8 @@ public class KVMFencerTest {
HostVO host = Mockito.mock(HostVO.class);
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getDataCenterId()).thenReturn(1l);
Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getStatus()).thenReturn(Status.Down);
Mockito.when(host.getId()).thenReturn(1l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@ -94,12 +102,16 @@ public class KVMFencerTest {
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
Mockito.when(host.getDataCenterId()).thenReturn(1l);
Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(1l);
HostVO secondHost = Mockito.mock(HostVO.class);
Mockito.when(secondHost.getClusterId()).thenReturn(1l);
Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
Mockito.when(secondHost.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(2l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@ -118,12 +130,16 @@ public class KVMFencerTest {
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
Mockito.when(host.getDataCenterId()).thenReturn(1l);
Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(1l);
HostVO secondHost = Mockito.mock(HostVO.class);
Mockito.when(secondHost.getClusterId()).thenReturn(1l);
Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
Mockito.when(secondHost.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(2l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@ -141,12 +157,16 @@ public class KVMFencerTest {
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
Mockito.when(host.getDataCenterId()).thenReturn(1l);
Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(1l);
HostVO secondHost = Mockito.mock(HostVO.class);
Mockito.when(secondHost.getClusterId()).thenReturn(1l);
Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
Mockito.when(secondHost.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(2l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@ -165,6 +185,8 @@ public class KVMFencerTest {
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.Any);
Mockito.when(host.getStatus()).thenReturn(Status.Down);
Mockito.when(host.getId()).thenReturn(1l);
Mockito.when(host.getDataCenterId()).thenReturn(1l);
Mockito.when(host.getPodId()).thenReturn(1l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
Mockito.when(resourceManager.listAllHostsInCluster(1l)).thenReturn(Collections.singletonList(host));