CLOUDSTACK-8628: Send an alert when fencing a KVM host failed

Also change the logging a bit so that you get useful logs when
not running on DEBUG level

Signed-off-by: Wido den Hollander <wido@widodh.nl>

This closes #580

server/src/com/cloud/ha/KVMFencer.java

@@ -26,6 +26,7 @@ import javax.naming.ConfigurationException;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.AgentManager;
+import com.cloud.alert.AlertManager;
 import com.cloud.agent.api.FenceAnswer;
 import com.cloud.agent.api.FenceCommand;
 import com.cloud.exception.AgentUnavailableException;
@@ -48,6 +49,8 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
     @Inject
     AgentManager _agentMgr;
     @Inject
+    AlertManager _alertMgr;
+    @Inject
     ResourceManager _resourceMgr;
 
     @Override
@@ -75,18 +78,22 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
     @Override
     public Boolean fenceOff(VirtualMachine vm, Host host) {
         if (host.getHypervisorType() != HypervisorType.KVM && host.getHypervisorType() != HypervisorType.LXC) {
-            s_logger.debug("Don't know how to fence non kvm hosts " + host.getHypervisorType());
+            s_logger.warn("Don't know how to fence non kvm hosts " + host.getHypervisorType());
             return null;
         }
 
         List<HostVO> hosts = _resourceMgr.listAllHostsInCluster(host.getClusterId());
         FenceCommand fence = new FenceCommand(vm, host);
 
+        int i = 0;
         for (HostVO h : hosts) {
             if (h.getHypervisorType() == HypervisorType.KVM || h.getHypervisorType() == HypervisorType.LXC) {
                 if (h.getStatus() != Status.Up) {
                     continue;
                 }
+
+                i++;
+
                 if (h.getId() == host.getId()) {
                     continue;
                 }
@@ -94,14 +101,10 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
                 try {
                     answer = (FenceAnswer)_agentMgr.send(h.getId(), fence);
                 } catch (AgentUnavailableException e) {
-                    if (s_logger.isDebugEnabled()) {
+                    s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable");
-                        s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
-                    }
                     continue;
                 } catch (OperationTimedoutException e) {
-                    if (s_logger.isDebugEnabled()) {
+                    s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable");
-                        s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
-                    }
                     continue;
                 }
                 if (answer != null && answer.getResult()) {
@@ -110,9 +113,12 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
             }
         }
 
-        if (s_logger.isDebugEnabled()) {
+        _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(),
-            s_logger.debug("Unable to fence off " + vm.toString() + " on " + host.toString());
+                            "Unable to fence off host: " + host.getId(),
-        }
+                            "Fencing off host " + host.getId() + " did not succeed after asking " + i + " hosts. " +
+                            "Check Agent logs for more information.");
+
+        s_logger.error("Unable to fence off " + vm.toString() + " on " + host.toString());
 
         return false;
     }

server/test/com/cloud/ha/KVMFencerTest.java

@@ -31,6 +31,7 @@ import org.mockito.Mockito;
 import org.mockito.runners.MockitoJUnitRunner;
 
 import com.cloud.agent.AgentManager;
+import com.cloud.alert.AlertManager;
 import com.cloud.agent.api.FenceAnswer;
 import com.cloud.agent.api.FenceCommand;
 import com.cloud.exception.AgentUnavailableException;
@@ -50,6 +51,8 @@ public class KVMFencerTest {
     @Mock
     AgentManager agentManager;
     @Mock
+    AlertManager alertMgr;
+    @Mock
     ResourceManager resourceManager;
 
     KVMFencer fencer;
@@ -58,6 +61,7 @@ public class KVMFencerTest {
     public void setup() {
         fencer = new KVMFencer();
         fencer._agentMgr = agentManager;
+        fencer._alertMgr = alertMgr;
         fencer._hostDao = hostDao;
         fencer._resourceMgr = resourceManager;
     }
@@ -67,6 +71,8 @@ public class KVMFencerTest {
         HostVO host = Mockito.mock(HostVO.class);
         Mockito.when(host.getClusterId()).thenReturn(1l);
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         Mockito.when(host.getStatus()).thenReturn(Status.Up);
         Mockito.when(host.getId()).thenReturn(1l);
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -80,6 +86,8 @@ public class KVMFencerTest {
         HostVO host = Mockito.mock(HostVO.class);
         Mockito.when(host.getClusterId()).thenReturn(1l);
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         Mockito.when(host.getStatus()).thenReturn(Status.Down);
         Mockito.when(host.getId()).thenReturn(1l);
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -94,12 +102,16 @@ public class KVMFencerTest {
         Mockito.when(host.getClusterId()).thenReturn(1l);
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(host.getStatus()).thenReturn(Status.Up);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(1l);
 
         HostVO secondHost = Mockito.mock(HostVO.class);
         Mockito.when(secondHost.getClusterId()).thenReturn(1l);
         Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+        Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+        Mockito.when(secondHost.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(2l);
 
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -118,12 +130,16 @@ public class KVMFencerTest {
         Mockito.when(host.getClusterId()).thenReturn(1l);
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(host.getStatus()).thenReturn(Status.Up);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(1l);
 
         HostVO secondHost = Mockito.mock(HostVO.class);
         Mockito.when(secondHost.getClusterId()).thenReturn(1l);
         Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+        Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+        Mockito.when(secondHost.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(2l);
 
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -141,12 +157,16 @@ public class KVMFencerTest {
         Mockito.when(host.getClusterId()).thenReturn(1l);
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(host.getStatus()).thenReturn(Status.Up);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(1l);
 
         HostVO secondHost = Mockito.mock(HostVO.class);
         Mockito.when(secondHost.getClusterId()).thenReturn(1l);
         Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
         Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+        Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+        Mockito.when(secondHost.getPodId()).thenReturn(1l);
         Mockito.when(host.getId()).thenReturn(2l);
 
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -165,6 +185,8 @@ public class KVMFencerTest {
         Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.Any);
         Mockito.when(host.getStatus()).thenReturn(Status.Down);
         Mockito.when(host.getId()).thenReturn(1l);
+        Mockito.when(host.getDataCenterId()).thenReturn(1l);
+        Mockito.when(host.getPodId()).thenReturn(1l);
         VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
 
         Mockito.when(resourceManager.listAllHostsInCluster(1l)).thenReturn(Collections.singletonList(host));