Set host into ErrorInMaintenance in case of failure trying to enter Maintenance mode

server/src/com/cloud/resource/ResourceManagerImpl.java

@@ -30,6 +30,7 @@ import java.util.Random;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.vm.dao.UserVmDetailsDao;
 import org.apache.commons.lang.ObjectUtils;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
@@ -54,6 +55,8 @@ import org.apache.commons.collections.CollectionUtils;
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
+import com.cloud.agent.api.GetVncPortCommand;
+import com.cloud.agent.api.GetVncPortAnswer;
 import com.cloud.agent.api.GetGPUStatsAnswer;
 import com.cloud.agent.api.GetGPUStatsCommand;
 import com.cloud.agent.api.GetHostStatsAnswer;
@@ -252,6 +255,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
     private ConfigurationManager _configMgr;
     @Inject
     private ClusterVSMMapDao _clusterVSMMapDao;
+    @Inject
+    private UserVmDetailsDao userVmDetailsDao;
 
     private final long _nodeId = ManagementServerNode.getManagementServerId();
 
@@ -1287,6 +1292,68 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
         }
     }
 
+    /**
+     * Add VNC details as user VM details for each VM in 'vms' (KVM hosts only)
+     */
+    private void setKVMVncAccess(long hostId, List<VMInstanceVO> vms) {
+        for (VMInstanceVO vm : vms) {
+            GetVncPortAnswer vmVncPortAnswer = (GetVncPortAnswer) _agentMgr.easySend(hostId, new GetVncPortCommand(vm.getId(), vm.getInstanceName()));
+            if (vmVncPortAnswer != null) {
+                userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.address", vmVncPortAnswer.getAddress(), true);
+                userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.port", String.valueOf(vmVncPortAnswer.getPort()), true);
+            }
+        }
+    }
+
+    /**
+     * Configure VNC access for host VMs which have failed migrating to another host while trying to enter Maintenance mode
+     */
+    private void configureVncAccessForKVMHostFailedMigrations(HostVO host, List<VMInstanceVO> failedMigrations) {
+        if (host.getHypervisorType().equals(HypervisorType.KVM)) {
+            _agentMgr.pullAgentOutMaintenance(host.getId());
+            setKVMVncAccess(host.getId(), failedMigrations);
+            _agentMgr.pullAgentToMaintenance(host.getId());
+        }
+    }
+
+    /**
+     * Set host into ErrorInMaintenance state, as errors occurred during VM migrations. Do the following:
+     * - Cancel scheduled migrations for those which have already failed
+     * - Configure VNC access for VMs (KVM hosts only)
+     */
+    private boolean setHostIntoErrorInMaintenance(HostVO host, List<VMInstanceVO> failedMigrations) throws NoTransitionException {
+        s_logger.debug("Unable to migrate " + failedMigrations.size() + " VM(s) from host " + host.getUuid());
+        _haMgr.cancelScheduledMigrations(host);
+        configureVncAccessForKVMHostFailedMigrations(host, failedMigrations);
+        resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId);
+        return false;
+    }
+
+    /**
+     * Safely transit host into Maintenance mode
+     */
+    private boolean setHostIntoMaintenance(HostVO host) throws NoTransitionException {
+        s_logger.debug("Host " + host.getUuid() + " entering in Maintenance");
+        resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId);
+        ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
+                EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE,
+                "completed maintenance for host " + host.getId(), 0);
+        return true;
+    }
+
+    /**
+     * Return true if host goes into Maintenance mode, only when:
+     * - No Running, Migrating or Failed migrations (host_id = last_host_id) for the host
+     */
+    private boolean isHostInMaintenance(HostVO host, List<VMInstanceVO> runningVms, List<VMInstanceVO> migratingVms, List<VMInstanceVO> failedMigrations) throws NoTransitionException {
+        if (CollectionUtils.isEmpty(runningVms) && CollectionUtils.isEmpty(migratingVms)) {
+            return CollectionUtils.isEmpty(failedMigrations) ?
+                    setHostIntoMaintenance(host) :
+                    setHostIntoErrorInMaintenance(host, failedMigrations);
+        }
+        return false;
+    }
+
     @Override
     public boolean checkAndMaintain(final long hostId) {
         boolean hostInMaintenance = false;
@@ -1296,18 +1363,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
             if (host.getType() != Host.Type.Storage) {
                 final List<VMInstanceVO> vos = _vmDao.listByHostId(hostId);
                 final List<VMInstanceVO> vosMigrating = _vmDao.listVmsMigratingFromHost(hostId);
-                final List<VMInstanceVO> failedMigratedVms = _vmDao.listNonMigratingVmsByHostEqualsLastHost(hostId);
+                final List<VMInstanceVO> failedVmMigrations = _vmDao.listNonMigratingVmsByHostEqualsLastHost(hostId);
-                if (vos.isEmpty() && vosMigrating.isEmpty()) {
+
-                    if (!failedMigratedVms.isEmpty()) {
+                hostInMaintenance = isHostInMaintenance(host, vos, vosMigrating, failedVmMigrations);
-                        s_logger.debug("Unable to migrate " + failedMigratedVms.size() + " VM(s) from host " + host.getUuid());
-                        resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId);
-                    } else {
-                        s_logger.debug("Host " + host.getUuid() + " entering in Maintenance");
-                        resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId);
-                        hostInMaintenance = true;
-                        ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE, "completed maintenance for host " + hostId, 0);
-                    }
-                }
             }
         } catch (final NoTransitionException e) {
             s_logger.debug("Cannot transmit host " + host.getId() + "to Maintenance state", e);

server/src/com/cloud/servlet/ConsoleProxyServlet.java

@@ -35,6 +35,7 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpSession;
 
+import com.cloud.resource.ResourceState;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
@@ -418,7 +419,14 @@ public class ConsoleProxyServlet extends HttpServlet {
         StringBuffer sb = new StringBuffer(rootUrl);
         String host = hostVo.getPrivateIpAddress();
 
-        Pair<String, Integer> portInfo = _ms.getVncPort(vm);
+        Pair<String, Integer> portInfo;
+        if (hostVo.getResourceState().equals(ResourceState.ErrorInMaintenance)) {
+            UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.address");
+            UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.port");
+            portInfo = new Pair<>(detailAddress.getValue(), Integer.valueOf(detailPort.getValue()));
+        } else {
+            portInfo = _ms.getVncPort(vm);
+        }
         if (s_logger.isDebugEnabled())
             s_logger.debug("Port info " + portInfo.first());