From a22ab69bb6c1e45ce43d4c00520d4ea8656d21f2 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 12 Jun 2018 09:42:09 -0300 Subject: [PATCH] Set host into ErrorInMaintenance in case of failure trying to enter Maintenance mode --- .../cloud/resource/ResourceManagerImpl.java | 82 ++++++++++++++++--- .../cloud/servlet/ConsoleProxyServlet.java | 10 ++- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 25bfc4c9c67..8eb53e62587 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -30,6 +30,7 @@ import java.util.Random; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.vm.dao.UserVmDetailsDao; import org.apache.commons.lang.ObjectUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -54,6 +55,8 @@ import org.apache.commons.collections.CollectionUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; +import com.cloud.agent.api.GetVncPortCommand; +import com.cloud.agent.api.GetVncPortAnswer; import com.cloud.agent.api.GetGPUStatsAnswer; import com.cloud.agent.api.GetGPUStatsCommand; import com.cloud.agent.api.GetHostStatsAnswer; @@ -252,6 +255,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, private ConfigurationManager _configMgr; @Inject private ClusterVSMMapDao _clusterVSMMapDao; + @Inject + private UserVmDetailsDao userVmDetailsDao; private final long _nodeId = ManagementServerNode.getManagementServerId(); @@ -1287,6 +1292,68 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } } + /** + * Add VNC details as user VM details for each VM in 'vms' (KVM hosts only) + */ + private void setKVMVncAccess(long hostId, List vms) { + for (VMInstanceVO vm : vms) { + GetVncPortAnswer vmVncPortAnswer = (GetVncPortAnswer) _agentMgr.easySend(hostId, new GetVncPortCommand(vm.getId(), vm.getInstanceName())); + if (vmVncPortAnswer != null) { + userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.address", vmVncPortAnswer.getAddress(), true); + userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.port", String.valueOf(vmVncPortAnswer.getPort()), true); + } + } + } + + /** + * Configure VNC access for host VMs which have failed migrating to another host while trying to enter Maintenance mode + */ + private void configureVncAccessForKVMHostFailedMigrations(HostVO host, List failedMigrations) { + if (host.getHypervisorType().equals(HypervisorType.KVM)) { + _agentMgr.pullAgentOutMaintenance(host.getId()); + setKVMVncAccess(host.getId(), failedMigrations); + _agentMgr.pullAgentToMaintenance(host.getId()); + } + } + + /** + * Set host into ErrorInMaintenance state, as errors occurred during VM migrations. Do the following: + * - Cancel scheduled migrations for those which have already failed + * - Configure VNC access for VMs (KVM hosts only) + */ + private boolean setHostIntoErrorInMaintenance(HostVO host, List failedMigrations) throws NoTransitionException { + s_logger.debug("Unable to migrate " + failedMigrations.size() + " VM(s) from host " + host.getUuid()); + _haMgr.cancelScheduledMigrations(host); + configureVncAccessForKVMHostFailedMigrations(host, failedMigrations); + resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId); + return false; + } + + /** + * Safely transit host into Maintenance mode + */ + private boolean setHostIntoMaintenance(HostVO host) throws NoTransitionException { + s_logger.debug("Host " + host.getUuid() + " entering in Maintenance"); + resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE, + "completed maintenance for host " + host.getId(), 0); + return true; + } + + /** + * Return true if host goes into Maintenance mode, only when: + * - No Running, Migrating or Failed migrations (host_id = last_host_id) for the host + */ + private boolean isHostInMaintenance(HostVO host, List runningVms, List migratingVms, List failedMigrations) throws NoTransitionException { + if (CollectionUtils.isEmpty(runningVms) && CollectionUtils.isEmpty(migratingVms)) { + return CollectionUtils.isEmpty(failedMigrations) ? + setHostIntoMaintenance(host) : + setHostIntoErrorInMaintenance(host, failedMigrations); + } + return false; + } + @Override public boolean checkAndMaintain(final long hostId) { boolean hostInMaintenance = false; @@ -1296,18 +1363,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (host.getType() != Host.Type.Storage) { final List vos = _vmDao.listByHostId(hostId); final List vosMigrating = _vmDao.listVmsMigratingFromHost(hostId); - final List failedMigratedVms = _vmDao.listNonMigratingVmsByHostEqualsLastHost(hostId); - if (vos.isEmpty() && vosMigrating.isEmpty()) { - if (!failedMigratedVms.isEmpty()) { - s_logger.debug("Unable to migrate " + failedMigratedVms.size() + " VM(s) from host " + host.getUuid()); - resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId); - } else { - s_logger.debug("Host " + host.getUuid() + " entering in Maintenance"); - resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId); - hostInMaintenance = true; - ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE, "completed maintenance for host " + hostId, 0); - } - } + final List failedVmMigrations = _vmDao.listNonMigratingVmsByHostEqualsLastHost(hostId); + + hostInMaintenance = isHostInMaintenance(host, vos, vosMigrating, failedVmMigrations); } } catch (final NoTransitionException e) { s_logger.debug("Cannot transmit host " + host.getId() + "to Maintenance state", e); diff --git a/server/src/com/cloud/servlet/ConsoleProxyServlet.java b/server/src/com/cloud/servlet/ConsoleProxyServlet.java index cc788c7b118..8cfaa9fd69b 100644 --- a/server/src/com/cloud/servlet/ConsoleProxyServlet.java +++ b/server/src/com/cloud/servlet/ConsoleProxyServlet.java @@ -35,6 +35,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; +import com.cloud.resource.ResourceState; import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -418,7 +419,14 @@ public class ConsoleProxyServlet extends HttpServlet { StringBuffer sb = new StringBuffer(rootUrl); String host = hostVo.getPrivateIpAddress(); - Pair portInfo = _ms.getVncPort(vm); + Pair portInfo; + if (hostVo.getResourceState().equals(ResourceState.ErrorInMaintenance)) { + UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.address"); + UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.port"); + portInfo = new Pair<>(detailAddress.getValue(), Integer.valueOf(detailPort.getValue())); + } else { + portInfo = _ms.getVncPort(vm); + } if (s_logger.isDebugEnabled()) s_logger.debug("Port info " + portInfo.first());