/** * Copyright (C) 2010 Cloud.com, Inc. All rights reserved. * * This software is licensed under the GNU General Public License v3 or later. * * It is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or any later version. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . * */ package com.cloud.consoleproxy; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.SocketChannel; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Date; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.ejb.Local; import javax.naming.ConfigurationException; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.AgentControlAnswer; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckVirtualMachineAnswer; import com.cloud.agent.api.CheckVirtualMachineCommand; import com.cloud.agent.api.Command; import com.cloud.agent.api.ConsoleAccessAuthenticationAnswer; import com.cloud.agent.api.ConsoleAccessAuthenticationCommand; import com.cloud.agent.api.ConsoleProxyLoadReportCommand; import com.cloud.agent.api.MigrateCommand; import com.cloud.agent.api.PrepareForMigrationCommand; import com.cloud.agent.api.RebootCommand; import com.cloud.agent.api.StartConsoleProxyAnswer; import com.cloud.agent.api.StartConsoleProxyCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StopAnswer; import com.cloud.agent.api.StopCommand; import com.cloud.agent.api.proxy.ConsoleProxyLoadAnswer; import com.cloud.async.AsyncJobExecutor; import com.cloud.async.AsyncJobManager; import com.cloud.async.AsyncJobVO; import com.cloud.async.BaseAsyncJobExecutor; import com.cloud.cluster.ClusterManager; import com.cloud.configuration.Config; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; import com.cloud.dc.Vlan.VlanType; import com.cloud.dc.VlanVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; import com.cloud.dc.dao.VlanDao; import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeploymentPlan; import com.cloud.domain.DomainVO; import com.cloud.event.EventState; import com.cloud.event.EventTypes; import com.cloud.event.EventVO; import com.cloud.event.dao.EventDao; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.StorageUnavailableException; import com.cloud.ha.HighAvailabilityManager; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.info.ConsoleProxyConnectionInfo; import com.cloud.info.ConsoleProxyInfo; import com.cloud.info.ConsoleProxyLoadInfo; import com.cloud.info.ConsoleProxyStatus; import com.cloud.info.RunningHostCountInfo; import com.cloud.info.RunningHostInfoAgregator; import com.cloud.info.RunningHostInfoAgregator.ZoneHostInfo; import com.cloud.maid.StackMaid; import com.cloud.network.IpAddrAllocator; import com.cloud.network.IpAddrAllocator.networkInfo; import com.cloud.network.NetworkConfigurationVO; import com.cloud.network.NetworkManager; import com.cloud.network.dao.IPAddressDao; import com.cloud.offering.NetworkOffering; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.GuestOSVO; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePoolVO; import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.User; import com.cloud.user.dao.AccountDao; import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.component.Adapters; import com.cloud.utils.component.ComponentLocator; import com.cloud.utils.component.Inject; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.Transaction; import com.cloud.utils.events.SubscriptionMgr; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExecutionException; import com.cloud.utils.net.NetUtils; import com.cloud.vm.ConsoleProxyVO; import com.cloud.vm.NicProfile; import com.cloud.vm.State; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Event; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VmManager; import com.cloud.vm.dao.ConsoleProxyDao; import com.cloud.vm.dao.VMInstanceDao; import com.google.gson.Gson; import com.google.gson.GsonBuilder; // // Possible console proxy state transition cases // Creating -> Destroyed // Creating -> Stopped --> Starting -> Running // HA -> Stopped -> Starting -> Running // Migrating -> Running (if previous state is Running before it enters into Migrating state // Migrating -> Stopped (if previous state is not Running before it enters into Migrating state) // Running -> HA (if agent lost connection) // Stopped -> Destroyed // // Creating state indicates of record creating and IP address allocation are ready, it is a transient // state which will soon be switching towards Running if everything goes well. // Stopped state indicates the readiness of being able to start (has storage and IP resources allocated) // Starting state can only be entered from Stopped states // // Starting, HA, Migrating, Creating and Running state are all counted as "Open" for available capacity calculation // because sooner or later, it will be driven into Running state // @Local(value = { ConsoleProxyManager.class }) public class ConsoleProxyManagerImpl implements ConsoleProxyManager, VirtualMachineManager, AgentHook { private static final Logger s_logger = Logger.getLogger(ConsoleProxyManagerImpl.class); private static final int DEFAULT_FIND_HOST_RETRY_COUNT = 2; private static final int DEFAULT_CAPACITY_SCAN_INTERVAL = 30000; // 30 // seconds private static final int EXECUTOR_SHUTDOWN_TIMEOUT = 1000; // 1 second private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 3; // 3 // seconds private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC = 180; // 3 // minutes private static final int API_WAIT_TIMEOUT = 5000; // 5 seconds (in // milliseconds) private static final int STARTUP_DELAY = 60000; // 60 seconds private int _consoleProxyPort = ConsoleProxyManager.DEFAULT_PROXY_VNC_PORT; private int _consoleProxyUrlPort = ConsoleProxyManager.DEFAULT_PROXY_URL_PORT; private String _mgmt_host; private int _mgmt_port = 8250; private String _name; private Adapters _consoleProxyAllocators; @Inject private ConsoleProxyDao _consoleProxyDao; @Inject private DataCenterDao _dcDao; @Inject private VlanDao _vlanDao; @Inject private VMTemplateDao _templateDao; @Inject private IPAddressDao _ipAddressDao; @Inject private VolumeDao _volsDao; @Inject private HostPodDao _podDao; @Inject private HostDao _hostDao; @Inject private ConfigurationDao _configDao; @Inject private VMInstanceDao _instanceDao; @Inject private AccountDao _accountDao; @Inject private VMTemplateHostDao _vmTemplateHostDao; @Inject private AgentManager _agentMgr; @Inject private StorageManager _storageMgr; @Inject private HighAvailabilityManager _haMgr; @Inject NetworkManager _networkMgr; @Inject AccountManager _accountMgr; @Inject private EventDao _eventDao; @Inject GuestOSDao _guestOSDao = null; @Inject ServiceOfferingDao _offeringDao; @Inject NetworkOfferingDao _networkOfferingDao; private IpAddrAllocator _IpAllocator; private ConsoleProxyListener _listener; private ServiceOfferingVO _serviceOffering; private int _networkRate; private int _multicastRate; private VMTemplateVO _template; NetworkOfferingVO _publicNetworkOffering; NetworkOfferingVO _managementNetworkOffering; NetworkOfferingVO _linkLocalNetworkOffering; @Inject private AsyncJobManager _asyncMgr; @Inject private VmManager _vmMgr; private final ScheduledExecutorService _capacityScanScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("CP-Scan")); private final ExecutorService _requestHandlerScheduler = Executors.newCachedThreadPool(new NamedThreadFactory("Request-handler")); private long _capacityScanInterval = DEFAULT_CAPACITY_SCAN_INTERVAL; private int _capacityPerProxy = ConsoleProxyManager.DEFAULT_PROXY_CAPACITY; private int _standbyCapacity = ConsoleProxyManager.DEFAULT_STANDBY_CAPACITY; private int _proxyRamSize; private int _find_host_retry = DEFAULT_FIND_HOST_RETRY_COUNT; private int _ssh_retry; private int _ssh_sleep; private boolean _use_lvm; private boolean _use_storage_vm; private String _domain; private String _instance; // private String _privateNetmask; private int _proxyCmdPort = DEFAULT_PROXY_CMD_PORT; private int _proxySessionTimeoutValue = DEFAULT_PROXY_SESSION_TIMEOUT; private boolean _sslEnabled = false; private final GlobalLock _capacityScanLock = GlobalLock.getInternLock(getCapacityScanLockName()); private final GlobalLock _allocProxyLock = GlobalLock.getInternLock(getAllocProxyLockName()); @Override public ConsoleProxyInfo assignProxy(final long dataCenterId, final long vmId) { final Pair result = new Pair(this, null); _requestHandlerScheduler.execute(new Runnable() { @Override public void run() { Transaction txn = Transaction.open(Transaction.CLOUD_DB); try { ConsoleProxyVO proxy = doAssignProxy(dataCenterId, vmId); synchronized (result) { result.second(proxy); result.notifyAll(); } } catch (Throwable e) { s_logger.warn("Unexpected exception " + e.getMessage(), e); } finally { StackMaid.current().exitCleanup(); txn.close(); } } }); synchronized (result) { try { result.wait(API_WAIT_TIMEOUT); } catch (InterruptedException e) { s_logger.info("Waiting for console proxy assignment is interrupted"); } } ConsoleProxyVO proxy = result.second(); if (proxy == null) return null; return new ConsoleProxyInfo(proxy.isSslEnabled(), proxy.getPublicIpAddress(), _consoleProxyPort, proxy.getPort(), _configDao.getValue("consoleproxy.url.domain")); } public ConsoleProxyVO doAssignProxy(long dataCenterId, long vmId) { ConsoleProxyVO proxy = null; VMInstanceVO vm = _instanceDao.findById(vmId); if (vm == null) { s_logger.warn("VM " + vmId + " no longer exists, return a null proxy for vm:" + vmId); return null; } Boolean[] proxyFromStoppedPool = new Boolean[1]; if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { try { proxy = getOrAllocProxyResource(dataCenterId, vmId, proxyFromStoppedPool); } finally { _allocProxyLock.unlock(); } } else { s_logger.error("Unable to acquire synchronization lock to get/allocate proxy resource for vm :" + vmId + ". Previous console proxy allocation is taking too long"); } if (proxy == null) { s_logger.warn("Unable to find or allocate console proxy resource"); return null; } long proxyVmId = proxy.getId(); GlobalLock proxyLock = GlobalLock.getInternLock(getProxyLockName(proxyVmId)); try { if (proxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { try { proxy = startProxy(proxyVmId, 0); if (proxy == null) { // // We had a situation with multi-pod configuration, // where // storage allocation of the console proxy VM may // succeed, but later-on starting of it // may fail because of running out of computing resource // (CPU/memory). We // currently don't support moving storage to another pod // on the fly, to deal // with the situation we will destroy this proxy VM and // let it the whole proxy VM // creation process re-start again, by hoping that new // storage and computing // resource may be allocated and assigned in another pod // if (s_logger.isInfoEnabled()) s_logger.info("Unable to start console proxy, proxy vm Id : " + proxyVmId + " will recycle it and restart a new one"); destroyProxy(proxyVmId, 0); return null; } else { if (s_logger.isTraceEnabled()) s_logger.trace("Console proxy " + proxy.getName() + " is started"); // if it is a new assignment or a changed assignment, // update the // record if (vm.getProxyId() == null || vm.getProxyId().longValue() != proxy.getId()) _instanceDao.updateProxyId(vmId, proxy.getId(), DateUtil.currentGMTTime()); proxy.setSslEnabled(_sslEnabled); if (_sslEnabled) proxy.setPort(443); else proxy.setPort(80); return proxy; } } finally { proxyLock.unlock(); } } else { s_logger.error("Unable to acquire synchronization lock to start console proxy " + proxyVmId + " for vm: " + vmId + ". It takes too long to start the proxy"); return null; } } finally { proxyLock.releaseRef(); } } private ConsoleProxyVO getOrAllocProxyResource(long dataCenterId, long vmId, Boolean[] proxyFromStoppedPool) { ConsoleProxyVO proxy = null; VMInstanceVO vm = this._instanceDao.findById(vmId); if (vm != null && vm.getState() != State.Running) { if (s_logger.isInfoEnabled()) s_logger.info("Detected that vm : " + vmId + " is not currently at running state, we will fail the proxy assignment for it"); return null; } if (vm != null && vm.getProxyId() != null) { proxy = _consoleProxyDao.findById(vm.getProxyId()); if (proxy != null) { if (!isInAssignableState(proxy)) { if (s_logger.isInfoEnabled()) s_logger.info("A previous assigned proxy is not assignable now, reassign console proxy for user vm : " + vmId); proxy = null; } else { if (_consoleProxyDao.getProxyActiveLoad(proxy.getId()) < _capacityPerProxy || hasPreviousSession(proxy, vm)) { if (s_logger.isTraceEnabled()) s_logger.trace("Assign previous allocated console proxy for user vm : " + vmId); if (proxy.getActiveSession() >= _capacityPerProxy) s_logger.warn("Assign overloaded proxy to user VM as previous session exists, user vm : " + vmId); } else { proxy = null; } } } } if (proxy == null) proxy = assignProxyFromRunningPool(dataCenterId); if (proxy == null) { if (s_logger.isInfoEnabled()) s_logger.info("No running console proxy is available, check to see if we can bring up a stopped one for data center : " + dataCenterId); proxy = assignProxyFromStoppedPool(dataCenterId); if (proxy == null) { if (s_logger.isInfoEnabled()) s_logger.info("No stopped console proxy is available, need to allocate a new console proxy for data center : " + dataCenterId); proxy = startNew(dataCenterId); } else { if (s_logger.isInfoEnabled()) s_logger.info("Found a stopped console proxy, bring it up to running pool. proxy vm id : " + proxy.getId() + ", data center : " + dataCenterId); proxyFromStoppedPool[0] = new Boolean(true); } } return proxy; } private static boolean isInAssignableState(ConsoleProxyVO proxy) { // console proxies that are in states of being able to serve user VM State state = proxy.getState(); if (state == State.Running || state == State.Starting || state == State.Creating || state == State.Migrating) return true; return false; } private boolean hasPreviousSession(ConsoleProxyVO proxy, VMInstanceVO vm) { ConsoleProxyStatus status = null; try { GsonBuilder gb = new GsonBuilder(); gb.setVersion(1.3); Gson gson = gb.create(); byte[] details = proxy.getSessionDetails(); status = gson.fromJson(details != null ? new String(details, Charset.forName("US-ASCII")) : null, ConsoleProxyStatus.class); } catch (Throwable e) { s_logger.warn("Unable to parse proxy session details : " + proxy.getSessionDetails()); } if (status != null && status.getConnections() != null) { ConsoleProxyConnectionInfo[] connections = status.getConnections(); for (int i = 0; i < connections.length; i++) { long taggedVmId = 0; if (connections[i].tag != null) { try { taggedVmId = Long.parseLong(connections[i].tag); } catch (NumberFormatException e) { s_logger.warn("Unable to parse console proxy connection info passed through tag: " + connections[i].tag, e); } } if (taggedVmId == vm.getId()) return true; } // // even if we are not in the list, it may because we haven't // received load-update yet // wait until session time // if (DateUtil.currentGMTTime().getTime() - vm.getProxyAssignTime().getTime() < _proxySessionTimeoutValue) return true; return false; } else { s_logger.error("No proxy load info on an overloaded proxy ?"); return false; } } @Override public ConsoleProxyVO startProxy(long proxyVmId, long startEventId) { try { return start(proxyVmId, startEventId); } catch (StorageUnavailableException e) { s_logger.warn("Exception while trying to start console proxy", e); return null; } catch (InsufficientCapacityException e) { s_logger.warn("Exception while trying to start console proxy", e); return null; } catch (ConcurrentOperationException e) { s_logger.warn("Exception while trying to start console proxy", e); return null; } } public ConsoleProxyVO start2(long proxyVmId, long startEventId) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException { ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId); DeploymentPlan plan = new DataCenterDeployment(proxy.getDataCenterId(), 1); return _vmMgr.start(proxy, plan); } @Override @DB public ConsoleProxyVO start(long proxyId, long startEventId) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException { AsyncJobExecutor asyncExecutor = BaseAsyncJobExecutor.getCurrentExecutor(); if (asyncExecutor != null) { AsyncJobVO job = asyncExecutor.getJob(); if (s_logger.isInfoEnabled()) s_logger.info("Start console proxy " + proxyId + ", update async job-" + job.getId()); _asyncMgr.updateAsyncJobAttachment(job.getId(), "console_proxy", proxyId); } ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyId); if (proxy == null || proxy.getRemoved() != null) { s_logger.debug("proxy is not found: " + proxyId); return null; } /* * // don't insert event here, it may be called multiple times! * saveStartedEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, * EventTypes.EVENT_PROXY_START, "Starting console proxy with Id: " + * proxyId, startEventId); */ if (s_logger.isTraceEnabled()) { s_logger.trace("Starting console proxy if it is not started, proxy vm id : " + proxyId); } for (int i = 0; i < 2; i++) { State state = proxy.getState(); if (state == State.Starting /* || state == State.Migrating */) { if (s_logger.isDebugEnabled()) s_logger.debug("Waiting console proxy to be ready, proxy vm id : " + proxyId + " proxy VM state : " + state.toString()); if (proxy.getPrivateIpAddress() == null || connect(proxy.getPrivateIpAddress(), _proxyCmdPort) != null) { if (proxy.getPrivateIpAddress() == null) s_logger.warn("Retruning a proxy that is being started but private IP has not been allocated yet, proxy vm id : " + proxyId); else s_logger.warn("Waiting console proxy to be ready timed out, proxy vm id : " + proxyId); // TODO, it is very tricky here, if the startup process // takes too long and it timed out here, // we may give back a proxy that is not fully ready for // functioning } return proxy; } if (state == State.Running) { if (s_logger.isTraceEnabled()) s_logger.trace("Console proxy is already started: " + proxy.getName()); return proxy; } DataCenterVO dc = _dcDao.findById(proxy.getDataCenterId()); HostPodVO pod = _podDao.findById(proxy.getPodId()); List sps = _storageMgr.getStoragePoolsForVm(proxy.getId()); StoragePoolVO sp = sps.get(0); // FIXME HashSet avoid = new HashSet(); HostVO routingHost = (HostVO) _agentMgr.findHost(Host.Type.Routing, dc, pod, sp, _serviceOffering, _template, proxy, null, avoid); if (routingHost == null) { if (s_logger.isDebugEnabled()) { String msg = "Unable to find a routing host for " + proxy.toString() + " in pod " + pod.getId(); s_logger.debug(msg); throw new CloudRuntimeException(msg); } } // to ensure atomic state transition to Starting state if (!_consoleProxyDao.updateIf(proxy, Event.StartRequested, routingHost.getId())) { if (s_logger.isDebugEnabled()) { ConsoleProxyVO temp = _consoleProxyDao.findById(proxyId); s_logger.debug("Unable to start console proxy " + proxy.getName() + " because it is not in a startable state : " + ((temp != null) ? temp.getState().toString() : "null")); } continue; } try { Answer answer = null; int retry = _find_host_retry; // Console proxy VM will be running at routing hosts as routing // hosts have public access to outside network do { if (s_logger.isDebugEnabled()) { s_logger.debug("Trying to start console proxy on host " + routingHost.getName()); } String privateIpAddress = allocPrivateIpAddress(proxy.getDataCenterId(), routingHost.getPodId(), proxy.getId(), proxy.getPrivateMacAddress()); if (privateIpAddress == null && (_IpAllocator != null && !_IpAllocator.exteralIpAddressAllocatorEnabled())) { String msg = "Unable to allocate private ip addresses for " + proxy.getName() + " in pod " + pod.getId(); s_logger.debug(msg); throw new CloudRuntimeException(msg); } proxy.setPrivateIpAddress(privateIpAddress); String guestIpAddress = _dcDao.allocateLinkLocalPrivateIpAddress(proxy.getDataCenterId(), routingHost.getPodId(), proxy.getId()); proxy.setGuestIpAddress(guestIpAddress); _consoleProxyDao.updateIf(proxy, Event.OperationRetry, routingHost.getId()); proxy = _consoleProxyDao.findById(proxy.getId()); List vols = _storageMgr.prepare(proxy, routingHost); if (vols == null || vols.size() == 0) { String msg = "Unable to prepare storage for " + proxy.getName() + " in pod " + pod.getId(); s_logger.debug(msg); throw new CloudRuntimeException(msg); } // Determine the VM's OS description String guestOSDescription; GuestOSVO guestOS = _guestOSDao.findById(proxy.getGuestOSId()); if (guestOS == null) { String msg = "Could not find guest OS description for OSId " + proxy.getGuestOSId() + " for vm: " + proxy.getName(); s_logger.debug(msg); throw new CloudRuntimeException(msg); } else { guestOSDescription = guestOS.getDisplayName(); } // _storageMgr.share(proxy, vols, null, true); // carry the console proxy port info over so that we don't // need to configure agent on this StartConsoleProxyCommand cmdStart = new StartConsoleProxyCommand(_networkRate, _multicastRate, _proxyCmdPort, proxy, proxy.getName(), "", vols, Integer.toString(_consoleProxyPort), Integer.toString(_consoleProxyUrlPort), _mgmt_host, _mgmt_port, _sslEnabled, guestOSDescription); if (s_logger.isDebugEnabled()) s_logger.debug("Sending start command for console proxy " + proxy.getName() + " to " + routingHost.getName()); try { answer = _agentMgr.send(routingHost.getId(), cmdStart); s_logger.debug("StartConsoleProxy Answer: " + (answer != null ? answer : "null")); if (s_logger.isDebugEnabled()) s_logger.debug("Received answer on starting console proxy " + proxy.getName() + " on " + routingHost.getName()); if (answer != null && answer.getResult()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Console proxy " + proxy.getName() + " started on " + routingHost.getName()); } if (answer instanceof StartConsoleProxyAnswer) { StartConsoleProxyAnswer rAnswer = (StartConsoleProxyAnswer) answer; if (rAnswer.getPrivateIpAddress() != null) { proxy.setPrivateIpAddress(rAnswer.getPrivateIpAddress()); } if (rAnswer.getPrivateMacAddress() != null) { proxy.setPrivateMacAddress(rAnswer.getPrivateMacAddress()); } } final EventVO event = new EventVO(); event.setUserId(User.UID_SYSTEM); event.setAccountId(Account.ACCOUNT_ID_SYSTEM); event.setType(EventTypes.EVENT_PROXY_START); event.setLevel(EventVO.LEVEL_INFO); event.setStartId(startEventId); event.setDescription("Console proxy started - " + proxy.getName()); _eventDao.persist(event); break; } s_logger.debug("Unable to start " + proxy.toString() + " on host " + routingHost.toString() + " due to " + answer.getDetails()); } catch (OperationTimedoutException e) { if (e.isActive()) { s_logger.debug("Unable to start vm " + proxy.getName() + " due to operation timed out and it is active so scheduling a restart."); _haMgr.scheduleRestart(proxy, true); return null; } } catch (AgentUnavailableException e) { s_logger.debug("Agent " + routingHost.toString() + " was unavailable to start VM " + proxy.getName()); } avoid.add(routingHost); proxy.setPrivateIpAddress(null); freePrivateIpAddress(privateIpAddress, proxy.getDataCenterId(), proxy.getId()); proxy.setGuestIpAddress(null); _dcDao.releaseLinkLocalPrivateIpAddress(guestIpAddress, proxy.getDataCenterId(), proxy.getId()); _storageMgr.unshare(proxy, vols, routingHost); } while (--retry > 0 && (routingHost = (HostVO) _agentMgr .findHost(Host.Type.Routing, dc, pod, sp, _serviceOffering, _template, proxy, null, avoid)) != null); if (routingHost == null || retry <= 0) { SubscriptionMgr.getInstance().notifySubscribers( ConsoleProxyManager.ALERT_SUBJECT, this, new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_START_FAILURE, proxy.getDataCenterId(), proxy.getId(), proxy, "Unable to find a routing host to run")); final EventVO event = new EventVO(); event.setUserId(User.UID_SYSTEM); event.setAccountId(Account.ACCOUNT_ID_SYSTEM); event.setType(EventTypes.EVENT_PROXY_START); event.setLevel(EventVO.LEVEL_ERROR); event.setStartId(startEventId); event.setDescription("Starting console proxy failed due to unable to find a host - " + proxy.getName()); _eventDao.persist(event); throw new ExecutionException("Couldn't find a routingHost to run console proxy"); } _consoleProxyDao.updateIf(proxy, Event.OperationSucceeded, routingHost.getId()); if (s_logger.isDebugEnabled()) { s_logger.debug("Console proxy is now started, vm id : " + proxy.getId()); } // If starting the console proxy failed due to the external // firewall not being reachable, send an alert. if (answer != null && answer.getDetails() != null && answer.getDetails().equals("firewall")) { SubscriptionMgr.getInstance().notifySubscribers( ConsoleProxyManager.ALERT_SUBJECT, this, new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_FIREWALL_ALERT, proxy.getDataCenterId(), proxy.getId(), proxy, null)); } SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this, new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_UP, proxy.getDataCenterId(), proxy.getId(), proxy, null)); return proxy; } catch (Throwable thr) { s_logger.warn("Unexpected exception: ", thr); SubscriptionMgr.getInstance().notifySubscribers( ConsoleProxyManager.ALERT_SUBJECT, this, new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_START_FAILURE, proxy.getDataCenterId(), proxy.getId(), proxy, "Unexpected exception: " + thr.getMessage())); /* * final EventVO event = new EventVO(); * event.setUserId(User.UID_SYSTEM); * event.setAccountId(Account.ACCOUNT_ID_SYSTEM); * event.setType(EventTypes.EVENT_PROXY_START); * event.setLevel(EventVO.LEVEL_ERROR); * event.setStartId(startEventId); event.setDescription( * "Starting console proxy failed due to unhandled exception - " * + proxy.getName()); _eventDao.persist(event); */ Transaction txn = Transaction.currentTxn(); try { txn.start(); String privateIpAddress = proxy.getPrivateIpAddress(); if (privateIpAddress != null) { proxy.setPrivateIpAddress(null); freePrivateIpAddress(privateIpAddress, proxy.getDataCenterId(), proxy.getId()); } _consoleProxyDao.updateIf(proxy, Event.OperationFailed, null); txn.commit(); } catch (Exception e) { s_logger.error("Caught exception during error recovery"); } if (thr instanceof StorageUnavailableException) { throw (StorageUnavailableException) thr; } else if (thr instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) thr; } else if (thr instanceof ExecutionException) { s_logger.error("Error while starting console proxy due to " + thr.getMessage()); } else { s_logger.error("Error while starting console proxy ", thr); } return null; } } s_logger.warn("Starting console proxy encounters non-startable situation"); return null; } public ConsoleProxyVO assignProxyFromRunningPool(long dataCenterId) { if (s_logger.isTraceEnabled()) s_logger.trace("Assign console proxy from running pool for request from data center : " + dataCenterId); ConsoleProxyAllocator allocator = getCurrentAllocator(); assert (allocator != null); List runningList = _consoleProxyDao.getProxyListInStates(dataCenterId, State.Running); if (runningList != null && runningList.size() > 0) { if (s_logger.isTraceEnabled()) { s_logger.trace("Running proxy pool size : " + runningList.size()); for (ConsoleProxyVO proxy : runningList) s_logger.trace("Running proxy instance : " + proxy.getName()); } List> l = _consoleProxyDao.getProxyLoadMatrix(); Map loadInfo = new HashMap(); if (l != null) { for (Pair p : l) { loadInfo.put(p.first(), p.second()); if (s_logger.isTraceEnabled()) { s_logger.trace("Running proxy instance allocation load { proxy id : " + p.first() + ", load : " + p.second() + "}"); } } } return allocator.allocProxy(runningList, loadInfo, dataCenterId); } else { if (s_logger.isTraceEnabled()) s_logger.trace("Empty running proxy pool for now in data center : " + dataCenterId); } return null; } public ConsoleProxyVO assignProxyFromStoppedPool(long dataCenterId) { List l = _consoleProxyDao.getProxyListInStates(dataCenterId, State.Creating, State.Starting, State.Stopped, State.Migrating); if (l != null && l.size() > 0) return l.get(0); return null; } public ConsoleProxyVO startNewConsoleProxy(long dataCenterId) { if (s_logger.isDebugEnabled()) s_logger.debug("Assign console proxy from a newly started instance for request from data center : " + dataCenterId); Map context = createProxyInstance(dataCenterId); long proxyVmId = (Long) context.get("proxyVmId"); if (proxyVmId == 0) { if (s_logger.isTraceEnabled()) s_logger.trace("Creating proxy instance failed, data center id : " + dataCenterId); // release critical system resource on failure if (context.get("publicIpAddress") != null) freePublicIpAddress((String) context.get("publicIpAddress"), dataCenterId, 0); return null; } ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId); //allocProxyStorage(dataCenterId, proxyVmId); if (proxy != null) { SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this, new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATED, dataCenterId, proxy.getId(), proxy, null)); return proxy; } else { if (s_logger.isDebugEnabled()) s_logger.debug("Unable to allocate console proxy storage, remove the console proxy record from DB, proxy id: " + proxyVmId); SubscriptionMgr.getInstance().notifySubscribers( ConsoleProxyManager.ALERT_SUBJECT, this, new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE, dataCenterId, proxyVmId, null, "Unable to allocate storage")); destroyProxyDBOnly(proxyVmId); } return null; } public ConsoleProxyVO startNew(long dataCenterId) { if (s_logger.isDebugEnabled()) s_logger.debug("Assign console proxy from a newly started instance for request from data center : " + dataCenterId); Map context = createProxyInstance(dataCenterId); long proxyVmId = (Long) context.get("proxyVmId"); if (proxyVmId == 0) { if (s_logger.isTraceEnabled()) s_logger.trace("Creating proxy instance failed, data center id : " + dataCenterId); // release critical system resource on failure if (context.get("publicIpAddress") != null) freePublicIpAddress((String) context.get("publicIpAddress"), dataCenterId, 0); return null; } ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId); //allocProxyStorage(dataCenterId, proxyVmId); if (proxy != null) { SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this, new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATED, dataCenterId, proxy.getId(), proxy, null)); return proxy; } else { if (s_logger.isDebugEnabled()) s_logger.debug("Unable to allocate console proxy storage, remove the console proxy record from DB, proxy id: " + proxyVmId); SubscriptionMgr.getInstance().notifySubscribers( ConsoleProxyManager.ALERT_SUBJECT, this, new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE, dataCenterId, proxyVmId, null, "Unable to allocate storage")); destroyProxyDBOnly(proxyVmId); } return null; } @DB protected Map createProxyInstance(long dataCenterId) { Map context = new HashMap(); String publicIpAddress = null; Transaction txn = Transaction.currentTxn(); try { DataCenterVO dc = _dcDao.findById(dataCenterId); assert (dc != null); context.put("dc", dc); // this will basically allocate the pod based on data center id as // we use system user id here Set avoidPods = new HashSet(); Pair pod = null; networkInfo publicIpAndVlan = null; // About MAC address allocation // MAC address used by User VM is inherited from DomR MAC address, // with the least 16 bits overrided. to avoid // potential conflicts, domP will mask bit 31 // String[] macAddresses = _dcDao.getNextAvailableMacAddressPair(dataCenterId, (1L << 31)); String privateMacAddress = macAddresses[0]; String publicMacAddress = macAddresses[1]; macAddresses = _dcDao.getNextAvailableMacAddressPair(dataCenterId, (1L << 31)); String guestMacAddress = macAddresses[0]; while ((pod = _agentMgr.findPod(_template, _serviceOffering, dc, Account.ACCOUNT_ID_SYSTEM, avoidPods)) != null) { publicIpAndVlan = allocPublicIpAddress(dataCenterId, pod.first().getId(), publicMacAddress); if (publicIpAndVlan == null) { s_logger.warn("Unable to allocate public IP address for console proxy vm in data center : " + dataCenterId + ", pod=" + pod.first().getId()); avoidPods.add(pod.first().getId()); } else { break; } } if (pod == null || publicIpAndVlan == null) { String msg = "Unable to allocate pod for console proxy vm in data center : " + dataCenterId; s_logger.warn(msg); throw new CloudRuntimeException(msg); } long id = _consoleProxyDao.getNextInSequence(Long.class, "id"); context.put("publicIpAddress", publicIpAndVlan._ipAddr); context.put("pod", pod); if (s_logger.isDebugEnabled()) { s_logger.debug("Pod allocated " + pod.first().getName()); } String cidrNetmask = NetUtils.getCidrNetmask(pod.first().getCidrSize()); // Find the VLAN ID, VLAN gateway, and VLAN netmask for // publicIpAddress publicIpAddress = publicIpAndVlan._ipAddr; String vlanGateway = publicIpAndVlan._gateWay; String vlanNetmask = publicIpAndVlan._netMask; AccountVO systemAccount = _accountMgr.getSystemAccount(); txn.start(); ConsoleProxyVO proxy; String name = VirtualMachineName.getConsoleProxyName(id, _instance).intern(); proxy = new ConsoleProxyVO(id, _serviceOffering.getId(), name, guestMacAddress, null, NetUtils.getLinkLocalNetMask(), privateMacAddress, null, cidrNetmask, _template.getId(), _template.getGuestOSId(), publicMacAddress, publicIpAddress, vlanNetmask, publicIpAndVlan._vlanDbId, publicIpAndVlan._vlanid, pod.first().getId(), dataCenterId, systemAccount.getDomainId(), systemAccount.getId(), vlanGateway, null, dc.getDns1(), dc.getDns2(), _domain, _proxyRamSize, 0); proxy.setLastHostId(pod.second()); proxy = _consoleProxyDao.persist(proxy); long proxyVmId = proxy.getId(); final EventVO event = new EventVO(); event.setUserId(User.UID_SYSTEM); event.setAccountId(Account.ACCOUNT_ID_SYSTEM); event.setType(EventTypes.EVENT_PROXY_CREATE); event.setLevel(EventVO.LEVEL_INFO); event.setDescription("New console proxy created - " + proxy.getName()); _eventDao.persist(event); txn.commit(); context.put("proxyVmId", proxyVmId); return context; } catch (Throwable e) { s_logger.error("Unexpected exception : ", e); context.put("proxyVmId", (long) 0); return context; } } protected Map createProxyInstance2(long dataCenterId) { long id = _consoleProxyDao.getNextInSequence(Long.class, "id"); String name = VirtualMachineName.getConsoleProxyName(id, _instance); DataCenterVO dc = _dcDao.findById(dataCenterId); AccountVO systemAcct = _accountMgr.getSystemAccount(); DataCenterDeployment plan = new DataCenterDeployment(dataCenterId, 1); List defaultOffering = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemVmPublicNetwork); List offerings = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemVmControlNetwork, NetworkOfferingVO.SystemVmManagementNetwork); List> networks = new ArrayList>(offerings.size() + 1); NicProfile defaultNic = new NicProfile(); defaultNic.setDefaultNic(true); defaultNic.setDeviceId(2); networks.add(new Pair(_networkMgr.setupNetworkConfiguration(systemAcct, defaultOffering.get(0), plan), defaultNic)); for (NetworkOfferingVO offering : offerings) { networks.add(new Pair(_networkMgr.setupNetworkConfiguration(systemAcct, offering, plan), null)); } ConsoleProxyVO proxy = new ConsoleProxyVO(id, _serviceOffering.getId(), name, _template.getId(), _template.getGuestOSId(), dataCenterId, systemAcct.getDomainId(), systemAcct.getId(), 0); proxy = _consoleProxyDao.persist(proxy); try { VirtualMachineProfile vmProfile = _vmMgr.allocate(proxy, _template, _serviceOffering, networks, plan, systemAcct); } catch (InsufficientCapacityException e) { s_logger.warn("InsufficientCapacity", e); throw new CloudRuntimeException("Insufficient capacity exception", e); } catch (StorageUnavailableException e) { s_logger.warn("Unable to contact storage", e); throw new CloudRuntimeException("Unable to contact storage", e); } Map context = new HashMap(); context.put("dc", dc); // context.put("publicIpAddress", publicIpAndVlan._ipAddr); HostPodVO pod = _podDao.findById(proxy.getPodId()); context.put("pod", pod); context.put("proxyVmId", proxy.getId()); return context; } @DB protected ConsoleProxyVO allocProxyStorage(long dataCenterId, long proxyVmId) { ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId); assert (proxy != null); DataCenterVO dc = _dcDao.findById(dataCenterId); HostPodVO pod = _podDao.findById(proxy.getPodId()); final AccountVO account = _accountDao.findById(Account.ACCOUNT_ID_SYSTEM); try { List vols = _storageMgr.create(account, proxy, _template, dc, pod, _serviceOffering, null, 0); if (vols == null) { s_logger.error("Unable to alloc storage for console proxy"); return null; } Transaction txn = Transaction.currentTxn(); txn.start(); // update pool id ConsoleProxyVO vo = _consoleProxyDao.findById(proxy.getId()); _consoleProxyDao.update(proxy.getId(), vo); // kick the state machine _consoleProxyDao.updateIf(proxy, Event.OperationSucceeded, null); txn.commit(); return proxy; } catch (StorageUnavailableException e) { s_logger.error("Unable to alloc storage for console proxy: ", e); return null; } catch (ExecutionException e) { s_logger.error("Unable to alloc storage for console proxy: ", e); return null; } } private networkInfo allocPublicIpAddress(long dcId, long podId, String macAddr) { if (_IpAllocator != null && _IpAllocator.exteralIpAddressAllocatorEnabled()) { IpAddrAllocator.IpAddr ip = _IpAllocator.getPublicIpAddress(macAddr, dcId, podId); networkInfo net = new networkInfo(ip.ipaddr, ip.netMask, ip.gateway, null, "untagged"); return net; } Pair ipAndVlan = _vlanDao.assignIpAddress(dcId, Account.ACCOUNT_ID_SYSTEM, DomainVO.ROOT_DOMAIN, VlanType.VirtualNetwork, true); if (ipAndVlan == null) { s_logger.debug("Unable to get public ip address (type=Virtual) for console proxy vm for data center : " + dcId); ipAndVlan = _vlanDao.assignPodDirectAttachIpAddress(dcId, podId, Account.ACCOUNT_ID_SYSTEM, DomainVO.ROOT_DOMAIN); if (ipAndVlan == null) s_logger.debug("Unable to get public ip address (type=DirectAttach) for console proxy vm for data center : " + dcId); } if (ipAndVlan != null) { VlanVO vlan = ipAndVlan.second(); networkInfo net = new networkInfo(ipAndVlan.first(), vlan.getVlanNetmask(), vlan.getVlanGateway(), vlan.getId(), vlan.getVlanId()); return net; } return null; } private String allocPrivateIpAddress(Long dcId, Long podId, Long proxyId, String macAddr) { if (_IpAllocator != null && _IpAllocator.exteralIpAddressAllocatorEnabled()) { return _IpAllocator.getPrivateIpAddress(macAddr, dcId, podId).ipaddr; } else { return _dcDao.allocatePrivateIpAddress(dcId, podId, proxyId); } } private void freePrivateIpAddress(String ipAddress, Long dcId, Long podId) { if (_IpAllocator != null && _IpAllocator.exteralIpAddressAllocatorEnabled()) { _IpAllocator.releasePrivateIpAddress(ipAddress, dcId, podId); } else { _dcDao.releasePrivateIpAddress(ipAddress, dcId, podId); } } private void freePublicIpAddress(String ipAddress, long dcId, long podId) { if (_IpAllocator != null && _IpAllocator.exteralIpAddressAllocatorEnabled()) { _IpAllocator.releasePublicIpAddress(ipAddress, dcId, podId); } else { _ipAddressDao.unassignIpAddress(ipAddress); } } private ConsoleProxyAllocator getCurrentAllocator() { // for now, only one adapter is supported Enumeration it = _consoleProxyAllocators.enumeration(); if (it.hasMoreElements()) return it.nextElement(); return null; } protected String connect(String ipAddress, int port) { for (int i = 0; i <= _ssh_retry; i++) { SocketChannel sch = null; try { if (s_logger.isDebugEnabled()) { s_logger.debug("Trying to connect to " + ipAddress); } sch = SocketChannel.open(); sch.configureBlocking(true); sch.socket().setSoTimeout(5000); InetSocketAddress addr = new InetSocketAddress(ipAddress, port); sch.connect(addr); return null; } catch (IOException e) { if (s_logger.isDebugEnabled()) { s_logger.debug("Could not connect to " + ipAddress); } } finally { if (sch != null) { try { sch.close(); } catch (IOException e) { } } } try { Thread.sleep(_ssh_sleep); } catch (InterruptedException ex) { } } s_logger.debug("Unable to logon to " + ipAddress); return "Unable to connect"; } public void onLoadAnswer(ConsoleProxyLoadAnswer answer) { if (answer.getDetails() == null) return; ConsoleProxyStatus status = null; try { GsonBuilder gb = new GsonBuilder(); gb.setVersion(1.3); Gson gson = gb.create(); status = gson.fromJson(answer.getDetails(), ConsoleProxyStatus.class); } catch (Throwable e) { s_logger.warn("Unable to parse load info from proxy, proxy vm id : " + answer.getProxyVmId() + ", info : " + answer.getDetails()); } if (status != null) { int count = 0; if (status.getConnections() != null) count = status.getConnections().length; byte[] details = null; if (answer.getDetails() != null) details = answer.getDetails().getBytes(Charset.forName("US-ASCII")); _consoleProxyDao.update(answer.getProxyVmId(), count, DateUtil.currentGMTTime(), details); } else { if (s_logger.isTraceEnabled()) s_logger.trace("Unable to get console proxy load info, id : " + answer.getProxyVmId()); _consoleProxyDao.update(answer.getProxyVmId(), 0, DateUtil.currentGMTTime(), null); // TODO : something is wrong with the VM, restart it? } } @Override public void onLoadReport(ConsoleProxyLoadReportCommand cmd) { if (cmd.getLoadInfo() == null) return; ConsoleProxyStatus status = null; try { GsonBuilder gb = new GsonBuilder(); gb.setVersion(1.3); Gson gson = gb.create(); status = gson.fromJson(cmd.getLoadInfo(), ConsoleProxyStatus.class); } catch (Throwable e) { s_logger.warn("Unable to parse load info from proxy, proxy vm id : " + cmd.getProxyVmId() + ", info : " + cmd.getLoadInfo()); } if (status != null) { int count = 0; if (status.getConnections() != null) count = status.getConnections().length; byte[] details = null; if (cmd.getLoadInfo() != null) details = cmd.getLoadInfo().getBytes(Charset.forName("US-ASCII")); _consoleProxyDao.update(cmd.getProxyVmId(), count, DateUtil.currentGMTTime(), details); } else { if (s_logger.isTraceEnabled()) s_logger.trace("Unable to get console proxy load info, id : " + cmd.getProxyVmId()); _consoleProxyDao.update(cmd.getProxyVmId(), 0, DateUtil.currentGMTTime(), null); } } @Override public AgentControlAnswer onConsoleAccessAuthentication(ConsoleAccessAuthenticationCommand cmd) { long vmId = 0; if (cmd.getVmId() != null && cmd.getVmId().isEmpty()) { if (s_logger.isTraceEnabled()) s_logger.trace("Invalid vm id sent from proxy(happens when proxy session has terminated)"); return new ConsoleAccessAuthenticationAnswer(cmd, false); } try { vmId = Long.parseLong(cmd.getVmId()); } catch (NumberFormatException e) { s_logger.error("Invalid vm id " + cmd.getVmId() + " sent from console access authentication", e); return new ConsoleAccessAuthenticationAnswer(cmd, false); } // TODO authentication channel between console proxy VM and management // server needs to be secured, // the data is now being sent through private network, but this is // apparently not enough VMInstanceVO vm = _instanceDao.findById(vmId); if (vm == null) { return new ConsoleAccessAuthenticationAnswer(cmd, false); } if (vm.getHostId() == null) { s_logger.warn("VM " + vmId + " lost host info, failed authentication request"); return new ConsoleAccessAuthenticationAnswer(cmd, false); } HostVO host = _hostDao.findById(vm.getHostId()); if (host == null) { s_logger.warn("VM " + vmId + "'s host does not exist, fail authentication request"); return new ConsoleAccessAuthenticationAnswer(cmd, false); } String sid = cmd.getSid(); if (sid == null || !sid.equals(vm.getVncPassword())) { s_logger.warn("sid " + sid + " in url does not match stored sid " + vm.getVncPassword()); return new ConsoleAccessAuthenticationAnswer(cmd, false); } return new ConsoleAccessAuthenticationAnswer(cmd, true); } private ConsoleProxyVO findConsoleProxyByHost(HostVO host) throws NumberFormatException { String name = host.getName(); long proxyVmId = 0; ConsoleProxyVO proxy = null; if (name != null && name.startsWith("v-")) { String[] tokens = name.split("-"); proxyVmId = Long.parseLong(tokens[1]); proxy = this._consoleProxyDao.findById(proxyVmId); } return proxy; } @Override public void onAgentConnect(HostVO host, StartupCommand cmd) { if (host.getType() == Type.ConsoleProxy) { // TODO we can use this event to mark the proxy is up and // functioning instead of // pinging the console proxy VM command port // // for now, just log a message if (s_logger.isInfoEnabled()) s_logger.info("Console proxy agent is connected. proxy: " + host.getName()); /* update public/private ip address */ if (_IpAllocator != null && _IpAllocator.exteralIpAddressAllocatorEnabled()) { try { ConsoleProxyVO console = findConsoleProxyByHost(host); if (console == null) { s_logger.debug("Can't find console proxy "); return; } console.setPrivateIpAddress(cmd.getPrivateIpAddress()); console.setPrivateNetmask(cmd.getPrivateNetmask()); console.setPublicIpAddress(cmd.getPublicIpAddress()); console.setPublicNetmask(cmd.getPublicNetmask()); _consoleProxyDao.persist(console); } catch (NumberFormatException e) { } } } } @Override public void onAgentDisconnect(long agentId, com.cloud.host.Status state) { if (state == com.cloud.host.Status.Alert || state == com.cloud.host.Status.Disconnected) { // be it either in alert or in disconnected state, the agent process // may be gone in the VM, // we will be reacting to stop the corresponding VM and let the scan // process to HostVO host = _hostDao.findById(agentId); if (host.getType() == Type.ConsoleProxy) { String name = host.getName(); if (s_logger.isInfoEnabled()) s_logger.info("Console proxy agent disconnected, proxy: " + name); if (name != null && name.startsWith("v-")) { String[] tokens = name.split("-"); long proxyVmId = 0; try { proxyVmId = Long.parseLong(tokens[1]); } catch (NumberFormatException e) { s_logger.error("Unexpected exception " + e.getMessage(), e); return; } final ConsoleProxyVO proxy = this._consoleProxyDao.findById(proxyVmId); if (proxy != null) { Long hostId = proxy.getHostId(); // Disable this feature for now, as it conflicts with // the case of allowing user to reboot console proxy // when rebooting happens, we will receive disconnect // here and we can't enter into stopping process, // as when the rebooted one comes up, it will kick off a // newly started one and trigger the process // continue on forever /* * _capacityScanScheduler.execute(new Runnable() { * public void run() { if(s_logger.isInfoEnabled()) * s_logger.info("Stop console proxy " + proxy.getName() * + * " VM because of that the agent running inside it has disconnected" * ); stopProxy(proxy.getId()); } }); */ } else { if (s_logger.isInfoEnabled()) s_logger.info("Console proxy agent disconnected but corresponding console proxy VM no longer exists in DB, proxy: " + name); } } else { assert (false) : "Invalid console proxy name: " + name; } } } } private void checkPendingProxyVMs() { // drive state to change away from transient states List l = _consoleProxyDao.getProxyListInStates(State.Creating); if (l != null && l.size() > 0) { for (ConsoleProxyVO proxy : l) { if (proxy.getLastUpdateTime() == null || (proxy.getLastUpdateTime() != null && System.currentTimeMillis() - proxy.getLastUpdateTime().getTime() > 60000)) { try { ConsoleProxyVO readyProxy = null; if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { try { readyProxy = allocProxyStorage(proxy.getDataCenterId(), proxy.getId()); } finally { _allocProxyLock.unlock(); } if (readyProxy != null) { GlobalLock proxyLock = GlobalLock.getInternLock(getProxyLockName(readyProxy.getId())); try { if (proxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { try { readyProxy = start(readyProxy.getId(), 0); } finally { proxyLock.unlock(); } } else { if (s_logger.isInfoEnabled()) s_logger.info("Unable to acquire synchronization lock to start console proxy : " + readyProxy.getName()); } } finally { proxyLock.releaseRef(); } } } else { if (s_logger.isInfoEnabled()) s_logger.info("Unable to acquire synchronization lock to allocate proxy storage, wait for next turn"); } } catch (StorageUnavailableException e) { s_logger.warn("Storage unavailable", e); } catch (InsufficientCapacityException e) { s_logger.warn("insuffiient capacity", e); } catch (ConcurrentOperationException e) { s_logger.debug("Concurrent operation: " + e.getMessage()); } } } } } private Runnable getCapacityScanTask() { return new Runnable() { @Override public void run() { Transaction txn = Transaction.open(Transaction.CLOUD_DB); try { reallyRun(); } catch (Throwable e) { s_logger.warn("Unexpected exception " + e.getMessage(), e); } finally { StackMaid.current().exitCleanup(); txn.close(); } } private void reallyRun() { if (s_logger.isTraceEnabled()) s_logger.trace("Begin console proxy capacity scan"); // config var for consoleproxy.restart check String restart = _configDao.getValue("consoleproxy.restart"); if (restart != null && restart.equalsIgnoreCase("false")) { s_logger.debug("Capacity scan disabled purposefully, consoleproxy.restart = false. This happens when the primarystorage is in maintenance mode"); return; } Map zoneHostInfoMap = getZoneHostInfo(); if (isServiceReady(zoneHostInfoMap)) { if (s_logger.isTraceEnabled()) s_logger.trace("Service is ready, check to see if we need to allocate standby capacity"); if (!_capacityScanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { if (s_logger.isTraceEnabled()) s_logger.trace("Capacity scan lock is used by others, skip and wait for my turn"); return; } if (s_logger.isTraceEnabled()) s_logger.trace("*** Begining capacity scan... ***"); try { checkPendingProxyVMs(); // scan default data center first long defaultId = 0; // proxy count info by data-centers (zone-id, zone-name, // count) List l = _consoleProxyDao.getDatacenterProxyLoadMatrix(); // running VM session count by data-centers (zone-id, // zone-name, count) List listVmCounts = _consoleProxyDao.getDatacenterSessionLoadMatrix(); // indexing load info by data-center id Map mapVmCounts = new HashMap(); if (listVmCounts != null) for (ConsoleProxyLoadInfo info : listVmCounts) mapVmCounts.put(info.getId(), info); for (ConsoleProxyLoadInfo info : l) { if (info.getName().equals(_instance)) { ConsoleProxyLoadInfo vmInfo = mapVmCounts.get(info.getId()); if (!checkCapacity(info, vmInfo != null ? vmInfo : new ConsoleProxyLoadInfo())) { if (isZoneReady(zoneHostInfoMap, info.getId())) { allocCapacity(info.getId()); } else { if (s_logger.isTraceEnabled()) s_logger.trace("Zone " + info.getId() + " is not ready to alloc standy console proxy"); } } defaultId = info.getId(); break; } } // scan rest of data-centers for (ConsoleProxyLoadInfo info : l) { if (info.getId() != defaultId) { ConsoleProxyLoadInfo vmInfo = mapVmCounts.get(info.getId()); if (!checkCapacity(info, vmInfo != null ? vmInfo : new ConsoleProxyLoadInfo())) { if (isZoneReady(zoneHostInfoMap, info.getId())) { allocCapacity(info.getId()); } else { if (s_logger.isTraceEnabled()) s_logger.trace("Zone " + info.getId() + " is not ready to alloc standy console proxy"); } } } } if (s_logger.isTraceEnabled()) s_logger.trace("*** Stop capacity scan ***"); } finally { _capacityScanLock.unlock(); } } else { if (s_logger.isTraceEnabled()) s_logger.trace("Service is not ready for capacity preallocation, wait for next time"); } if (s_logger.isTraceEnabled()) s_logger.trace("End of console proxy capacity scan"); } }; } private boolean checkCapacity(ConsoleProxyLoadInfo proxyCountInfo, ConsoleProxyLoadInfo vmCountInfo) { if (proxyCountInfo.getCount() * _capacityPerProxy - vmCountInfo.getCount() <= _standbyCapacity) return false; return true; } private void allocCapacity(long dataCenterId) { if (s_logger.isTraceEnabled()) s_logger.trace("Allocate console proxy standby capacity for data center : " + dataCenterId); boolean proxyFromStoppedPool = false; ConsoleProxyVO proxy = assignProxyFromStoppedPool(dataCenterId); if (proxy == null) { if (s_logger.isInfoEnabled()) s_logger.info("No stopped console proxy is available, need to allocate a new console proxy"); if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { try { proxy = startNew(dataCenterId); } finally { _allocProxyLock.unlock(); } } else { if (s_logger.isInfoEnabled()) s_logger.info("Unable to acquire synchronization lock to allocate proxy resource for standby capacity, wait for next scan"); return; } } else { if (s_logger.isInfoEnabled()) s_logger.info("Found a stopped console proxy, bring it up to running pool. proxy vm id : " + proxy.getId()); proxyFromStoppedPool = true; } if (proxy != null) { long proxyVmId = proxy.getId(); GlobalLock proxyLock = GlobalLock.getInternLock(getProxyLockName(proxyVmId)); try { if (proxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { try { proxy = startProxy(proxyVmId, 0); } finally { proxyLock.unlock(); } } else { if (s_logger.isInfoEnabled()) s_logger.info("Unable to acquire synchronization lock to start proxy for standby capacity, proxy vm id : " + proxy.getId()); return; } } finally { proxyLock.releaseRef(); } if (proxy == null) { if (s_logger.isInfoEnabled()) s_logger.info("Unable to start console proxy for standby capacity, proxy vm Id : " + proxyVmId + ", will recycle it and start a new one"); if (proxyFromStoppedPool) destroyProxy(proxyVmId, 0); } else { if (s_logger.isInfoEnabled()) s_logger.info("Console proxy " + proxy.getName() + " is started"); } } } public boolean isServiceReady(Map zoneHostInfoMap) { for (ZoneHostInfo zoneHostInfo : zoneHostInfoMap.values()) { if (isZoneHostReady(zoneHostInfo)) { if (s_logger.isInfoEnabled()) s_logger.info("Zone " + zoneHostInfo.getDcId() + " is ready to launch"); return true; } } return false; } public boolean isZoneReady(Map zoneHostInfoMap, long dataCenterId) { ZoneHostInfo zoneHostInfo = zoneHostInfoMap.get(dataCenterId); if (zoneHostInfo != null && isZoneHostReady(zoneHostInfo)) { VMTemplateVO template = _templateDao.findConsoleProxyTemplate(); HostVO secondaryStorageHost = _storageMgr.getSecondaryStorageHost(dataCenterId); boolean templateReady = false; if (template != null && secondaryStorageHost != null) { VMTemplateHostVO templateHostRef = _vmTemplateHostDao.findByHostTemplate(secondaryStorageHost.getId(), template.getId()); templateReady = (templateHostRef != null) && (templateHostRef.getDownloadState() == Status.DOWNLOADED); } if (templateReady) { List> l = _consoleProxyDao.getDatacenterStoragePoolHostInfo(dataCenterId, _use_lvm); if (l != null && l.size() > 0 && l.get(0).second().intValue() > 0) { return true; } else { if (s_logger.isTraceEnabled()) s_logger.trace("Primary storage is not ready, wait until it is ready to launch console proxy"); } } else { if (s_logger.isTraceEnabled()) s_logger.trace("Zone host is ready, but console proxy template is not ready"); } } return false; } private boolean isZoneHostReady(ZoneHostInfo zoneHostInfo) { int expectedFlags = 0; if (_use_storage_vm) expectedFlags = RunningHostInfoAgregator.ZoneHostInfo.ROUTING_HOST_MASK; else expectedFlags = RunningHostInfoAgregator.ZoneHostInfo.ALL_HOST_MASK; return (zoneHostInfo.getFlags() & expectedFlags) == expectedFlags; } private synchronized Map getZoneHostInfo() { Date cutTime = DateUtil.currentGMTTime(); List l = _hostDao.getRunningHostCounts(new Date(cutTime.getTime() - ClusterManager.DEFAULT_HEARTBEAT_THRESHOLD)); RunningHostInfoAgregator aggregator = new RunningHostInfoAgregator(); if (l.size() > 0) for (RunningHostCountInfo countInfo : l) aggregator.aggregate(countInfo); return aggregator.getZoneHostInfoMap(); } @Override public String getName() { return _name; } @Override public boolean start() { if (s_logger.isInfoEnabled()) s_logger.info("Start console proxy manager"); return true; } @Override public boolean stop() { if (s_logger.isInfoEnabled()) s_logger.info("Stop console proxy manager"); _capacityScanScheduler.shutdownNow(); try { _capacityScanScheduler.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { } _capacityScanLock.releaseRef(); _allocProxyLock.releaseRef(); return true; } @Override public Command cleanup(ConsoleProxyVO vm, String vmName) { if (vmName != null) { return new StopCommand(vm, vmName, VirtualMachineName.getVnet(vmName)); } else if (vm != null) { ConsoleProxyVO vo = vm; return new StopCommand(vo, null); } else { throw new CloudRuntimeException("Shouldn't even be here!"); } } @Override public void completeStartCommand(ConsoleProxyVO vm) { _consoleProxyDao.updateIf(vm, Event.AgentReportRunning, vm.getHostId()); } @Override public void completeStopCommand(ConsoleProxyVO vm) { completeStopCommand(vm, Event.AgentReportStopped); } @DB protected void completeStopCommand(ConsoleProxyVO proxy, Event ev) { Transaction txn = Transaction.currentTxn(); try { txn.start(); String privateIpAddress = proxy.getPrivateIpAddress(); if (privateIpAddress != null) { proxy.setPrivateIpAddress(null); freePrivateIpAddress(privateIpAddress, proxy.getDataCenterId(), proxy.getId()); } String guestIpAddress = proxy.getGuestIpAddress(); if (guestIpAddress != null) { proxy.setGuestIpAddress(null); _dcDao.releaseLinkLocalPrivateIpAddress(guestIpAddress, proxy.getDataCenterId(), proxy.getId()); } if (!_consoleProxyDao.updateIf(proxy, ev, null)) { s_logger.debug("Unable to update the console proxy"); return; } txn.commit(); } catch (Exception e) { s_logger.error("Unable to complete stop command due to ", e); } if (_storageMgr.unshare(proxy, null) == null) { s_logger.warn("Unable to set share to false for " + proxy.getId()); } } @Override public ConsoleProxyVO get(long id) { return _consoleProxyDao.findById(id); } @Override public Long convertToId(String vmName) { if (!VirtualMachineName.isValidConsoleProxyName(vmName, _instance)) { return null; } return VirtualMachineName.getConsoleProxyId(vmName); } @Override public boolean stopProxy(long proxyVmId, long startEventId) { AsyncJobExecutor asyncExecutor = BaseAsyncJobExecutor.getCurrentExecutor(); if (asyncExecutor != null) { AsyncJobVO job = asyncExecutor.getJob(); if (s_logger.isInfoEnabled()) s_logger.info("Stop console proxy " + proxyVmId + ", update async job-" + job.getId()); _asyncMgr.updateAsyncJobAttachment(job.getId(), "console_proxy", proxyVmId); } ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId); if (proxy == null) { if (s_logger.isDebugEnabled()) s_logger.debug("Stopping console proxy failed: console proxy " + proxyVmId + " no longer exists"); return false; } /* * saveStartedEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, * EventTypes.EVENT_PROXY_STOP, "Stopping console proxy with Id: " + * proxyVmId, startEventId); */ try { return stop(proxy, startEventId); } catch (AgentUnavailableException e) { if (s_logger.isDebugEnabled()) s_logger.debug("Stopping console proxy " + proxy.getName() + " faled : exception " + e.toString()); return false; } } @Override public boolean rebootProxy(long proxyVmId, long startEventId) { AsyncJobExecutor asyncExecutor = BaseAsyncJobExecutor.getCurrentExecutor(); if (asyncExecutor != null) { AsyncJobVO job = asyncExecutor.getJob(); if (s_logger.isInfoEnabled()) s_logger.info("Reboot console proxy " + proxyVmId + ", update async job-" + job.getId()); _asyncMgr.updateAsyncJobAttachment(job.getId(), "console_proxy", proxyVmId); } final ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId); if (proxy == null || proxy.getState() == State.Destroyed) { return false; } /* * saveStartedEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, * EventTypes.EVENT_PROXY_REBOOT, "Rebooting console proxy with Id: " + * proxyVmId, startEventId); */ if (proxy.getState() == State.Running && proxy.getHostId() != null) { final RebootCommand cmd = new RebootCommand(proxy.getInstanceName()); final Answer answer = _agentMgr.easySend(proxy.getHostId(), cmd); if (answer != null) { if (s_logger.isDebugEnabled()) s_logger.debug("Successfully reboot console proxy " + proxy.getName()); SubscriptionMgr.getInstance() .notifySubscribers( ConsoleProxyManager.ALERT_SUBJECT, this, new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_REBOOTED, proxy.getDataCenterId(), proxy.getId(), proxy, null)); final EventVO event = new EventVO(); event.setUserId(User.UID_SYSTEM); event.setAccountId(Account.ACCOUNT_ID_SYSTEM); event.setType(EventTypes.EVENT_PROXY_REBOOT); event.setLevel(EventVO.LEVEL_INFO); event.setStartId(startEventId); event.setDescription("Console proxy rebooted - " + proxy.getName()); _eventDao.persist(event); return true; } else { if (s_logger.isDebugEnabled()) s_logger.debug("failed to reboot console proxy : " + proxy.getName()); final EventVO event = new EventVO(); event.setUserId(User.UID_SYSTEM); event.setAccountId(Account.ACCOUNT_ID_SYSTEM); event.setType(EventTypes.EVENT_PROXY_REBOOT); event.setLevel(EventVO.LEVEL_ERROR); event.setStartId(startEventId); event.setDescription("Rebooting console proxy failed - " + proxy.getName()); _eventDao.persist(event); return false; } } else { return startProxy(proxyVmId, 0) != null; } } @Override public boolean destroy(ConsoleProxyVO proxy) throws AgentUnavailableException { return destroyProxy(proxy.getId(), 0); } @Override @DB public boolean destroyProxy(long vmId, long startEventId) { AsyncJobExecutor asyncExecutor = BaseAsyncJobExecutor.getCurrentExecutor(); if (asyncExecutor != null) { AsyncJobVO job = asyncExecutor.getJob(); if (s_logger.isInfoEnabled()) s_logger.info("Destroy console proxy " + vmId + ", update async job-" + job.getId()); _asyncMgr.updateAsyncJobAttachment(job.getId(), "console_proxy", vmId); } ConsoleProxyVO vm = _consoleProxyDao.findById(vmId); if (vm == null || vm.getState() == State.Destroyed) { if (s_logger.isDebugEnabled()) { s_logger.debug("Unable to find vm or vm is destroyed: " + vmId); } return true; } /* * saveStartedEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, * EventTypes.EVENT_PROXY_DESTROY, "Destroying console proxy with Id: " * + vmId, startEventId); */ if (s_logger.isDebugEnabled()) { s_logger.debug("Destroying console proxy vm " + vmId); } if (!_consoleProxyDao.updateIf(vm, Event.DestroyRequested, null)) { s_logger.debug("Unable to destroy the vm because it is not in the correct state: " + vmId); return false; } Transaction txn = Transaction.currentTxn(); List vols = null; try { vols = _volsDao.findByInstance(vmId); if (vols.size() != 0) { _storageMgr.destroy(vm, vols); } return true; } finally { try { txn.start(); // release critical system resources used by the VM before we // delete them if (vm.getPublicIpAddress() != null) freePublicIpAddress(vm.getPublicIpAddress(), vm.getDataCenterId(), vm.getPodId()); vm.setPublicIpAddress(null); _consoleProxyDao.remove(vm.getId()); final EventVO event = new EventVO(); event.setUserId(User.UID_SYSTEM); event.setAccountId(Account.ACCOUNT_ID_SYSTEM); event.setType(EventTypes.EVENT_PROXY_DESTROY); event.setLevel(EventVO.LEVEL_INFO); event.setStartId(startEventId); event.setDescription("Console proxy destroyed - " + vm.getName()); _eventDao.persist(event); txn.commit(); } catch (Exception e) { s_logger.error("Caught this error: ", e); txn.rollback(); return false; } finally { s_logger.debug("console proxy vm is destroyed : " + vm.getName()); } } } @DB public boolean destroyProxyDBOnly(long vmId) { Transaction txn = Transaction.currentTxn(); try { txn.start(); _volsDao.deleteVolumesByInstance(vmId); ConsoleProxyVO proxy = _consoleProxyDao.findById(vmId); if (proxy != null) { if (proxy.getPublicIpAddress() != null) freePublicIpAddress(proxy.getPublicIpAddress(), proxy.getDataCenterId(), proxy.getPodId()); _consoleProxyDao.remove(vmId); final EventVO event = new EventVO(); event.setUserId(User.UID_SYSTEM); event.setAccountId(Account.ACCOUNT_ID_SYSTEM); event.setType(EventTypes.EVENT_PROXY_DESTROY); event.setLevel(EventVO.LEVEL_INFO); event.setDescription("Console proxy destroyed - " + proxy.getName()); _eventDao.persist(event); } txn.commit(); return true; } catch (Exception e) { s_logger.error("Caught this error: ", e); txn.rollback(); return false; } finally { s_logger.debug("console proxy vm is destroyed from DB : " + vmId); } } @Override public boolean stop(ConsoleProxyVO proxy, long startEventId) throws AgentUnavailableException { if (!_consoleProxyDao.updateIf(proxy, Event.StopRequested, proxy.getHostId())) { s_logger.debug("Unable to stop console proxy: " + proxy.toString()); return false; } // IPAddressVO ip = _ipAddressDao.findById(proxy.getPublicIpAddress()); // VlanVO vlan = _vlanDao.findById(new Long(ip.getVlanDbId())); GlobalLock proxyLock = GlobalLock.getInternLock(getProxyLockName(proxy.getId())); try { if (proxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { try { StopCommand cmd = new StopCommand(proxy, true, Integer.toString(_consoleProxyPort), Integer.toString(_consoleProxyUrlPort), proxy.getPublicIpAddress()); try { Long proxyHostId = proxy.getHostId(); if (proxyHostId == null) { s_logger.debug("Unable to stop due to proxy " + proxy.getId() + " as host is no longer available, proxy may already have been stopped"); return false; } StopAnswer answer = (StopAnswer) _agentMgr.send(proxyHostId, cmd); if (answer == null || !answer.getResult()) { s_logger.debug("Unable to stop due to " + (answer == null ? "answer is null" : answer.getDetails())); final EventVO event = new EventVO(); event.setUserId(User.UID_SYSTEM); event.setAccountId(Account.ACCOUNT_ID_SYSTEM); event.setType(EventTypes.EVENT_PROXY_STOP); event.setLevel(EventVO.LEVEL_ERROR); event.setStartId(startEventId); event.setDescription("Stopping console proxy failed due to negative answer from agent - " + proxy.getName()); _eventDao.persist(event); return false; } completeStopCommand(proxy, Event.OperationSucceeded); SubscriptionMgr.getInstance().notifySubscribers( ConsoleProxyManager.ALERT_SUBJECT, this, new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_DOWN, proxy.getDataCenterId(), proxy.getId(), proxy, null)); final EventVO event = new EventVO(); event.setUserId(User.UID_SYSTEM); event.setAccountId(Account.ACCOUNT_ID_SYSTEM); event.setType(EventTypes.EVENT_PROXY_STOP); event.setLevel(EventVO.LEVEL_INFO); event.setStartId(startEventId); event.setDescription("Console proxy stopped - " + proxy.getName()); _eventDao.persist(event); return true; } catch (OperationTimedoutException e) { final EventVO event = new EventVO(); event.setUserId(User.UID_SYSTEM); event.setAccountId(Account.ACCOUNT_ID_SYSTEM); event.setType(EventTypes.EVENT_PROXY_STOP); event.setLevel(EventVO.LEVEL_ERROR); event.setStartId(startEventId); event.setDescription("Stopping console proxy failed due to operation time out - " + proxy.getName()); _eventDao.persist(event); throw new AgentUnavailableException(proxy.getHostId()); } } finally { proxyLock.unlock(); } } else { s_logger.debug("Unable to acquire console proxy lock : " + proxy.toString()); return false; } } finally { proxyLock.releaseRef(); } } @Override public boolean migrate(ConsoleProxyVO proxy, HostVO host) { HostVO fromHost = _hostDao.findById(proxy.getId()); if (!_consoleProxyDao.updateIf(proxy, Event.MigrationRequested, proxy.getHostId())) { s_logger.debug("State for " + proxy.toString() + " has changed so migration can not take place."); return false; } MigrateCommand cmd = new MigrateCommand(proxy.getInstanceName(), host.getPrivateIpAddress(), false); Answer answer = _agentMgr.easySend(fromHost.getId(), cmd); if (answer == null) { return false; } _storageMgr.unshare(proxy, fromHost); return true; } @Override public boolean completeMigration(ConsoleProxyVO proxy, HostVO host) throws AgentUnavailableException, OperationTimedoutException { CheckVirtualMachineCommand cvm = new CheckVirtualMachineCommand(proxy.getInstanceName()); CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) _agentMgr.send(host.getId(), cvm); if (!answer.getResult()) { s_logger.debug("Unable to complete migration for " + proxy.getId()); _consoleProxyDao.updateIf(proxy, Event.AgentReportStopped, null); return false; } State state = answer.getState(); if (state == State.Stopped) { s_logger.warn("Unable to complete migration as we can not detect it on " + host.getId()); _consoleProxyDao.updateIf(proxy, Event.AgentReportStopped, null); return false; } _consoleProxyDao.updateIf(proxy, Event.OperationSucceeded, host.getId()); return true; } @Override public HostVO prepareForMigration(ConsoleProxyVO proxy) throws StorageUnavailableException { VMTemplateVO template = _templateDao.findById(proxy.getTemplateId()); long routerId = proxy.getId(); boolean mirroredVols = proxy.isMirroredVols(); DataCenterVO dc = _dcDao.findById(proxy.getDataCenterId()); HostPodVO pod = _podDao.findById(proxy.getPodId()); List sps = _storageMgr.getStoragePoolsForVm(proxy.getId()); StoragePoolVO sp = sps.get(0); // FIXME List vols = _volsDao.findCreatedByInstance(routerId); String[] storageIps = new String[2]; VolumeVO vol = vols.get(0); storageIps[0] = vol.getHostIp(); if (mirroredVols && (vols.size() == 2)) { storageIps[1] = vols.get(1).getHostIp(); } PrepareForMigrationCommand cmd = new PrepareForMigrationCommand(proxy.getName(), null, storageIps, vols, mirroredVols); HostVO routingHost = null; HashSet avoid = new HashSet(); HostVO fromHost = _hostDao.findById(proxy.getHostId()); if (fromHost.getClusterId() == null) { s_logger.debug("The host is not in a cluster"); return null; } avoid.add(fromHost); while ((routingHost = (HostVO) _agentMgr.findHost(Host.Type.Routing, dc, pod, sp, _serviceOffering, template, proxy, fromHost, avoid)) != null) { avoid.add(routingHost); if (s_logger.isDebugEnabled()) { s_logger.debug("Trying to migrate router to host " + routingHost.getName()); } if (!_storageMgr.share(proxy, vols, routingHost, false)) { s_logger.warn("Can not share " + proxy.getName()); throw new StorageUnavailableException("Can not share " + proxy.getName(), vol); } Answer answer = _agentMgr.easySend(routingHost.getId(), cmd); if (answer != null && answer.getResult()) { return routingHost; } _storageMgr.unshare(proxy, vols, routingHost); } return null; } private String getCapacityScanLockName() { // to improve security, it may be better to return a unique mashed // name(for example MD5 hashed) return "consoleproxy.capacity.scan"; } private String getAllocProxyLockName() { // to improve security, it may be better to return a unique mashed // name(for example MD5 hashed) return "consoleproxy.alloc"; } private String getProxyLockName(long id) { return "consoleproxy." + id; } private Long saveStartedEvent(Long userId, Long accountId, String type, String description, long startEventId) { EventVO event = new EventVO(); event.setUserId(userId); event.setAccountId(accountId); event.setType(type); event.setState(EventState.Started); event.setDescription(description); event.setStartId(startEventId); event = _eventDao.persist(event); if (event != null) return event.getId(); return null; } @Override public boolean configure(String name, Map params) throws ConfigurationException { if (s_logger.isInfoEnabled()) s_logger.info("Start configuring console proxy manager : " + name); _name = name; ComponentLocator locator = ComponentLocator.getCurrentLocator(); ConfigurationDao configDao = locator.getDao(ConfigurationDao.class); if (configDao == null) { throw new ConfigurationException("Unable to get the configuration dao."); } Map configs = configDao.getConfiguration("management-server", params); _proxyRamSize = NumbersUtil.parseInt(configs.get("consoleproxy.ram.size"), DEFAULT_PROXY_VM_RAMSIZE); String value = configs.get("start.retry"); _find_host_retry = NumbersUtil.parseInt(value, DEFAULT_FIND_HOST_RETRY_COUNT); value = configs.get("consoleproxy.cmd.port"); _proxyCmdPort = NumbersUtil.parseInt(value, DEFAULT_PROXY_CMD_PORT); value = configs.get("consoleproxy.sslEnabled"); if (value != null && value.equalsIgnoreCase("true")) _sslEnabled = true; value = configs.get("consoleproxy.capacityscan.interval"); _capacityScanInterval = NumbersUtil.parseLong(value, DEFAULT_CAPACITY_SCAN_INTERVAL); _capacityPerProxy = NumbersUtil.parseInt(configs.get("consoleproxy.session.max"), DEFAULT_PROXY_CAPACITY); _standbyCapacity = NumbersUtil.parseInt(configs.get("consoleproxy.capacity.standby"), DEFAULT_STANDBY_CAPACITY); _proxySessionTimeoutValue = NumbersUtil.parseInt(configs.get("consoleproxy.session.timeout"), DEFAULT_PROXY_SESSION_TIMEOUT); value = configs.get("consoleproxy.port"); if (value != null) _consoleProxyPort = NumbersUtil.parseInt(value, ConsoleProxyManager.DEFAULT_PROXY_VNC_PORT); value = configs.get("consoleproxy.url.port"); if (value != null) _consoleProxyUrlPort = NumbersUtil.parseInt(value, ConsoleProxyManager.DEFAULT_PROXY_URL_PORT); value = configs.get("system.vm.use.local.storage"); if (value != null && value.equalsIgnoreCase("true")) _use_lvm = true; value = configs.get("secondary.storage.vm"); if (value != null && value.equalsIgnoreCase("true")) _use_storage_vm = true; if (s_logger.isInfoEnabled()) { s_logger.info("Console proxy max session soft limit : " + _capacityPerProxy); s_logger.info("Console proxy standby capacity : " + _standbyCapacity); } _domain = configs.get("domain"); if (_domain == null) { _domain = "foo.com"; } _instance = configs.get("instance.name"); if (_instance == null) { _instance = "DEFAULT"; } value = (String) params.get("ssh.sleep"); _ssh_sleep = NumbersUtil.parseInt(value, 5) * 1000; value = (String) params.get("ssh.retry"); _ssh_retry = NumbersUtil.parseInt(value, 3); Map agentMgrConfigs = configDao.getConfiguration("AgentManager", params); _mgmt_host = agentMgrConfigs.get("host"); if (_mgmt_host == null) { s_logger.warn("Critical warning! Please configure your management server host address right after you have started your management server and then restart it, otherwise you won't be able to do console access"); } value = agentMgrConfigs.get("port"); _mgmt_port = NumbersUtil.parseInt(value, 8250); _consoleProxyAllocators = locator.getAdapters(ConsoleProxyAllocator.class); if (_consoleProxyAllocators == null || !_consoleProxyAllocators.isSet()) { throw new ConfigurationException("Unable to get proxy allocators"); } _listener = new ConsoleProxyListener(this); _agentMgr.registerForHostEvents(_listener, true, true, false); Adapters ipAllocators = locator.getAdapters(IpAddrAllocator.class); if (ipAllocators != null && ipAllocators.isSet()) { Enumeration it = ipAllocators.enumeration(); _IpAllocator = it.nextElement(); } HighAvailabilityManager haMgr = locator.getManager(HighAvailabilityManager.class); if (haMgr != null) { haMgr.registerHandler(VirtualMachine.Type.ConsoleProxy, this); } boolean useLocalStorage = Boolean.parseBoolean((String) params.get(Config.SystemVMUseLocalStorage.key())); String networkRateStr = _configDao.getValue("network.throttling.rate"); String multicastRateStr = _configDao.getValue("multicast.throttling.rate"); _networkRate = ((networkRateStr == null) ? 200 : Integer.parseInt(networkRateStr)); _multicastRate = ((multicastRateStr == null) ? 10 : Integer.parseInt(multicastRateStr)); _serviceOffering = new ServiceOfferingVO("Fake Offering For DomP", 1, _proxyRamSize, 0, 0, 0, false, null, NetworkOffering.GuestIpType.Virtualized, useLocalStorage, true, null); _serviceOffering.setUniqueName("Cloud.com-ConsoleProxy"); _serviceOffering = _offeringDao.persistSystemServiceOffering(_serviceOffering); _template = _templateDao.findConsoleProxyTemplate(); if (_template == null) { throw new ConfigurationException("Unable to find the template for console proxy VMs"); } _capacityScanScheduler.scheduleAtFixedRate(getCapacityScanTask(), STARTUP_DELAY, _capacityScanInterval, TimeUnit.MILLISECONDS); if (s_logger.isInfoEnabled()) s_logger.info("Console Proxy Manager is configured."); return true; } protected ConsoleProxyManagerImpl() { } }