// Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. package com.cloud.network.router; import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TimeZone; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.ejb.Local; import javax.naming.ConfigurationException; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.AgentManager.OnError; import com.cloud.agent.Listener; import com.cloud.agent.api.AgentControlAnswer; import com.cloud.agent.api.AgentControlCommand; import com.cloud.agent.api.Answer; import com.cloud.agent.api.BumpUpPriorityCommand; import com.cloud.agent.api.CheckRouterAnswer; import com.cloud.agent.api.CheckRouterCommand; import com.cloud.agent.api.CheckS2SVpnConnectionsAnswer; import com.cloud.agent.api.CheckS2SVpnConnectionsCommand; import com.cloud.agent.api.Command; import com.cloud.agent.api.GetDomRVersionAnswer; import com.cloud.agent.api.GetDomRVersionCmd; import com.cloud.agent.api.ModifySshKeysCommand; import com.cloud.agent.api.NetworkUsageAnswer; import com.cloud.agent.api.NetworkUsageCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StopAnswer; import com.cloud.agent.api.check.CheckSshAnswer; import com.cloud.agent.api.check.CheckSshCommand; import com.cloud.agent.api.routing.DhcpEntryCommand; import com.cloud.agent.api.routing.IpAssocCommand; import com.cloud.agent.api.routing.LoadBalancerConfigCommand; import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.agent.api.routing.RemoteAccessVpnCfgCommand; import com.cloud.agent.api.routing.SavePasswordCommand; import com.cloud.agent.api.routing.SetFirewallRulesCommand; import com.cloud.agent.api.routing.SetPortForwardingRulesCommand; import com.cloud.agent.api.routing.SetPortForwardingRulesVpcCommand; import com.cloud.agent.api.routing.SetStaticNatRulesCommand; import com.cloud.agent.api.routing.VmDataCommand; import com.cloud.agent.api.routing.VpnUsersCfgCommand; import com.cloud.agent.api.to.FirewallRuleTO; import com.cloud.agent.api.to.IpAddressTO; import com.cloud.agent.api.to.LoadBalancerTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.PortForwardingRuleTO; import com.cloud.agent.api.to.StaticNatRuleTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.agent.manager.Commands; import com.cloud.alert.AlertManager; import com.cloud.api.commands.UpgradeRouterCmd; import com.cloud.cluster.ManagementServerHostVO; import com.cloud.cluster.ManagementServerNode; import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; import com.cloud.configuration.ZoneConfig; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenter.NetworkType; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; import com.cloud.dc.Pod; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; import com.cloud.dc.dao.VlanDao; import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.ConnectionException; import com.cloud.exception.InsufficientAddressCapacityException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InsufficientVirtualNetworkCapcityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.StorageUnavailableException; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.IPAddressVO; import com.cloud.network.IpAddress; import com.cloud.network.LoadBalancerVO; import com.cloud.network.Network; import com.cloud.network.Network.GuestType; import com.cloud.network.Network.Provider; import com.cloud.network.Network.Service; import com.cloud.network.NetworkManager; import com.cloud.network.NetworkVO; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.IsolationType; import com.cloud.network.Networks.TrafficType; import com.cloud.network.PhysicalNetworkServiceProvider; import com.cloud.network.PublicIpAddress; import com.cloud.network.RemoteAccessVpn; import com.cloud.network.Site2SiteCustomerGateway; import com.cloud.network.Site2SiteVpnConnection; import com.cloud.network.Site2SiteVpnConnectionVO; import com.cloud.network.SshKeysDistriMonitor; import com.cloud.network.VirtualNetworkApplianceService; import com.cloud.network.VirtualRouterProvider; import com.cloud.network.VirtualRouterProvider.VirtualRouterProviderType; import com.cloud.network.VpnUser; import com.cloud.network.VpnUserVO; import com.cloud.network.addr.PublicIp; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVMMapDao; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; import com.cloud.network.dao.RemoteAccessVpnDao; import com.cloud.network.dao.Site2SiteCustomerGatewayDao; import com.cloud.network.dao.Site2SiteVpnConnectionDao; import com.cloud.network.dao.Site2SiteVpnGatewayDao; import com.cloud.network.dao.VirtualRouterProviderDao; import com.cloud.network.dao.VpnUserDao; import com.cloud.network.lb.LoadBalancingRule; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; import com.cloud.network.lb.LoadBalancingRulesManager; import com.cloud.network.router.VirtualRouter.RedundantState; import com.cloud.network.router.VirtualRouter.Role; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRule.Purpose; import com.cloud.network.rules.PortForwardingRule; import com.cloud.network.rules.RulesManager; import com.cloud.network.rules.StaticNat; import com.cloud.network.rules.StaticNatImpl; import com.cloud.network.rules.StaticNatRule; import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.vpn.Site2SiteVpnManager; import com.cloud.offering.ServiceOffering; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.resource.ResourceManager; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.GuestOSVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.User; import com.cloud.user.UserContext; import com.cloud.user.UserStatisticsVO; import com.cloud.user.UserStatsLogVO; import com.cloud.user.UserVO; import com.cloud.user.dao.UserDao; import com.cloud.user.dao.UserStatisticsDao; import com.cloud.user.dao.UserStatsLogDao; import com.cloud.uservm.UserVm; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.PasswordGenerator; import com.cloud.utils.StringUtils; import com.cloud.utils.component.ComponentLocator; import com.cloud.utils.component.Inject; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.Transaction; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.MacAddress; import com.cloud.utils.net.NetUtils; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.Nic; import com.cloud.vm.NicProfile; import com.cloud.vm.NicVO; import com.cloud.vm.ReservationContext; import com.cloud.vm.ReservationContextImpl; import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineGuru; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VirtualMachineProfile.Param; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; /** * VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack. */ @Local(value = { VirtualNetworkApplianceManager.class, VirtualNetworkApplianceService.class }) public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplianceManager, VirtualNetworkApplianceService, VirtualMachineGuru, Listener { private static final Logger s_logger = Logger.getLogger(VirtualNetworkApplianceManagerImpl.class); String _name; @Inject DataCenterDao _dcDao = null; @Inject VlanDao _vlanDao = null; @Inject FirewallRulesDao _rulesDao = null; @Inject LoadBalancerDao _loadBalancerDao = null; @Inject LoadBalancerVMMapDao _loadBalancerVMMapDao = null; @Inject IPAddressDao _ipAddressDao = null; @Inject VMTemplateDao _templateDao = null; @Inject DomainRouterDao _routerDao = null; @Inject UserDao _userDao = null; @Inject UserStatisticsDao _userStatsDao = null; @Inject HostDao _hostDao = null; @Inject ConfigurationDao _configDao; @Inject HostPodDao _podDao = null; @Inject UserStatsLogDao _userStatsLogDao = null; @Inject AgentManager _agentMgr; @Inject AlertManager _alertMgr; @Inject AccountManager _accountMgr; @Inject ConfigurationManager _configMgr; @Inject ServiceOfferingDao _serviceOfferingDao = null; @Inject UserVmDao _userVmDao; @Inject UserStatisticsDao _statsDao = null; @Inject NetworkOfferingDao _networkOfferingDao = null; @Inject GuestOSDao _guestOSDao = null; @Inject NetworkManager _networkMgr; @Inject VirtualMachineManager _itMgr; @Inject VpnUserDao _vpnUsersDao; @Inject RemoteAccessVpnDao _remoteAccessVpnDao; @Inject RulesManager _rulesMgr; @Inject NetworkDao _networkDao; @Inject LoadBalancingRulesManager _lbMgr; @Inject PortForwardingRulesDao _pfRulesDao; @Inject RemoteAccessVpnDao _vpnDao; @Inject NicDao _nicDao; @Inject VolumeDao _volumeDao = null; @Inject UserVmDetailsDao _vmDetailsDao; @Inject ClusterDao _clusterDao; @Inject ResourceManager _resourceMgr; @Inject PhysicalNetworkServiceProviderDao _physicalProviderDao; @Inject VirtualRouterProviderDao _vrProviderDao; @Inject ManagementServerHostDao _msHostDao; @Inject Site2SiteCustomerGatewayDao _s2sCustomerGatewayDao; @Inject Site2SiteVpnGatewayDao _s2sVpnGatewayDao; @Inject Site2SiteVpnConnectionDao _s2sVpnConnectionDao; @Inject Site2SiteVpnManager _s2sVpnMgr; int _routerRamSize; int _routerCpuMHz; int _retry = 2; String _instance; String _mgmt_host; String _mgmt_cidr; int _routerStatsInterval = 300; int _routerCheckInterval = 30; protected ServiceOfferingVO _offering; private String _dnsBasicZoneUpdates = "all"; private boolean _disable_rp_filter = false; int _routerExtraPublicNics = 2; private int _usageAggregationRange = 1440; private String _usageTimeZone = "GMT"; private final long mgmtSrvrId = MacAddress.getMacAddress().toLong(); private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 5; // 5 seconds ScheduledExecutorService _executor; ScheduledExecutorService _checkExecutor; ScheduledExecutorService _networkStatsUpdateExecutor; Account _systemAcct; @Override public boolean sendSshKeysToHost(Long hostId, String pubKey, String prvKey) { ModifySshKeysCommand cmd = new ModifySshKeysCommand(pubKey, prvKey); final Answer answer = _agentMgr.easySend(hostId, cmd); if (answer != null) { return true; } else { return false; } } @Override public VirtualRouter destroyRouter(final long routerId) throws ResourceUnavailableException, ConcurrentOperationException { UserContext context = UserContext.current(); User user = _accountMgr.getActiveUser(context.getCallerUserId()); if (s_logger.isDebugEnabled()) { s_logger.debug("Attempting to destroy router " + routerId); } DomainRouterVO router = _routerDao.findById(routerId); if (router == null) { return null; } _accountMgr.checkAccess(context.getCaller(), null, true, router); boolean result = _itMgr.expunge(router, user, _accountMgr.getAccount(router.getAccountId())); if (result) { return router; } return null; } @Override @DB public VirtualRouter upgradeRouter(UpgradeRouterCmd cmd) { Long routerId = cmd.getId(); Long serviceOfferingId = cmd.getServiceOfferingId(); Account caller = UserContext.current().getCaller(); DomainRouterVO router = _routerDao.findById(routerId); if (router == null) { throw new InvalidParameterValueException("Unable to find router with id " + routerId); } _accountMgr.checkAccess(caller, null, true, router); if (router.getServiceOfferingId() == serviceOfferingId) { s_logger.debug("Router: " + routerId + "already has service offering: " + serviceOfferingId); return _routerDao.findById(routerId); } ServiceOffering newServiceOffering = _configMgr.getServiceOffering(serviceOfferingId); if (newServiceOffering == null) { throw new InvalidParameterValueException("Unable to find service offering with id " + serviceOfferingId); } // check if it is a system service offering, if yes return with error as it cannot be used for user vms if (!newServiceOffering.getSystemUse()) { throw new InvalidParameterValueException("Cannot upgrade router vm to a non system service offering " + serviceOfferingId); } // Check that the router is stopped if (!router.getState().equals(State.Stopped)) { s_logger.warn("Unable to upgrade router " + router.toString() + " in state " + router.getState()); throw new InvalidParameterValueException("Unable to upgrade router " + router.toString() + " in state " + router.getState() + "; make sure the router is stopped and not in an error state before upgrading."); } ServiceOfferingVO currentServiceOffering = _serviceOfferingDao.findById(router.getServiceOfferingId()); // Check that the service offering being upgraded to has the same storage pool preference as the VM's current service // offering if (currentServiceOffering.getUseLocalStorage() != newServiceOffering.getUseLocalStorage()) { throw new InvalidParameterValueException("Can't upgrade, due to new local storage status : " + newServiceOffering.getUseLocalStorage() + " is different from " + "curruent local storage status: " + currentServiceOffering.getUseLocalStorage()); } router.setServiceOfferingId(serviceOfferingId); if (_routerDao.update(routerId, router)) { return _routerDao.findById(routerId); } else { throw new CloudRuntimeException("Unable to upgrade router " + routerId); } } @Override public boolean savePasswordToRouter(Network network, NicProfile nic, VirtualMachineProfile profile, List routers) throws ResourceUnavailableException { if (routers == null || routers.isEmpty()) { s_logger.warn("Unable save password, router doesn't exist in network " + network.getId()); throw new CloudRuntimeException("Unable to save password to router"); } UserVm userVm = profile.getVirtualMachine(); String password = (String) profile.getParameter(Param.VmPassword); String encodedPassword = PasswordGenerator.rot13(password); DataCenter dc = _dcDao.findById(userVm.getDataCenterIdToDeployIn()); boolean result = true; for (VirtualRouter router : routers) { boolean sendPassword = true; if (dc.getNetworkType() == NetworkType.Basic && userVm.getPodIdToDeployIn().longValue() != router.getPodIdToDeployIn().longValue()) { sendPassword = false; } if (sendPassword) { Commands cmds = new Commands(OnError.Continue); SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), userVm.getHostName()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("password", cmd); result = result && sendCommandsToRouter(router, cmds); } } return result; } @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_STOP, eventDescription = "stopping router Vm", async = true) public VirtualRouter stopRouter(long routerId, boolean forced) throws ResourceUnavailableException, ConcurrentOperationException { UserContext context = UserContext.current(); Account account = context.getCaller(); // verify parameters DomainRouterVO router = _routerDao.findById(routerId); if (router == null) { throw new InvalidParameterValueException("Unable to find router by id " + routerId + "."); } _accountMgr.checkAccess(account, null, true, router); UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); VirtualRouter virtualRouter = stop(router, forced, user, account); if(virtualRouter == null){ throw new CloudRuntimeException("Failed to stop router with id " + routerId); } return virtualRouter; } @DB public void processStopOrRebootAnswer(final DomainRouterVO router, Answer answer) { final Transaction txn = Transaction.currentTxn(); try { txn.start(); //FIXME!!! - UserStats command should grab bytesSent/Received for all guest interfaces of the VR List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (Long guestNtwkId : routerGuestNtwkIds) { final UserStatisticsVO userStats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterIdToDeployIn(), guestNtwkId, null, router.getId(), router.getType().toString()); if (userStats != null) { final long currentBytesRcvd = userStats.getCurrentBytesReceived(); userStats.setCurrentBytesReceived(0); userStats.setNetBytesReceived(userStats.getNetBytesReceived() + currentBytesRcvd); final long currentBytesSent = userStats.getCurrentBytesSent(); userStats.setCurrentBytesSent(0); userStats.setNetBytesSent(userStats.getNetBytesSent() + currentBytesSent); _userStatsDao.update(userStats.getId(), userStats); s_logger.debug("Successfully updated user statistics as a part of domR " + router + " reboot/stop"); } else { s_logger.warn("User stats were not created for account " + router.getAccountId() + " and dc " + router.getDataCenterIdToDeployIn()); } } txn.commit(); } catch (final Exception e) { txn.rollback(); throw new CloudRuntimeException("Problem updating stats after reboot/stop ", e); } } @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_REBOOT, eventDescription = "rebooting router Vm", async = true) public VirtualRouter rebootRouter(long routerId, boolean reprogramNetwork) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { Account caller = UserContext.current().getCaller(); // verify parameters DomainRouterVO router = _routerDao.findById(routerId); if (router == null) { throw new InvalidParameterValueException("Unable to find domain router with id " + routerId + "."); } _accountMgr.checkAccess(caller, null, true, router); // Can reboot domain router only in Running state if (router == null || router.getState() != State.Running) { s_logger.warn("Unable to reboot, virtual router is not in the right state " + router.getState()); throw new ResourceUnavailableException("Unable to reboot domR, it is not in right state " + router.getState(), DataCenter.class, router.getDataCenterIdToDeployIn()); } UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); s_logger.debug("Stopping and starting router " + router + " as a part of router reboot"); if (stop(router, false, user, caller) != null) { return startRouter(routerId, reprogramNetwork); } else { throw new CloudRuntimeException("Failed to reboot router " + router); } } @Override public boolean configure(final String name, final Map params) throws ConfigurationException { _name = name; _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor")); _checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor")); _networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater")); final ComponentLocator locator = ComponentLocator.getCurrentLocator(); final Map configs = _configDao.getConfiguration("AgentManager", params); _mgmt_host = configs.get("host"); _routerRamSize = NumbersUtil.parseInt(configs.get("router.ram.size"), DEFAULT_ROUTER_VM_RAMSIZE); _routerCpuMHz = NumbersUtil.parseInt(configs.get("router.cpu.mhz"), DEFAULT_ROUTER_CPU_MHZ); _routerExtraPublicNics = NumbersUtil.parseInt(_configDao.getValue(Config.RouterExtraPublicNics.key()), 2); String value = configs.get("start.retry"); _retry = NumbersUtil.parseInt(value, 2); value = configs.get("router.stats.interval"); _routerStatsInterval = NumbersUtil.parseInt(value, 300); value = configs.get("router.check.interval"); _routerCheckInterval = NumbersUtil.parseInt(value, 30); _instance = configs.get("instance.name"); if (_instance == null) { _instance = "DEFAULT"; } String rpValue = configs.get("network.disable.rpfilter"); if (rpValue != null && rpValue.equalsIgnoreCase("true")) { _disable_rp_filter = true; } _dnsBasicZoneUpdates = String.valueOf(_configDao.getValue(Config.DnsBasicZoneUpdates.key())); s_logger.info("Router configurations: " + "ramsize=" + _routerRamSize); final UserStatisticsDao statsDao = locator.getDao(UserStatisticsDao.class); if (statsDao == null) { throw new ConfigurationException("Unable to get " + UserStatisticsDao.class.getName()); } _agentMgr.registerForHostEvents(new SshKeysDistriMonitor(_agentMgr, _hostDao, _configDao), true, false, false); _itMgr.registerGuru(VirtualMachine.Type.DomainRouter, this); boolean useLocalStorage = Boolean.parseBoolean(configs.get(Config.SystemVMUseLocalStorage.key())); _offering = new ServiceOfferingVO("System Offering For Software Router", 1, _routerRamSize, _routerCpuMHz, null, null, true, null, useLocalStorage, true, null, true, VirtualMachine.Type.DomainRouter, true); _offering.setUniqueName(ServiceOffering.routerDefaultOffUniqueName); _offering = _serviceOfferingDao.persistSystemServiceOffering(_offering); // this can sometimes happen, if DB is manually or programmatically manipulated if(_offering == null) { String msg = "Data integrity problem : System Offering For Software router VM has been removed?"; s_logger.error(msg); throw new ConfigurationException(msg); } _systemAcct = _accountMgr.getSystemAccount(); String aggregationRange = configs.get("usage.stats.job.aggregation.range"); _usageAggregationRange = NumbersUtil.parseInt(aggregationRange, 1440); _usageTimeZone = configs.get("usage.aggregation.timezone"); if(_usageTimeZone == null){ _usageTimeZone = "GMT"; } _agentMgr.registerForHostEvents(this, true, false, false); s_logger.info("DomainRouterManager is configured."); return true; } @Override public String getName() { return _name; } @Override public boolean start() { if (_routerStatsInterval > 0){ _executor.scheduleAtFixedRate(new NetworkUsageTask(), _routerStatsInterval, _routerStatsInterval, TimeUnit.SECONDS); }else{ s_logger.debug("router.stats.interval - " + _routerStatsInterval+ " so not scheduling the router stats thread"); } //Schedule Network stats update task TimeZone usageTimezone = TimeZone.getTimeZone(_usageTimeZone); Calendar cal = Calendar.getInstance(usageTimezone); cal.setTime(new Date()); long endDate = 0; int HOURLY_TIME = 60; final int DAILY_TIME = 60 * 24; if (_usageAggregationRange == DAILY_TIME) { cal.roll(Calendar.DAY_OF_YEAR, false); cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); cal.roll(Calendar.DAY_OF_YEAR, true); cal.add(Calendar.MILLISECOND, -1); endDate = cal.getTime().getTime(); } else if (_usageAggregationRange == HOURLY_TIME) { cal.roll(Calendar.HOUR_OF_DAY, false); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); cal.roll(Calendar.HOUR_OF_DAY, true); cal.add(Calendar.MILLISECOND, -1); endDate = cal.getTime().getTime(); } else { endDate = cal.getTime().getTime(); } _networkStatsUpdateExecutor.scheduleAtFixedRate(new NetworkStatsUpdateTask(), (endDate - System.currentTimeMillis()), (_usageAggregationRange * 60 * 1000), TimeUnit.MILLISECONDS); if (_routerCheckInterval > 0) { _checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _routerCheckInterval, _routerCheckInterval, TimeUnit.SECONDS); } else { s_logger.debug("router.check.interval - " + _routerCheckInterval+ " so not scheduling the redundant router checking thread"); } return true; } @Override public boolean stop() { return true; } protected VirtualNetworkApplianceManagerImpl() { } @Override public Long convertToId(final String vmName) { if (!VirtualMachineName.isValidRouterName(vmName, _instance)) { return null; } return VirtualMachineName.getRouterId(vmName); } private VmDataCommand generateVmDataCommand(VirtualRouter router, String vmPrivateIpAddress, String userData, String serviceOffering, String zoneName, String guestIpAddress, String vmName, String vmInstanceName, long vmId, String publicKey, long guestNetworkId) { VmDataCommand cmd = new VmDataCommand(vmPrivateIpAddress, vmName); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmd.addVmData("userdata", "user-data", userData); cmd.addVmData("metadata", "service-offering", StringUtils.unicodeEscape(serviceOffering)); cmd.addVmData("metadata", "availability-zone", StringUtils.unicodeEscape(zoneName)); cmd.addVmData("metadata", "local-ipv4", guestIpAddress); cmd.addVmData("metadata", "local-hostname", StringUtils.unicodeEscape(vmName)); if (dcVo.getNetworkType() == NetworkType.Basic) { cmd.addVmData("metadata", "public-ipv4", guestIpAddress); cmd.addVmData("metadata", "public-hostname", StringUtils.unicodeEscape(vmName)); } else { if (router.getPublicIpAddress() == null) { cmd.addVmData("metadata", "public-ipv4", guestIpAddress); } else { cmd.addVmData("metadata", "public-ipv4", router.getPublicIpAddress()); } cmd.addVmData("metadata", "public-hostname", router.getPublicIpAddress()); } cmd.addVmData("metadata", "instance-id", vmInstanceName); cmd.addVmData("metadata", "vm-id", String.valueOf(vmId)); cmd.addVmData("metadata", "public-keys", publicKey); String cloudIdentifier = _configDao.getValue("cloud.identifier"); if (cloudIdentifier == null) { cloudIdentifier = ""; } else { cloudIdentifier = "CloudStack-{" + cloudIdentifier + "}"; } cmd.addVmData("metadata", "cloud-identifier", cloudIdentifier); return cmd; } protected class NetworkUsageTask implements Runnable { public NetworkUsageTask() { } @Override public void run() { try{ final List routers = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId); s_logger.debug("Found " + routers.size() + " running routers. "); for (DomainRouterVO router : routers) { String privateIP = router.getPrivateIpAddress(); if (privateIP != null) { List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (Long guestNtwkId : routerGuestNtwkIds) { boolean forVpc = router.getVpcId() != null; Network guestNtwk = _networkMgr.getNetwork(guestNtwkId); Nic guestNic = _nicDao.findByInstanceIdAndNetworkId(guestNtwk.getId(), router.getId()); NicProfile guestNicProfile = new NicProfile(guestNic, guestNtwk, guestNic.getBroadcastUri(), guestNic.getIsolationUri(), _networkMgr.getNetworkRate(guestNtwk.getId(), router.getId()), _networkMgr.isSecurityGroupSupportedInNetwork(guestNtwk), _networkMgr.getNetworkTag(router.getHypervisorType(), guestNtwk)); final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, _itMgr.toNicTO(guestNicProfile, router.getHypervisorType())); UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), router.getDataCenterIdToDeployIn(), guestNtwkId, null, router.getId(), router.getType().toString()); NetworkUsageAnswer answer = null; try { answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); } catch (Exception e) { s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); continue; } if (answer != null) { if (!answer.getResult()) { s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); continue; } Transaction txn = Transaction.open(Transaction.CLOUD_DB); try { if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); continue; } txn.start(); UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), router.getDataCenterIdToDeployIn(), guestNtwkId, null, router.getId(), router.getType().toString()); if (stats == null) { s_logger.warn("unable to find stats for account: " + router.getAccountId()); continue; } if(previousStats != null && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); continue; } if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answer.getRouterName()+" Reported: " + answer.getBytesReceived() + " Stored: " + stats.getCurrentBytesReceived()); } stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } stats.setCurrentBytesReceived(answer.getBytesReceived()); if (stats.getCurrentBytesSent() > answer.getBytesSent()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answer.getRouterName()+" Reported: " + answer.getBytesSent() + " Stored: " + stats.getCurrentBytesSent()); } stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); } stats.setCurrentBytesSent(answer.getBytesSent()); _statsDao.update(stats.getId(), stats); txn.commit(); } catch (Exception e) { txn.rollback(); s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); } finally { txn.close(); } } } } } } catch (Exception e) { s_logger.warn("Error while collecting network stats", e); } } } protected class NetworkStatsUpdateTask implements Runnable { public NetworkStatsUpdateTask() { } @Override public void run() { GlobalLock scanLock = GlobalLock.getInternLock("network.stats"); try { if(scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { //Check for ownership //msHost in UP state with min id should run the job ManagementServerHostVO msHost = _msHostDao.findOneInUpState(new Filter(ManagementServerHostVO.class, "id", true, 0L, 1L)); if(msHost == null || (msHost.getMsid() != mgmtSrvrId)){ s_logger.debug("Skipping aggregate network stats update"); scanLock.unlock(); return; } Transaction txn = Transaction.open(Transaction.CLOUD_DB); try { txn.start(); //get all stats with delta > 0 List updatedStats = _statsDao.listUpdatedStats(); Date updatedTime = new Date(); for(UserStatisticsVO stat : updatedStats){ //update agg bytes stat.setAggBytesReceived(stat.getCurrentBytesReceived() + stat.getNetBytesReceived()); stat.setAggBytesSent(stat.getCurrentBytesSent() + stat.getNetBytesSent()); _userStatsDao.update(stat.getId(), stat); //insert into op_user_stats_log UserStatsLogVO statsLog = new UserStatsLogVO(stat.getId(), stat.getNetBytesReceived(), stat.getNetBytesSent(), stat.getCurrentBytesReceived(), stat.getCurrentBytesSent(), stat.getAggBytesReceived(), stat.getAggBytesSent(), updatedTime); _userStatsLogDao.persist(statsLog); } s_logger.debug("Successfully updated aggregate network stats"); txn.commit(); } catch (Exception e){ txn.rollback(); s_logger.debug("Failed to update aggregate network stats", e); } finally { scanLock.unlock(); txn.close(); } } } catch (Exception e){ s_logger.debug("Exception while trying to acquire network stats lock", e); } finally { scanLock.releaseRef(); } } } @DB protected void updateSite2SiteVpnConnectionState(List routers) { for (DomainRouterVO router : routers) { List conns = _s2sVpnMgr.getConnectionsForRouter(router); if (conns == null || conns.isEmpty()) { continue; } if (router.getState() != State.Running) { for (Site2SiteVpnConnectionVO conn : conns) { if (conn.getState() != Site2SiteVpnConnection.State.Error) { conn.setState(Site2SiteVpnConnection.State.Disconnected); _s2sVpnConnectionDao.persist(conn); } } continue; } List ipList = new ArrayList(); for (Site2SiteVpnConnectionVO conn : conns) { if (conn.getState() != Site2SiteVpnConnection.State.Connected && conn.getState() != Site2SiteVpnConnection.State.Disconnected) { continue; } Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); ipList.add(gw.getGatewayIp()); } String privateIP = router.getPrivateIpAddress(); HostVO host = _hostDao.findById(router.getHostId()); if (host == null || host.getStatus() != Status.Up) { continue; } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { /* Only cover hosts managed by this management server */ continue; } else if (privateIP != null) { final CheckS2SVpnConnectionsCommand command = new CheckS2SVpnConnectionsCommand(ipList); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); command.setWait(30); final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); CheckS2SVpnConnectionsAnswer answer = null; if (origAnswer instanceof CheckS2SVpnConnectionsAnswer) { answer = (CheckS2SVpnConnectionsAnswer)origAnswer; } else { s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); continue; } if (!answer.getResult()) { s_logger.warn("Unable to update router " + router.getHostName() + "'s VPN connection status"); continue; } for (Site2SiteVpnConnectionVO conn : conns) { Site2SiteVpnConnectionVO lock = _s2sVpnConnectionDao.acquireInLockTable(conn.getId()); if (lock == null) { throw new CloudRuntimeException("Unable to acquire lock on " + lock); } try { if (conn.getState() != Site2SiteVpnConnection.State.Connected && conn.getState() != Site2SiteVpnConnection.State.Disconnected) { continue; } Site2SiteVpnConnection.State oldState = conn.getState(); Site2SiteCustomerGateway gw = _s2sCustomerGatewayDao.findById(conn.getCustomerGatewayId()); if (answer.isConnected(gw.getGatewayIp())) { conn.setState(Site2SiteVpnConnection.State.Connected); } else { conn.setState(Site2SiteVpnConnection.State.Disconnected); } _s2sVpnConnectionDao.persist(conn); if (oldState != conn.getState()) { String title = "Site-to-site Vpn Connection to " + gw.getName() + " just switch from " + oldState + " to " + conn.getState(); String context = "Site-to-site Vpn Connection to " + gw.getName() + " on router " + router.getHostName() + "(id: " + router.getId() + ") " + " just switch from " + oldState + " to " + conn.getState(); s_logger.info(context); _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); } } finally { _s2sVpnConnectionDao.releaseFromLockTable(lock.getId()); } } } } } protected void updateRoutersRedundantState(List routers) { boolean updated = false; for (DomainRouterVO router : routers) { updated = false; if (!router.getIsRedundantRouter()) { continue; } RedundantState prevState = router.getRedundantState(); if (router.getState() != State.Running) { router.setRedundantState(RedundantState.UNKNOWN); router.setIsPriorityBumpUp(false); updated = true; } else { String privateIP = router.getPrivateIpAddress(); HostVO host = _hostDao.findById(router.getHostId()); if (host == null || host.getStatus() != Status.Up) { router.setRedundantState(RedundantState.UNKNOWN); updated = true; } else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) { /* Only cover hosts managed by this management server */ continue; } else if (privateIP != null) { final CheckRouterCommand command = new CheckRouterCommand(); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); command.setWait(60); final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); CheckRouterAnswer answer = null; if (origAnswer instanceof CheckRouterAnswer) { answer = (CheckRouterAnswer)origAnswer; } else { s_logger.warn("Unable to update router " + router.getHostName() + "'s status"); } RedundantState state = RedundantState.UNKNOWN; boolean isBumped = router.getIsPriorityBumpUp(); if (answer != null && answer.getResult()) { state = answer.getState(); isBumped = answer.isBumped(); } router.setRedundantState(state); router.setIsPriorityBumpUp(isBumped); updated = true; } } if (updated) { Transaction txn = Transaction.open(Transaction.CLOUD_DB); try { txn.start(); _routerDao.update(router.getId(), router); txn.commit(); } catch (Exception e) { txn.rollback(); s_logger.warn("Unable to update router status for account: " + router.getAccountId()); } finally { txn.close(); } } RedundantState currState = router.getRedundantState(); if (prevState != currState) { String title = "Redundant virtual router " + router.getInstanceName() + " just switch from " + prevState + " to " + currState; String context = "Redundant virtual router (name: " + router.getHostName() + ", id: " + router.getId() + ") " + " just switch from " + prevState + " to " + currState; s_logger.info(context); if (currState == RedundantState.MASTER) { _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); } } } } //Ensure router status is update to date before execute this function. The function would try best to recover all routers except MASTER protected void recoverRedundantNetwork(DomainRouterVO masterRouter, DomainRouterVO backupRouter) { UserContext context = UserContext.current(); context.setAccountId(1); if (masterRouter.getState() == State.Running && backupRouter.getState() == State.Running) { HostVO masterHost = _hostDao.findById(masterRouter.getHostId()); HostVO backupHost = _hostDao.findById(backupRouter.getHostId()); if (masterHost.getStatus() == Status.Up && backupHost.getStatus() == Status.Up) { String title = "Reboot " + backupRouter.getInstanceName() + " to ensure redundant virtual routers work"; if (s_logger.isDebugEnabled()) { s_logger.debug(title); } _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, backupRouter.getDataCenterIdToDeployIn(), backupRouter.getPodIdToDeployIn(), title, title); try { rebootRouter(backupRouter.getId(), false); } catch (ConcurrentOperationException e) { s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); } catch (ResourceUnavailableException e) { s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); } catch (InsufficientCapacityException e) { s_logger.warn("Fail to reboot " + backupRouter.getInstanceName(), e); } } } } private int getRealPriority(DomainRouterVO router) { int priority = router.getPriority(); if (router.getIsPriorityBumpUp()) { priority += DEFAULT_DELTA; } return priority; } protected class CheckRouterTask implements Runnable { public CheckRouterTask() { } /* * In order to make fail-over works well at any time, we have to ensure: * 1. Backup router's priority = Master's priority - DELTA + 1 * 2. Backup router's priority hasn't been bumped up. */ private void checkSanity(List routers) { Set checkedNetwork = new HashSet(); for (DomainRouterVO router : routers) { if (!router.getIsRedundantRouter()) { continue; } List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (Long routerGuestNtwkId : routerGuestNtwkIds) { if (checkedNetwork.contains(routerGuestNtwkId)) { continue; } checkedNetwork.add(routerGuestNtwkId); List checkingRouters = _routerDao.listByNetworkAndRole(routerGuestNtwkId, Role.VIRTUAL_ROUTER); if (checkingRouters.size() != 2) { continue; } DomainRouterVO masterRouter = null; DomainRouterVO backupRouter = null; for (DomainRouterVO r : checkingRouters) { if (r.getRedundantState() == RedundantState.MASTER) { if (masterRouter == null) { masterRouter = r; } else { //Duplicate master! We give up, until the admin fix duplicate MASTER issue break; } } else if (r.getRedundantState() == RedundantState.BACKUP) { if (backupRouter == null) { backupRouter = r; } else { break; } } } if (masterRouter != null && backupRouter != null) { if (getRealPriority(masterRouter) - DEFAULT_DELTA + 1 != getRealPriority(backupRouter) || backupRouter.getIsPriorityBumpUp()) { recoverRedundantNetwork(masterRouter, backupRouter); } } } } } private void checkDuplicateMaster(List routers) { Map networkRouterMaps = new HashMap(); for (DomainRouterVO router : routers) { List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (Long routerGuestNtwkId : routerGuestNtwkIds) { if (router.getRedundantState() == RedundantState.MASTER) { if (networkRouterMaps.containsKey(routerGuestNtwkId)) { DomainRouterVO dupRouter = networkRouterMaps.get(routerGuestNtwkId); String title = "More than one redundant virtual router is in MASTER state! Router " + router.getHostName() + " and router " + dupRouter.getHostName(); String context = "Virtual router (name: " + router.getHostName() + ", id: " + router.getId() + " and router (name: " + dupRouter.getHostName() + ", id: " + router.getId() + ") are both in MASTER state! If the problem persist, restart both of routers. "; _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context); _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, dupRouter.getDataCenterIdToDeployIn(), dupRouter.getPodIdToDeployIn(), title, context); } else { networkRouterMaps.put(routerGuestNtwkId, router); } } } } } @Override public void run() { try { final List routers = _routerDao.listIsolatedByHostId(null); s_logger.debug("Found " + routers.size() + " routers. "); updateRoutersRedundantState(routers); updateSite2SiteVpnConnectionState(routers); /* FIXME assumed the a pair of redundant routers managed by same mgmt server, * then the update above can get the latest status */ checkDuplicateMaster(routers); checkSanity(routers); } catch (Exception ex) { s_logger.error("Fail to complete the CheckRouterTask! ", ex); } } } private final int DEFAULT_PRIORITY = 100; private final int DEFAULT_DELTA = 2; protected int getUpdatedPriority(Network guestNetwork, List routers, DomainRouterVO exclude) throws InsufficientVirtualNetworkCapcityException { int priority; if (routers.size() == 0) { priority = DEFAULT_PRIORITY; } else { int maxPriority = 0; for (DomainRouterVO r : routers) { if (!r.getIsRedundantRouter()) { throw new CloudRuntimeException("Redundant router is mixed with single router in one network!"); } //FIXME Assume the maxPriority one should be running or just created. if (r.getId() != exclude.getId() && getRealPriority(r) > maxPriority) { maxPriority = getRealPriority(r); } } if (maxPriority == 0) { return DEFAULT_PRIORITY; } if (maxPriority < 20) { s_logger.error("Current maximum priority is too low!"); throw new InsufficientVirtualNetworkCapcityException("Current maximum priority is too low as " + maxPriority + "!", guestNetwork.getId()); } else if (maxPriority > 200) { s_logger.error("Too many times fail-over happened! Current maximum priority is too high as " + maxPriority + "!"); throw new InsufficientVirtualNetworkCapcityException("Too many times fail-over happened! Current maximum priority is too high as " + maxPriority + "!", guestNetwork.getId()); } priority = maxPriority - DEFAULT_DELTA + 1; } return priority; } /* * Ovm won't support any system. So we have to choose a partner cluster in the same pod to start domain router for us */ private HypervisorType getClusterToStartDomainRouterForOvm(long podId) { List clusters = _clusterDao.listByPodId(podId); for (ClusterVO cv : clusters) { if (cv.getHypervisorType() == HypervisorType.Ovm || cv.getHypervisorType() == HypervisorType.BareMetal) { continue; } List hosts = _resourceMgr.listAllHostsInCluster(cv.getId()); if (hosts == null || hosts.isEmpty()) { continue; } for (HostVO h : hosts) { if (h.getStatus() == Status.Up) { s_logger.debug("Pick up host that has hypervisor type " + h.getHypervisorType() + " in cluster " + cv.getId() + " to start domain router for OVM"); return h.getHypervisorType(); } } } String errMsg = "Cannot find an available cluster in Pod " + podId + " to start domain router for Ovm. \n Ovm won't support any system vm including domain router, " + "please make sure you have a cluster with hypervisor type of any of xenserver/KVM/Vmware in the same pod" + " with Ovm cluster. And there is at least one host in UP status in that cluster."; throw new CloudRuntimeException(errMsg); } @DB protected List findOrDeployVirtualRouterInGuestNetwork(Network guestNetwork, DeployDestination dest, Account owner, boolean isRedundant, Map params) throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException { assert guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup || guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: " + guestNetwork; assert guestNetwork.getTrafficType() == TrafficType.Guest; //1) Get deployment plan and find out the list of routers boolean isPodBased = (dest.getDataCenter().getNetworkType() == NetworkType.Basic || _networkMgr.areServicesSupportedInNetwork(guestNetwork.getId(), Service.SecurityGroup)) && guestNetwork.getTrafficType() == TrafficType.Guest; Pair> planAndRouters = getDeploymentPlanAndRouters(isPodBased, dest, guestNetwork.getId()); DeploymentPlan plan = planAndRouters.first(); List routers = planAndRouters.second(); //2) Figure out required routers count int routerCount = 1; if (isRedundant) { routerCount = 2; } /* If it is the single router network, then keep it untouched */ for (DomainRouterVO router : routers) { if (!router.getIsRedundantRouter() || isPodBased) { routerCount = 1; break; } } /* If old network is redundant but new is single router, then routers.size() = 2 but routerCount = 1 */ if (routers.size() >= routerCount) { return routers; } if (routers.size() >= 5) { s_logger.error("Too much redundant routers!"); } Network network = _networkDao.acquireInLockTable(guestNetwork.getId()); if (network == null) { throw new ConcurrentOperationException("Unable to lock network " + guestNetwork.getId()); } try { //Check if providers are supported in the physical networks VirtualRouterProviderType type = VirtualRouterProviderType.VirtualRouter; Long physicalNetworkId = _networkMgr.getPhysicalNetworkId(network); PhysicalNetworkServiceProvider provider = _physicalProviderDao.findByServiceProvider(physicalNetworkId, type.toString()); if (provider == null) { throw new CloudRuntimeException("Cannot find service provider " + type.toString() + " in physical network " + physicalNetworkId); } VirtualRouterProvider vrProvider = _vrProviderDao.findByNspIdAndType(provider.getId(), type); if (vrProvider == null) { throw new CloudRuntimeException("Cannot find virtual router provider " + type.toString()+ " as service provider " + provider.getId()); } if (_networkMgr.isNetworkSystem(guestNetwork) || guestNetwork.getGuestType() == Network.GuestType.Shared) { owner = _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM); } //Check if public network has to be set on VR boolean publicNetwork = false; if (_networkMgr.isProviderSupportServiceInNetwork(guestNetwork.getId(), Service.SourceNat, Provider.VirtualRouter)) { publicNetwork = true; } if (isRedundant && !publicNetwork) { s_logger.error("Didn't support redundant virtual router without public network!"); return null; } Long offeringId = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()).getServiceOfferingId(); if (offeringId == null) { offeringId = _offering.getId(); } PublicIp sourceNatIp = null; if (publicNetwork) { sourceNatIp = _networkMgr.assignSourceNatIpAddressToGuestNetwork(owner, guestNetwork); } //3) deploy virtual router(s) int count = routerCount - routers.size(); for (int i = 0; i < count; i++) { List> networks = createRouterNetworks(owner, isRedundant, plan, guestNetwork, new Pair(publicNetwork, sourceNatIp)); DomainRouterVO router = deployRouter(owner, dest, plan, params, isRedundant, vrProvider, offeringId, null, networks); _routerDao.addRouterToGuestNetwork(router, network); routers.add(router); } } finally { if (network != null) { _networkDao.releaseFromLockTable(network.getId()); } } return routers; } protected DomainRouterVO deployRouter(Account owner, DeployDestination dest, DeploymentPlan plan, Map params, boolean isRedundant, VirtualRouterProvider vrProvider, long svcOffId, Long vpcId, List> networks) throws ConcurrentOperationException, InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, StorageUnavailableException, ResourceUnavailableException { long id = _routerDao.getNextInSequence(Long.class, "id"); if (s_logger.isDebugEnabled()) { s_logger.debug("Creating the router " + id + " in datacenter " + dest.getDataCenter()); } ServiceOfferingVO routerOffering = _serviceOfferingDao.findById(svcOffId); // Router is the network element, we don't know the hypervisor type yet. //Try to allocate the domR twice using diff hypervisors, and when failed both times, throw the exception up List supportedHypervisors = new ArrayList(); HypervisorType defaults = _resourceMgr.getDefaultHypervisor(dest.getDataCenter().getId()); if (defaults != HypervisorType.None) { supportedHypervisors.add(defaults); } if (dest.getCluster() != null) { if (dest.getCluster().getHypervisorType() == HypervisorType.Ovm) { supportedHypervisors.add(getClusterToStartDomainRouterForOvm(dest.getCluster().getPodId())); } else { supportedHypervisors.add(dest.getCluster().getHypervisorType()); } } else { supportedHypervisors = _resourceMgr.getSupportedHypervisorTypes(dest.getDataCenter().getId(), true, plan.getPodId()); } if (supportedHypervisors.isEmpty()) { if (plan.getPodId() != null) { throw new InsufficientServerCapacityException("Unable to create virtual router, " + "there are no clusters in the pod ", Pod.class, plan.getPodId()); } throw new InsufficientServerCapacityException("Unable to create virtual router, " + "there are no clusters in the zone ", DataCenter.class, dest.getDataCenter().getId()); } int allocateRetry = 0; int startRetry = 0; DomainRouterVO router = null; for (Iterator iter = supportedHypervisors.iterator();iter.hasNext();) { HypervisorType hType = iter.next(); try { s_logger.debug("Allocating the domR with the hypervisor type " + hType); VMTemplateVO template = _templateDao.findRoutingTemplate(hType); if (template == null) { s_logger.debug(hType + " won't support system vm, skip it"); continue; } boolean offerHA = routerOffering.getOfferHA(); /* We don't provide HA to redundant router VMs, admin should own it all, and redundant router themselves are HA */ if (isRedundant) { offerHA = false; } router = new DomainRouterVO(id, routerOffering.getId(), vrProvider.getId(), VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), isRedundant, 0, false, RedundantState.UNKNOWN, offerHA, false, vpcId); router.setRole(Role.VIRTUAL_ROUTER); router = _itMgr.allocate(router, template, routerOffering, networks, plan, null, owner); } catch (InsufficientCapacityException ex) { if (allocateRetry < 2 && iter.hasNext()) { s_logger.debug("Failed to allocate the domR with hypervisor type " + hType + ", retrying one more time"); continue; } else { throw ex; } } finally { allocateRetry++; } try { router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); break; } catch (InsufficientCapacityException ex) { if (startRetry < 2 && iter.hasNext()) { s_logger.debug("Failed to start the domR " + router + " with hypervisor type " + hType + ", " + "destroying it and recreating one more time"); //destroy the router destroyRouter(router.getId()); continue; } else { throw ex; } } finally { startRetry++; } } return router; } protected List> createRouterNetworks(Account owner, boolean isRedundant, DeploymentPlan plan, Network guestNetwork, Pair publicNetwork) throws ConcurrentOperationException, InsufficientAddressCapacityException { boolean setupPublicNetwork = false; if (publicNetwork != null) { setupPublicNetwork = publicNetwork.first(); } //Form networks List> networks = new ArrayList>(3); //1) Guest network boolean hasGuestNetwork = false; if (guestNetwork != null) { s_logger.debug("Adding nic for Virtual Router in Guest network " + guestNetwork); String defaultNetworkStartIp = null; if (guestNetwork.getCidr() != null && !setupPublicNetwork) { String startIp = _networkMgr.getStartIpAddress(guestNetwork.getId()); if (startIp != null && _ipAddressDao.findByIpAndSourceNetworkId(guestNetwork.getId(), startIp).getAllocatedTime() == null) { defaultNetworkStartIp = startIp; } else if (s_logger.isDebugEnabled()){ s_logger.debug("First ip " + startIp + " in network id=" + guestNetwork.getId() + " is already allocated, can't use it for domain router; will get random ip address from the range"); } } NicProfile gatewayNic = new NicProfile(defaultNetworkStartIp); if (setupPublicNetwork) { if (isRedundant) { gatewayNic.setIp4Address(_networkMgr.acquireGuestIpAddress(guestNetwork, null)); } else { gatewayNic.setIp4Address(guestNetwork.getGateway()); } gatewayNic.setBroadcastUri(guestNetwork.getBroadcastUri()); gatewayNic.setBroadcastType(guestNetwork.getBroadcastDomainType()); gatewayNic.setIsolationUri(guestNetwork.getBroadcastUri()); gatewayNic.setMode(guestNetwork.getMode()); String gatewayCidr = guestNetwork.getCidr(); gatewayNic.setNetmask(NetUtils.getCidrNetmask(gatewayCidr)); } else { gatewayNic.setDefaultNic(true); } networks.add(new Pair((NetworkVO) guestNetwork, gatewayNic)); hasGuestNetwork = true; } //2) Control network s_logger.debug("Adding nic for Virtual Router in Control network "); List offerings = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemControlNetwork); NetworkOfferingVO controlOffering = offerings.get(0); NetworkVO controlConfig = _networkMgr.setupNetwork(_systemAcct, controlOffering, plan, null, null, false).get(0); networks.add(new Pair(controlConfig, null)); //3) Public network if (setupPublicNetwork) { PublicIp sourceNatIp = publicNetwork.second(); s_logger.debug("Adding nic for Virtual Router in Public network "); //if source nat service is supported by the network, get the source nat ip address NicProfile defaultNic = new NicProfile(); defaultNic.setDefaultNic(true); defaultNic.setIp4Address(sourceNatIp.getAddress().addr()); defaultNic.setGateway(sourceNatIp.getGateway()); defaultNic.setNetmask(sourceNatIp.getNetmask()); defaultNic.setMacAddress(sourceNatIp.getMacAddress()); defaultNic.setBroadcastType(BroadcastDomainType.Vlan); defaultNic.setBroadcastUri(BroadcastDomainType.Vlan.toUri(sourceNatIp.getVlanTag())); defaultNic.setIsolationUri(IsolationType.Vlan.toUri(sourceNatIp.getVlanTag())); if (hasGuestNetwork) { defaultNic.setDeviceId(2); } NetworkOfferingVO publicOffering = _networkMgr.getSystemAccountNetworkOfferings(NetworkOfferingVO.SystemPublicNetwork).get(0); List publicNetworks = _networkMgr.setupNetwork(_systemAcct, publicOffering, plan, null, null, false); networks.add(new Pair(publicNetworks.get(0), defaultNic)); } return networks; } protected Pair> getDeploymentPlanAndRouters(boolean isPodBased, DeployDestination dest, long guestNetworkId) { long dcId = dest.getDataCenter().getId(); List routers = null; DeploymentPlan plan = new DataCenterDeployment(dcId); if (isPodBased) { Pod pod = dest.getPod(); Long podId = null; if (pod != null) { podId = pod.getId(); } else { throw new CloudRuntimeException("Pod id is expected in deployment destination"); } routers = _routerDao.listByNetworkAndPodAndRole(guestNetworkId, podId, Role.VIRTUAL_ROUTER); plan = new DataCenterDeployment(dcId, podId, null, null, null, null); } else { routers = _routerDao.listByNetworkAndRole(guestNetworkId, Role.VIRTUAL_ROUTER); } return new Pair>(plan, routers); } private DomainRouterVO startVirtualRouter(DomainRouterVO router, User user, Account caller, Map params) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { if (router.getRole() != Role.VIRTUAL_ROUTER || !router.getIsRedundantRouter()) { return this.start(router, user, caller, params, null); } if (router.getState() == State.Running) { s_logger.debug("Redundant router " + router.getInstanceName() + " is already running!"); return router; } DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null); DomainRouterVO result = null; assert router.getIsRedundantRouter(); List routerList = _routerDao.findBy(router.getAccountId(), router.getDataCenterIdToDeployIn()); DomainRouterVO routerToBeAvoid = null; for (DomainRouterVO rrouter : routerList) { if (rrouter.getHostId() != null && rrouter.getIsRedundantRouter() && rrouter.getState() == State.Running) { if (routerToBeAvoid != null) { throw new ResourceUnavailableException("Try to start router " + router.getInstanceName() + "(" + router.getId() + ")" + ", but there are already two redundant routers with IP " + router.getPublicIpAddress() + ", they are " + rrouter.getInstanceName() + "(" + rrouter.getId() + ") and " + routerToBeAvoid.getInstanceName() + "(" + routerToBeAvoid.getId() + ")", DataCenter.class, rrouter.getDataCenterIdToDeployIn()); } routerToBeAvoid = rrouter; } } if (routerToBeAvoid == null) { return this.start(router, user, caller, params, null); } // We would try best to deploy the router to another place int retryIndex = 5; ExcludeList[] avoids = new ExcludeList[5]; avoids[0] = new ExcludeList(); avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn()); avoids[1] = new ExcludeList(); avoids[1].addCluster(_hostDao.findById(routerToBeAvoid.getHostId()).getClusterId()); avoids[2] = new ExcludeList(); List volumes = _volumeDao.findByInstanceAndType(routerToBeAvoid.getId(), Type.ROOT); if (volumes != null && volumes.size() != 0) { avoids[2].addPool(volumes.get(0).getPoolId()); } avoids[2].addHost(routerToBeAvoid.getHostId()); avoids[3] = new ExcludeList(); avoids[3].addHost(routerToBeAvoid.getHostId()); avoids[4] = new ExcludeList(); for (int i = 0; i < retryIndex; i++) { if (s_logger.isTraceEnabled()) { s_logger.trace("Try to deploy redundant virtual router:" + router.getHostName() + ", for " + i + " time"); } plan.setAvoids(avoids[i]); try { result = this.start(router, user, caller, params, plan); } catch (InsufficientServerCapacityException ex) { result = null; } if (result != null) { break; } } return result; } @Override public List deployVirtualRouterInGuestNetwork(Network guestNetwork, DeployDestination dest, Account owner, Map params, boolean isRedundant) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { List routers = findOrDeployVirtualRouterInGuestNetwork (guestNetwork, dest, owner, isRedundant, params); return startRouters(params, routers); } protected List startRouters(Map params, List routers) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { List runningRouters = null; if (routers != null) { runningRouters = new ArrayList(); } for (DomainRouterVO router : routers) { boolean skip = false; State state = router.getState(); if (router.getHostId() != null && state != State.Running) { HostVO host = _hostDao.findById(router.getHostId()); if (host == null || host.getStatus() != Status.Up) { skip = true; } } if (!skip) { if (state != State.Running) { router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), params); } if (router != null) { runningRouters.add(router); } } } return runningRouters; } @Override public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) { boolean dnsProvided = true; boolean dhcpProvided = true; boolean publicNetwork = false; DataCenterVO dc = _dcDao.findById(dest.getDataCenter().getId()); _dcDao.loadDetails(dc); //1) Set router details DomainRouterVO router = profile.getVirtualMachine(); Map details = _vmDetailsDao.findDetails(router.getId()); router.setDetails(details); //2) Prepare boot loader elements related with Control network StringBuilder buf = profile.getBootArgsBuilder(); buf.append(" template=domP"); buf.append(" name=").append(profile.getHostName()); if (Boolean.valueOf(_configDao.getValue("system.vm.random.password"))) { buf.append(" vmpassword=").append(_configDao.getValue("system.vm.password")); } NicProfile controlNic = null; String defaultDns1 = null; String defaultDns2 = null; for (NicProfile nic : profile.getNics()) { int deviceId = nic.getDeviceId(); buf.append(" eth").append(deviceId).append("ip=").append(nic.getIp4Address()); buf.append(" eth").append(deviceId).append("mask=").append(nic.getNetmask()); if (nic.isDefaultNic()) { buf.append(" gateway=").append(nic.getGateway()); defaultDns1 = nic.getDns1(); defaultDns2 = nic.getDns2(); } if (nic.getTrafficType() == TrafficType.Management) { buf.append(" localgw=").append(dest.getPod().getGateway()); } else if (nic.getTrafficType() == TrafficType.Control) { controlNic = nic; // DOMR control command is sent over management server in VMware if (dest.getHost().getHypervisorType() == HypervisorType.VMware) { if (s_logger.isInfoEnabled()) { s_logger.info("Check if we need to add management server explicit route to DomR. pod cidr: " + dest.getPod().getCidrAddress() + "/" + dest.getPod().getCidrSize() + ", pod gateway: " + dest.getPod().getGateway() + ", management host: " + _mgmt_host); } if (s_logger.isInfoEnabled()) { s_logger.info("Add management server explicit route to DomR."); } // always add management explicit route, for basic networking setup, DomR may have two interfaces while both // are on the same subnet _mgmt_cidr = _configDao.getValue(Config.ManagementNetwork.key()); if (NetUtils.isValidCIDR(_mgmt_cidr)) { buf.append(" mgmtcidr=").append(_mgmt_cidr); buf.append(" localgw=").append(dest.getPod().getGateway()); } if (dc.getNetworkType() == NetworkType.Basic) { // ask domR to setup SSH on guest network buf.append(" sshonguest=true"); } } } else if (nic.getTrafficType() == TrafficType.Guest) { dnsProvided = _networkMgr.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dns, Provider.VirtualRouter); dhcpProvided = _networkMgr.isProviderSupportServiceInNetwork(nic.getNetworkId(), Service.Dhcp, Provider.VirtualRouter); //build bootloader parameter for the guest buf.append(createGuestBootLoadArgs(nic, defaultDns1, defaultDns2, router)); } else if (nic.getTrafficType() == TrafficType.Public) { publicNetwork = true; } } if (controlNic == null) { throw new CloudRuntimeException("Didn't start a control port"); } String rpValue = _configDao.getValue(Config.NetworkRouterRpFilter.key()); if (rpValue != null && rpValue.equalsIgnoreCase("true")) { _disable_rp_filter = true; }else { _disable_rp_filter = false; } String rpFilter = " "; String type = null; if (router.getVpcId() != null) { type = "vpcrouter"; if (_disable_rp_filter) { rpFilter=" disable_rp_filter=true"; } } else if (!publicNetwork) { type = "dhcpsrvr"; } else { type = "router"; if (_disable_rp_filter) { rpFilter=" disable_rp_filter=true"; } } if (_disable_rp_filter) { rpFilter=" disable_rp_filter=true"; } buf.append(" type=" + type + rpFilter); String domain_suffix = dc.getDetail(ZoneConfig.DnsSearchOrder.getName()); if (domain_suffix != null) { buf.append(" dnssearchorder=").append(domain_suffix); } if (profile.getHypervisorType() == HypervisorType.VMware) { buf.append(" extra_pubnics=" + _routerExtraPublicNics); } /* If virtual router didn't provide DNS service but provide DHCP service, we need to override the DHCP response * to return DNS server rather than * virtual router itself. */ if (dnsProvided || dhcpProvided) { if (defaultDns1 != null) { buf.append(" dns1=").append(defaultDns1); } if (defaultDns2 != null) { buf.append(" dns2=").append(defaultDns2); } boolean useExtDns = !dnsProvided; /* For backward compatibility */ String use_external_dns = _configDao.getValue(Config.UseExternalDnsServers.key()); if (use_external_dns != null && use_external_dns.equals("true")) { useExtDns = true; } if (useExtDns) { buf.append(" useextdns=true"); } } if (s_logger.isDebugEnabled()) { s_logger.debug("Boot Args for " + profile + ": " + buf.toString()); } return true; } protected StringBuilder createGuestBootLoadArgs(NicProfile guestNic, String defaultDns1, String defaultDns2, DomainRouterVO router) { long guestNetworkId = guestNic.getNetworkId(); NetworkVO guestNetwork = _networkDao.findById(guestNetworkId); String dhcpRange = null; DataCenterVO dc = _dcDao.findById(guestNetwork.getDataCenterId()); StringBuilder buf = new StringBuilder(); boolean isRedundant = router.getIsRedundantRouter(); if (isRedundant) { buf.append(" redundant_router=1"); List routers = _routerDao.listByNetworkAndRole(guestNetwork.getId(), Role.VIRTUAL_ROUTER); try { int priority = getUpdatedPriority(guestNetwork, routers, router); router.setPriority(priority); } catch (InsufficientVirtualNetworkCapcityException e) { s_logger.error("Failed to get update priority!", e); throw new CloudRuntimeException("Failed to get update priority!"); } Network net = _networkMgr.getNetwork(guestNic.getNetworkId()); buf.append(" guestgw=").append(net.getGateway()); String brd = NetUtils.long2Ip(NetUtils.ip2Long(guestNic.getIp4Address()) | ~NetUtils.ip2Long(guestNic.getNetmask())); buf.append(" guestbrd=").append(brd); buf.append(" guestcidrsize=").append(NetUtils.getCidrSize(guestNic.getNetmask())); buf.append(" router_pr=").append(router.getPriority()); } //setup network domain String domain = guestNetwork.getNetworkDomain(); if (domain != null) { buf.append(" domain=" + domain); } //setup dhcp range if (dc.getNetworkType() == NetworkType.Basic) { if (guestNic.isDefaultNic()) { long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); if (cidr != null) { dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); } } } else if (dc.getNetworkType() == NetworkType.Advanced) { String cidr = guestNetwork.getCidr(); if (cidr != null) { dhcpRange = NetUtils.getDhcpRange(cidr); } } if (dhcpRange != null) { buf.append(" dhcprange=" + dhcpRange); } return buf; } protected String getGuestDhcpRange(NicProfile guestNic, Network guestNetwork, DataCenter dc) { String dhcpRange = null; //setup dhcp range if (dc.getNetworkType() == NetworkType.Basic) { long cidrSize = NetUtils.getCidrSize(guestNic.getNetmask()); String cidr = NetUtils.getCidrSubNet(guestNic.getGateway(), cidrSize); if (cidr != null) { dhcpRange = NetUtils.getIpRangeStartIpFromCidr(cidr, cidrSize); } } else if (dc.getNetworkType() == NetworkType.Advanced) { String cidr = guestNetwork.getCidr(); if (cidr != null) { dhcpRange = NetUtils.getDhcpRange(cidr); } } return dhcpRange; } @Override public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) throws ResourceUnavailableException { DomainRouterVO router = profile.getVirtualMachine(); List nics = profile.getNics(); for (NicProfile nic : nics) { if (nic.getTrafficType() == TrafficType.Public) { router.setPublicIpAddress(nic.getIp4Address()); router.setPublicNetmask(nic.getNetmask()); router.setPublicMacAddress(nic.getMacAddress()); } else if (nic.getTrafficType() == TrafficType.Control) { router.setPrivateIpAddress(nic.getIp4Address()); router.setPrivateMacAddress(nic.getMacAddress()); } } _routerDao.update(router.getId(), router); finalizeCommandsOnStart(cmds, profile); return true; } @Override public boolean finalizeCommandsOnStart(Commands cmds, VirtualMachineProfile profile) { DomainRouterVO router = profile.getVirtualMachine(); NicProfile controlNic = getControlNic(profile); if (controlNic == null) { s_logger.error("Control network doesn't exist for the router " + router); return false; } finalizeSshAndVersionAndNetworkUsageOnStart(cmds, profile, router, controlNic); // restart network if restartNetwork = false is not specified in profile parameters boolean reprogramGuestNtwks = true; if (profile.getParameter(Param.ReProgramGuestNetworks) != null && (Boolean) profile.getParameter(Param.ReProgramGuestNetworks) == false) { reprogramGuestNtwks = false; } VirtualRouterProvider vrProvider = _vrProviderDao.findById(router.getElementId()); if (vrProvider == null) { throw new CloudRuntimeException("Cannot find related virtual router provider of router: " + router.getHostName()); } Provider provider = Network.Provider.getProvider(vrProvider.getType().toString()); if (provider == null) { throw new CloudRuntimeException("Cannot find related provider of virtual router provider: " + vrProvider.getType().toString()); } List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (Long guestNetworkId : routerGuestNtwkIds) { if (reprogramGuestNtwks) { finalizeIpAssocForNetwork(cmds, router, provider, guestNetworkId, null); finalizeNetworkRulesForNetwork(cmds, router, provider, guestNetworkId); } finalizeUserDataAndDhcpOnStart(cmds, router, provider, guestNetworkId); } return true; } protected NicProfile getControlNic(VirtualMachineProfile profile) { DomainRouterVO router = profile.getVirtualMachine(); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); NicProfile controlNic = null; if (profile.getHypervisorType() == HypervisorType.VMware && dcVo.getNetworkType() == NetworkType.Basic) { // TODO this is a ugly to test hypervisor type here // for basic network mode, we will use the guest NIC for control NIC for (NicProfile nic : profile.getNics()) { if (nic.getTrafficType() == TrafficType.Guest && nic.getIp4Address() != null) { controlNic = nic; } } } else { for (NicProfile nic : profile.getNics()) { if (nic.getTrafficType() == TrafficType.Control && nic.getIp4Address() != null) { controlNic = nic; } } } return controlNic; } protected void finalizeSshAndVersionAndNetworkUsageOnStart(Commands cmds, VirtualMachineProfile profile, DomainRouterVO router, NicProfile controlNic) { cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922)); // Update router template/scripts version final GetDomRVersionCmd command = new GetDomRVersionCmd(); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIp4Address()); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); cmds.addCommand("getDomRVersion", command); // Network usage command to create iptables rules boolean forVpc = profile.getVirtualMachine().getVpcId() != null; cmds.addCommand("networkUsage", new NetworkUsageCommand(controlNic.getIp4Address(), router.getHostName(), "create", forVpc)); } protected void finalizeUserDataAndDhcpOnStart(Commands cmds, DomainRouterVO router, Provider provider, Long guestNetworkId) { if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Dhcp, provider)) { // Resend dhcp s_logger.debug("Reapplying dhcp entries as a part of domR " + router + " start..."); createDhcpEntryCommandsForVMs(router, cmds, guestNetworkId); } if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.UserData, provider)) { // Resend user data s_logger.debug("Reapplying vm data (userData and metaData) entries as a part of domR " + router + " start..."); createVmDataCommandForVMs(router, cmds, guestNetworkId); } } protected void finalizeNetworkRulesForNetwork(Commands cmds, DomainRouterVO router, Provider provider, Long guestNetworkId) { s_logger.debug("Resending ipAssoc, port forwarding, load balancing rules as a part of Virtual router start"); ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); if (publicIps != null && !publicIps.isEmpty()) { List vpns = new ArrayList(); List pfRules = new ArrayList(); List staticNatFirewallRules = new ArrayList(); List staticNats = new ArrayList(); List firewallRules = new ArrayList(); //Get information about all the rules (StaticNats and StaticNatRules; PFVPN to reapply on domR start) for (PublicIpAddress ip : publicIps) { if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.PortForwarding, provider)) { pfRules.addAll(_pfRulesDao.listForApplication(ip.getId())); } if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { staticNatFirewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.StaticNat)); } if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { firewallRules.addAll(_rulesDao.listByIpAndPurpose(ip.getId(), Purpose.Firewall)); } if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Vpn, provider)) { RemoteAccessVpn vpn = _vpnDao.findById(ip.getId()); if (vpn != null) { vpns.add(vpn); } } if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.StaticNat, provider)) { if (ip.isOneToOneNat()) { String dstIp = _networkMgr.getIpInNetwork(ip.getAssociatedWithVmId(), guestNetworkId); StaticNatImpl staticNat = new StaticNatImpl(ip.getAccountId(), ip.getDomainId(), guestNetworkId, ip.getId(), dstIp, false); staticNats.add(staticNat); } } } //Re-apply static nats s_logger.debug("Found " + staticNats.size() + " static nat(s) to apply as a part of domR " + router + " start."); if (!staticNats.isEmpty()) { createApplyStaticNatCommands(staticNats, router, cmds, guestNetworkId); } //Re-apply firewall rules s_logger.debug("Found " + staticNats.size() + " firewall rule(s) to apply as a part of domR " + router + " start."); if (!firewallRules.isEmpty()) { createFirewallRulesCommands(firewallRules, router, cmds, guestNetworkId); } // Re-apply port forwarding rules s_logger.debug("Found " + pfRules.size() + " port forwarding rule(s) to apply as a part of domR " + router + " start."); if (!pfRules.isEmpty()) { createApplyPortForwardingRulesCommands(pfRules, router, cmds, guestNetworkId); } // Re-apply static nat rules s_logger.debug("Found " + staticNatFirewallRules.size() + " static nat rule(s) to apply as a part of domR " + router + " start."); if (!staticNatFirewallRules.isEmpty()) { List staticNatRules = new ArrayList(); for (FirewallRule rule : staticNatFirewallRules) { staticNatRules.add(_rulesMgr.buildStaticNatRule(rule, false)); } createApplyStaticNatRulesCommands(staticNatRules, router, cmds, guestNetworkId); } // Re-apply vpn rules s_logger.debug("Found " + vpns.size() + " vpn(s) to apply as a part of domR " + router + " start."); if (!vpns.isEmpty()) { for (RemoteAccessVpn vpn : vpns) { createApplyVpnCommands(vpn, router, cmds); } } List lbs = _loadBalancerDao.listByNetworkId(guestNetworkId); List lbRules = new ArrayList(); if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Lb, provider)) { // Re-apply load balancing rules for (LoadBalancerVO lb : lbs) { List dstList = _lbMgr.getExistingDestinations(lb.getId()); List policyList = _lbMgr.getStickinessPolicies(lb.getId()); LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList); lbRules.add(loadBalancing); } } s_logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of domR " + router + " start."); if (!lbRules.isEmpty()) { createApplyLoadBalancingRulesCommands(lbRules, router, cmds, guestNetworkId); } } } protected void finalizeIpAssocForNetwork(Commands cmds, VirtualRouter router, Provider provider, Long guestNetworkId, Map vlanMacAddress) { ArrayList publicIps = getPublicIpsToApply(router, provider, guestNetworkId); if (publicIps != null && !publicIps.isEmpty()) { s_logger.debug("Found " + publicIps.size() + " ip(s) to apply as a part of domR " + router + " start."); // Re-apply public ip addresses - should come before PF/LB/VPN if (_networkMgr.isProviderSupportServiceInNetwork(guestNetworkId, Service.Firewall, provider)) { createAssociateIPCommands(router, publicIps, cmds, 0); } } } protected ArrayList getPublicIpsToApply(VirtualRouter router, Provider provider, Long guestNetworkId, com.cloud.network.IpAddress.State... skipInStates) { long ownerId = router.getAccountId(); final List userIps = _networkMgr.listPublicIpsAssignedToGuestNtwk(ownerId, guestNetworkId, null); List allPublicIps = new ArrayList(); if (userIps != null && !userIps.isEmpty()) { boolean addIp = true; for (IPAddressVO userIp : userIps) { if (skipInStates != null) { for (IpAddress.State stateToSkip : skipInStates) { if (userIp.getState() == stateToSkip) { s_logger.debug("Skipping ip address " + userIp + " in state " + userIp.getState()); addIp = false; break; } } } if (addIp) { PublicIp publicIp = new PublicIp(userIp, _vlanDao.findById(userIp.getVlanId()), NetUtils.createSequenceBasedMacAddress(userIp.getMacAddress())); allPublicIps.add(publicIp); } } } //Get public Ips that should be handled by router Network network = _networkDao.findById(guestNetworkId); Map> ipToServices = _networkMgr.getIpToServices(allPublicIps, false, false); Map> providerToIpList = _networkMgr.getProviderToIpList(network, ipToServices); // Only cover virtual router for now, if ELB use it this need to be modified ArrayList publicIps = providerToIpList.get(provider); return publicIps; } @Override public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Commands cmds, ReservationContext context) { DomainRouterVO router = profile.getVirtualMachine(); boolean result = true; Answer answer = cmds.getAnswer("checkSsh"); if (answer != null && answer instanceof CheckSshAnswer) { CheckSshAnswer sshAnswer = (CheckSshAnswer) answer; if (sshAnswer == null || !sshAnswer.getResult()) { s_logger.warn("Unable to ssh to the VM: " + sshAnswer.getDetails()); result = false; } } else { result = false; } if (result == false) { return result; } //Get guest networks info List guestNetworks = new ArrayList(); List routerNics = _nicDao.listByVmId(profile.getId()); for (Nic routerNic : routerNics) { Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); if (network.getTrafficType() == TrafficType.Guest) { guestNetworks.add(network); } } answer = cmds.getAnswer("getDomRVersion"); if (answer != null && answer instanceof GetDomRVersionAnswer) { GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer)answer; if (answer == null || !answer.getResult()) { s_logger.warn("Unable to get the template/scripts version of router " + router.getInstanceName() + " due to: " + versionAnswer.getDetails()); result = false; } else { router.setTemplateVersion(versionAnswer.getTemplateVersion()); router.setScriptsVersion(versionAnswer.getScriptsVersion()); router = _routerDao.persist(router, guestNetworks); } } else { result = false; } return result; } @Override public void finalizeStop(VirtualMachineProfile profile, StopAnswer answer) { if (answer != null) { VMInstanceVO vm = profile.getVirtualMachine(); DomainRouterVO domR = _routerDao.findById(vm.getId()); processStopOrRebootAnswer(domR, answer); } } @Override public void finalizeExpunge(DomainRouterVO vm) { } @Override public boolean startRemoteAccessVpn(Network network, RemoteAccessVpn vpn, List routers) throws ResourceUnavailableException { if (routers == null || routers.isEmpty()) { s_logger.warn("Failed to start remote access VPN: no router found for account and zone"); throw new ResourceUnavailableException("Failed to start remote access VPN: no router found for account and zone", DataCenter.class, network.getDataCenterId()); } for (VirtualRouter router : routers) { if (router.getState() != State.Running) { s_logger.warn("Failed to start remote access VPN: router not in right state " + router.getState()); throw new ResourceUnavailableException("Failed to start remote access VPN: router not in right state " + router.getState(), DataCenter.class, network.getDataCenterId()); } Commands cmds = new Commands(OnError.Stop); createApplyVpnCommands(vpn, router, cmds); try { _agentMgr.send(router.getHostId(), cmds); } catch (OperationTimedoutException e) { s_logger.debug("Failed to start remote access VPN: ", e); throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); } Answer answer = cmds.getAnswer("users"); if (!answer.getResult()) { s_logger.error("Unable to start vpn: unable add users to vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " + answer.getDetails()); throw new ResourceUnavailableException("Unable to start vpn: Unable to add users to vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn()); } answer = cmds.getAnswer("startVpn"); if (!answer.getResult()) { s_logger.error("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " + answer.getDetails()); throw new ResourceUnavailableException("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn()); } } return true; } @Override public boolean deleteRemoteAccessVpn(Network network, RemoteAccessVpn vpn, List routers) throws ResourceUnavailableException { if (routers == null || routers.isEmpty()) { s_logger.warn("Failed to delete remote access VPN: no router found for account and zone"); throw new ResourceUnavailableException("Failed to delete remote access VPN", DataCenter.class, network.getDataCenterId()); } boolean result = true; for (VirtualRouter router : routers) { if (router.getState() == State.Running) { Commands cmds = new Commands(OnError.Continue); IpAddress ip = _networkMgr.getIp(vpn.getServerAddressId()); RemoteAccessVpnCfgCommand removeVpnCmd = new RemoteAccessVpnCfgCommand(false, ip.getAddress().addr(), vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey()); removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); removeVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); removeVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(removeVpnCmd); result = result && sendCommandsToRouter(router, cmds); } else if (router.getState() == State.Stopped) { s_logger.debug("Router " + router + " is in Stopped state, not sending deleteRemoteAccessVpn command to it"); continue; } else { s_logger.warn("Failed to delete remote access VPN: domR " + router + " is not in right state " + router.getState()); throw new ResourceUnavailableException("Failed to delete remote access VPN: domR is not in right state " + router.getState(), DataCenter.class, network.getDataCenterId()); } } return result; } private DomainRouterVO start(DomainRouterVO router, User user, Account caller, Map params, DeploymentPlan planToDeploy) throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { s_logger.debug("Starting router " + router); if (_itMgr.start(router, params, user, caller, planToDeploy) != null) { // We don't want the failure of VPN Connection affect the status of router, so we try to make connection only after router start successfully Long vpcId = router.getVpcId(); if (vpcId != null) { _s2sVpnMgr.reconnectDisconnectedVpnByVpc(vpcId); } return _routerDao.findById(router.getId()); } else { return null; } } @Override public DomainRouterVO stop(VirtualRouter router, boolean forced, User user, Account caller) throws ConcurrentOperationException, ResourceUnavailableException { s_logger.debug("Stopping router " + router); try { if (_itMgr.advanceStop((DomainRouterVO) router, forced, user, caller)) { return _routerDao.findById(router.getId()); } else { return null; } } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Unable to stop " + router, e); } } @Override public boolean applyDhcpEntry(Network network, final NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, List routers) throws ResourceUnavailableException { _userVmDao.loadDetails((UserVmVO) profile.getVirtualMachine()); final VirtualMachineProfile updatedProfile = profile; final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); final Long podId = isZoneBasic ? dest.getPod().getId() : null; boolean podLevelException = false; //for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; so throwing exception with Pod scope if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { podLevelException = true; } return applyRules(network, routers, "dhcp entry", podLevelException, podId, true, new RuleApplier() { @Override public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { //for basic zone, send dhcp/dns information to all routers in the basic network only when _dnsBasicZoneUpdates is set to "all" value Commands cmds = new Commands(OnError.Stop); if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue() && _dnsBasicZoneUpdates.equalsIgnoreCase("pod"))) { NicVO nicVo = _nicDao.findById(nic.getId()); createDhcpEntryCommand(router, updatedProfile.getVirtualMachine(), nicVo, cmds); return sendCommandsToRouter(router, cmds); } return true; } }); } private String findDefaultDnsIp(long userVmId) { NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); //check if DNS provider is the domR if (!_networkMgr.isProviderSupportServiceInNetwork(defaultNic.getNetworkId(), Service.Dns, Provider.VirtualRouter)) { return null; } NetworkOfferingVO offering = _networkOfferingDao.findById(_networkDao.findById(defaultNic.getNetworkId()).getNetworkOfferingId()); if (offering.getRedundantRouter()) { return findGatewayIp(userVmId); } //find domR's nic in the network NicVO domrDefaultNic = _nicDao.findByNetworkIdAndType(defaultNic.getNetworkId(), VirtualMachine.Type.DomainRouter); return domrDefaultNic.getIp4Address(); } private String findGatewayIp(long userVmId) { NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId); return defaultNic.getGateway(); } @Override public boolean applyUserData(Network network, final NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, List routers) throws ResourceUnavailableException { _userVmDao.loadDetails((UserVmVO) profile.getVirtualMachine()); final VirtualMachineProfile updatedProfile = profile; final boolean isZoneBasic = (dest.getDataCenter().getNetworkType() == NetworkType.Basic); final Long podId = isZoneBasic ? dest.getPod().getId() : null; boolean podLevelException = false; //for user vm in Basic zone we should try to re-deploy vm in a diff pod if it fails to deploy in original pod; so throwing exception with Pod scope if (isZoneBasic && podId != null && updatedProfile.getVirtualMachine().getType() == VirtualMachine.Type.User && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Shared) { podLevelException = true; } return applyRules(network, routers, "userdata and password entry", podLevelException, podId, false, new RuleApplier() { @Override public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { //for basic zone, send vm data/password information only to the router in the same pod Commands cmds = new Commands(OnError.Stop); if (!(isZoneBasic && router.getPodIdToDeployIn().longValue() != podId.longValue())) { NicVO nicVo = _nicDao.findById(nic.getId()); createPasswordCommand(router, updatedProfile, nicVo, cmds); createVmDataCommand(router, updatedProfile.getVirtualMachine(), nicVo, updatedProfile.getVirtualMachine().getDetail("SSH.PublicKey"), cmds); return sendCommandsToRouter(router, cmds); } return true; } }); } @Override public DomainRouterVO persist(DomainRouterVO router) { DomainRouterVO virtualRouter = _routerDao.persist(router); return virtualRouter; } @Override //FIXME add partial success and STOP state support public String[] applyVpnUsers(Network network, List users, List routers) throws ResourceUnavailableException { if (routers == null || routers.isEmpty()) { s_logger.warn("Failed to add/remove VPN users: no router found for account and zone"); throw new ResourceUnavailableException("Unable to assign ip addresses, domR doesn't exist for network " + network.getId(), DataCenter.class, network.getDataCenterId()); } boolean agentResults = true; for (DomainRouterVO router : routers) { if (router.getState() != State.Running) { s_logger.warn("Failed to add/remove VPN users: router not in running state"); throw new ResourceUnavailableException("Unable to assign ip addresses, domR is not in right state " + router.getState(), DataCenter.class, network.getDataCenterId()); } Commands cmds = new Commands(OnError.Continue); List addUsers = new ArrayList(); List removeUsers = new ArrayList(); for (VpnUser user : users) { if (user.getState() == VpnUser.State.Add || user.getState() == VpnUser.State.Active) { addUsers.add(user); } else if (user.getState() == VpnUser.State.Revoke) { removeUsers.add(user); } } VpnUsersCfgCommand cmd = new VpnUsersCfgCommand(addUsers, removeUsers); cmd.setAccessDetail(NetworkElementCommand.ACCOUNT_ID, String.valueOf(router.getAccountId())); cmd.setAccessDetail(NetworkElementCommand.GUEST_NETWORK_CIDR, network.getCidr()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(network.getId(), router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(cmd); // Currently we receive just one answer from the agent. In the future we have to parse individual answers and set // results accordingly boolean agentResult = sendCommandsToRouter(router, cmds); agentResults = agentResults && agentResult; } String[] result = new String[users.size()]; for (int i = 0; i < result.length; i++) { if (agentResults) { result[i] = null; } else { result[i] = String.valueOf(agentResults); } } return result; } @Override public DomainRouterVO findById(long id) { return _routerDao.findById(id); } @Override public DomainRouterVO findByName(String name) { if (!VirtualMachineName.isValidRouterName(name)) { return null; } return _routerDao.findById(VirtualMachineName.getRouterId(name)); } @Override @ActionEvent(eventType = EventTypes.EVENT_ROUTER_START, eventDescription = "starting router Vm", async = true) public VirtualRouter startRouter(long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException{ return startRouter(id, true); } @Override public VirtualRouter startRouter(long routerId, boolean reprogramNetwork) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException { Account caller = UserContext.current().getCaller(); User callerUser = _accountMgr.getActiveUser(UserContext.current().getCallerUserId()); // verify parameters DomainRouterVO router = _routerDao.findById(routerId); if (router == null) { throw new InvalidParameterValueException("Unable to find router by id " + routerId + "."); } _accountMgr.checkAccess(caller, null, true, router); Account owner = _accountMgr.getAccount(router.getAccountId()); // Check if all networks are implemented for the domR; if not - implement them DataCenter dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); HostPodVO pod = null; if (router.getPodIdToDeployIn() != null) { pod = _podDao.findById(router.getPodIdToDeployIn()); } DeployDestination dest = new DeployDestination(dc, pod, null, null); ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); List nics = _nicDao.listByVmId(routerId); for (NicVO nic : nics) { if (!_networkMgr.startNetwork(nic.getNetworkId(), dest, context)) { s_logger.warn("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); throw new CloudRuntimeException("Failed to start network id=" + nic.getNetworkId() + " as a part of domR start"); } } UserVO user = _userDao.findById(UserContext.current().getCallerUserId()); Map params = new HashMap(); if (reprogramNetwork) { params.put(Param.ReProgramGuestNetworks, true); } else { params.put(Param.ReProgramGuestNetworks, false); } VirtualRouter virtualRouter = startVirtualRouter(router, user, caller, params); if(virtualRouter == null){ throw new CloudRuntimeException("Failed to start router with id " + routerId); } return virtualRouter; } private void createAssociateIPCommands(final VirtualRouter router, final List ips, Commands cmds, long vmId) { // Ensure that in multiple vlans case we first send all ip addresses of vlan1, then all ip addresses of vlan2, etc.. Map> vlanIpMap = new HashMap>(); for (final PublicIpAddress ipAddress : ips) { String vlanTag = ipAddress.getVlanTag(); ArrayList ipList = vlanIpMap.get(vlanTag); if (ipList == null) { ipList = new ArrayList(); } //domR doesn't support release for sourceNat IP address; so reset the state if (ipAddress.isSourceNat() && ipAddress.getState() == IpAddress.State.Releasing) { ipAddress.setState(IpAddress.State.Allocated); } ipList.add(ipAddress); vlanIpMap.put(vlanTag, ipList); } for (Map.Entry> vlanAndIp : vlanIpMap.entrySet()) { List ipAddrList = vlanAndIp.getValue(); // Source nat ip address should always be sent first Collections.sort(ipAddrList, new Comparator() { @Override public int compare(PublicIpAddress o1, PublicIpAddress o2) { boolean s1 = o1.isSourceNat(); boolean s2 = o2.isSourceNat(); return (s1 ^ s2) ? ((s1 ^ true) ? 1 : -1) : 0; } }); // Get network rate - required for IpAssoc Integer networkRate = _networkMgr.getNetworkRate(ipAddrList.get(0).getNetworkId(), router.getId()); Network network = _networkMgr.getNetwork(ipAddrList.get(0).getNetworkId()); IpAddressTO[] ipsToSend = new IpAddressTO[ipAddrList.size()]; int i = 0; boolean firstIP = true; for (final PublicIpAddress ipAddr : ipAddrList) { boolean add = (ipAddr.getState() == IpAddress.State.Releasing ? false : true); boolean sourceNat = ipAddr.isSourceNat(); /* enable sourceNAT for the first ip of the public interface */ if (firstIP) { sourceNat = true; } String vlanId = ipAddr.getVlanTag(); String vlanGateway = ipAddr.getGateway(); String vlanNetmask = ipAddr.getNetmask(); String vifMacAddress = ipAddr.getMacAddress(); String vmGuestAddress = null; IpAddressTO ip = new IpAddressTO(ipAddr.getAccountId(), ipAddr.getAddress().addr(), add, firstIP, sourceNat, vlanId, vlanGateway, vlanNetmask, vifMacAddress, vmGuestAddress, networkRate, ipAddr.isOneToOneNat()); ip.setTrafficType(network.getTrafficType()); ip.setNetworkName(_networkMgr.getNetworkTag(router.getHypervisorType(), network)); ipsToSend[i++] = ip; /* send the firstIP = true for the first Add, this is to create primary on interface*/ if (!firstIP || add) { firstIP = false; } } IpAssocCommand cmd = new IpAssocCommand(ipsToSend); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(ipAddrList.get(0).getNetworkId(), router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("IPAssocCommand", cmd); } } private void createApplyPortForwardingRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { List rulesTO = null; if (rules != null) { rulesTO = new ArrayList(); for (PortForwardingRule rule : rules) { IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); PortForwardingRuleTO ruleTO = new PortForwardingRuleTO(rule, null, sourceIp.getAddress().addr()); rulesTO.add(ruleTO); } } SetPortForwardingRulesCommand cmd = null; if (router.getVpcId() != null) { cmd = new SetPortForwardingRulesVpcCommand(rulesTO); } else { cmd = new SetPortForwardingRulesCommand(rulesTO); } cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(cmd); } private void createApplyStaticNatRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { List rulesTO = null; if (rules != null) { rulesTO = new ArrayList(); for (StaticNatRule rule : rules) { IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); StaticNatRuleTO ruleTO = new StaticNatRuleTO(rule, null, sourceIp.getAddress().addr(), rule.getDestIpAddress()); rulesTO.add(ruleTO); } } SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(cmd); } private void createApplyLoadBalancingRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()]; int i = 0; for (LoadBalancingRule rule : rules) { boolean revoked = (rule.getState().equals(FirewallRule.State.Revoke)); String protocol = rule.getProtocol(); String algorithm = rule.getAlgorithm(); String srcIp = _networkMgr.getIp(rule.getSourceIpAddressId()).getAddress().addr(); int srcPort = rule.getSourcePortStart(); List destinations = rule.getDestinations(); List stickinessPolicies = rule.getStickinessPolicies(); LoadBalancerTO lb = new LoadBalancerTO(srcIp, srcPort, protocol, algorithm, revoked, false, destinations, stickinessPolicies); lbs[i++] = lb; } String routerPublicIp = null; if (router instanceof DomainRouterVO) { DomainRouterVO domr = _routerDao.findById(router.getId()); routerPublicIp = domr.getPublicIpAddress(); } Network guestNetwork = _networkMgr.getNetwork(guestNetworkId); Nic nic = _nicDao.findByInstanceIdAndNetworkId(guestNetwork.getId(), router.getId()); NicProfile nicProfile = new NicProfile(nic, guestNetwork, nic.getBroadcastUri(), nic.getIsolationUri(), _networkMgr.getNetworkRate(guestNetwork.getId(), router.getId()), _networkMgr.isSecurityGroupSupportedInNetwork(guestNetwork), _networkMgr.getNetworkTag(router.getHypervisorType(), guestNetwork)); LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs,routerPublicIp, getRouterIpInNetwork(guestNetworkId, router.getId()),router.getPrivateIpAddress(), _itMgr.toNicTO(nicProfile, router.getHypervisorType()), router.getVpcId()); cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key()); cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key()); cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key()); cmd.lbStatsPort = _configDao.getValue(Config.NetworkLBHaproxyStatsPort.key()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(cmd); } private void createApplyVpnCommands(RemoteAccessVpn vpn, VirtualRouter router, Commands cmds) { List vpnUsers = _vpnUsersDao.listByAccount(vpn.getAccountId()); List addUsers = new ArrayList(); List removeUsers = new ArrayList(); for (VpnUser user : vpnUsers) { if (user.getState() == VpnUser.State.Add) { addUsers.add(user); } else if (user.getState() == VpnUser.State.Revoke) { removeUsers.add(user); } } VpnUsersCfgCommand addUsersCmd = new VpnUsersCfgCommand(addUsers, removeUsers); addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(vpn.getNetworkId(), router.getId())); addUsersCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); IpAddress ip = _networkMgr.getIp(vpn.getServerAddressId()); RemoteAccessVpnCfgCommand startVpnCmd = new RemoteAccessVpnCfgCommand(true, ip.getAddress().addr(), vpn.getLocalIp(), vpn.getIpRange(), vpn.getIpsecPresharedKey()); startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(vpn.getNetworkId(), router.getId())); startVpnCmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); startVpnCmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("users", addUsersCmd); cmds.addCommand("startVpn", startVpnCmd); } private void createPasswordCommand(VirtualRouter router, VirtualMachineProfile profile, NicVO nic, Commands cmds) { String password = (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); // password should be set only on default network element if (password != null && nic.isDefaultNic()) { final String encodedPassword = PasswordGenerator.rot13(password); SavePasswordCommand cmd = new SavePasswordCommand(encodedPassword, nic.getIp4Address(), profile.getVirtualMachine().getHostName()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("password", cmd); } } private void createVmDataCommand(VirtualRouter router, UserVm vm, NicVO nic, String publicKey, Commands cmds) { String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getServiceOfferingId()).getDisplayText(); String zoneName = _dcDao.findById(router.getDataCenterIdToDeployIn()).getName(); cmds.addCommand("vmdata", generateVmDataCommand(router, nic.getIp4Address(), vm.getUserData(), serviceOffering, zoneName, nic.getIp4Address(), vm.getHostName(), vm.getInstanceName(), vm.getId(), publicKey, nic.getNetworkId())); } private void createVmDataCommandForVMs(DomainRouterVO router, Commands cmds, long guestNetworkId) { List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); DataCenterVO dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); for (UserVmVO vm : vms) { boolean createVmData = true; if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue()) { createVmData = false; } if (createVmData) { NicVO nic = _nicDao.findByInstanceIdAndNetworkId(guestNetworkId, vm.getId()); if (nic != null) { s_logger.debug("Creating user data entry for vm " + vm + " on domR " + router); createVmDataCommand(router, vm, nic, null, cmds); } } } } private void createDhcpEntryCommand(VirtualRouter router, UserVm vm, NicVO nic, Commands cmds) { DhcpEntryCommand dhcpCommand = new DhcpEntryCommand(nic.getMacAddress(), nic.getIp4Address(), vm.getHostName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); String gatewayIp = findGatewayIp(vm.getId()); if (!gatewayIp.equals(nic.getGateway())) { GuestOSVO guestOS = _guestOSDao.findById(vm.getGuestOSId()); // Don't set dhcp:router option for non-default nic on CentOS/RHEL, because they would set routing on wrong interface // This is tricky, we may need to update this when we have more information on various OS's behavior if (guestOS.getDisplayName().startsWith("CentOS") || guestOS.getDisplayName().startsWith("Red Hat Enterprise")) { gatewayIp = "0.0.0.0"; } } dhcpCommand.setDefaultRouter(gatewayIp); dhcpCommand.setDefaultDns(findDefaultDnsIp(vm.getId())); dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); dhcpCommand.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(nic.getNetworkId(), router.getId())); dhcpCommand.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand("dhcp", dhcpCommand); } private void createDhcpEntryCommandsForVMs(DomainRouterVO router, Commands cmds, long guestNetworkId) { List vms = _userVmDao.listByNetworkIdAndStates(guestNetworkId, State.Running, State.Migrating, State.Stopping); DataCenterVO dc = _dcDao.findById(router.getDataCenterIdToDeployIn()); for (UserVmVO vm : vms) { boolean createDhcp = true; if (dc.getNetworkType() == NetworkType.Basic && router.getPodIdToDeployIn().longValue() != vm.getPodIdToDeployIn().longValue() && _dnsBasicZoneUpdates.equalsIgnoreCase("pod")) { createDhcp = false; } if (createDhcp) { NicVO nic = _nicDao.findByInstanceIdAndNetworkId(guestNetworkId, vm.getId()); if (nic != null) { s_logger.debug("Creating dhcp entry for vm " + vm + " on domR " + router + "."); createDhcpEntryCommand(router, vm, nic, cmds); } } } } protected boolean sendCommandsToRouter(final VirtualRouter router, Commands cmds) throws AgentUnavailableException { Answer[] answers = null; try { answers = _agentMgr.send(router.getHostId(), cmds); } catch (OperationTimedoutException e) { s_logger.warn("Timed Out", e); throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); } if (answers == null) { return false; } if (answers.length != cmds.size()) { return false; } // FIXME: Have to return state for individual command in the future boolean result = true; if (answers.length > 0) { for (Answer answer : answers) { if (!answer.getResult()) { result = false; break; } } } return result; } protected void handleSingleWorkingRedundantRouter(List connectedRouters, List disconnectedRouters, String reason) throws ResourceUnavailableException { if (connectedRouters.isEmpty() || disconnectedRouters.isEmpty()) { return; } if (connectedRouters.size() != 1 || disconnectedRouters.size() != 1) { s_logger.warn("How many redundant routers do we have?? "); return; } if (!connectedRouters.get(0).getIsRedundantRouter()) { throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", DataCenter.class, connectedRouters.get(0).getDataCenterIdToDeployIn()); } if (!disconnectedRouters.get(0).getIsRedundantRouter()) { throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn()); } DomainRouterVO connectedRouter = (DomainRouterVO)connectedRouters.get(0); DomainRouterVO disconnectedRouter = (DomainRouterVO)disconnectedRouters.get(0); if (s_logger.isDebugEnabled()) { s_logger.debug("About to stop the router " + disconnectedRouter.getInstanceName() + " due to: " + reason); } String title = "Virtual router " + disconnectedRouter.getInstanceName() + " would be stopped after connecting back, due to " + reason; String context = "Virtual router (name: " + disconnectedRouter.getInstanceName() + ", id: " + disconnectedRouter.getId() + ") would be stopped after connecting back, due to: " + reason; _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, disconnectedRouter.getDataCenterIdToDeployIn(), disconnectedRouter.getPodIdToDeployIn(), title, context); disconnectedRouter.setStopPending(true); disconnectedRouter = _routerDao.persist(disconnectedRouter); int connRouterPR = getRealPriority(connectedRouter); int disconnRouterPR = getRealPriority(disconnectedRouter); if (connRouterPR < disconnRouterPR) { //connRouterPR < disconnRouterPR, they won't equal at anytime if (!connectedRouter.getIsPriorityBumpUp()) { final BumpUpPriorityCommand command = new BumpUpPriorityCommand(); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(connectedRouter.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, connectedRouter.getInstanceName()); final Answer answer = _agentMgr.easySend(connectedRouter.getHostId(), command); if (!answer.getResult()) { s_logger.error("Failed to bump up " + connectedRouter.getInstanceName() + "'s priority! " + answer.getDetails()); } } else { String t = "Can't bump up virtual router " + connectedRouter.getInstanceName() + "'s priority due to it's already bumped up!"; _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, connectedRouter.getDataCenterIdToDeployIn(), connectedRouter.getPodIdToDeployIn(), t, t); } } } @Override public boolean associatePublicIP(Network network, final List ipAddress, List routers) throws ResourceUnavailableException { if (ipAddress == null || ipAddress.isEmpty()) { s_logger.debug("No ip association rules to be applied for network " + network.getId()); return true; } return applyRules(network, routers, "ip association", false, null, false, new RuleApplier() { @Override public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { Commands cmds = new Commands(OnError.Continue); createAssociateIPCommands(router, ipAddress, cmds, 0); return sendCommandsToRouter(router, cmds); } }); } @Override public boolean applyFirewallRules(Network network, final List rules, List routers) throws ResourceUnavailableException { if (rules == null || rules.isEmpty()) { s_logger.debug("No firewall rules to be applied for network " + network.getId()); return true; } return applyRules(network, routers, "firewall rules", false, null, false, new RuleApplier() { @Override public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { if (rules.get(0).getPurpose() == Purpose.LoadBalancing) { // for load balancer we have to resend all lb rules for the network List lbs = _loadBalancerDao.listByNetworkId(network.getId()); List lbRules = new ArrayList(); for (LoadBalancerVO lb : lbs) { List dstList = _lbMgr.getExistingDestinations(lb.getId()); List policyList = _lbMgr.getStickinessPolicies(lb.getId()); LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList,policyList); lbRules.add(loadBalancing); } return sendLBRules(router, lbRules, network.getId()); } else if (rules.get(0).getPurpose() == Purpose.PortForwarding) { return sendPortForwardingRules(router, (List) rules, network.getId()); } else if (rules.get(0).getPurpose() == Purpose.StaticNat) { return sendStaticNatRules(router, (List) rules, network.getId()); } else if (rules.get(0).getPurpose() == Purpose.Firewall) { return sendFirewallRules(router, (List) rules, network.getId()); } else { s_logger.warn("Unable to apply rules of purpose: " + rules.get(0).getPurpose()); return false; } } }); } protected boolean sendLBRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { Commands cmds = new Commands(OnError.Continue); createApplyLoadBalancingRulesCommands(rules, router, cmds, guestNetworkId); return sendCommandsToRouter(router, cmds); } protected boolean sendPortForwardingRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { Commands cmds = new Commands(OnError.Continue); createApplyPortForwardingRulesCommands(rules, router, cmds, guestNetworkId); return sendCommandsToRouter(router, cmds); } protected boolean sendStaticNatRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { Commands cmds = new Commands(OnError.Continue); createApplyStaticNatRulesCommands(rules, router, cmds, guestNetworkId); return sendCommandsToRouter(router, cmds); } @Override public List getRoutersForNetwork(long networkId) { List routers = _routerDao.findByNetwork(networkId); List vrs = new ArrayList(routers.size()); for (DomainRouterVO router : routers) { vrs.add(router); } return vrs; } private void createFirewallRulesCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { List rulesTO = null; if (rules != null) { rulesTO = new ArrayList(); for (FirewallRule rule : rules) { IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); FirewallRuleTO ruleTO = new FirewallRuleTO(rule, null, sourceIp.getAddress().addr()); rulesTO.add(ruleTO); } } SetFirewallRulesCommand cmd = new SetFirewallRulesCommand(rulesTO); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(cmd); } protected boolean sendFirewallRules(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { Commands cmds = new Commands(OnError.Continue); createFirewallRulesCommands(rules, router, cmds, guestNetworkId); return sendCommandsToRouter(router, cmds); } @Override public String getDnsBasicZoneUpdate() { return _dnsBasicZoneUpdates; } protected interface RuleApplier { boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException; } protected boolean applyRules(Network network, List routers, String typeString, boolean isPodLevelException, Long podId, boolean failWhenDisconnect, RuleApplier applier) throws ResourceUnavailableException { if (routers == null || routers.isEmpty()) { s_logger.warn("Unable to apply " + typeString + ", virtual router doesn't exist in the network " + network.getId()); throw new ResourceUnavailableException("Unable to apply " + typeString , DataCenter.class, network.getDataCenterId()); } DataCenter dc = _dcDao.findById(network.getDataCenterId()); boolean isZoneBasic = (dc.getNetworkType() == NetworkType.Basic); // isPodLevelException and podId is only used for basic zone assert !((!isZoneBasic && isPodLevelException) || (isZoneBasic && isPodLevelException && podId == null)); List connectedRouters = new ArrayList(); List disconnectedRouters = new ArrayList(); boolean result = true; String msg = "Unable to apply " + typeString + " on disconnected router "; for (VirtualRouter router : routers) { if (router.getState() == State.Running) { s_logger.debug("Applying " + typeString + " in network " + network); if (router.isStopPending()) { if (_hostDao.findById(router.getHostId()).getStatus() == Status.Up) { throw new ResourceUnavailableException("Unable to process due to the stop pending router " + router.getInstanceName() + " haven't been stopped after it's host coming back!", DataCenter.class, router.getDataCenterIdToDeployIn()); } s_logger.debug("Router " + router.getInstanceName() + " is stop pending, so not sending apply " + typeString + " commands to the backend"); continue; } try { result = applier.execute(network, router); connectedRouters.add(router); } catch (AgentUnavailableException e) { s_logger.warn(msg + router.getInstanceName(), e); disconnectedRouters.add(router); } //If rules fail to apply on one domR and not due to disconnection, no need to proceed with the rest if (!result) { if (isZoneBasic && isPodLevelException) { throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", Pod.class, podId); } throw new ResourceUnavailableException("Unable to apply " + typeString + " on router ", DataCenter.class, router.getDataCenterIdToDeployIn()); } } else if (router.getState() == State.Stopped || router.getState() == State.Stopping) { s_logger.debug("Router " + router.getInstanceName() + " is in " + router.getState() + ", so not sending apply " + typeString + " commands to the backend"); } else { s_logger.warn("Unable to apply " + typeString +", virtual router is not in the right state " + router.getState()); if (isZoneBasic && isPodLevelException) { throw new ResourceUnavailableException("Unable to apply " + typeString + ", virtual router is not in the right state", Pod.class, podId); } throw new ResourceUnavailableException("Unable to apply " + typeString + ", virtual router is not in the right state", DataCenter.class, router.getDataCenterIdToDeployIn()); } } if (!connectedRouters.isEmpty()) { if (!isZoneBasic && !disconnectedRouters.isEmpty() && disconnectedRouters.get(0).getIsRedundantRouter()) { // These disconnected redundant virtual routers are out of sync now, stop them for synchronization handleSingleWorkingRedundantRouter(connectedRouters, disconnectedRouters, msg); } } else if (!disconnectedRouters.isEmpty()) { for (VirtualRouter router : disconnectedRouters) { if (s_logger.isDebugEnabled()) { s_logger.debug(msg + router.getInstanceName() + "(" + router.getId() + ")"); } } if (isZoneBasic && isPodLevelException) { throw new ResourceUnavailableException(msg, Pod.class, podId); } throw new ResourceUnavailableException(msg, DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn()); } result = true; if (failWhenDisconnect) { result = !connectedRouters.isEmpty(); } return result; } @Override public boolean applyStaticNats(Network network, final List rules, List routers) throws ResourceUnavailableException { if (rules == null || rules.isEmpty()) { s_logger.debug("No static nat rules to be applied for network " + network.getId()); return true; } return applyRules(network, routers, "static nat rules", false, null, false, new RuleApplier() { @Override public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException { return applyStaticNat(router, rules, network.getId()); } }); } protected boolean applyStaticNat(VirtualRouter router, List rules, long guestNetworkId) throws ResourceUnavailableException { Commands cmds = new Commands(OnError.Continue); createApplyStaticNatCommands(rules, router, cmds, guestNetworkId); return sendCommandsToRouter(router, cmds); } private void createApplyStaticNatCommands(List rules, VirtualRouter router, Commands cmds, long guestNetworkId) { List rulesTO = null; if (rules != null) { rulesTO = new ArrayList(); for (StaticNat rule : rules) { IpAddress sourceIp = _networkMgr.getIp(rule.getSourceIpAddressId()); StaticNatRuleTO ruleTO = new StaticNatRuleTO(0, sourceIp.getAddress().addr(), null, null, rule.getDestIpAddress(), null, null, null, rule.isForRevoke(), false); rulesTO.add(ruleTO); } } SetStaticNatRulesCommand cmd = new SetStaticNatRulesCommand(rulesTO, router.getVpcId()); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, getRouterIpInNetwork(guestNetworkId, router.getId())); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); DataCenterVO dcVo = _dcDao.findById(router.getDataCenterIdToDeployIn()); cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); cmds.addCommand(cmd); } @Override public int getTimeout() { return -1; } @Override public boolean isRecurring() { return false; } @Override public boolean processAnswers(long agentId, long seq, Answer[] answers) { return false; } @Override public boolean processCommands(long agentId, long seq, Command[] commands) { return false; } @Override public void processConnect(HostVO host, StartupCommand cmd, boolean forRebalance) throws ConnectionException { UserContext context = UserContext.current(); context.setAccountId(1); List routers = _routerDao.listIsolatedByHostId(host.getId()); for (DomainRouterVO router : routers) { if (router.isStopPending()) { State state = router.getState(); if (state != State.Stopped && state != State.Destroyed) { try { stopRouter(router.getId(), false); } catch (ResourceUnavailableException e) { s_logger.warn("Fail to stop router " + router.getInstanceName(), e); throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); } catch (ConcurrentOperationException e) { s_logger.warn("Fail to stop router " + router.getInstanceName(), e); throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName()); } } router.setStopPending(false); router = _routerDao.persist(router); } } } @Override public AgentControlAnswer processControlCommand(long agentId, AgentControlCommand cmd) { return null; } @Override public boolean processDisconnect(long agentId, Status state) { return false; } @Override public boolean processTimeout(long agentId, long seq) { return false; } protected String getRouterControlIp(long routerId) { String routerControlIpAddress = null; List nics = _nicDao.listByVmId(routerId); for (NicVO n : nics) { NetworkVO nc = _networkDao.findById(n.getNetworkId()); if (nc.getTrafficType() == TrafficType.Control) { routerControlIpAddress = n.getIp4Address(); } } if(routerControlIpAddress == null) { s_logger.warn("Unable to find router's control ip in its attached NICs!. routerId: " + routerId); DomainRouterVO router = _routerDao.findById(routerId); return router.getPrivateIpAddress(); } return routerControlIpAddress; } protected String getRouterIpInNetwork(long networkId, long instanceId) { return _nicDao.getIpAddress(networkId, instanceId); } @Override public boolean plugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { //not supported throw new UnsupportedOperationException("Plug nic is not supported for vm of type " + vm.getType()); } @Override public boolean unplugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException { //not supported throw new UnsupportedOperationException("Unplug nic is not supported for vm of type " + vm.getType()); } @Override public void prepareStop(VirtualMachineProfile profile){ //Collect network usage before stopping Vm VMInstanceVO vm = profile.getVirtualMachine(); DomainRouterVO router = _routerDao.findById(vm.getId()); if(router == null){ return; } String privateIP = router.getPrivateIpAddress(); if (privateIP != null) { List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); for (Long guestNtwkId : routerGuestNtwkIds) { boolean forVpc = router.getVpcId() != null; Network guestNtwk = _networkMgr.getNetwork(guestNtwkId); Nic guestNic = _nicDao.findByInstanceIdAndNetworkId(guestNtwk.getId(), router.getId()); NicProfile guestNicProfile = new NicProfile(guestNic, guestNtwk, guestNic.getBroadcastUri(), guestNic.getIsolationUri(), _networkMgr.getNetworkRate(guestNtwk.getId(), router.getId()), _networkMgr.isSecurityGroupSupportedInNetwork(guestNtwk), _networkMgr.getNetworkTag(router.getHypervisorType(), guestNtwk)); final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, _itMgr.toNicTO(guestNicProfile, router.getHypervisorType())); UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), router.getDataCenterIdToDeployIn(), guestNtwkId, null, router.getId(), router.getType().toString()); NetworkUsageAnswer answer = null; try { answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); } catch (Exception e) { s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); continue; } if (answer != null) { if (!answer.getResult()) { s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); continue; } Transaction txn = Transaction.open(Transaction.CLOUD_DB); try { if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); continue; } txn.start(); UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), router.getDataCenterIdToDeployIn(), guestNtwkId, null, router.getId(), router.getType().toString()); if (stats == null) { s_logger.warn("unable to find stats for account: " + router.getAccountId()); continue; } if(previousStats != null && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); continue; } if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answer.getRouterName()+" Reported: " + answer.getBytesReceived() + " Stored: " + stats.getCurrentBytesReceived()); } stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } stats.setCurrentBytesReceived(answer.getBytesReceived()); if (stats.getCurrentBytesSent() > answer.getBytesSent()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answer.getRouterName()+" Reported: " + answer.getBytesSent() + " Stored: " + stats.getCurrentBytesSent()); } stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); } stats.setCurrentBytesSent(answer.getBytesSent()); _statsDao.update(stats.getId(), stats); txn.commit(); } catch (Exception e) { txn.rollback(); s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); } finally { txn.close(); } } } } } @Override public boolean recreateNeeded( VirtualMachineProfile profile, long hostId, Commands cmds, ReservationContext context) { //asssume that if failed to ssh into router, meaning router is crashed CheckSshAnswer answer = (CheckSshAnswer) cmds.getAnswer("checkSsh"); if (answer == null || !answer.getResult()) { return true; } return false; } }