From 75f89c7676ac04821436e4b1bff820a17797bf46 Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Thu, 6 Aug 2015 14:15:48 +0530 Subject: [PATCH] CLOUDSTACK-8751 Minimise network downtime during network updates when redundant VR is being used. database schema changes Made changes to the updateNetwork API. --- api/src/com/cloud/network/Network.java | 2 + api/src/com/cloud/network/NetworkService.java | 2 +- .../network/element/RedundantResource.java | 11 + .../cloud/network/router/VirtualRouter.java | 4 + .../apache/cloudstack/api/ApiConstants.java | 1 + .../network/UpdateNetworkCmdByAdmin.java | 2 +- .../user/network/UpdateNetworkCmd.java | 12 +- .../service/NetworkOrchestrationService.java | 6 + .../orchestration/NetworkOrchestrator.java | 53 +++- .../src/com/cloud/vm/DomainRouterVO.java | 14 + .../com/cloud/network/NetworkServiceImpl.java | 256 ++++++++++-------- .../network/element/VirtualRouterElement.java | 98 ++++++- .../VirtualNetworkApplianceManagerImpl.java | 5 + .../RouterDeploymentDefinition.java | 12 +- .../RouterDeploymentDefinitionBuilder.java | 4 + .../com/cloud/vpc/MockNetworkManagerImpl.java | 17 +- .../RouterDeploymentDefinitionTest.java | 2 +- .../RouterDeploymentDefinitionTestBase.java | 3 + setup/db/db/schema-452to460.sql | 2 + 19 files changed, 379 insertions(+), 127 deletions(-) create mode 100644 api/src/com/cloud/network/element/RedundantResource.java diff --git a/api/src/com/cloud/network/Network.java b/api/src/com/cloud/network/Network.java index 7cc5441603a..81447d6457c 100644 --- a/api/src/com/cloud/network/Network.java +++ b/api/src/com/cloud/network/Network.java @@ -41,6 +41,8 @@ public interface Network extends ControlledEntity, StateObject, I Shared, Isolated } + public String updatingInSequence ="updatingInSequence"; + public static class Service { private static List supportedServices = new ArrayList(); diff --git a/api/src/com/cloud/network/NetworkService.java b/api/src/com/cloud/network/NetworkService.java index c1b68eb3a46..e26db340ff4 100644 --- a/api/src/com/cloud/network/NetworkService.java +++ b/api/src/com/cloud/network/NetworkService.java @@ -77,7 +77,7 @@ public interface NetworkService { IpAddress getIp(long id); Network updateGuestNetwork(long networkId, String name, String displayText, Account callerAccount, User callerUser, String domainSuffix, Long networkOfferingId, - Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String newUUID); + Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String newUUID, boolean updateInSequence); PhysicalNetwork createPhysicalNetwork(Long zoneId, String vnetRange, String networkSpeed, List isolationMethods, String broadcastDomainRange, Long domainId, List tags, String name); diff --git a/api/src/com/cloud/network/element/RedundantResource.java b/api/src/com/cloud/network/element/RedundantResource.java new file mode 100644 index 00000000000..863c9cd330c --- /dev/null +++ b/api/src/com/cloud/network/element/RedundantResource.java @@ -0,0 +1,11 @@ +package com.cloud.network.element; + +import com.cloud.network.Network; + +/** + * Created by bharat on 11/08/15. + */ +public interface RedundantResource { + public void configureResource(Network network); + public int getResourceCount(Network network); +} diff --git a/api/src/com/cloud/network/router/VirtualRouter.java b/api/src/com/cloud/network/router/VirtualRouter.java index 0114a962146..060ef0fa141 100644 --- a/api/src/com/cloud/network/router/VirtualRouter.java +++ b/api/src/com/cloud/network/router/VirtualRouter.java @@ -26,6 +26,10 @@ public interface VirtualRouter extends VirtualMachine { VIRTUAL_ROUTER, LB, INTERNAL_LB_VM } + public enum UpdateState { + UPDATE_NEEDED, UPDATE_IN_PROGRESS, UPDATE_COMPLETE, UPDATE_FAILED + } + Role getRole(); boolean getIsRedundantRouter(); diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java index 1d0b4a3756a..0e838490eb0 100644 --- a/api/src/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/org/apache/cloudstack/api/ApiConstants.java @@ -272,6 +272,7 @@ public class ApiConstants { public static final String USERNAME = "username"; public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist"; public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork"; + public static final String Update_IN_SEQUENCE ="updateinsequence"; public static final String VALUE = "value"; public static final String VIRTUAL_MACHINE_ID = "virtualmachineid"; public static final String VIRTUAL_MACHINE_IDS = "virtualmachineids"; diff --git a/api/src/org/apache/cloudstack/api/command/admin/network/UpdateNetworkCmdByAdmin.java b/api/src/org/apache/cloudstack/api/command/admin/network/UpdateNetworkCmdByAdmin.java index 269f43ec959..f2c5119466c 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/network/UpdateNetworkCmdByAdmin.java +++ b/api/src/org/apache/cloudstack/api/command/admin/network/UpdateNetworkCmdByAdmin.java @@ -49,7 +49,7 @@ public class UpdateNetworkCmdByAdmin extends UpdateNetworkCmd { } Network result = _networkService.updateGuestNetwork(getId(), getNetworkName(), getDisplayText(), callerAccount, - callerUser, getNetworkDomain(), getNetworkOfferingId(), getChangeCidr(), getGuestVmCidr(), getDisplayNetwork(), getCustomId()); + callerUser, getNetworkDomain(), getNetworkOfferingId(), getChangeCidr(), getGuestVmCidr(), getDisplayNetwork(), getCustomId(), getUpdateInSequence()); if (result != null) { diff --git a/api/src/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java b/api/src/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java index 921e74b67b2..8ef9251e047 100644 --- a/api/src/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java @@ -75,6 +75,9 @@ public class UpdateNetworkCmd extends BaseAsyncCustomIdCmd { @Parameter(name = ApiConstants.GUEST_VM_CIDR, type = CommandType.STRING, description = "CIDR for guest VMs, CloudStack allocates IPs to guest VMs only from this CIDR") private String guestVmCidr; + @Parameter(name =ApiConstants.Update_IN_SEQUENCE, type=CommandType.BOOLEAN, description = "if true, we will update the routers one after the other. applicable only for redundant router based networks using virtual router as provider") + private Boolean updateInSequence; + @Parameter(name = ApiConstants.DISPLAY_NETWORK, type = CommandType.BOOLEAN, description = "an optional field, whether to the display the network to the end user or not.", authorized = {RoleType.Admin}) @@ -119,6 +122,13 @@ public class UpdateNetworkCmd extends BaseAsyncCustomIdCmd { return displayNetwork; } + public Boolean getUpdateInSequence(){ + if(updateInSequence ==null) + return false; + else + return updateInSequence; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -149,7 +159,7 @@ public class UpdateNetworkCmd extends BaseAsyncCustomIdCmd { Network result = _networkService.updateGuestNetwork(getId(), getNetworkName(), getDisplayText(), callerAccount, callerUser, getNetworkDomain(), getNetworkOfferingId(), - getChangeCidr(), getGuestVmCidr(), getDisplayNetwork(), getCustomId()); + getChangeCidr(), getGuestVmCidr(), getDisplayNetwork(), getCustomId(), getUpdateInSequence()); if (result != null) { NetworkResponse response = _responseGenerator.createNetworkResponse(ResponseView.Restricted, result); diff --git a/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index b71aa96d5cc..1e2761f274a 100644 --- a/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -224,4 +224,10 @@ public interface NetworkOrchestrationService { boolean resourceCountNeedsUpdate(NetworkOffering ntwkOff, ACLType aclType); void prepareAllNicsForMigration(VirtualMachineProfile vm, DeployDestination dest); + + boolean canUpdateInSequence(Network network); + + void configureUpdateInSequence(Network network); + + int getResourceCount(Network network); } diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 66185c60978..c0ea2f9b635 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -36,6 +36,12 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.network.Networks; + +import com.cloud.network.dao.NetworkDetailsDao; +import com.cloud.network.element.RedundantResource; +import com.cloud.vm.dao.DomainRouterDao; +import org.apache.log4j.Logger; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.cloud.entity.api.db.VMNetworkMapVO; @@ -50,7 +56,6 @@ import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.region.PortableIpDao; -import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; @@ -107,7 +112,6 @@ import com.cloud.network.NetworkMigrationResponder; import com.cloud.network.NetworkModel; import com.cloud.network.NetworkProfile; import com.cloud.network.NetworkStateListener; -import com.cloud.network.Networks; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.TrafficType; import com.cloud.network.PhysicalNetwork; @@ -265,9 +269,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra MessageBus _messageBus; @Inject VMNetworkMapDao _vmNetworkMapDao; + @Inject + DomainRouterDao _rotuerDao; List networkGurus; + public List getNetworkGurus() { return networkGurus; } @@ -350,6 +357,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra PortableIpDao _portableIpDao; @Inject ConfigDepot _configDepot; + @Inject + NetworkDetailsDao _networkDetailsDao; protected StateMachine2 _stateMachine; ScheduledExecutorService _executor; @@ -1271,6 +1280,46 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return true; } + @Override + public boolean canUpdateInSequence(Network network){ + List providers = getNetworkProviders(network.getId()); + //check if the there are no service provider other than virtualrouter. + for(Provider provider :providers){ + if(provider!=Provider.VirtualRouter) + throw new UnsupportedOperationException("Cannot update the network resources in sequence when providers other than virtualrouter are used"); + } + return true; + } + + @Override + public void configureUpdateInSequence(Network network) { + List providers = getNetworkProviders(network.getId()); + for (NetworkElement element : networkElements) { + if (providers.contains(element.getProvider())) { + if (element instanceof RedundantResource) { + ((RedundantResource) element).configureResource(network); + } + } + } + } + + @Override + public int getResourceCount(Network network){ + List providers = getNetworkProviders(network.getId()); + int resourceCount=0; + for (NetworkElement element : networkElements) { + if (providers.contains(element.getProvider())) { + //currently only one element implements the redundant resource interface + if (element instanceof RedundantResource) { + resourceCount= ((RedundantResource) element).getResourceCount(network); + break; + } + } + } + return resourceCount; + } + + @DB protected void updateNic(final NicVO nic, final long networkId, final int count) { Transaction.execute(new TransactionCallbackNoReturn() { diff --git a/engine/schema/src/com/cloud/vm/DomainRouterVO.java b/engine/schema/src/com/cloud/vm/DomainRouterVO.java index 2596d24a5fb..2a7aa49b6ed 100644 --- a/engine/schema/src/com/cloud/vm/DomainRouterVO.java +++ b/engine/schema/src/com/cloud/vm/DomainRouterVO.java @@ -69,6 +69,11 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter { @Column(name = "vpc_id") private Long vpcId; + + @Column(name= "update_state") + @Enumerated(EnumType.STRING) + private UpdateState updateState; + public DomainRouterVO(final long id, final long serviceOfferingId, final long elementId, final String name, final long templateId, final HypervisorType hypervisorType, final long guestOSId, final long domainId, final long accountId, final long userId, final boolean isRedundantRouter, final RedundantState redundantState, final boolean haEnabled, final boolean stopPending, final Long vpcId) { @@ -193,4 +198,13 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter { return vpcId; } + public UpdateState getUpdateState() { + return updateState; + } + + public void setUpdateState(UpdateState updateState) { + this.updateState = updateState; + } + + } diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index b6dac872f30..bb573decde2 100644 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -108,6 +108,8 @@ import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.LoadBalancerVMMapDao; import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkDetailVO; +import com.cloud.network.dao.NetworkDetailsDao; import com.cloud.network.dao.NetworkDomainDao; import com.cloud.network.dao.NetworkDomainVO; import com.cloud.network.dao.NetworkServiceMapDao; @@ -178,6 +180,7 @@ import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExceptionUtil; import com.cloud.utils.net.NetUtils; +import com.cloud.vm.DomainRouterVO; import com.cloud.vm.Nic; import com.cloud.vm.NicSecondaryIp; import com.cloud.vm.NicVO; @@ -187,6 +190,7 @@ import com.cloud.vm.SecondaryStorageVmVO; import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.NicSecondaryIpDao; import com.cloud.vm.dao.NicSecondaryIpVO; @@ -324,6 +328,12 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { @Inject MessageBus _messageBus; + @Inject + DomainRouterDao _routerDao; + + @Inject + NetworkDetailsDao _networkDetailsDao; + int _cidrLimit; boolean _allowSubdomainNetworkAccess; @@ -1992,8 +2002,7 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { @DB @ActionEvent(eventType = EventTypes.EVENT_NETWORK_UPDATE, eventDescription = "updating network", async = true) public Network updateGuestNetwork(final long networkId, String name, String displayText, Account callerAccount, User callerUser, String domainSuffix, - final Long networkOfferingId, Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String customId) { - + final Long networkOfferingId, Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String customId, boolean updateInSequence) { boolean restartNetwork = false; // verify input parameters @@ -2239,130 +2248,153 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { ReservationContext context = new ReservationContextImpl(null, null, callerUser, callerAccount); // 1) Shutdown all the elements and cleanup all the rules. Don't allow to shutdown network in intermediate // states - Shutdown and Implementing - boolean validStateToShutdown = (network.getState() == Network.State.Implemented || network.getState() == Network.State.Setup || network.getState() == Network.State.Allocated); - if (restartNetwork) { - if (validStateToShutdown) { - if (!changeCidr) { - s_logger.debug("Shutting down elements and resources for network id=" + networkId + " as a part of network update"); + List routers=null; + int resourceCount=1; + if(updateInSequence && restartNetwork && _networkOfferingDao.findById(network.getNetworkOfferingId()).getRedundantRouter() && networkOfferingId!=null && _networkOfferingDao.findById(networkOfferingId).getRedundantRouter() && network.getVpcId()==null) { + _networkMgr.canUpdateInSequence(network); + NetworkDetailVO networkDetail =new NetworkDetailVO(network.getId(),Network.updatingInSequence,"true",true); + _networkDetailsDao.persist(networkDetail); + _networkMgr.configureUpdateInSequence(network); + resourceCount=_networkMgr.getResourceCount(network); + } - if (!_networkMgr.shutdownNetworkElementsAndResources(context, true, network)) { - s_logger.warn("Failed to shutdown the network elements and resources as a part of network restart: " + network); - CloudRuntimeException ex = new CloudRuntimeException("Failed to shutdown the network elements and resources as a part of update to network of specified id"); + boolean validStateToShutdown = (network.getState() == Network.State.Implemented || network.getState() == Network.State.Setup || network.getState() == Network.State.Allocated); + try { + + do { + if (restartNetwork) { + if (validStateToShutdown) { + if (!changeCidr) { + s_logger.debug("Shutting down elements and resources for network id=" + networkId + " as a part of network update"); + + if (!_networkMgr.shutdownNetworkElementsAndResources(context, true, network)) { + s_logger.warn("Failed to shutdown the network elements and resources as a part of network restart: " + network); + CloudRuntimeException ex = new CloudRuntimeException("Failed to shutdown the network elements and resources as a part of update to network of specified id"); + ex.addProxyObject(network.getUuid(), "networkId"); + throw ex; + } + } else { + // We need to shutdown the network, since we want to re-implement the network. + s_logger.debug("Shutting down network id=" + networkId + " as a part of network update"); + + //check if network has reservation + if (NetUtils.isNetworkAWithinNetworkB(network.getCidr(), network.getNetworkCidr())) { + s_logger.warn("Existing IP reservation will become ineffective for the network with id = " + networkId + + " You need to reapply reservation after network reimplementation."); + //set cidr to the newtork cidr + network.setCidr(network.getNetworkCidr()); + //set networkCidr to null to bring network back to no IP reservation state + network.setNetworkCidr(null); + } + + if (!_networkMgr.shutdownNetwork(network.getId(), context, true)) { + s_logger.warn("Failed to shutdown the network as a part of update to network with specified id"); + CloudRuntimeException ex = new CloudRuntimeException("Failed to shutdown the network as a part of update of specified network id"); + ex.addProxyObject(network.getUuid(), "networkId"); + throw ex; + } + } + } else { + CloudRuntimeException ex = new CloudRuntimeException( + "Failed to shutdown the network elements and resources as a part of update to network with specified id; network is in wrong state: " + network.getState()); ex.addProxyObject(network.getUuid(), "networkId"); throw ex; } + } + + // 2) Only after all the elements and rules are shutdown properly, update the network VO + // get updated network + Network.State networkState = _networksDao.findById(networkId).getState(); + boolean validStateToImplement = (networkState == Network.State.Implemented || networkState == Network.State.Setup || networkState == Network.State.Allocated); + if (restartNetwork && !validStateToImplement) { + CloudRuntimeException ex = new CloudRuntimeException( + "Failed to implement the network elements and resources as a part of update to network with specified id; network is in wrong state: " + networkState); + ex.addProxyObject(network.getUuid(), "networkId"); + throw ex; + } + + if (networkOfferingId != null) { + if (networkOfferingChanged) { + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + network.setNetworkOfferingId(networkOfferingId); + _networksDao.update(networkId, network, newSvcProviders); + // get all nics using this network + // log remove usage events for old offering + // log assign usage events for new offering + List nics = _nicDao.listByNetworkId(networkId); + for (NicVO nic : nics) { + long vmId = nic.getInstanceId(); + VMInstanceVO vm = _vmDao.findById(vmId); + if (vm == null) { + s_logger.error("Vm for nic " + nic.getId() + " not found with Vm Id:" + vmId); + continue; + } + long isDefault = (nic.isDefaultNic()) ? 1 : 0; + String nicIdString = Long.toString(nic.getId()); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString, + oldNetworkOfferingId, null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay()); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString, + networkOfferingId, null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay()); + } + } + }); + } else { + network.setNetworkOfferingId(networkOfferingId); + _networksDao.update(networkId, network, + _networkMgr.finalizeServicesAndProvidersForNetwork(_entityMgr.findById(NetworkOffering.class, networkOfferingId), network.getPhysicalNetworkId())); + } } else { - // We need to shutdown the network, since we want to re-implement the network. - s_logger.debug("Shutting down network id=" + networkId + " as a part of network update"); + _networksDao.update(networkId, network); + } - //check if network has reservation - if (NetUtils.isNetworkAWithinNetworkB(network.getCidr(), network.getNetworkCidr())) { - s_logger.warn("Existing IP reservation will become ineffective for the network with id = " + networkId - + " You need to reapply reservation after network reimplementation."); - //set cidr to the newtork cidr - network.setCidr(network.getNetworkCidr()); - //set networkCidr to null to bring network back to no IP reservation state - network.setNetworkCidr(null); - } - - if (!_networkMgr.shutdownNetwork(network.getId(), context, true)) { - s_logger.warn("Failed to shutdown the network as a part of update to network with specified id"); - CloudRuntimeException ex = new CloudRuntimeException("Failed to shutdown the network as a part of update of specified network id"); - ex.addProxyObject(network.getUuid(), "networkId"); - throw ex; + // 3) Implement the elements and rules again + if (restartNetwork) { + if (network.getState() != Network.State.Allocated) { + DeployDestination dest = new DeployDestination(_dcDao.findById(network.getDataCenterId()), null, null, null); + s_logger.debug("Implementing the network " + network + " elements and resources as a part of network update"); + try { + if (!changeCidr) { + _networkMgr.implementNetworkElementsAndResources(dest, context, network, _networkOfferingDao.findById(network.getNetworkOfferingId())); + } else { + _networkMgr.implementNetwork(network.getId(), dest, context); + } + } catch (Exception ex) { + s_logger.warn("Failed to implement network " + network + " elements and resources as a part of network update due to ", ex); + CloudRuntimeException e = new CloudRuntimeException("Failed to implement network (with specified id) elements and resources as a part of network update"); + e.addProxyObject(network.getUuid(), "networkId"); + throw e; + } } } - } else { - CloudRuntimeException ex = new CloudRuntimeException( - "Failed to shutdown the network elements and resources as a part of update to network with specified id; network is in wrong state: " + network.getState()); - ex.addProxyObject(network.getUuid(), "networkId"); - throw ex; - } - } - // 2) Only after all the elements and rules are shutdown properly, update the network VO - // get updated network - Network.State networkState = _networksDao.findById(networkId).getState(); - boolean validStateToImplement = (networkState == Network.State.Implemented || networkState == Network.State.Setup || networkState == Network.State.Allocated); - if (restartNetwork && !validStateToImplement) { - CloudRuntimeException ex = new CloudRuntimeException( - "Failed to implement the network elements and resources as a part of update to network with specified id; network is in wrong state: " + networkState); - ex.addProxyObject(network.getUuid(), "networkId"); - throw ex; - } - - if (networkOfferingId != null) { - if (networkOfferingChanged) { - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - network.setNetworkOfferingId(networkOfferingId); - _networksDao.update(networkId, network, newSvcProviders); - // get all nics using this network - // log remove usage events for old offering - // log assign usage events for new offering - List nics = _nicDao.listByNetworkId(networkId); - for (NicVO nic : nics) { - long vmId = nic.getInstanceId(); - VMInstanceVO vm = _vmDao.findById(vmId); - if (vm == null) { - s_logger.error("Vm for nic " + nic.getId() + " not found with Vm Id:" + vmId); - continue; + // 4) if network has been upgraded from a non persistent ntwk offering to a persistent ntwk offering, + // implement the network if its not already + if (networkOfferingChanged && !oldNtwkOff.getIsPersistent() && networkOffering.getIsPersistent()) { + if (network.getState() == Network.State.Allocated) { + try { + DeployDestination dest = new DeployDestination(_dcDao.findById(network.getDataCenterId()), null, null, null); + _networkMgr.implementNetwork(network.getId(), dest, context); + } catch (Exception ex) { + s_logger.warn("Failed to implement network " + network + " elements and resources as a part o" + "f network update due to ", ex); + CloudRuntimeException e = new CloudRuntimeException("Failed to implement network (with specified" + " id) elements and resources as a part of network update"); + e.addProxyObject(network.getUuid(), "networkId"); + throw e; + } } - long isDefault = (nic.isDefaultNic()) ? 1 : 0; - String nicIdString = Long.toString(nic.getId()); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString, - oldNetworkOfferingId, null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay()); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString, - networkOfferingId, null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay()); } - } - }); - } else { - network.setNetworkOfferingId(networkOfferingId); - _networksDao.update(networkId, network, - _networkMgr.finalizeServicesAndProvidersForNetwork(_entityMgr.findById(NetworkOffering.class, networkOfferingId), network.getPhysicalNetworkId())); - } - } else { - _networksDao.update(networkId, network); - } - - // 3) Implement the elements and rules again - if (restartNetwork) { - if (network.getState() != Network.State.Allocated) { - DeployDestination dest = new DeployDestination(_dcDao.findById(network.getDataCenterId()), null, null, null); - s_logger.debug("Implementing the network " + network + " elements and resources as a part of network update"); - try { - if (!changeCidr) { - _networkMgr.implementNetworkElementsAndResources(dest, context, network, _networkOfferingDao.findById(network.getNetworkOfferingId())); - } else { - _networkMgr.implementNetwork(network.getId(), dest, context); - } - } catch (Exception ex) { - s_logger.warn("Failed to implement network " + network + " elements and resources as a part of network update due to ", ex); - CloudRuntimeException e = new CloudRuntimeException("Failed to implement network (with specified id) elements and resources as a part of network update"); - e.addProxyObject(network.getUuid(), "networkId"); - throw e; + resourceCount--; + } while(updateInSequence && resourceCount>0); + }catch (Exception exception){ + throw new CloudRuntimeException("failed to update network "+network.getUuid()+"due to "+exception.getMessage()); + }finally { + if(updateInSequence){ + if( _networkDetailsDao.findDetail(networkId,Network.updatingInSequence)!=null){ + _networkDetailsDao.removeDetail(networkId,Network.updatingInSequence); } } } - - // 4) if network has been upgraded from a non persistent ntwk offering to a persistent ntwk offering, - // implement the network if its not already - if (networkOfferingChanged && !oldNtwkOff.getIsPersistent() && networkOffering.getIsPersistent()) { - if (network.getState() == Network.State.Allocated) { - try { - DeployDestination dest = new DeployDestination(_dcDao.findById(network.getDataCenterId()), null, null, null); - _networkMgr.implementNetwork(network.getId(), dest, context); - } catch (Exception ex) { - s_logger.warn("Failed to implement network " + network + " elements and resources as a part o" + "f network update due to ", ex); - CloudRuntimeException e = new CloudRuntimeException("Failed to implement network (with specified" + " id) elements and resources as a part of network update"); - e.addProxyObject(network.getUuid(), "networkId"); - throw e; - } - } - } - return getNetwork(network.getId()); } diff --git a/server/src/com/cloud/network/element/VirtualRouterElement.java b/server/src/com/cloud/network/element/VirtualRouterElement.java index d802188e4c4..03d19588af3 100644 --- a/server/src/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/com/cloud/network/element/VirtualRouterElement.java @@ -24,6 +24,9 @@ import java.util.Set; import javax.inject.Inject; +import com.cloud.network.dao.NetworkDetailVO; +import com.cloud.network.dao.NetworkDetailsDao; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.agent.api.to.LoadBalancerTO; import com.cloud.configuration.ConfigurationManager; import com.cloud.dc.DataCenter; @@ -109,7 +112,7 @@ import org.cloud.network.router.deployment.RouterDeploymentDefinitionBuilder; public class VirtualRouterElement extends AdapterBase implements VirtualRouterElementService, DhcpServiceProvider, UserDataServiceProvider, SourceNatServiceProvider, StaticNatServiceProvider, FirewallServiceProvider, LoadBalancingServiceProvider, PortForwardingServiceProvider, RemoteAccessVPNServiceProvider, IpDeployer, -NetworkMigrationResponder, AggregatedCommandExecutor { +NetworkMigrationResponder, AggregatedCommandExecutor, RedundantResource { private static final Logger s_logger = Logger.getLogger(VirtualRouterElement.class); public static final AutoScaleCounterType AutoScaleCounterCpu = new AutoScaleCounterType("cpu"); public static final AutoScaleCounterType AutoScaleCounterMemory = new AutoScaleCounterType("memory"); @@ -159,6 +162,9 @@ NetworkMigrationResponder, AggregatedCommandExecutor { @Inject NetworkTopologyContext networkTopologyContext; + @Inject + NetworkDetailsDao _networkDetailsDao; + @Inject protected RouterDeploymentDefinitionBuilder routerDeploymentDefinitionBuilder; @@ -262,7 +268,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor { public boolean applyFWRules(final Network network, final List rules) throws ResourceUnavailableException { boolean result = true; if (canHandle(network, Service.Firewall)) { - final List routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER); + final List routers = getRouters(network); if (routers == null || routers.isEmpty()) { s_logger.debug("Virtual router elemnt doesn't need to apply firewall rules on the backend; virtual " + "router doesn't exist in the network " + network.getId()); return true; @@ -407,7 +413,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor { return false; } - final List routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER); + final List routers = getRouters(network); if (routers == null || routers.isEmpty()) { s_logger.debug("Virtual router elemnt doesn't need to apply lb rules on the backend; virtual " + "router doesn't exist in the network " + network.getId()); return true; @@ -498,7 +504,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor { } boolean result = true; if (canHandle) { - final List routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER); + final List routers = getRouters(network); if (routers == null || routers.isEmpty()) { s_logger.debug("Virtual router elemnt doesn't need to associate ip addresses on the backend; virtual " + "router doesn't exist in the network " + network.getId()); return true; @@ -657,7 +663,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor { public boolean applyStaticNats(final Network network, final List rules) throws ResourceUnavailableException { boolean result = true; if (canHandle(network, Service.StaticNat)) { - final List routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER); + final List routers = getRouters(network); if (routers == null || routers.isEmpty()) { s_logger.debug("Virtual router elemnt doesn't need to apply static nat on the backend; virtual " + "router doesn't exist in the network " + network.getId()); return true; @@ -673,6 +679,46 @@ NetworkMigrationResponder, AggregatedCommandExecutor { return result; } + public List getRouters(Network network){ + List routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER); + if (routers !=null && routers.isEmpty()) { + return null; + } + NetworkDetailVO updateInSequence=_networkDetailsDao.findDetail(network.getId(), Network.updatingInSequence); + if(network.isRedundant() && updateInSequence!=null && "true".equalsIgnoreCase(updateInSequence.getValue())){ + List masterRouters=new ArrayList(); + int noOfrouters=routers.size(); + while (noOfrouters>0){ + DomainRouterVO router = routers.get(0); + if(router.getUpdateState()== VirtualRouter.UpdateState.UPDATE_IN_PROGRESS){ + ArrayList routerList = new ArrayList(); + routerList.add(router); + return routerList; + } + if(router.getUpdateState()== VirtualRouter.UpdateState.UPDATE_COMPLETE) { + routers.remove(router); + noOfrouters--; + continue; + } + if(router.getRedundantState()!=VirtualRouter.RedundantState.BACKUP) { + masterRouters.add(router); + routers.remove(router); + } + noOfrouters--; + } + if(routers.size()==0 && masterRouters.size()==0){ + return null; + } + if(routers.size()==0 && masterRouters.size()!=0){ + routers=masterRouters; + } + routers=routers.subList(0,1); + routers.get(0).setUpdateState(VirtualRouter.UpdateState.UPDATE_IN_PROGRESS); + _routerDao.persist(routers.get(0)); + } + return routers; + } + @Override public boolean shutdown(final Network network, final ReservationContext context, final boolean cleanup) throws ConcurrentOperationException, ResourceUnavailableException { final List routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER); @@ -1031,7 +1077,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor { List routers; if (publicNetwork) { - routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER); + routers = getRouters(network); } else { if (isPodBased && dest.getPod() != null) { final Long podId = dest.getPod().getId(); @@ -1228,7 +1274,27 @@ NetworkMigrationResponder, AggregatedCommandExecutor { throw new ResourceUnavailableException("Can't find at least one router!", DataCenter.class, network.getDataCenterId()); } - return _routerMgr.completeAggregatedExecution(network, routers); + NetworkDetailVO networkDetail=_networkDetailsDao.findDetail(network.getId(), Network.updatingInSequence); + boolean updateInSequence= "true".equalsIgnoreCase((networkDetail!=null ? networkDetail.getValue() : null)); + if(updateInSequence){ + DomainRouterVO router=routers.get(0); + router.setUpdateState(VirtualRouter.UpdateState.UPDATE_COMPLETE); + _routerDao.persist(router); + } + boolean result=false; + try{ + result=_routerMgr.completeAggregatedExecution(network, routers); + } finally { + if(!result && updateInSequence) { + //fail the network update. even if one router fails we fail the network update. + List routerList = _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER); + for (DomainRouterVO router : routerList) { + router.setUpdateState(VirtualRouter.UpdateState.UPDATE_FAILED); + _routerDao.persist(router); + } + } + } + return result; } @Override @@ -1237,4 +1303,22 @@ NetworkMigrationResponder, AggregatedCommandExecutor { // lets not waste another command return true; } + + @Override + public void configureResource(Network network) { + NetworkDetailVO networkDetail=_networkDetailsDao.findDetail(network.getId(), Network.updatingInSequence); + if(networkDetail==null || !"true".equalsIgnoreCase(networkDetail.getValue())) + throw new CloudRuntimeException("failed to configure the resource, network update is not in progress."); + Listrouters = _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER); + for(DomainRouterVO router : routers){ + router.setUpdateState(VirtualRouter.UpdateState.UPDATE_NEEDED); + _routerDao.persist(router); + } + } + + @Override + public int getResourceCount(Network network) { + return _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER).size(); + } + } diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 3332393f652..b6c8cf091ae 100644 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -2215,6 +2215,11 @@ Configurable, StateListener params) { @@ -410,7 +414,13 @@ public class RouterDeploymentDefinition { // Don't start the router as we are holding the network lock that // needs to be released at the end of router allocation final DomainRouterVO router = nwHelper.deployRouter(this, false); - + //check if the network update is in progress. + //if update is in progress add the update_pending flag to DomainRouterVO. + NetworkDetailVO detail =networkDetailsDao.findDetail(guestNetwork.getId(),Network.updatingInSequence); + if("true".equalsIgnoreCase(detail!=null ? detail.getValue() : null)) { + router.setUpdateState(VirtualRouter.UpdateState.UPDATE_IN_PROGRESS); + routerDao.persist(router); + } if (router != null) { routerDao.addRouterToGuestNetwork(router, guestNetwork); //Fix according to changes by Sheng Yang in commit ID cb4513379996b262ae378daf00c6388c6b7313cf diff --git a/server/src/org/cloud/network/router/deployment/RouterDeploymentDefinitionBuilder.java b/server/src/org/cloud/network/router/deployment/RouterDeploymentDefinitionBuilder.java index 3ba4fad77de..3765537a148 100644 --- a/server/src/org/cloud/network/router/deployment/RouterDeploymentDefinitionBuilder.java +++ b/server/src/org/cloud/network/router/deployment/RouterDeploymentDefinitionBuilder.java @@ -22,6 +22,7 @@ import java.util.Map; import javax.inject.Inject; +import com.cloud.network.dao.NetworkDetailsDao; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -96,6 +97,8 @@ public class RouterDeploymentDefinitionBuilder { private VpcManager vpcMgr; @Inject private VlanDao vlanDao; + @Inject + private NetworkDetailsDao networkDetailsDao; @Autowired @Qualifier("networkHelper") @@ -133,6 +136,7 @@ public class RouterDeploymentDefinitionBuilder { routerDeploymentDefinition.ipv6Dao = ipv6Dao; routerDeploymentDefinition.ipAddressDao = ipAddressDao; routerDeploymentDefinition.serviceOfferingId = offeringId; + routerDeploymentDefinition.networkDetailsDao = networkDetailsDao; routerDeploymentDefinition.nwHelper = nwHelper; diff --git a/server/test/com/cloud/vpc/MockNetworkManagerImpl.java b/server/test/com/cloud/vpc/MockNetworkManagerImpl.java index a5d8a1ab584..3e80865572b 100644 --- a/server/test/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/test/com/cloud/vpc/MockNetworkManagerImpl.java @@ -247,7 +247,7 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches */ @Override public Network updateGuestNetwork(long networkId, String name, String displayText, Account callerAccount, User callerUser, String domainSuffix, - Long networkOfferingId, Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String newUUID) { + Long networkOfferingId, Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String newUUID,boolean updateInSequence) { // TODO Auto-generated method stub return null; } @@ -841,6 +841,21 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches return; } + @Override + public boolean canUpdateInSequence(Network network) { + return false; + } + + @Override + public void configureUpdateInSequence(Network network) { + return; + } + + @Override + public int getResourceCount(Network network) { + return 0; + } + @Override public void prepareNicForMigration(VirtualMachineProfile vm, DeployDestination dest) { // TODO Auto-generated method stub diff --git a/server/test/org/cloud/network/router/deployment/RouterDeploymentDefinitionTest.java b/server/test/org/cloud/network/router/deployment/RouterDeploymentDefinitionTest.java index eff16c16b8d..6195387dbe2 100644 --- a/server/test/org/cloud/network/router/deployment/RouterDeploymentDefinitionTest.java +++ b/server/test/org/cloud/network/router/deployment/RouterDeploymentDefinitionTest.java @@ -714,7 +714,7 @@ public class RouterDeploymentDefinitionTest extends RouterDeploymentDefinitionTe final DomainRouterVO routerVO2 = mock(DomainRouterVO.class); when(mockNetworkHelper.deployRouter(deploymentUT, false)) .thenReturn(routerVO1).thenReturn(routerVO2); - + when(networkDetailsDao.findById(anyLong())).thenReturn(null); // Execute deploymentUT.deployAllVirtualRouters(); diff --git a/server/test/org/cloud/network/router/deployment/RouterDeploymentDefinitionTestBase.java b/server/test/org/cloud/network/router/deployment/RouterDeploymentDefinitionTestBase.java index 4225083ca2b..626c2d7acc6 100644 --- a/server/test/org/cloud/network/router/deployment/RouterDeploymentDefinitionTestBase.java +++ b/server/test/org/cloud/network/router/deployment/RouterDeploymentDefinitionTestBase.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.cloud.network.dao.NetworkDetailsDao; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; @@ -79,6 +80,8 @@ public class RouterDeploymentDefinitionTestBase { @Mock protected NetworkHelper mockNetworkHelper; @Mock + protected NetworkDetailsDao networkDetailsDao; + @Mock protected VpcNetworkHelperImpl vpcNwHelper; @Mock protected VMInstanceDao mockVmDao; diff --git a/setup/db/db/schema-452to460.sql b/setup/db/db/schema-452to460.sql index dfb629f95a0..e05ad6d255d 100644 --- a/setup/db/db/schema-452to460.sql +++ b/setup/db/db/schema-452to460.sql @@ -420,3 +420,5 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'KVM', 'default', 'CentOS 7', 246, utc_timestamp(), 0); UPDATE `cloud`.`hypervisor_capabilities` SET `max_data_volumes_limit` = '32' WHERE `hypervisor_capabilities`.`hypervisor_type` = 'KVM'; +ALTER TABLE `cloud`.`domain_router` ADD COLUMN update_state varchar(64) DEFAULT NULL; +