CLOUDSTACK-8751

Minimise network downtime during network updates when redundant VR is being used.
database schema changes
Made changes to the updateNetwork API.

api/src/com/cloud/network/Network.java

@@ -41,6 +41,8 @@ public interface Network extends ControlledEntity, StateObject<Network.State>, I
         Shared, Isolated
     }
 
+    public String updatingInSequence ="updatingInSequence";
+
     public static class Service {
         private static List<Service> supportedServices = new ArrayList<Service>();
 

api/src/com/cloud/network/NetworkService.java

@@ -77,7 +77,7 @@ public interface NetworkService {
     IpAddress getIp(long id);
 
     Network updateGuestNetwork(long networkId, String name, String displayText, Account callerAccount, User callerUser, String domainSuffix, Long networkOfferingId,
-        Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String newUUID);
+        Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String newUUID, boolean updateInSequence);
 
     PhysicalNetwork createPhysicalNetwork(Long zoneId, String vnetRange, String networkSpeed, List<String> isolationMethods, String broadcastDomainRange, Long domainId,
         List<String> tags, String name);

api/src/com/cloud/network/element/RedundantResource.java

@@ -0,0 +1,11 @@
+package com.cloud.network.element;
+
+import com.cloud.network.Network;
+
+/**
+ * Created by bharat on 11/08/15.
+ */
+public interface RedundantResource {
+     public void configureResource(Network network);
+     public int getResourceCount(Network network);
+}

api/src/com/cloud/network/router/VirtualRouter.java

@@ -26,6 +26,10 @@ public interface VirtualRouter extends VirtualMachine {
         VIRTUAL_ROUTER, LB, INTERNAL_LB_VM
     }
 
+    public enum UpdateState {
+        UPDATE_NEEDED, UPDATE_IN_PROGRESS, UPDATE_COMPLETE, UPDATE_FAILED
+    }
+
     Role getRole();
 
     boolean getIsRedundantRouter();

api/src/org/apache/cloudstack/api/ApiConstants.java

@@ -272,6 +272,7 @@ public class ApiConstants {
     public static final String USERNAME = "username";
     public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist";
     public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork";
+    public static final String Update_IN_SEQUENCE ="updateinsequence";
     public static final String VALUE = "value";
     public static final String VIRTUAL_MACHINE_ID = "virtualmachineid";
     public static final String VIRTUAL_MACHINE_IDS = "virtualmachineids";

api/src/org/apache/cloudstack/api/command/admin/network/UpdateNetworkCmdByAdmin.java

@@ -49,7 +49,7 @@ public class UpdateNetworkCmdByAdmin extends UpdateNetworkCmd {
         }
 
         Network result = _networkService.updateGuestNetwork(getId(), getNetworkName(), getDisplayText(), callerAccount,
-                callerUser, getNetworkDomain(), getNetworkOfferingId(), getChangeCidr(), getGuestVmCidr(), getDisplayNetwork(), getCustomId());
+                callerUser, getNetworkDomain(), getNetworkOfferingId(), getChangeCidr(), getGuestVmCidr(), getDisplayNetwork(), getCustomId(), getUpdateInSequence());
 
 
         if (result != null) {

api/src/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java

@@ -75,6 +75,9 @@ public class UpdateNetworkCmd extends BaseAsyncCustomIdCmd {
     @Parameter(name = ApiConstants.GUEST_VM_CIDR, type = CommandType.STRING, description = "CIDR for guest VMs, CloudStack allocates IPs to guest VMs only from this CIDR")
     private String guestVmCidr;
 
+    @Parameter(name =ApiConstants.Update_IN_SEQUENCE, type=CommandType.BOOLEAN, description = "if true, we will update the routers one after the other. applicable only for redundant router based networks using virtual router as provider")
+    private Boolean updateInSequence;
+
     @Parameter(name = ApiConstants.DISPLAY_NETWORK,
                type = CommandType.BOOLEAN,
  description = "an optional field, whether to the display the network to the end user or not.", authorized = {RoleType.Admin})
@@ -119,6 +122,13 @@ public class UpdateNetworkCmd extends BaseAsyncCustomIdCmd {
         return displayNetwork;
     }
 
+    public Boolean getUpdateInSequence(){
+        if(updateInSequence ==null)
+            return false;
+        else
+            return updateInSequence;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -149,7 +159,7 @@ public class UpdateNetworkCmd extends BaseAsyncCustomIdCmd {
 
         Network result =
             _networkService.updateGuestNetwork(getId(), getNetworkName(), getDisplayText(), callerAccount, callerUser, getNetworkDomain(), getNetworkOfferingId(),
-                getChangeCidr(), getGuestVmCidr(), getDisplayNetwork(), getCustomId());
+                getChangeCidr(), getGuestVmCidr(), getDisplayNetwork(), getCustomId(), getUpdateInSequence());
 
         if (result != null) {
             NetworkResponse response = _responseGenerator.createNetworkResponse(ResponseView.Restricted, result);

engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java

@@ -224,4 +224,10 @@ public interface NetworkOrchestrationService {
     boolean resourceCountNeedsUpdate(NetworkOffering ntwkOff, ACLType aclType);
 
     void prepareAllNicsForMigration(VirtualMachineProfile vm, DeployDestination dest);
+
+    boolean canUpdateInSequence(Network network);
+
+    void configureUpdateInSequence(Network network);
+
+    int getResourceCount(Network network);
 }

engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -36,6 +36,12 @@ import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.network.Networks;
+
+import com.cloud.network.dao.NetworkDetailsDao;
+import com.cloud.network.element.RedundantResource;
+import com.cloud.vm.dao.DomainRouterDao;
+import org.apache.log4j.Logger;
 import org.apache.cloudstack.acl.ControlledEntity.ACLType;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.cloud.entity.api.db.VMNetworkMapVO;
@@ -50,7 +56,6 @@ import org.apache.cloudstack.framework.messagebus.MessageBus;
 import org.apache.cloudstack.framework.messagebus.PublishScope;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.region.PortableIpDao;
-import org.apache.log4j.Logger;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.Listener;
@@ -107,7 +112,6 @@ import com.cloud.network.NetworkMigrationResponder;
 import com.cloud.network.NetworkModel;
 import com.cloud.network.NetworkProfile;
 import com.cloud.network.NetworkStateListener;
-import com.cloud.network.Networks;
 import com.cloud.network.Networks.BroadcastDomainType;
 import com.cloud.network.Networks.TrafficType;
 import com.cloud.network.PhysicalNetwork;
@@ -265,9 +269,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     MessageBus _messageBus;
     @Inject
     VMNetworkMapDao _vmNetworkMapDao;
+    @Inject
+    DomainRouterDao _rotuerDao;
 
     List<NetworkGuru> networkGurus;
 
+
     public List<NetworkGuru> getNetworkGurus() {
         return networkGurus;
     }
@@ -350,6 +357,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     PortableIpDao _portableIpDao;
     @Inject
     ConfigDepot _configDepot;
+    @Inject
+    NetworkDetailsDao _networkDetailsDao;
 
     protected StateMachine2<Network.State, Network.Event, Network> _stateMachine;
     ScheduledExecutorService _executor;
@@ -1271,6 +1280,46 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         return true;
     }
 
+    @Override
+    public boolean canUpdateInSequence(Network network){
+        List<Provider> providers = getNetworkProviders(network.getId());
+        //check if the there are no service provider other than virtualrouter.
+        for(Provider provider :providers){
+            if(provider!=Provider.VirtualRouter)
+                throw new UnsupportedOperationException("Cannot update the network resources in sequence when providers other than virtualrouter are used");
+        }
+        return true;
+    }
+
+    @Override
+    public void configureUpdateInSequence(Network network) {
+        List<Provider> providers = getNetworkProviders(network.getId());
+        for (NetworkElement element : networkElements) {
+            if (providers.contains(element.getProvider())) {
+                if (element instanceof RedundantResource) {
+                    ((RedundantResource) element).configureResource(network);
+                }
+            }
+        }
+    }
+
+    @Override
+    public int getResourceCount(Network network){
+        List<Provider> providers = getNetworkProviders(network.getId());
+        int resourceCount=0;
+        for (NetworkElement element : networkElements) {
+            if (providers.contains(element.getProvider())) {
+                //currently only one element implements the redundant resource interface
+                if (element instanceof RedundantResource) {
+                    resourceCount= ((RedundantResource) element).getResourceCount(network);
+                    break;
+                    }
+                }
+            }
+        return resourceCount;
+        }
+
+
     @DB
     protected void updateNic(final NicVO nic, final long networkId, final int count) {
         Transaction.execute(new TransactionCallbackNoReturn() {

engine/schema/src/com/cloud/vm/DomainRouterVO.java

@@ -69,6 +69,11 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter {
     @Column(name = "vpc_id")
     private Long vpcId;
 
+
+    @Column(name= "update_state")
+    @Enumerated(EnumType.STRING)
+    private UpdateState updateState;
+
     public DomainRouterVO(final long id, final long serviceOfferingId, final long elementId, final String name, final long templateId, final HypervisorType hypervisorType, final long guestOSId, final long domainId,
             final long accountId, final long userId, final boolean isRedundantRouter, final RedundantState redundantState, final boolean haEnabled, final boolean stopPending,
             final Long vpcId) {
@@ -193,4 +198,13 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter {
         return vpcId;
     }
 
+    public UpdateState getUpdateState() {
+        return updateState;
+    }
+
+    public void setUpdateState(UpdateState updateState) {
+        this.updateState = updateState;
+    }
+
+
 }

server/src/com/cloud/network/NetworkServiceImpl.java

@@ -108,6 +108,8 @@ import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.IPAddressVO;
 import com.cloud.network.dao.LoadBalancerVMMapDao;
 import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkDetailVO;
+import com.cloud.network.dao.NetworkDetailsDao;
 import com.cloud.network.dao.NetworkDomainDao;
 import com.cloud.network.dao.NetworkDomainVO;
 import com.cloud.network.dao.NetworkServiceMapDao;
@@ -178,6 +180,7 @@ import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.exception.ExceptionUtil;
 import com.cloud.utils.net.NetUtils;
+import com.cloud.vm.DomainRouterVO;
 import com.cloud.vm.Nic;
 import com.cloud.vm.NicSecondaryIp;
 import com.cloud.vm.NicVO;
@@ -187,6 +190,7 @@ import com.cloud.vm.SecondaryStorageVmVO;
 import com.cloud.vm.UserVmVO;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.NicDao;
 import com.cloud.vm.dao.NicSecondaryIpDao;
 import com.cloud.vm.dao.NicSecondaryIpVO;
@@ -324,6 +328,12 @@ public class NetworkServiceImpl extends ManagerBase implements  NetworkService {
     @Inject
     MessageBus _messageBus;
 
+    @Inject
+    DomainRouterDao _routerDao;
+
+    @Inject
+    NetworkDetailsDao _networkDetailsDao;
+
     int _cidrLimit;
     boolean _allowSubdomainNetworkAccess;
 
@@ -1992,8 +2002,7 @@ public class NetworkServiceImpl extends ManagerBase implements  NetworkService {
     @DB
     @ActionEvent(eventType = EventTypes.EVENT_NETWORK_UPDATE, eventDescription = "updating network", async = true)
     public Network updateGuestNetwork(final long networkId, String name, String displayText, Account callerAccount, User callerUser, String domainSuffix,
-            final Long networkOfferingId, Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String customId) {
+            final Long networkOfferingId, Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String customId, boolean updateInSequence) {
-
         boolean restartNetwork = false;
 
         // verify input parameters
@@ -2239,130 +2248,153 @@ public class NetworkServiceImpl extends ManagerBase implements  NetworkService {
         ReservationContext context = new ReservationContextImpl(null, null, callerUser, callerAccount);
         // 1) Shutdown all the elements and cleanup all the rules. Don't allow to shutdown network in intermediate
         // states - Shutdown and Implementing
-        boolean validStateToShutdown = (network.getState() == Network.State.Implemented || network.getState() == Network.State.Setup || network.getState() == Network.State.Allocated);
+        List<DomainRouterVO> routers=null;
-        if (restartNetwork) {
+        int resourceCount=1;
-            if (validStateToShutdown) {
+        if(updateInSequence && restartNetwork && _networkOfferingDao.findById(network.getNetworkOfferingId()).getRedundantRouter() && networkOfferingId!=null && _networkOfferingDao.findById(networkOfferingId).getRedundantRouter() && network.getVpcId()==null) {
-                if (!changeCidr) {
+            _networkMgr.canUpdateInSequence(network);
-                    s_logger.debug("Shutting down elements and resources for network id=" + networkId + " as a part of network update");
+            NetworkDetailVO networkDetail =new NetworkDetailVO(network.getId(),Network.updatingInSequence,"true",true);
+            _networkDetailsDao.persist(networkDetail);
+            _networkMgr.configureUpdateInSequence(network);
+            resourceCount=_networkMgr.getResourceCount(network);
+        }
 
-                    if (!_networkMgr.shutdownNetworkElementsAndResources(context, true, network)) {
+        boolean validStateToShutdown = (network.getState() == Network.State.Implemented || network.getState() == Network.State.Setup || network.getState() == Network.State.Allocated);
-                        s_logger.warn("Failed to shutdown the network elements and resources as a part of network restart: " + network);
+        try {
-                        CloudRuntimeException ex = new CloudRuntimeException("Failed to shutdown the network elements and resources as a part of update to network of specified id");
+
+            do {
+                if (restartNetwork) {
+                    if (validStateToShutdown) {
+                        if (!changeCidr) {
+                            s_logger.debug("Shutting down elements and resources for network id=" + networkId + " as a part of network update");
+
+                            if (!_networkMgr.shutdownNetworkElementsAndResources(context, true, network)) {
+                                s_logger.warn("Failed to shutdown the network elements and resources as a part of network restart: " + network);
+                                CloudRuntimeException ex = new CloudRuntimeException("Failed to shutdown the network elements and resources as a part of update to network of specified id");
+                                ex.addProxyObject(network.getUuid(), "networkId");
+                                throw ex;
+                            }
+                        } else {
+                            // We need to shutdown the network, since we want to re-implement the network.
+                            s_logger.debug("Shutting down network id=" + networkId + " as a part of network update");
+
+                            //check if network has reservation
+                            if (NetUtils.isNetworkAWithinNetworkB(network.getCidr(), network.getNetworkCidr())) {
+                                s_logger.warn("Existing IP reservation will become ineffective for the network with id =  " + networkId
+                                        + " You need to reapply reservation after network reimplementation.");
+                                //set cidr to the newtork cidr
+                                network.setCidr(network.getNetworkCidr());
+                                //set networkCidr to null to bring network back to no IP reservation state
+                                network.setNetworkCidr(null);
+                            }
+
+                            if (!_networkMgr.shutdownNetwork(network.getId(), context, true)) {
+                                s_logger.warn("Failed to shutdown the network as a part of update to network with specified id");
+                                CloudRuntimeException ex = new CloudRuntimeException("Failed to shutdown the network as a part of update of specified network id");
+                                ex.addProxyObject(network.getUuid(), "networkId");
+                                throw ex;
+                            }
+                        }
+                    } else {
+                        CloudRuntimeException ex = new CloudRuntimeException(
+                                "Failed to shutdown the network elements and resources as a part of update to network with specified id; network is in wrong state: " + network.getState());
                         ex.addProxyObject(network.getUuid(), "networkId");
                         throw ex;
                     }
+                }
+
+                // 2) Only after all the elements and rules are shutdown properly, update the network VO
+                // get updated network
+                Network.State networkState = _networksDao.findById(networkId).getState();
+                boolean validStateToImplement = (networkState == Network.State.Implemented || networkState == Network.State.Setup || networkState == Network.State.Allocated);
+                if (restartNetwork && !validStateToImplement) {
+                    CloudRuntimeException ex = new CloudRuntimeException(
+                            "Failed to implement the network elements and resources as a part of update to network with specified id; network is in wrong state: " + networkState);
+                    ex.addProxyObject(network.getUuid(), "networkId");
+                    throw ex;
+                }
+
+                if (networkOfferingId != null) {
+                    if (networkOfferingChanged) {
+                        Transaction.execute(new TransactionCallbackNoReturn() {
+                            @Override
+                            public void doInTransactionWithoutResult(TransactionStatus status) {
+                                network.setNetworkOfferingId(networkOfferingId);
+                                _networksDao.update(networkId, network, newSvcProviders);
+                                // get all nics using this network
+                                // log remove usage events for old offering
+                                // log assign usage events for new offering
+                                List<NicVO> nics = _nicDao.listByNetworkId(networkId);
+                                for (NicVO nic : nics) {
+                                    long vmId = nic.getInstanceId();
+                                    VMInstanceVO vm = _vmDao.findById(vmId);
+                                    if (vm == null) {
+                                        s_logger.error("Vm for nic " + nic.getId() + " not found with Vm Id:" + vmId);
+                                        continue;
+                                    }
+                                    long isDefault = (nic.isDefaultNic()) ? 1 : 0;
+                                    String nicIdString = Long.toString(nic.getId());
+                                    UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString,
+                                            oldNetworkOfferingId, null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay());
+                                    UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString,
+                                            networkOfferingId, null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay());
+                                }
+                            }
+                        });
+                    } else {
+                        network.setNetworkOfferingId(networkOfferingId);
+                        _networksDao.update(networkId, network,
+                                _networkMgr.finalizeServicesAndProvidersForNetwork(_entityMgr.findById(NetworkOffering.class, networkOfferingId), network.getPhysicalNetworkId()));
+                    }
                 } else {
-                    // We need to shutdown the network, since we want to re-implement the network.
+                    _networksDao.update(networkId, network);
-                    s_logger.debug("Shutting down network id=" + networkId + " as a part of network update");
+                }
 
-                    //check if network has reservation
+                // 3) Implement the elements and rules again
-                    if (NetUtils.isNetworkAWithinNetworkB(network.getCidr(), network.getNetworkCidr())) {
+                if (restartNetwork) {
-                        s_logger.warn("Existing IP reservation will become ineffective for the network with id =  " + networkId
+                    if (network.getState() != Network.State.Allocated) {
-                                + " You need to reapply reservation after network reimplementation.");
+                        DeployDestination dest = new DeployDestination(_dcDao.findById(network.getDataCenterId()), null, null, null);
-                        //set cidr to the newtork cidr
+                        s_logger.debug("Implementing the network " + network + " elements and resources as a part of network update");
-                        network.setCidr(network.getNetworkCidr());
+                        try {
-                        //set networkCidr to null to bring network back to no IP reservation state
+                            if (!changeCidr) {
-                        network.setNetworkCidr(null);
+                                _networkMgr.implementNetworkElementsAndResources(dest, context, network, _networkOfferingDao.findById(network.getNetworkOfferingId()));
-                    }
+                            } else {
-
+                                _networkMgr.implementNetwork(network.getId(), dest, context);
-                    if (!_networkMgr.shutdownNetwork(network.getId(), context, true)) {
+                            }
-                        s_logger.warn("Failed to shutdown the network as a part of update to network with specified id");
+                        } catch (Exception ex) {
-                        CloudRuntimeException ex = new CloudRuntimeException("Failed to shutdown the network as a part of update of specified network id");
+                            s_logger.warn("Failed to implement network " + network + " elements and resources as a part of network update due to ", ex);
-                        ex.addProxyObject(network.getUuid(), "networkId");
+                            CloudRuntimeException e = new CloudRuntimeException("Failed to implement network (with specified id) elements and resources as a part of network update");
-                        throw ex;
+                            e.addProxyObject(network.getUuid(), "networkId");
+                            throw e;
+                        }
                     }
                 }
-            } else {
-                CloudRuntimeException ex = new CloudRuntimeException(
-                        "Failed to shutdown the network elements and resources as a part of update to network with specified id; network is in wrong state: " + network.getState());
-                ex.addProxyObject(network.getUuid(), "networkId");
-                throw ex;
-            }
-        }
 
-        // 2) Only after all the elements and rules are shutdown properly, update the network VO
+                // 4) if network has been upgraded from a non persistent ntwk offering to a persistent ntwk offering,
-        // get updated network
+                // implement the network if its not already
-        Network.State networkState = _networksDao.findById(networkId).getState();
+                if (networkOfferingChanged && !oldNtwkOff.getIsPersistent() && networkOffering.getIsPersistent()) {
-        boolean validStateToImplement = (networkState == Network.State.Implemented || networkState == Network.State.Setup || networkState == Network.State.Allocated);
+                    if (network.getState() == Network.State.Allocated) {
-        if (restartNetwork && !validStateToImplement) {
+                        try {
-            CloudRuntimeException ex = new CloudRuntimeException(
+                            DeployDestination dest = new DeployDestination(_dcDao.findById(network.getDataCenterId()), null, null, null);
-                    "Failed to implement the network elements and resources as a part of update to network with specified id; network is in wrong state: " + networkState);
+                            _networkMgr.implementNetwork(network.getId(), dest, context);
-            ex.addProxyObject(network.getUuid(), "networkId");
+                        } catch (Exception ex) {
-            throw ex;
+                            s_logger.warn("Failed to implement network " + network + " elements and resources as a part o" + "f network update due to ", ex);
-        }
+                            CloudRuntimeException e = new CloudRuntimeException("Failed to implement network (with specified" + " id) elements and resources as a part of network update");
-
+                            e.addProxyObject(network.getUuid(), "networkId");
-        if (networkOfferingId != null) {
+                            throw e;
-            if (networkOfferingChanged) {
+                        }
-                Transaction.execute(new TransactionCallbackNoReturn() {
-                    @Override
-                    public void doInTransactionWithoutResult(TransactionStatus status) {
-                network.setNetworkOfferingId(networkOfferingId);
-                _networksDao.update(networkId, network, newSvcProviders);
-                // get all nics using this network
-                // log remove usage events for old offering
-                // log assign usage events for new offering
-                List<NicVO> nics = _nicDao.listByNetworkId(networkId);
-                for (NicVO nic : nics) {
-                    long vmId = nic.getInstanceId();
-                    VMInstanceVO vm = _vmDao.findById(vmId);
-                    if (vm == null) {
-                        s_logger.error("Vm for nic " + nic.getId() + " not found with Vm Id:" + vmId);
-                        continue;
                     }
-                    long isDefault = (nic.isDefaultNic()) ? 1 : 0;
-                    String nicIdString = Long.toString(nic.getId());
-                            UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString,
-                                    oldNetworkOfferingId, null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay());
-                            UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), nicIdString,
-                                    networkOfferingId, null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay());
                 }
-                    }
+                resourceCount--;
-                });
+            } while(updateInSequence && resourceCount>0);
-            }   else {
+        }catch (Exception exception){
-                network.setNetworkOfferingId(networkOfferingId);
+             throw new CloudRuntimeException("failed to update network "+network.getUuid()+"due to "+exception.getMessage());
-                _networksDao.update(networkId, network,
+        }finally {
-                        _networkMgr.finalizeServicesAndProvidersForNetwork(_entityMgr.findById(NetworkOffering.class, networkOfferingId), network.getPhysicalNetworkId()));
+            if(updateInSequence){
-            }
+                if( _networkDetailsDao.findDetail(networkId,Network.updatingInSequence)!=null){
-        }   else {
+                    _networkDetailsDao.removeDetail(networkId,Network.updatingInSequence);
-            _networksDao.update(networkId, network);
-        }
-
-        // 3) Implement the elements and rules again
-        if (restartNetwork) {
-            if (network.getState() != Network.State.Allocated) {
-                DeployDestination dest = new DeployDestination(_dcDao.findById(network.getDataCenterId()), null, null, null);
-                s_logger.debug("Implementing the network " + network + " elements and resources as a part of network update");
-                try {
-                    if (!changeCidr) {
-                        _networkMgr.implementNetworkElementsAndResources(dest, context, network, _networkOfferingDao.findById(network.getNetworkOfferingId()));
-                    } else {
-                        _networkMgr.implementNetwork(network.getId(), dest, context);
-                    }
-                } catch (Exception ex) {
-                    s_logger.warn("Failed to implement network " + network + " elements and resources as a part of network update due to ", ex);
-                    CloudRuntimeException e = new CloudRuntimeException("Failed to implement network (with specified id) elements and resources as a part of network update");
-                    e.addProxyObject(network.getUuid(), "networkId");
-                    throw e;
                 }
             }
         }
-
-        // 4) if network has been upgraded from a non persistent ntwk offering to a persistent ntwk offering,
-        // implement the network if its not already
-        if (networkOfferingChanged && !oldNtwkOff.getIsPersistent() && networkOffering.getIsPersistent()) {
-            if (network.getState() == Network.State.Allocated) {
-                try {
-                    DeployDestination dest = new DeployDestination(_dcDao.findById(network.getDataCenterId()), null, null, null);
-                    _networkMgr.implementNetwork(network.getId(), dest, context);
-                } catch (Exception ex) {
-                    s_logger.warn("Failed to implement network " + network + " elements and resources as a part o" + "f network update due to ", ex);
-                    CloudRuntimeException e = new CloudRuntimeException("Failed to implement network (with specified" + " id) elements and resources as a part of network update");
-                    e.addProxyObject(network.getUuid(), "networkId");
-                    throw e;
-                }
-            }
-        }
-
         return getNetwork(network.getId());
     }
 

server/src/com/cloud/network/element/VirtualRouterElement.java

@@ -24,6 +24,9 @@ import java.util.Set;
 
 import javax.inject.Inject;
 
+import com.cloud.network.dao.NetworkDetailVO;
+import com.cloud.network.dao.NetworkDetailsDao;
+import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.agent.api.to.LoadBalancerTO;
 import com.cloud.configuration.ConfigurationManager;
 import com.cloud.dc.DataCenter;
@@ -109,7 +112,7 @@ import org.cloud.network.router.deployment.RouterDeploymentDefinitionBuilder;
 
 public class VirtualRouterElement extends AdapterBase implements VirtualRouterElementService, DhcpServiceProvider, UserDataServiceProvider, SourceNatServiceProvider,
 StaticNatServiceProvider, FirewallServiceProvider, LoadBalancingServiceProvider, PortForwardingServiceProvider, RemoteAccessVPNServiceProvider, IpDeployer,
-NetworkMigrationResponder, AggregatedCommandExecutor {
+NetworkMigrationResponder, AggregatedCommandExecutor, RedundantResource {
     private static final Logger s_logger = Logger.getLogger(VirtualRouterElement.class);
     public static final AutoScaleCounterType AutoScaleCounterCpu = new AutoScaleCounterType("cpu");
     public static final AutoScaleCounterType AutoScaleCounterMemory = new AutoScaleCounterType("memory");
@@ -159,6 +162,9 @@ NetworkMigrationResponder, AggregatedCommandExecutor {
     @Inject
     NetworkTopologyContext networkTopologyContext;
 
+    @Inject
+    NetworkDetailsDao _networkDetailsDao;
+
     @Inject
     protected RouterDeploymentDefinitionBuilder routerDeploymentDefinitionBuilder;
 
@@ -262,7 +268,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor {
     public boolean applyFWRules(final Network network, final List<? extends FirewallRule> rules) throws ResourceUnavailableException {
         boolean result = true;
         if (canHandle(network, Service.Firewall)) {
-            final List<DomainRouterVO> routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER);
+            final List<DomainRouterVO> routers = getRouters(network);
             if (routers == null || routers.isEmpty()) {
                 s_logger.debug("Virtual router elemnt doesn't need to apply firewall rules on the backend; virtual " + "router doesn't exist in the network " + network.getId());
                 return true;
@@ -407,7 +413,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor {
                 return false;
             }
 
-            final List<DomainRouterVO> routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER);
+            final List<DomainRouterVO> routers = getRouters(network);
             if (routers == null || routers.isEmpty()) {
                 s_logger.debug("Virtual router elemnt doesn't need to apply lb rules on the backend; virtual " + "router doesn't exist in the network " + network.getId());
                 return true;
@@ -498,7 +504,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor {
         }
         boolean result = true;
         if (canHandle) {
-            final List<DomainRouterVO> routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER);
+            final List<DomainRouterVO> routers = getRouters(network);
             if (routers == null || routers.isEmpty()) {
                 s_logger.debug("Virtual router elemnt doesn't need to associate ip addresses on the backend; virtual " + "router doesn't exist in the network " + network.getId());
                 return true;
@@ -657,7 +663,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor {
     public boolean applyStaticNats(final Network network, final List<? extends StaticNat> rules) throws ResourceUnavailableException {
         boolean result = true;
         if (canHandle(network, Service.StaticNat)) {
-            final List<DomainRouterVO> routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER);
+            final List<DomainRouterVO> routers = getRouters(network);
             if (routers == null || routers.isEmpty()) {
                 s_logger.debug("Virtual router elemnt doesn't need to apply static nat on the backend; virtual " + "router doesn't exist in the network " + network.getId());
                 return true;
@@ -673,6 +679,46 @@ NetworkMigrationResponder, AggregatedCommandExecutor {
         return result;
     }
 
+    public List<DomainRouterVO> getRouters(Network network){
+        List<DomainRouterVO> routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER);
+        if (routers !=null && routers.isEmpty()) {
+            return null;
+        }
+        NetworkDetailVO updateInSequence=_networkDetailsDao.findDetail(network.getId(), Network.updatingInSequence);
+        if(network.isRedundant() && updateInSequence!=null && "true".equalsIgnoreCase(updateInSequence.getValue())){
+            List<DomainRouterVO> masterRouters=new ArrayList<DomainRouterVO>();
+            int noOfrouters=routers.size();
+            while (noOfrouters>0){
+                DomainRouterVO router = routers.get(0);
+                if(router.getUpdateState()== VirtualRouter.UpdateState.UPDATE_IN_PROGRESS){
+                    ArrayList<DomainRouterVO> routerList = new ArrayList<DomainRouterVO>();
+                    routerList.add(router);
+                    return routerList;
+                }
+                if(router.getUpdateState()== VirtualRouter.UpdateState.UPDATE_COMPLETE) {
+                    routers.remove(router);
+                    noOfrouters--;
+                    continue;
+                }
+                if(router.getRedundantState()!=VirtualRouter.RedundantState.BACKUP) {
+                    masterRouters.add(router);
+                    routers.remove(router);
+                }
+                noOfrouters--;
+            }
+            if(routers.size()==0 && masterRouters.size()==0){
+                return null;
+            }
+            if(routers.size()==0 && masterRouters.size()!=0){
+                routers=masterRouters;
+            }
+            routers=routers.subList(0,1);
+            routers.get(0).setUpdateState(VirtualRouter.UpdateState.UPDATE_IN_PROGRESS);
+            _routerDao.persist(routers.get(0));
+        }
+        return routers;
+    }
+
     @Override
     public boolean shutdown(final Network network, final ReservationContext context, final boolean cleanup) throws ConcurrentOperationException, ResourceUnavailableException {
         final List<DomainRouterVO> routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER);
@@ -1031,7 +1077,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor {
         List<DomainRouterVO> routers;
 
         if (publicNetwork) {
-            routers = _routerDao.listByNetworkAndRole(network.getId(), Role.VIRTUAL_ROUTER);
+            routers = getRouters(network);
         } else {
             if (isPodBased && dest.getPod() != null) {
                 final Long podId = dest.getPod().getId();
@@ -1228,7 +1274,27 @@ NetworkMigrationResponder, AggregatedCommandExecutor {
             throw new ResourceUnavailableException("Can't find at least one router!", DataCenter.class, network.getDataCenterId());
         }
 
-        return _routerMgr.completeAggregatedExecution(network, routers);
+        NetworkDetailVO networkDetail=_networkDetailsDao.findDetail(network.getId(), Network.updatingInSequence);
+        boolean updateInSequence= "true".equalsIgnoreCase((networkDetail!=null ? networkDetail.getValue() : null));
+        if(updateInSequence){
+            DomainRouterVO router=routers.get(0);
+            router.setUpdateState(VirtualRouter.UpdateState.UPDATE_COMPLETE);
+            _routerDao.persist(router);
+        }
+        boolean result=false;
+        try{
+            result=_routerMgr.completeAggregatedExecution(network, routers);
+        } finally {
+            if(!result && updateInSequence) {
+                //fail the network update. even if one router fails we fail the network update.
+                List<DomainRouterVO> routerList = _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER);
+                for (DomainRouterVO router : routerList) {
+                    router.setUpdateState(VirtualRouter.UpdateState.UPDATE_FAILED);
+                    _routerDao.persist(router);
+                }
+            }
+        }
+        return result;
     }
 
     @Override
@@ -1237,4 +1303,22 @@ NetworkMigrationResponder, AggregatedCommandExecutor {
         // lets not waste another command
         return true;
     }
+
+    @Override
+    public void configureResource(Network network) {
+        NetworkDetailVO networkDetail=_networkDetailsDao.findDetail(network.getId(), Network.updatingInSequence);
+        if(networkDetail==null || !"true".equalsIgnoreCase(networkDetail.getValue()))
+            throw new CloudRuntimeException("failed to configure the resource, network update is not in progress.");
+        List<DomainRouterVO>routers = _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER);
+        for(DomainRouterVO router : routers){
+            router.setUpdateState(VirtualRouter.UpdateState.UPDATE_NEEDED);
+            _routerDao.persist(router);
+        }
+    }
+
+    @Override
+    public int getResourceCount(Network network) {
+        return _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER).size();
+    }
+
 }

server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -2215,6 +2215,11 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
 
         // verify parameters
         DomainRouterVO router = _routerDao.findById(routerId);
+        //clean up the update_state feild
+        if(router.getUpdateState()== VirtualRouter.UpdateState.UPDATE_FAILED){
+            router.setUpdateState(null);
+            _routerDao.update(router.getId(),router);
+        }
         if (router == null) {
             throw new InvalidParameterValueException("Unable to find router by id " + routerId + ".");
         }

server/src/org/cloud/network/router/deployment/RouterDeploymentDefinition.java

@@ -20,6 +20,9 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.network.dao.NetworkDetailVO;
+import com.cloud.network.dao.NetworkDetailsDao;
+import com.cloud.network.router.VirtualRouter;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
 import org.apache.log4j.Logger;
 
@@ -106,6 +109,7 @@ public class RouterDeploymentDefinition {
     protected Long tableLockId;
     protected boolean isPublicNetwork;
     protected PublicIp sourceNatIp;
+    protected NetworkDetailsDao networkDetailsDao;
 
     protected RouterDeploymentDefinition(final Network guestNetwork, final DeployDestination dest,
             final Account owner, final Map<Param, Object> params) {
@@ -410,7 +414,13 @@ public class RouterDeploymentDefinition {
             // Don't start the router as we are holding the network lock that
             // needs to be released at the end of router allocation
             final DomainRouterVO router = nwHelper.deployRouter(this, false);
-
+            //check if the network update is in progress.
+            //if update is in progress add the update_pending flag to DomainRouterVO.
+            NetworkDetailVO detail =networkDetailsDao.findDetail(guestNetwork.getId(),Network.updatingInSequence);
+            if("true".equalsIgnoreCase(detail!=null ? detail.getValue() : null)) {
+                router.setUpdateState(VirtualRouter.UpdateState.UPDATE_IN_PROGRESS);
+                routerDao.persist(router);
+            }
             if (router != null) {
                 routerDao.addRouterToGuestNetwork(router, guestNetwork);
                 //Fix according to changes by Sheng Yang in commit ID cb4513379996b262ae378daf00c6388c6b7313cf

server/src/org/cloud/network/router/deployment/RouterDeploymentDefinitionBuilder.java

@@ -22,6 +22,7 @@ import java.util.Map;
 
 import javax.inject.Inject;
 
+import com.cloud.network.dao.NetworkDetailsDao;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -96,6 +97,8 @@ public class RouterDeploymentDefinitionBuilder {
     private VpcManager vpcMgr;
     @Inject
     private VlanDao vlanDao;
+    @Inject
+    private NetworkDetailsDao networkDetailsDao;
 
     @Autowired
     @Qualifier("networkHelper")
@@ -133,6 +136,7 @@ public class RouterDeploymentDefinitionBuilder {
         routerDeploymentDefinition.ipv6Dao = ipv6Dao;
         routerDeploymentDefinition.ipAddressDao = ipAddressDao;
         routerDeploymentDefinition.serviceOfferingId = offeringId;
+        routerDeploymentDefinition.networkDetailsDao = networkDetailsDao;
 
         routerDeploymentDefinition.nwHelper = nwHelper;
 

server/test/com/cloud/vpc/MockNetworkManagerImpl.java

@@ -247,7 +247,7 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches
      */
     @Override
     public Network updateGuestNetwork(long networkId, String name, String displayText, Account callerAccount, User callerUser, String domainSuffix,
-        Long networkOfferingId, Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String newUUID) {
+        Long networkOfferingId, Boolean changeCidr, String guestVmCidr, Boolean displayNetwork, String newUUID,boolean updateInSequence) {
         // TODO Auto-generated method stub
         return null;
     }
@@ -841,6 +841,21 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches
         return;
     }
 
+    @Override
+    public boolean canUpdateInSequence(Network network) {
+        return false;
+    }
+
+    @Override
+    public void configureUpdateInSequence(Network network) {
+        return;
+    }
+
+    @Override
+    public int getResourceCount(Network network) {
+        return 0;
+    }
+
     @Override
     public void prepareNicForMigration(VirtualMachineProfile vm, DeployDestination dest) {
         // TODO Auto-generated method stub

server/test/org/cloud/network/router/deployment/RouterDeploymentDefinitionTest.java

@@ -714,7 +714,7 @@ public class RouterDeploymentDefinitionTest extends RouterDeploymentDefinitionTe
         final DomainRouterVO routerVO2 = mock(DomainRouterVO.class);
         when(mockNetworkHelper.deployRouter(deploymentUT, false))
         .thenReturn(routerVO1).thenReturn(routerVO2);
-
+        when(networkDetailsDao.findById(anyLong())).thenReturn(null);
         // Execute
         deploymentUT.deployAllVirtualRouters();
 

server/test/org/cloud/network/router/deployment/RouterDeploymentDefinitionTestBase.java

@@ -23,6 +23,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.network.dao.NetworkDetailsDao;
 import org.junit.runner.RunWith;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
@@ -79,6 +80,8 @@ public class RouterDeploymentDefinitionTestBase {
     @Mock
     protected NetworkHelper mockNetworkHelper;
     @Mock
+    protected NetworkDetailsDao networkDetailsDao;
+    @Mock
     protected VpcNetworkHelperImpl vpcNwHelper;
     @Mock
     protected VMInstanceDao mockVmDao;

setup/db/db/schema-452to460.sql

@@ -420,3 +420,5 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis
 INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'KVM', 'default', 'CentOS 7', 246, utc_timestamp(), 0);
 
 UPDATE  `cloud`.`hypervisor_capabilities` SET  `max_data_volumes_limit` =  '32' WHERE  `hypervisor_capabilities`.`hypervisor_type` =  'KVM';
+ALTER TABLE `cloud`.`domain_router` ADD COLUMN  update_state varchar(64) DEFAULT NULL;
+