CLOUDSTACK-6047: Enable VR aggregation commands for VR start/reboot

api/src/com/cloud/network/element/AggregatedCommandExecutor.java

@@ -15,17 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package com.cloud.agent.api.routing;
+package com.cloud.network.element;
 
-public class StartAggregationCommand extends NetworkElementCommand{
+import com.cloud.deploy.DeployDestination;
-    protected StartAggregationCommand() {
+import com.cloud.exception.ResourceUnavailableException;
-        super();
+import com.cloud.network.Network;
-    }
 
-    public StartAggregationCommand(String name, String ip, String guestIp) {
+public interface AggregatedCommandExecutor {
-        super();
+    public boolean prepareAggregatedExecution(Network network, DeployDestination dest) throws ResourceUnavailableException;
-        this.setAccessDetail(NetworkElementCommand.ROUTER_NAME, name);
+    public boolean completeAggregatedExecution(Network network, DeployDestination dest) throws ResourceUnavailableException;
-        this.setAccessDetail(NetworkElementCommand.ROUTER_IP, ip);
+    public boolean cleanupAggregatedExecution(Network network, DeployDestination dest) throws ResourceUnavailableException;
-        this.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, guestIp);
-    }
 }

core/src/com/cloud/agent/api/routing/AggregationControlCommand.java

@@ -17,15 +17,28 @@
 
 package com.cloud.agent.api.routing;
 
-public class FinishAggregationCommand extends NetworkElementCommand{
+public class AggregationControlCommand extends NetworkElementCommand{
-    protected FinishAggregationCommand() {
+    public enum Action {
+        Start,
+        Finish,
+        Cleanup,
+    }
+
+    private Action action;
+
+    protected AggregationControlCommand() {
         super();
     }
 
-    public FinishAggregationCommand(String name, String ip, String guestIp) {
+    public AggregationControlCommand(Action action, String name, String ip, String guestIp) {
         super();
+        this.action = action;
         this.setAccessDetail(NetworkElementCommand.ROUTER_NAME, name);
         this.setAccessDetail(NetworkElementCommand.ROUTER_IP, ip);
         this.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, guestIp);
     }
+
+    public Action getAction() {
+        return action;
+    }
 }

core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java

@@ -25,11 +25,12 @@ import com.cloud.agent.api.CheckS2SVpnConnectionsCommand;
 import com.cloud.agent.api.GetDomRVersionAnswer;
 import com.cloud.agent.api.GetDomRVersionCmd;
 import com.cloud.agent.api.SetupGuestNetworkCommand;
+import com.cloud.agent.api.routing.AggregationControlCommand;
+import com.cloud.agent.api.routing.AggregationControlCommand.Action;
 import com.cloud.agent.api.routing.CreateIpAliasCommand;
 import com.cloud.agent.api.routing.DeleteIpAliasCommand;
 import com.cloud.agent.api.routing.DhcpEntryCommand;
 import com.cloud.agent.api.routing.DnsMasqConfigCommand;
-import com.cloud.agent.api.routing.FinishAggregationCommand;
 import com.cloud.agent.api.routing.GroupAnswer;
 import com.cloud.agent.api.routing.IpAliasTO;
 import com.cloud.agent.api.routing.IpAssocCommand;
@@ -47,7 +48,6 @@ import com.cloud.agent.api.routing.SetSourceNatCommand;
 import com.cloud.agent.api.routing.SetStaticNatRulesCommand;
 import com.cloud.agent.api.routing.SetStaticRouteCommand;
 import com.cloud.agent.api.routing.Site2SiteVpnCfgCommand;
-import com.cloud.agent.api.routing.StartAggregationCommand;
 import com.cloud.agent.api.routing.VmDataCommand;
 import com.cloud.agent.api.routing.VpnUsersCfgCommand;
 import com.cloud.agent.api.to.DhcpTO;
@@ -163,10 +163,8 @@ public class VirtualRoutingResource {
                 return executeQueryCommand(cmd);
             }
 
-            if (cmd instanceof StartAggregationCommand) {
+            if (cmd instanceof AggregationControlCommand) {
-                return execute((StartAggregationCommand)cmd);
+                return execute((AggregationControlCommand)cmd);
-            } else if (cmd instanceof FinishAggregationCommand) {
-                return execute((FinishAggregationCommand)cmd);
             }
 
             if (_vrAggregateCommandsSet.containsKey(routerName)) {
@@ -1032,15 +1030,6 @@ public class VirtualRoutingResource {
         return false;
     }
 
-    private Answer execute(StartAggregationCommand cmd) {
-        // Access IP would be used as ID for router
-        String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
-        assert routerName != null;
-        Queue<NetworkElementCommand> queue = new LinkedBlockingQueue<>();
-        _vrAggregateCommandsSet.put(routerName, queue);
-        return new Answer(cmd);
-    }
-
     private List<ConfigItem> generateCommandCfg(NetworkElementCommand cmd) {
         List<ConfigItem> cfg;
         if (cmd instanceof SetPortForwardingRulesVpcCommand) {
@@ -1091,10 +1080,19 @@ public class VirtualRoutingResource {
         return cfg;
     }
 
-    private Answer execute(FinishAggregationCommand cmd) {
+    private Answer execute(AggregationControlCommand cmd) {
+        Action action = cmd.getAction();
         String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
         assert routerName != null;
         assert cmd.getRouterAccessIp() != null;
+
+        if (action == Action.Start) {
+            assert (!_vrAggregateCommandsSet.containsKey(routerName));
+
+            Queue<NetworkElementCommand> queue = new LinkedBlockingQueue<>();
+            _vrAggregateCommandsSet.put(routerName, queue);
+            return new Answer(cmd);
+        } else if (action == Action.Finish) {
             Queue<NetworkElementCommand> queue = _vrAggregateCommandsSet.get(routerName);
             try {
                 StringBuilder sb = new StringBuilder();
@@ -1136,6 +1134,16 @@ public class VirtualRoutingResource {
                 queue.clear();
                 _vrAggregateCommandsSet.remove(routerName);
             }
+        } else if (action == Action.Cleanup) {
+            assert (_vrAggregateCommandsSet.containsKey(routerName));
+            Queue<NetworkElementCommand> queue = _vrAggregateCommandsSet.get(routerName);
+            if (queue != null) {
+                queue.clear();
             }
+            _vrAggregateCommandsSet.remove(routerName);
 
+            return new Answer(cmd);
+        }
+        return new Answer(cmd, false, "Fail to recongize aggregation action " + action.toString());
+    }
 }

core/test/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java

@@ -19,11 +19,12 @@ package com.cloud.agent.resource.virtualnetwork;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.BumpUpPriorityCommand;
 import com.cloud.agent.api.SetupGuestNetworkCommand;
+import com.cloud.agent.api.routing.AggregationControlCommand;
+import com.cloud.agent.api.routing.AggregationControlCommand.Action;
 import com.cloud.agent.api.routing.CreateIpAliasCommand;
 import com.cloud.agent.api.routing.DeleteIpAliasCommand;
 import com.cloud.agent.api.routing.DhcpEntryCommand;
 import com.cloud.agent.api.routing.DnsMasqConfigCommand;
-import com.cloud.agent.api.routing.FinishAggregationCommand;
 import com.cloud.agent.api.routing.GroupAnswer;
 import com.cloud.agent.api.routing.IpAliasTO;
 import com.cloud.agent.api.routing.IpAssocCommand;
@@ -41,7 +42,6 @@ import com.cloud.agent.api.routing.SetSourceNatCommand;
 import com.cloud.agent.api.routing.SetStaticNatRulesCommand;
 import com.cloud.agent.api.routing.SetStaticRouteCommand;
 import com.cloud.agent.api.routing.Site2SiteVpnCfgCommand;
-import com.cloud.agent.api.routing.StartAggregationCommand;
 import com.cloud.agent.api.routing.VmDataCommand;
 import com.cloud.agent.api.routing.VpnUsersCfgCommand;
 import com.cloud.agent.api.to.DhcpTO;
@@ -84,7 +84,6 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer {
     NetworkElementCommand _currentCmd;
     int _count;
     String _file;
-    boolean _aggregated = false;
 
     String ROUTERIP = "169.254.3.4";
     String ROUTERGUESTIP = "10.200.1.1";
@@ -138,8 +137,8 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer {
     }
 
     private void verifyFile(NetworkElementCommand cmd, String path, String filename, String content) {
-        if (cmd instanceof FinishAggregationCommand) {
+        if (cmd instanceof AggregationControlCommand) {
-            verifyFile((FinishAggregationCommand)cmd, path, filename, content);
+            verifyFile((AggregationControlCommand)cmd, path, filename, content);
         } else if (cmd instanceof LoadBalancerConfigCommand) {
             verifyFile((LoadBalancerConfigCommand)cmd, path, filename, content);
         }
@@ -190,10 +189,8 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer {
             verifyArgs((IpAssocCommand)cmd, script, args);
         }
 
-        if (cmd instanceof StartAggregationCommand) {
+        if (cmd instanceof AggregationControlCommand) {
-            return;
+            verifyArgs((AggregationControlCommand)cmd, script, args);
-        } else if (cmd instanceof FinishAggregationCommand) {
-            verifyArgs((FinishAggregationCommand)cmd, script, args);
         }
     }
 
@@ -948,7 +945,7 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer {
     @Test
     public void testAggregationCommands() {
         List<NetworkElementCommand> cmds = new LinkedList<>();
-        StartAggregationCommand startCmd = new StartAggregationCommand(ROUTERNAME, ROUTERIP, ROUTERGUESTIP);
+        AggregationControlCommand startCmd = new AggregationControlCommand(Action.Start, ROUTERNAME, ROUTERIP, ROUTERGUESTIP);
         cmds.add(startCmd);
         cmds.add(generateIpAssocCommand());
         cmds.add(generateIpAssocVpcCommand());
@@ -979,26 +976,22 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer {
         cmds.add(generateSavePasswordCommand());
         cmds.add(generateVmDataCommand());
 
-        FinishAggregationCommand finishCmd = new FinishAggregationCommand(ROUTERNAME, ROUTERIP, ROUTERGUESTIP);
+        AggregationControlCommand finishCmd = new AggregationControlCommand(Action.Finish, ROUTERNAME, ROUTERIP, ROUTERGUESTIP);
         cmds.add(finishCmd);
 
         for (NetworkElementCommand cmd : cmds) {
             Answer answer = _resource.executeRequest(cmd);
-            if (!(cmd instanceof FinishAggregationCommand)) {
             assertTrue(answer.getResult());
-            } else {
-
-            }
         }
     }
 
-    private void verifyArgs(FinishAggregationCommand cmd, String script, String args) {
+    private void verifyArgs(AggregationControlCommand cmd, String script, String args) {
         assertEquals(script, VRScripts.VR_CFG);
         assertTrue(args.startsWith("-c /var/cache/cloud/VR-"));
         assertTrue(args.endsWith(".cfg"));
     }
 
-    protected void verifyFile(FinishAggregationCommand cmd, String path, String filename, String content) {
+    protected void verifyFile(AggregationControlCommand cmd, String path, String filename, String content) {
         assertEquals(path, "/var/cache/cloud/");
         assertTrue(filename.startsWith("VR-"));
         assertTrue(filename.endsWith(".cfg"));

engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -16,37 +16,6 @@
 // under the License.
 package org.apache.cloudstack.engine.orchestration;
 
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import javax.ejb.Local;
-import javax.inject.Inject;
-import javax.naming.ConfigurationException;
-
-import org.apache.cloudstack.acl.ControlledEntity.ACLType;
-import org.apache.cloudstack.context.CallContext;
-import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
-import org.apache.cloudstack.framework.config.ConfigDepot;
-import org.apache.cloudstack.framework.config.ConfigKey;
-import org.apache.cloudstack.framework.config.Configurable;
-import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-import org.apache.cloudstack.managed.context.ManagedContextRunnable;
-import org.apache.cloudstack.region.PortableIpDao;
-import org.apache.log4j.Logger;
-
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.Listener;
 import com.cloud.agent.api.AgentControlAnswer;
@@ -127,6 +96,7 @@ import com.cloud.network.dao.PhysicalNetworkServiceProviderDao;
 import com.cloud.network.dao.PhysicalNetworkTrafficTypeDao;
 import com.cloud.network.dao.PhysicalNetworkTrafficTypeVO;
 import com.cloud.network.dao.PhysicalNetworkVO;
+import com.cloud.network.element.AggregatedCommandExecutor;
 import com.cloud.network.element.DhcpServiceProvider;
 import com.cloud.network.element.IpDeployer;
 import com.cloud.network.element.LoadBalancingServiceProvider;
@@ -199,6 +169,35 @@ import com.cloud.vm.dao.NicSecondaryIpDao;
 import com.cloud.vm.dao.NicSecondaryIpVO;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.VMInstanceDao;
+import org.apache.cloudstack.acl.ControlledEntity.ACLType;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.cloudstack.framework.config.ConfigDepot;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.region.PortableIpDao;
+import org.apache.log4j.Logger;
+
+import javax.ejb.Local;
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
 /**
  * NetworkManagerImpl implements NetworkManager.
@@ -1068,6 +1067,13 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             }
         }
 
+        for (NetworkElement element : networkElements) {
+            if ((element instanceof AggregatedCommandExecutor) && (providersToImplement.contains(element.getProvider()))) {
+                ((AggregatedCommandExecutor)element).prepareAggregatedExecution(network, dest);
+            }
+        }
+
+        try {
             // reapply all the firewall/staticNat/lb rules
             s_logger.debug("Reprogramming network " + network + " as a part of network implement");
             if (!reprogramNetworkRules(network.getId(), CallContext.current().getCallingAccount(), network)) {
@@ -1078,6 +1084,25 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 ex.addProxyObject(_entityMgr.findById(DataCenter.class, network.getDataCenterId()).getUuid());
                 throw ex;
             }
+            for (NetworkElement element : networkElements) {
+                if ((element instanceof AggregatedCommandExecutor) && (providersToImplement.contains(element.getProvider()))) {
+                    if (!((AggregatedCommandExecutor)element).completeAggregatedExecution(network, dest)) {
+                        s_logger.warn("Failed to re-program the network as a part of network " + network + " implement due to aggregated commands execution failure!");
+                        // see DataCenterVO.java
+                        ResourceUnavailableException ex = new ResourceUnavailableException("Unable to apply network rules as a part of network " + network + " implement", DataCenter.class,
+                                network.getDataCenterId());
+                        ex.addProxyObject(_entityMgr.findById(DataCenter.class, network.getDataCenterId()).getUuid());
+                        throw ex;
+                    }
+                }
+            }
+        } finally {
+            for (NetworkElement element : networkElements) {
+                if ((element instanceof AggregatedCommandExecutor) && (providersToImplement.contains(element.getProvider()))) {
+                    ((AggregatedCommandExecutor)element).cleanupAggregatedExecution(network, dest);
+                }
+            }
+        }
     }
 
     // This method re-programs the rules/ips for existing network

server/src/com/cloud/network/element/VirtualRouterElement.java

@@ -16,26 +16,6 @@
 // under the License.
 package com.cloud.network.element;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.ejb.Local;
-import javax.inject.Inject;
-
-import org.apache.log4j.Logger;
-
-import com.google.gson.Gson;
-
-import org.apache.cloudstack.api.command.admin.router.ConfigureOvsElementCmd;
-import org.apache.cloudstack.api.command.admin.router.ConfigureVirtualRouterElementCmd;
-import org.apache.cloudstack.api.command.admin.router.CreateVirtualRouterElementCmd;
-import org.apache.cloudstack.api.command.admin.router.ListOvsElementsCmd;
-import org.apache.cloudstack.api.command.admin.router.ListVirtualRouterElementsCmd;
-import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-
 import com.cloud.agent.api.to.LoadBalancerTO;
 import com.cloud.configuration.ConfigurationManager;
 import com.cloud.dc.DataCenter;
@@ -101,6 +81,22 @@ import com.cloud.vm.VirtualMachine.State;
 import com.cloud.vm.VirtualMachineProfile;
 import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.UserVmDao;
+import com.google.gson.Gson;
+import org.apache.cloudstack.api.command.admin.router.ConfigureOvsElementCmd;
+import org.apache.cloudstack.api.command.admin.router.ConfigureVirtualRouterElementCmd;
+import org.apache.cloudstack.api.command.admin.router.CreateVirtualRouterElementCmd;
+import org.apache.cloudstack.api.command.admin.router.ListOvsElementsCmd;
+import org.apache.cloudstack.api.command.admin.router.ListVirtualRouterElementsCmd;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.log4j.Logger;
+
+import javax.ejb.Local;
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 @Local(value = {NetworkElement.class, FirewallServiceProvider.class,
     DhcpServiceProvider.class, UserDataServiceProvider.class,
@@ -110,7 +106,7 @@ import com.cloud.vm.dao.UserVmDao;
 public class VirtualRouterElement extends AdapterBase implements VirtualRouterElementService, DhcpServiceProvider,
         UserDataServiceProvider, SourceNatServiceProvider, StaticNatServiceProvider, FirewallServiceProvider,
         LoadBalancingServiceProvider, PortForwardingServiceProvider, RemoteAccessVPNServiceProvider, IpDeployer,
-        NetworkMigrationResponder {
+        NetworkMigrationResponder, AggregatedCommandExecutor {
     private static final Logger s_logger = Logger.getLogger(VirtualRouterElement.class);
     public static final AutoScaleCounterType AutoScaleCounterCpu = new AutoScaleCounterType("cpu");
     public static final AutoScaleCounterType AutoScaleCounterMemory = new AutoScaleCounterType("memory");
@@ -1091,4 +1087,37 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl
             _userVmMgr.setupVmForPvlan(true, userVm.getHostId(), nic);
         }
     }
+
+    @Override
+    public boolean prepareAggregatedExecution(Network network, DeployDestination dest) throws ResourceUnavailableException {
+        List<DomainRouterVO> routers = getRouters(network, dest);
+
+        if ((routers == null) || (routers.size() == 0)) {
+            throw new ResourceUnavailableException("Can't find at least one router!", DataCenter.class, network.getDataCenterId());
+        }
+
+        return _routerMgr.prepareAggregatedExecution(network, routers);
+    }
+
+    @Override
+    public boolean completeAggregatedExecution(Network network, DeployDestination dest) throws ResourceUnavailableException {
+        List<DomainRouterVO> routers = getRouters(network, dest);
+
+        if ((routers == null) || (routers.size() == 0)) {
+            throw new ResourceUnavailableException("Can't find at least one router!", DataCenter.class, network.getDataCenterId());
+        }
+
+        return _routerMgr.completeAggregatedExecution(network, routers);
+    }
+
+    @Override
+    public boolean cleanupAggregatedExecution(Network network, DeployDestination dest) throws ResourceUnavailableException {
+        List<DomainRouterVO> routers = getRouters(network, dest);
+
+        if ((routers == null) || (routers.size() == 0)) {
+            throw new ResourceUnavailableException("Can't find at least one router!", DataCenter.class, network.getDataCenterId());
+        }
+
+        return _routerMgr.cleanupAggregatedExecution(network, routers);
+    }
 }

server/src/com/cloud/network/router/VirtualNetworkApplianceManager.java

@@ -16,12 +16,8 @@
 // under the License.
 package com.cloud.network.router;
 
-import java.util.List;
-import java.util.Map;
-
-import org.apache.cloudstack.framework.config.ConfigKey;
-
 import com.cloud.deploy.DeployDestination;
+import com.cloud.exception.AgentUnavailableException;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ResourceUnavailableException;
@@ -39,6 +35,10 @@ import com.cloud.utils.component.Manager;
 import com.cloud.vm.DomainRouterVO;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.VirtualMachineProfile;
+import org.apache.cloudstack.framework.config.ConfigKey;
+
+import java.util.List;
+import java.util.Map;
 
 /**
  * NetworkManager manages the network for the different end users.
@@ -129,4 +129,10 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA
     boolean removeDhcpSupportForSubnet(Network network, List<DomainRouterVO> routers) throws ResourceUnavailableException;
 
     boolean setupDhcpForPvlan(boolean add, DomainRouterVO router, Long hostId, NicProfile nic);
+
+    public boolean prepareAggregatedExecution(Network network, List<DomainRouterVO> routers) throws AgentUnavailableException;
+
+    public boolean completeAggregatedExecution(Network network, List<DomainRouterVO> routers) throws AgentUnavailableException;
+
+    public boolean cleanupAggregatedExecution(Network network, List<DomainRouterVO> routers) throws AgentUnavailableException;
 }

server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -17,47 +17,6 @@
 
 package com.cloud.network.router;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import javax.ejb.Local;
-import javax.inject.Inject;
-import javax.naming.ConfigurationException;
-
-import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd;
-import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
-import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
-import org.apache.cloudstack.config.ApiServiceConfiguration;
-import org.apache.cloudstack.context.CallContext;
-import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
-import org.apache.cloudstack.framework.config.ConfigDepot;
-import org.apache.cloudstack.framework.config.ConfigKey;
-import org.apache.cloudstack.framework.config.Configurable;
-import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-import org.apache.cloudstack.framework.jobs.AsyncJobManager;
-import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
-import org.apache.cloudstack.managed.context.ManagedContextRunnable;
-import org.apache.cloudstack.utils.identity.ManagementServerNode;
-import org.apache.log4j.Logger;
-
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.Listener;
 import com.cloud.agent.api.AgentControlAnswer;
@@ -78,6 +37,8 @@ import com.cloud.agent.api.PvlanSetupCommand;
 import com.cloud.agent.api.StartupCommand;
 import com.cloud.agent.api.check.CheckSshAnswer;
 import com.cloud.agent.api.check.CheckSshCommand;
+import com.cloud.agent.api.routing.AggregationControlCommand;
+import com.cloud.agent.api.routing.AggregationControlCommand.Action;
 import com.cloud.agent.api.routing.CreateIpAliasCommand;
 import com.cloud.agent.api.routing.DeleteIpAliasCommand;
 import com.cloud.agent.api.routing.DhcpEntryCommand;
@@ -277,6 +238,45 @@ import com.cloud.vm.dao.NicIpAliasVO;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
+import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd;
+import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
+import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
+import org.apache.cloudstack.config.ApiServiceConfiguration;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.cloudstack.framework.config.ConfigDepot;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.framework.jobs.AsyncJobManager;
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.utils.identity.ManagementServerNode;
+import org.apache.log4j.Logger;
+
+import javax.ejb.Local;
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
 /**
  * VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack.
@@ -2320,12 +2320,20 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
 
         final List<Long> routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId());
         for (final Long guestNetworkId : routerGuestNtwkIds) {
+            AggregationControlCommand startCmd = new AggregationControlCommand(Action.Start, router.getInstanceName(), controlNic.getIp4Address(),
+                    getRouterIpInNetwork(guestNetworkId, router.getId()));
+            cmds.addCommand(startCmd);
+
             if (reprogramGuestNtwks) {
                 finalizeIpAssocForNetwork(cmds, router, provider, guestNetworkId, null);
                 finalizeNetworkRulesForNetwork(cmds, router, provider, guestNetworkId);
             }
 
             finalizeUserDataAndDhcpOnStart(cmds, router, provider, guestNetworkId);
+
+            AggregationControlCommand finishCmd = new AggregationControlCommand(Action.Finish, router.getInstanceName(), controlNic.getIp4Address(),
+                    getRouterIpInNetwork(guestNetworkId, router.getId()));
+            cmds.addCommand(finishCmd);
         }
 
 
@@ -2338,8 +2346,6 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
             finalizeMonitorServiceOnStrat(cmds, profile, router, provider, routerGuestNtwkIds.get(0), false);
         }
 
-
-
         return true;
     }
 
@@ -4259,4 +4265,31 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
             }
         }
     }
+
+    protected boolean aggregationExecution(AggregationControlCommand.Action action, Network network, List<DomainRouterVO> routers) throws AgentUnavailableException {
+        for (DomainRouterVO router : routers) {
+            AggregationControlCommand cmd = new AggregationControlCommand(action, router.getInstanceName(), getRouterControlIp(router.getId()),
+                    getRouterIpInNetwork(network.getId(), router.getId()));
+            Commands cmds = new Commands(cmd);
+            if (!sendCommandsToRouter(router, cmds)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public boolean prepareAggregatedExecution(Network network, List<DomainRouterVO> routers) throws AgentUnavailableException {
+        return aggregationExecution(Action.Start, network, routers);
+    }
+
+    @Override
+    public boolean completeAggregatedExecution(Network network, List<DomainRouterVO> routers) throws AgentUnavailableException {
+        return aggregationExecution(Action.Finish, network, routers);
+    }
+
+    @Override
+    public boolean cleanupAggregatedExecution(Network network, List<DomainRouterVO> routers) throws AgentUnavailableException {
+        return aggregationExecution(Action.Cleanup, network, routers);
+    }
 }

server/test/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java

@@ -17,17 +17,8 @@
 
 package com.cloud.vpc;
 
-import java.util.List;
-import java.util.Map;
-
-import javax.ejb.Local;
-import javax.naming.ConfigurationException;
-
-import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
-import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
-import org.springframework.stereotype.Component;
-
 import com.cloud.deploy.DeployDestination;
+import com.cloud.exception.AgentUnavailableException;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ResourceUnavailableException;
@@ -53,6 +44,14 @@ import com.cloud.vm.DomainRouterVO;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.VirtualMachineProfile;
 import com.cloud.vm.VirtualMachineProfile.Param;
+import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
+import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
+import org.springframework.stereotype.Component;
+
+import javax.ejb.Local;
+import javax.naming.ConfigurationException;
+import java.util.List;
+import java.util.Map;
 
 @Component
 @Local(value = {VpcVirtualNetworkApplianceManager.class, VpcVirtualNetworkApplianceService.class})
@@ -419,6 +418,21 @@ public class MockVpcVirtualNetworkApplianceManager extends ManagerBase implement
         return false;
     }
 
+    @Override
+    public boolean prepareAggregatedExecution(Network network, List<DomainRouterVO> routers) throws AgentUnavailableException {
+        return true;  //To change body of implemented methods use File | Settings | File Templates.
+    }
+
+    @Override
+    public boolean completeAggregatedExecution(Network network, List<DomainRouterVO> routers) throws AgentUnavailableException {
+        return true;  //To change body of implemented methods use File | Settings | File Templates.
+    }
+
+    @Override
+    public boolean cleanupAggregatedExecution(Network network, List<DomainRouterVO> routers) throws AgentUnavailableException {
+        return true;  //To change body of implemented methods use File | Settings | File Templates.
+    }
+
     @Override
     public boolean startRemoteAccessVpn(RemoteAccessVpn vpn, VirtualRouter router) throws ResourceUnavailableException {
         // TODO Auto-generated method stub