server: Support for persistence mode in L2 networks (#4561)

This PR aims at introducing persistence mode in L2 networks and enhancing the behavior in Isolated networks
Doc PR apache/cloudstack-documentation#183

Co-authored-by: Pearl Dsilva <pearl.dsilva@shapeblue.com>

core/src/main/java/com/cloud/agent/api/CleanupPersistentNetworkResourceAnswer.java

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.agent.api;
+
+public class CleanupPersistentNetworkResourceAnswer extends Answer {
+    public CleanupPersistentNetworkResourceAnswer() {
+    }
+
+    public CleanupPersistentNetworkResourceAnswer(CleanupPersistentNetworkResourceCommand cmd, boolean success, String result) {
+        super(cmd, success, result);
+    }
+}

core/src/main/java/com/cloud/agent/api/CleanupPersistentNetworkResourceCommand.java

@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.to.NicTO;
+
+public class CleanupPersistentNetworkResourceCommand extends Command {
+    NicTO nicTO;
+
+    protected CleanupPersistentNetworkResourceCommand() {}
+
+    public CleanupPersistentNetworkResourceCommand(NicTO nicTO) {
+        this.nicTO = nicTO;
+    }
+
+    public NicTO getNicTO() {
+        return nicTO;
+    }
+
+    public void setNicTO(NicTO nicTO) {
+        this.nicTO = nicTO;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

core/src/main/java/com/cloud/agent/api/MigrateCommand.java

@@ -40,6 +40,7 @@ public class MigrateCommand extends Command {
     private boolean executeInSequence = false;
     private List<MigrateDiskInfo> migrateDiskInfoList = new ArrayList<>();
     private Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
+    Map<String, Boolean> vlanToPersistenceMap = new HashMap<>();
 
     public Map<String, DpdkTO> getDpdkInterfaceMapping() {
         return dpdkInterfaceMapping;
@@ -49,6 +50,14 @@ public class MigrateCommand extends Command {
         this.dpdkInterfaceMapping = dpdkInterfaceMapping;
     }
 
+    public Map<String, Boolean> getVlanToPersistenceMap() {
+        return vlanToPersistenceMap;
+    }
+
+    public void setVlanToPersistenceMap(Map<String, Boolean> vlanToPersistenceMap) {
+        this.vlanToPersistenceMap = vlanToPersistenceMap;
+    }
+
     protected MigrateCommand() {
     }
 

core/src/main/java/com/cloud/agent/api/SetupPersistentNetworkAnswer.java

@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.agent.api;
+
+public class SetupPersistentNetworkAnswer extends Answer {
+    public SetupPersistentNetworkAnswer(){}
+
+    public SetupPersistentNetworkAnswer(SetupPersistentNetworkCommand cmd, boolean success, String result) {
+        super(cmd, success, result);
+    }
+}

core/src/main/java/com/cloud/agent/api/SetupPersistentNetworkCommand.java

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.to.NicTO;
+
+public class SetupPersistentNetworkCommand extends Command {
+
+    NicTO nic;
+
+    public SetupPersistentNetworkCommand(NicTO nic) {
+        this.nic = nic;
+    }
+
+    public NicTO getNic() {
+        return nic;
+    }
+
+    protected SetupPersistentNetworkCommand() {
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

core/src/main/java/com/cloud/agent/api/StopCommand.java

@@ -36,6 +36,7 @@ public class StopCommand extends RebootCommand {
     String controlIp = null;
     boolean forceStop = false;
     private Map<String, DpdkTO> dpdkInterfaceMapping;
+    Map<String, Boolean> vlanToPersistenceMap;
 
     public Map<String, DpdkTO> getDpdkInterfaceMapping() {
         return dpdkInterfaceMapping;
@@ -129,4 +130,12 @@ public class StopCommand extends RebootCommand {
     public List<Map<String, String>> getVolumesToDisconnect() {
         return volumesToDisconnect;
     }
+
+    public Map<String, Boolean> getVlanToPersistenceMap() {
+        return vlanToPersistenceMap;
+    }
+
+    public void setVlanToPersistenceMap(Map<String, Boolean> vlanToPersistenceMap) {
+        this.vlanToPersistenceMap = vlanToPersistenceMap;
+    }
 }

core/src/main/java/com/cloud/agent/api/UnPlugNicCommand.java

@@ -19,11 +19,14 @@
 
 package com.cloud.agent.api;
 
+import java.util.Map;
+
 import com.cloud.agent.api.to.NicTO;
 
 public class UnPlugNicCommand extends Command {
     NicTO nic;
     String instanceName;
+    Map<String, Boolean> vlanToPersistenceMap;
 
     public NicTO getNic() {
         return nic;
@@ -45,4 +48,12 @@ public class UnPlugNicCommand extends Command {
     public String getVmName() {
         return instanceName;
     }
+
+    public Map<String, Boolean> getVlanToPersistenceMap() {
+        return vlanToPersistenceMap;
+    }
+
+    public void setVlanToPersistenceMap(Map<String, Boolean> vlanToPersistenceMap) {
+        this.vlanToPersistenceMap = vlanToPersistenceMap;
+    }
 }

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -43,6 +43,11 @@ import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.api.query.dao.DomainRouterJoinDao;
+import com.cloud.api.query.dao.UserVmJoinDao;
+import com.cloud.api.query.vo.DomainRouterJoinVO;
+import com.cloud.api.query.vo.UserVmJoinVO;
+import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd;
@@ -143,7 +148,6 @@ import com.cloud.deploy.DeploymentPlan;
 import com.cloud.deploy.DeploymentPlanner;
 import com.cloud.deploy.DeploymentPlanner.ExcludeList;
 import com.cloud.deploy.DeploymentPlanningManager;
-import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao;
 import com.cloud.event.EventTypes;
 import com.cloud.event.UsageEventUtils;
 import com.cloud.event.UsageEventVO;
@@ -169,6 +173,7 @@ import com.cloud.hypervisor.HypervisorGuru;
 import com.cloud.hypervisor.HypervisorGuruManager;
 import com.cloud.network.Network;
 import com.cloud.network.NetworkModel;
+import com.cloud.network.Networks;
 import com.cloud.network.dao.NetworkDao;
 import com.cloud.network.dao.NetworkDetailVO;
 import com.cloud.network.dao.NetworkDetailsDao;
@@ -179,6 +184,8 @@ import com.cloud.offering.DiskOffering;
 import com.cloud.offering.DiskOfferingInfo;
 import com.cloud.offering.NetworkOffering;
 import com.cloud.offering.ServiceOffering;
+import com.cloud.offerings.NetworkOfferingVO;
+import com.cloud.offerings.dao.NetworkOfferingDao;
 import com.cloud.offerings.dao.NetworkOfferingDetailsDao;
 import com.cloud.org.Cluster;
 import com.cloud.resource.ResourceManager;
@@ -350,6 +357,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     private SecurityGroupManager _securityGroupManager;
     @Inject
     private UserVmDeployAsIsDetailsDao userVmDeployAsIsDetailsDao;
+    @Inject
+    private UserVmJoinDao userVmJoinDao;
+    @Inject
+    private NetworkOfferingDao networkOfferingDao;
+    @Inject
+    private DomainRouterJoinDao domainRouterJoinDao;
 
     VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this);
 
@@ -1247,6 +1260,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                             }
                             StopCommand stopCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
                             stopCmd.setControlIp(getControlNicIpForVM(vm));
+                            Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
+                            if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
+                                stopCmd.setVlanToPersistenceMap(vlanToPersistenceMap);
+                            }
                             final StopCommand cmd = stopCmd;
                             final Answer answer = _agentMgr.easySend(destHostId, cmd);
                             if (answer != null && answer instanceof StopAnswer) {
@@ -1646,7 +1663,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
 
     protected boolean sendStop(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final boolean force, final boolean checkBeforeCleanup) {
         final VirtualMachine vm = profile.getVirtualMachine();
+        Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
         StopCommand stpCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), checkBeforeCleanup);
+        if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
+            stpCmd.setVlanToPersistenceMap(vlanToPersistenceMap);
+        }
         stpCmd.setControlIp(getControlNicIpForVM(vm));
         stpCmd.setVolumesToDisconnect(getVolumesToDisconnect(vm));
         final StopCommand stop = stpCmd;
@@ -1847,6 +1868,64 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         advanceStop(vm, cleanUpEvenIfUnableToStop);
     }
 
+    private void updatePersistenceMap(Map<String, Boolean> vlanToPersistenceMap, NetworkVO networkVO) {
+        NetworkOfferingVO offeringVO = networkOfferingDao.findById(networkVO.getNetworkOfferingId());
+        if (offeringVO != null) {
+            Pair<String, Boolean> data = getVMNetworkDetails(networkVO, offeringVO.isPersistent());
+            Boolean shouldDeleteNwResource = (MapUtils.isNotEmpty(vlanToPersistenceMap) && data != null) ? vlanToPersistenceMap.get(data.first()) : null;
+            if (data != null && (shouldDeleteNwResource == null || shouldDeleteNwResource)) {
+                vlanToPersistenceMap.put(data.first(), data.second());
+            }
+        }
+    }
+
+    private Map<String, Boolean> getVlanToPersistenceMapForVM(long vmId) {
+        List<UserVmJoinVO> userVmJoinVOs = userVmJoinDao.searchByIds(vmId);
+        Map<String, Boolean> vlanToPersistenceMap = new HashMap<>();
+        if (userVmJoinVOs != null && !userVmJoinVOs.isEmpty()) {
+            for (UserVmJoinVO userVmJoinVO : userVmJoinVOs) {
+                NetworkVO networkVO = _networkDao.findById(userVmJoinVO.getNetworkId());
+                updatePersistenceMap(vlanToPersistenceMap, networkVO);
+            }
+        } else {
+            VMInstanceVO vmInstanceVO = _vmDao.findById(vmId);
+            if (vmInstanceVO != null && vmInstanceVO.getType() == VirtualMachine.Type.DomainRouter) {
+                DomainRouterJoinVO routerVO = domainRouterJoinDao.findById(vmId);
+                NetworkVO networkVO = _networkDao.findById(routerVO.getNetworkId());
+                updatePersistenceMap(vlanToPersistenceMap, networkVO);
+            }
+        }
+        return vlanToPersistenceMap;
+    }
+
+    /**
+     *
+     * @param networkVO - the network object used to determine the vlanId from the broadcast URI
+     * @param isPersistent - indicates if the corresponding network's network offering is Persistent
+     *
+     * @return <VlanId, ShouldKVMBridgeBeDeleted> - basically returns the vlan ID which is used to determine the
+     * bridge name for KVM hypervisor and based on the network and isolation type and persistent setting of the offering
+     * we decide whether the bridge is to be deleted (KVM) if the last VM in that host is destroyed / migrated
+     */
+    private Pair<String, Boolean> getVMNetworkDetails(NetworkVO networkVO, boolean isPersistent) {
+        URI broadcastUri = networkVO.getBroadcastUri();
+        if (broadcastUri != null) {
+            String scheme = broadcastUri.getScheme();
+            String vlanId = Networks.BroadcastDomainType.getValue(broadcastUri);
+            boolean shouldDelete = !((networkVO.getGuestType() == Network.GuestType.L2 || networkVO.getGuestType() == Network.GuestType.Isolated) &&
+                    (scheme != null && scheme.equalsIgnoreCase("vlan"))
+                    && isPersistent);
+            if (shouldDelete) {
+                int persistentNetworksCount = _networkDao.getOtherPersistentNetworksCount(networkVO.getId(), networkVO.getBroadcastUri().toString(), true);
+                if (persistentNetworksCount > 0) {
+                    shouldDelete = false;
+                }
+            }
+            return new Pair<>(vlanId, shouldDelete);
+        }
+        return null;
+    }
+
     private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnableToStop) throws AgentUnavailableException, OperationTimedoutException,
     ConcurrentOperationException {
         final State state = vm.getState();
@@ -1945,8 +2024,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
 
         vmGuru.prepareStop(profile);
 
+        Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
         final StopCommand stop = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false, cleanUpEvenIfUnableToStop);
         stop.setControlIp(getControlNicIpForVM(vm));
+        if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
+            stop.setVlanToPersistenceMap(vlanToPersistenceMap);
+        }
 
         boolean stopped = false;
         Answer answer = null;
@@ -2682,7 +2765,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         Map<String, DpdkTO> dpdkInterfaceMapping = null;
         try {
             final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
+            Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
             final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
+            if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
+                mc.setVlanToPersistenceMap(vlanToPersistenceMap);
+            }
 
             boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value();
             mc.setAutoConvergence(kvmAutoConvergence);
@@ -3525,6 +3612,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
             cmd.setDpdkInterfaceMapping(dpdkInterfaceMapping);
         }
+        Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
+        if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
+            cmd.setVlanToPersistenceMap(vlanToPersistenceMap);
+        }
         return cmd;
     }
 
@@ -3543,6 +3634,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
 
         StopCommand cmd = new StopCommand(vmName, getExecuteInSequence(null), false);
         cmd.setControlIp(getControlNicIpForVM(vm));
+        Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
+        if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
+            cmd.setVlanToPersistenceMap(vlanToPersistenceMap);
+        }
         return cmd;
     }
 
@@ -4329,8 +4424,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
 
         boolean migrated = false;
         try {
+            Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
             final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
             final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
+            if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
+                mc.setVlanToPersistenceMap(vlanToPersistenceMap);
+            }
 
             boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value();
             mc.setAutoConvergence(kvmAutoConvergence);
@@ -4487,6 +4586,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
             try {
                 final Commands cmds = new Commands(Command.OnError.Stop);
                 final UnPlugNicCommand unplugNicCmd = new UnPlugNicCommand(nic, vm.getName());
+                Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
+                if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
+                    unplugNicCmd.setVlanToPersistenceMap(vlanToPersistenceMap);
+                }
                 cmds.addCommand("unplugnic", unplugNicCmd);
                 _agentMgr.send(dest.getHost().getId(), cmds);
 

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -40,6 +40,12 @@ import java.util.stream.Collectors;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.agent.api.CleanupPersistentNetworkResourceAnswer;
+import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand;
+import com.cloud.agent.api.SetupPersistentNetworkAnswer;
+import com.cloud.agent.api.SetupPersistentNetworkCommand;
+import com.cloud.dc.ClusterVO;
+import com.cloud.dc.dao.ClusterDao;
 import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao;
 import org.apache.cloudstack.acl.ControlledEntity.ACLType;
 import org.apache.cloudstack.api.ApiConstants;
@@ -90,16 +96,19 @@ import com.cloud.deploy.DeploymentPlan;
 import com.cloud.domain.Domain;
 import com.cloud.event.EventTypes;
 import com.cloud.event.UsageEventUtils;
+import com.cloud.exception.AgentUnavailableException;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.ConnectionException;
 import com.cloud.exception.InsufficientAddressCapacityException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.InsufficientVirtualNetworkCapacityException;
 import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.exception.UnsupportedServiceException;
 import com.cloud.host.Host;
+import com.cloud.host.HostVO;
 import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
@@ -182,6 +191,7 @@ import com.cloud.offerings.NetworkOfferingVO;
 import com.cloud.offerings.dao.NetworkOfferingDao;
 import com.cloud.offerings.dao.NetworkOfferingDetailsDao;
 import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
+import com.cloud.resource.ResourceManager;
 import com.cloud.user.Account;
 import com.cloud.user.ResourceLimitService;
 import com.cloud.user.User;
@@ -305,6 +315,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     UserVmManager _userVmMgr;
     @Inject
     TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao;
+    @Inject
+    ResourceManager resourceManager;
 
     List<NetworkGuru> networkGurus;
 
@@ -385,6 +397,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     NetworkModel _networkModel;
     @Inject
     NicSecondaryIpDao _nicSecondaryIpDao;
+    @Inject
+    ClusterDao clusterDao;
 
     protected StateMachine2<Network.State, Network.Event, Network> _stateMachine;
     ScheduledExecutorService _executor;
@@ -1152,11 +1166,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
     boolean isNetworkImplemented(final NetworkVO network) {
         final Network.State state = network.getState();
+        final NetworkOfferingVO offeringVO = _networkOfferingDao.findById(network.getNetworkOfferingId());
         if (state == Network.State.Implemented) {
             return true;
         } else if (state == Network.State.Setup) {
             final DataCenterVO zone = _dcDao.findById(network.getDataCenterId());
-            if (!isSharedNetworkOfferingWithServices(network.getNetworkOfferingId()) || zone.getNetworkType() == NetworkType.Basic) {
+            if ((!isSharedNetworkOfferingWithServices(network.getNetworkOfferingId()) && !offeringVO.isPersistent()) || zone.getNetworkType() == NetworkType.Basic) {
                 return true;
             }
         }
@@ -1181,6 +1196,85 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         return implemented;
     }
 
+    /**
+     *
+     * Creates a dummy NicTO object which is used by the respective hypervisors to setup network elements / resources
+     * - bridges(KVM), VLANs(Xen) and portgroups(VMWare) for L2 network
+     */
+    private NicTO createNicTOFromNetworkAndOffering(NetworkVO networkVO, NetworkOfferingVO networkOfferingVO, HostVO hostVO) {
+        NicTO to = new NicTO();
+        to.setName(_networkModel.getNetworkTag(hostVO.getHypervisorType(), networkVO));
+        to.setBroadcastType(networkVO.getBroadcastDomainType());
+        to.setType(networkVO.getTrafficType());
+        to.setBroadcastUri(networkVO.getBroadcastUri());
+        to.setIsolationuri(networkVO.getBroadcastUri());
+        to.setNetworkRateMbps(_configMgr.getNetworkOfferingNetworkRate(networkOfferingVO.getId(), networkVO.getDataCenterId()));
+        to.setSecurityGroupEnabled(_networkModel.isSecurityGroupSupportedInNetwork(networkVO));
+        return to;
+    }
+
+    private Pair<Boolean, NicTO> isNtwConfiguredInCluster(HostVO hostVO, Map<Long, List<Long>> clusterToHostsMap, NetworkVO networkVO, NetworkOfferingVO networkOfferingVO) {
+        Long clusterId = hostVO.getClusterId();
+        List<Long> hosts = clusterToHostsMap.get(clusterId);
+        if (hosts == null) {
+            hosts = new ArrayList<>();
+        }
+        if (hostVO.getHypervisorType() == HypervisorType.KVM || hostVO.getHypervisorType() == HypervisorType.XenServer ) {
+            hosts.add(hostVO.getId());
+            clusterToHostsMap.put(clusterId, hosts);
+            return new Pair<>(false, createNicTOFromNetworkAndOffering(networkVO, networkOfferingVO, hostVO));
+        }
+        if (hosts != null && !hosts.isEmpty()) {
+            return new Pair<>(true, createNicTOFromNetworkAndOffering(networkVO, networkOfferingVO, hostVO));
+        }
+        hosts.add(hostVO.getId());
+        clusterToHostsMap.put(clusterId, hosts);
+        return new Pair<>(false, createNicTOFromNetworkAndOffering(networkVO, networkOfferingVO, hostVO));
+    }
+
+    private void setupPersistentNetwork(NetworkVO network, NetworkOfferingVO offering, Long dcId) throws AgentUnavailableException, OperationTimedoutException {
+        List<ClusterVO> clusterVOs = clusterDao.listClustersByDcId(dcId);
+        List<HostVO> hosts = resourceManager.listAllUpAndEnabledHostsInOneZoneByType(Host.Type.Routing, dcId);
+        Map<Long, List<Long>> clusterToHostsMap = new HashMap<>();
+
+        for (HostVO host : hosts) {
+            try {
+                Pair<Boolean, NicTO> networkCfgStateAndDetails = isNtwConfiguredInCluster(host, clusterToHostsMap, network, offering);
+                if (networkCfgStateAndDetails.first()) {
+                    continue;
+                }
+                NicTO to = networkCfgStateAndDetails.second();
+                SetupPersistentNetworkCommand cmd = new SetupPersistentNetworkCommand(to);
+                final SetupPersistentNetworkAnswer answer = (SetupPersistentNetworkAnswer) _agentMgr.send(host.getId(), cmd);
+
+                if (answer == null) {
+                    s_logger.warn("Unable to get an answer to the SetupPersistentNetworkCommand from agent:" + host.getId());
+                    clusterToHostsMap.get(host.getClusterId()).remove(host.getId());
+                    continue;
+                }
+
+                if (!answer.getResult()) {
+                    s_logger.warn("Unable to setup agent " + host.getId() + " due to " + answer.getDetails());
+                    clusterToHostsMap.get(host.getClusterId()).remove(host.getId());
+                }
+            } catch (Exception e) {
+                s_logger.warn("Failed to connect to host: "+ host.getName());
+            }
+        }
+        if (clusterToHostsMap.keySet().size() != clusterVOs.size()) {
+            s_logger.warn("Hosts on all clusters may not have been configured with network devices.");
+        }
+    }
+
+    private boolean networkMeetsPersistenceCriteria(NetworkVO network, NetworkOfferingVO offering, boolean cleanup) {
+        boolean criteriaMet = offering.isPersistent() &&
+                (network.getBroadcastUri() != null && BroadcastDomainType.getSchemeValue(network.getBroadcastUri()) == BroadcastDomainType.Vlan);
+        if (!cleanup) {
+            return criteriaMet && network.getGuestType() == GuestType.L2;
+        } else {
+            return criteriaMet && (network.getGuestType() == GuestType.L2 || network.getGuestType() == GuestType.Isolated);
+        }
+    }
     @Override
     @DB
     public Pair<NetworkGuru, NetworkVO> implementNetwork(final long networkId, final DeployDestination dest, final ReservationContext context) throws ConcurrentOperationException,
@@ -1239,6 +1333,10 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             // implement network elements and re-apply all the network rules
             implementNetworkElementsAndResources(dest, context, network, offering);
 
+            long dcId = dest.getDataCenter().getId();
+            if (networkMeetsPersistenceCriteria(network,offering, false)) {
+                setupPersistentNetwork(network, offering, dcId);
+            }
             if (isSharedNetworkWithServices(network)) {
                 network.setState(Network.State.Implemented);
             } else {
@@ -1251,10 +1349,10 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             return implemented;
         } catch (final NoTransitionException e) {
             s_logger.error(e.getMessage());
-            return null;
+            return new Pair<NetworkGuru, NetworkVO>(null, null);
-        } catch (final CloudRuntimeException e) {
+        } catch (final CloudRuntimeException | OperationTimedoutException e) {
             s_logger.error("Caught exception: " + e.getMessage());
-            return null;
+            return new Pair<NetworkGuru, NetworkVO>(null, null);
         } finally {
             if (implemented.first() == null) {
                 s_logger.debug("Cleaning up because we're unable to implement the network " + network);
@@ -2632,7 +2730,6 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                         userNetwork.setPvlanType(isolatedPvlanType);
                     }
                 }
-
                 final List<? extends Network> networks = setupNetwork(owner, ntwkOff, userNetwork, plan, name, displayText, true, domainId, aclType, subdomainAccessFinal, vpcId,
                         isDisplayNetworkEnabled);
                 Network network = null;
@@ -2868,6 +2965,34 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         return success;
     }
 
+    private void cleanupPersistentnNetworkResources(NetworkVO network) {
+        long networkOfferingId = network.getNetworkOfferingId();
+        NetworkOfferingVO offering = _networkOfferingDao.findById(networkOfferingId);
+        if (offering != null) {
+            if (networkMeetsPersistenceCriteria(network, offering, true) &&
+                    _networksDao.getOtherPersistentNetworksCount(network.getId(), network.getBroadcastUri().toString(), offering.isPersistent()) == 0) {
+                List<HostVO> hosts = resourceManager.listAllUpAndEnabledHostsInOneZoneByType(Host.Type.Routing, network.getDataCenterId());
+                for (HostVO host : hosts) {
+                    try {
+                        NicTO to = createNicTOFromNetworkAndOffering(network, offering, host);
+                        CleanupPersistentNetworkResourceCommand cmd = new CleanupPersistentNetworkResourceCommand(to);
+                        CleanupPersistentNetworkResourceAnswer answer = (CleanupPersistentNetworkResourceAnswer) _agentMgr.send(host.getId(), cmd);
+                        if (answer == null) {
+                            s_logger.warn("Unable to get an answer to the CleanupPersistentNetworkResourceCommand from agent:" + host.getId());
+                            continue;
+                        }
+
+                        if (!answer.getResult()) {
+                            s_logger.warn("Unable to setup agent " + host.getId() + " due to " + answer.getDetails());
+                        }
+                    } catch (Exception e) {
+                        s_logger.warn("Failed to cleanup network resources on host: "+ host.getName());
+                    }
+                }
+            }
+        }
+    }
+
     @Override
     @DB
     public boolean destroyNetwork(final long networkId, final ReservationContext context, final boolean forced) {
@@ -2907,6 +3032,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             }
         }
 
+        cleanupPersistentnNetworkResources(network);
+
         // Shutdown network first
         shutdownNetwork(networkId, context, false);
 
@@ -3069,7 +3196,6 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 s_logger.info("NetworkGarbageCollector uses '" + netGcWait + "' seconds for GC interval.");
 
                 for (final Long networkId : networkIds) {
-
                     if (!_networkModel.isNetworkReadyForGc(networkId)) {
                         continue;
                     }
@@ -3941,7 +4067,6 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     public NicProfile createNicForVm(final Network network, final NicProfile requested, final ReservationContext context, final VirtualMachineProfile vmProfile, final boolean prepare)
             throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException, InsufficientCapacityException,
             ResourceUnavailableException {
-
         final VirtualMachine vm = vmProfile.getVirtualMachine();
         final DataCenter dc = _entityMgr.findById(DataCenter.class, network.getDataCenterId());
         final Host host = _hostDao.findById(vm.getHostId());

engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java

@@ -45,6 +45,8 @@ public interface NetworkDao extends GenericDao<NetworkVO, Long>, StateDao<State,
 
     List<NetworkVO> getNetworksForOffering(long offeringId, long dataCenterId, long accountId);
 
+    int getOtherPersistentNetworksCount(long id, String broadcastURI, boolean isPersistent);
+
     @Override
     @Deprecated
     NetworkVO persist(NetworkVO vo);

engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java

@@ -79,6 +79,7 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
     SearchBuilder<NetworkVO> SourceNATSearch;
     GenericSearchBuilder<NetworkVO, Long> VpcNetworksCount;
     SearchBuilder<NetworkVO> OfferingAccountNetworkSearch;
+    SearchBuilder<NetworkVO> PersistentNetworkSearch;
 
     GenericSearchBuilder<NetworkVO, Long> GarbageCollectedSearch;
     SearchBuilder<NetworkVO> PrivateNetworkSearch;
@@ -181,6 +182,16 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
         CountBy.join("offerings", ntwkOffJoin, CountBy.entity().getNetworkOfferingId(), ntwkOffJoin.entity().getId(), JoinBuilder.JoinType.INNER);
         CountBy.done();
 
+        PersistentNetworkSearch = createSearchBuilder();
+        PersistentNetworkSearch.and("id", PersistentNetworkSearch.entity().getId(), Op.NEQ);
+        PersistentNetworkSearch.and("guestType", PersistentNetworkSearch.entity().getGuestType(), Op.IN);
+        PersistentNetworkSearch.and("broadcastUri", PersistentNetworkSearch.entity().getBroadcastUri(), Op.EQ);
+        PersistentNetworkSearch.and("removed", PersistentNetworkSearch.entity().getRemoved(), Op.NULL);
+        final SearchBuilder<NetworkOfferingVO> persistentNtwkOffJoin = _ntwkOffDao.createSearchBuilder();
+        persistentNtwkOffJoin.and("persistent", persistentNtwkOffJoin.entity().isPersistent(), Op.EQ);
+        PersistentNetworkSearch.join("persistent", persistentNtwkOffJoin, PersistentNetworkSearch.entity().getNetworkOfferingId(), persistentNtwkOffJoin.entity().getId(), JoinType.INNER);
+        PersistentNetworkSearch.done();
+
         PhysicalNetworkSearch = createSearchBuilder();
         PhysicalNetworkSearch.and("physicalNetworkId", PhysicalNetworkSearch.entity().getPhysicalNetworkId(), Op.EQ);
         PhysicalNetworkSearch.done();
@@ -390,6 +401,18 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
         return search(sc, null);
     }
 
+    @Override
+    public int getOtherPersistentNetworksCount(long id, String broadcastURI, boolean isPersistent) {
+        Object[] guestTypes = {"Isolated", "L2"};
+        final SearchCriteria<NetworkVO> sc = PersistentNetworkSearch.create();
+        sc.setParameters("id", id);
+        sc.setParameters("broadcastUri", broadcastURI);
+        sc.setParameters("guestType", guestTypes);
+        sc.setJoinParameters("persistent", "persistent", isPersistent);
+        List<NetworkVO> persistentNetworks = search(sc, null);
+        return persistentNetworks.size();
+    }
+
     @Override
     public String getNextAvailableMacAddress(final long networkConfigId, Integer zoneMacIdentifier) {
         final SequenceFetcher fetch = SequenceFetcher.getInstance();

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java

@@ -267,8 +267,8 @@ public class BridgeVifDriver extends VifDriverBase {
     }
 
     @Override
-    public void unplug(LibvirtVMDef.InterfaceDef iface) {
+    public void unplug(LibvirtVMDef.InterfaceDef iface, boolean deleteBr) {
-        deleteVnetBr(iface.getBrName());
+        deleteVnetBr(iface.getBrName(), deleteBr);
     }
 
     @Override
@@ -327,7 +327,7 @@ public class BridgeVifDriver extends VifDriverBase {
         }
     }
 
-    private void deleteVnetBr(String brName) {
+    private void deleteVnetBr(String brName, boolean deleteBr) {
         synchronized (_vnetBridgeMonitor) {
             String cmdout = Script.runSimpleBashScript("ls /sys/class/net/" + brName);
             if (cmdout == null)
@@ -376,6 +376,7 @@ public class BridgeVifDriver extends VifDriverBase {
             command.add("-v", vNetId);
             command.add("-p", pName);
             command.add("-b", brName);
+            command.add("-d", String.valueOf(deleteBr));
 
             final String result = command.execute();
             if (result != null) {
@@ -436,4 +437,22 @@ public class BridgeVifDriver extends VifDriverBase {
             return false;
         }
     }
+
+    @Override
+    public void deleteBr(NicTO nic) {
+        String vlanId = Networks.BroadcastDomainType.getValue(nic.getBroadcastUri());
+        String trafficLabel = nic.getName();
+        String pifName = _pifs.get(trafficLabel);
+        if (pifName == null) {
+            // if not found in bridge map, maybe traffic label refers to pif already?
+            File pif = new File("/sys/class/net/" + trafficLabel);
+            if (pif.isDirectory()) {
+                pifName = trafficLabel;
+            }
+        }
+        if (vlanId != null && pifName != null) {
+            String brName = generateVnetBrName(pifName, vlanId);
+            deleteVnetBr(brName, true);
+        }
+    }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/DirectVifDriver.java

@@ -62,7 +62,7 @@ public class DirectVifDriver extends VifDriverBase {
     }
 
     @Override
-    public void unplug(LibvirtVMDef.InterfaceDef iface) {
+    public void unplug(LibvirtVMDef.InterfaceDef iface, boolean deleteBr) {
         // not needed, libvirt will cleanup
     }
 
@@ -80,4 +80,7 @@ public class DirectVifDriver extends VifDriverBase {
     public void createControlNetwork(String privBrName) {
     }
 
+    @Override
+    public void deleteBr(NicTO nic) {
+    }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/IvsVifDriver.java

@@ -141,7 +141,7 @@ public class IvsVifDriver extends VifDriverBase {
     }
 
     @Override
-    public void unplug(InterfaceDef iface) {
+    public void unplug(InterfaceDef iface, boolean deleteBr) {
     }
 
     @Override
@@ -287,6 +287,10 @@ public class IvsVifDriver extends VifDriverBase {
         }
     }
 
+    @Override
+    public void deleteBr(NicTO nic) {
+    }
+
     private boolean isBridgeExists(String bridgeName) {
         File f = new File("/sys/devices/virtual/net/" + bridgeName);
         if (f.exists()) {

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -1870,7 +1870,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
                 // We don't know which "traffic type" is associated with
                 // each interface at this point, so inform all vif drivers
                 for (final VifDriver vifDriver : getAllVifDrivers()) {
-                    vifDriver.unplug(pluggedNic);
+                    vifDriver.unplug(pluggedNic, true);
                 }
             }
         }
@@ -3510,7 +3510,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         if (nics != null) {
             for (final InterfaceDef nic : nics) {
                 for (final VifDriver vifDriver : getAllVifDrivers()) {
-                    vifDriver.unplug(nic);
+                    vifDriver.unplug(nic, true);
                 }
             }
         }
@@ -4219,6 +4219,24 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         return vmsnapshots;
     }
 
+    public String getVlanIdFromBridgeName(String brName) {
+        if (org.apache.commons.lang.StringUtils.isNotBlank(brName)) {
+            String[] s = brName.split("-");
+            if (s.length > 1) {
+                return s[1];
+            }
+            return null;
+        }
+        return null;
+    }
+
+    public boolean shouldDeleteBridge(Map<String, Boolean> vlanToPersistenceMap, String vlanId) {
+        if (MapUtils.isNotEmpty(vlanToPersistenceMap) && vlanId != null && vlanToPersistenceMap.containsKey(vlanId)) {
+            return vlanToPersistenceMap.get(vlanId);
+        }
+        return true;
+    }
+
     private static String getTagValue(String tag, Element eElement) {
         NodeList nlList = eElement.getElementsByTagName(tag).item(0).getChildNodes();
         Node nValue = nlList.item(0);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java

@@ -194,7 +194,7 @@ public class OvsVifDriver extends VifDriverBase {
     }
 
     @Override
-    public void unplug(InterfaceDef iface) {
+    public void unplug(InterfaceDef iface, boolean deleteBr) {
         // Libvirt apparently takes care of this, see BridgeVifDriver unplug
         if (_libvirtComputingResource.dpdkSupport && StringUtils.isNotBlank(iface.getDpdkSourcePort())) {
             // If DPDK is enabled, we'll need to cleanup the port as libvirt won't
@@ -260,4 +260,8 @@ public class OvsVifDriver extends VifDriverBase {
             return false;
         }
     }
+
+    @Override
+    public void deleteBr(NicTO nic) {
+    }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java

@@ -34,7 +34,7 @@ public interface VifDriver {
 
     public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException;
 
-    public void unplug(LibvirtVMDef.InterfaceDef iface);
+    public void unplug(LibvirtVMDef.InterfaceDef iface, boolean delete);
 
     void attach(LibvirtVMDef.InterfaceDef iface);
 
@@ -44,4 +44,6 @@ public interface VifDriver {
 
     boolean isExistingBridge(String bridgeName);
 
+    void deleteBr(NicTO nic);
+
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriverBase.java

@@ -45,7 +45,7 @@ public abstract class VifDriverBase implements VifDriver {
     public abstract LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException;
 
     @Override
-    public abstract void unplug(LibvirtVMDef.InterfaceDef iface);
+    public abstract void unplug(LibvirtVMDef.InterfaceDef iface, boolean deleteBr);
 
     protected LibvirtVMDef.InterfaceDef.NicModel getGuestNicModel(String platformEmulator, String nicAdapter) {
         // if nicAdapter is found in ENUM, use it. Otherwise, match guest OS type as before

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCleanupPersistentNetworkResourceCommandWrapper.java

@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CleanupPersistentNetworkResourceAnswer;
+import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand;
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.hypervisor.kvm.resource.BridgeVifDriver;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.hypervisor.kvm.resource.VifDriver;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+
+@ResourceWrapper(handles = CleanupPersistentNetworkResourceCommand.class)
+public class LibvirtCleanupPersistentNetworkResourceCommandWrapper extends CommandWrapper<CleanupPersistentNetworkResourceCommand, Answer, LibvirtComputingResource> {
+    private static final Logger s_logger = Logger.getLogger(LibvirtCleanupPersistentNetworkResourceCommandWrapper.class);
+    @Override
+    public Answer execute(CleanupPersistentNetworkResourceCommand command, LibvirtComputingResource serverResource) {
+        NicTO nic = command.getNicTO();
+        VifDriver driver = serverResource.getVifDriver(nic.getType());
+        if (driver instanceof BridgeVifDriver) {
+            driver.deleteBr(nic);
+        }
+        return new CleanupPersistentNetworkResourceAnswer(command, true, "Successfully deleted bridge");
+    }
+}

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java

@@ -96,6 +96,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
     @Override
     public Answer execute(final MigrateCommand command, final LibvirtComputingResource libvirtComputingResource) {
         final String vmName = command.getVmName();
+        final Map<String, Boolean> vlanToPersistenceMap = command.getVlanToPersistenceMap();
         final String destinationUri = createMigrationURI(command.getDestinationIp(), libvirtComputingResource);
         final List<MigrateDiskInfo> migrateDiskInfoList = command.getMigrateDiskInfoList();
 
@@ -284,11 +285,12 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
         } else {
             libvirtComputingResource.destroyNetworkRulesForVM(conn, vmName);
             for (final InterfaceDef iface : ifaces) {
+                String vlanId = libvirtComputingResource.getVlanIdFromBridgeName(iface.getBrName());
                 // We don't know which "traffic type" is associated with
                 // each interface at this point, so inform all vif drivers
                 final List<VifDriver> allVifDrivers = libvirtComputingResource.getAllVifDrivers();
                 for (final VifDriver vifDriver : allVifDrivers) {
-                    vifDriver.unplug(iface);
+                    vifDriver.unplug(iface, libvirtComputingResource.shouldDeleteBridge(vlanToPersistenceMap, vlanId));
                 }
             }
         }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReplugNicCommandWrapper.java

@@ -90,7 +90,7 @@ public final class LibvirtReplugNicCommandWrapper extends CommandWrapper<ReplugN
             // We don't know which "traffic type" is associated with
             // each interface at this point, so inform all vif drivers
             for (final VifDriver vifDriver : libvirtComputingResource.getAllVifDrivers()) {
-                vifDriver.unplug(oldPluggedNic);
+                vifDriver.unplug(oldPluggedNic, true);
             }
 
             return new ReplugNicAnswer(command, true, "success");

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetupPersistentNetworkCommandWrapper.java

@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import org.apache.log4j.Logger;
+import org.libvirt.LibvirtException;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.SetupPersistentNetworkAnswer;
+import com.cloud.agent.api.SetupPersistentNetworkCommand;
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.exception.InternalErrorException;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.hypervisor.kvm.resource.VifDriver;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+
+
+@ResourceWrapper(handles = SetupPersistentNetworkCommand.class)
+public class LibvirtSetupPersistentNetworkCommandWrapper extends CommandWrapper<SetupPersistentNetworkCommand, Answer, LibvirtComputingResource> {
+    private static final Logger s_logger = Logger.getLogger(LibvirtSetupPersistentNetworkCommandWrapper.class);
+
+    @Override
+    public Answer execute(SetupPersistentNetworkCommand command, LibvirtComputingResource serverResource) {
+        NicTO nic = command.getNic();
+        VifDriver driver = serverResource.getVifDriver(nic.getType());
+        try {
+            driver.plug(nic, null, "", null);
+        } catch (InternalErrorException | LibvirtException e) {
+            return new SetupPersistentNetworkAnswer(command, false, e.getLocalizedMessage());
+        }
+
+        return new SetupPersistentNetworkAnswer(command, true, "Successfully setup persistent network");
+    }
+}

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java

@@ -56,7 +56,7 @@ public final class LibvirtStopCommandWrapper extends CommandWrapper<StopCommand,
     @Override
     public Answer execute(final StopCommand command, final LibvirtComputingResource libvirtComputingResource) {
         final String vmName = command.getVmName();
-
+        final Map<String, Boolean> vlanToPersistenceMap = command.getVlanToPersistenceMap();
         final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
 
         if (command.checkBeforeCleanup()) {
@@ -125,10 +125,11 @@ public final class LibvirtStopCommandWrapper extends CommandWrapper<StopCommand,
                     }
                 } else {
                     for (final InterfaceDef iface : ifaces) {
+                        String vlanId = libvirtComputingResource.getVlanIdFromBridgeName(iface.getBrName());
                         // We don't know which "traffic type" is associated with
                         // each interface at this point, so inform all vif drivers
                         for (final VifDriver vifDriver : libvirtComputingResource.getAllVifDrivers()) {
-                            vifDriver.unplug(iface);
+                            vifDriver.unplug(iface, libvirtComputingResource.shouldDeleteBridge(vlanToPersistenceMap, vlanId));
                         }
                     }
                 }
@@ -159,5 +160,4 @@ public final class LibvirtStopCommandWrapper extends CommandWrapper<StopCommand,
             s_logger.warn("Exception occurred when handling LibVirt VM onStop hook: {}", e);
         }
     }
-
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUnPlugNicCommandWrapper.java

@@ -20,6 +20,7 @@
 package com.cloud.hypervisor.kvm.resource.wrapper;
 
 import java.util.List;
+import java.util.Map;
 
 import org.apache.log4j.Logger;
 import org.libvirt.Connect;
@@ -45,6 +46,7 @@ public final class LibvirtUnPlugNicCommandWrapper extends CommandWrapper<UnPlugN
     public Answer execute(final UnPlugNicCommand command, final LibvirtComputingResource libvirtComputingResource) {
         final NicTO nic = command.getNic();
         final String vmName = command.getVmName();
+        final Map<String, Boolean> vlanToPersistenceMap = command.getVlanToPersistenceMap();
         Domain vm = null;
         try {
             final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
@@ -59,10 +61,11 @@ public final class LibvirtUnPlugNicCommandWrapper extends CommandWrapper<UnPlugN
                         libvirtComputingResource.destroyNetworkRulesForNic(conn, vmName, nic);
                     }
                     vm.detachDevice(pluggedNic.toString());
+                    String vlanId = libvirtComputingResource.getVlanIdFromBridgeName(pluggedNic.getBrName());
                     // We don't know which "traffic type" is associated with
                     // each interface at this point, so inform all vif drivers
                     for (final VifDriver vifDriver : libvirtComputingResource.getAllVifDrivers()) {
-                        vifDriver.unplug(pluggedNic);
+                        vifDriver.unplug(pluggedNic, libvirtComputingResource.shouldDeleteBridge(vlanToPersistenceMap, vlanId));
                     }
                     return new UnPlugNicAnswer(command, true, "success");
                 }

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java

@@ -3364,7 +3364,7 @@ public class LibvirtComputingResourceTest {
 
             when(libvirtComputingResource.getAllVifDrivers()).thenReturn(drivers);
 
-            doNothing().when(vifDriver).unplug(intDef);
+            doNothing().when(vifDriver).unplug(intDef, true);
 
         } catch (final LibvirtException e) {
             fail(e.getMessage());

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -48,6 +48,8 @@ import java.util.stream.Collectors;
 import javax.naming.ConfigurationException;
 import javax.xml.datatype.XMLGregorianCalendar;
 
+import com.cloud.agent.api.SetupPersistentNetworkAnswer;
+import com.cloud.agent.api.SetupPersistentNetworkCommand;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.storage.command.CopyCommand;
 import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
@@ -578,6 +580,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
                 answer = execute((PrepareUnmanageVMInstanceCommand) cmd);
             } else if (clz == ValidateVcenterDetailsCommand.class) {
                 answer = execute((ValidateVcenterDetailsCommand) cmd);
+            } else if (clz == SetupPersistentNetworkCommand.class) {
+                answer = execute((SetupPersistentNetworkCommand) cmd);
             } else {
                 answer = Answer.createUnsupportedCommandAnswer(cmd);
             }
@@ -618,6 +622,21 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
         return answer;
     }
 
+    private Answer execute(SetupPersistentNetworkCommand cmd) {
+        VmwareHypervisorHost host = getHyperHost(getServiceContext());
+        String hostname = null;
+        VmwareContext context = getServiceContext();
+        HostMO hostMO = new HostMO(context, host.getMor());
+
+        try {
+            prepareNetworkFromNicInfo(hostMO, cmd.getNic(), false, null);
+            hostname =  host.getHyperHostName();
+        } catch (Exception e) {
+            return new SetupPersistentNetworkAnswer(cmd, false, "failed to setup port-group due to: "+ e.getLocalizedMessage());
+        }
+        return new SetupPersistentNetworkAnswer(cmd, true, hostname);
+    }
+
     /**
      * Check if storage NFS version is already set or needs to be reconfigured.<br>
      * If _storageNfsVersion is not null -> nothing to do, version already set.<br>

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java

@@ -678,7 +678,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                         correctVif.destroy(conn);
 
                         // Disable the VLAN network if necessary
-                        disableVlanNetwork(conn, network);
+                        disableVlanNetwork(conn, network, true);
                     }
                 }
             }
@@ -1574,7 +1574,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         }
     }
 
-    public void disableVlanNetwork(final Connection conn, final Network network) {
+    public void disableVlanNetwork(final Connection conn, final Network network, boolean deleteVlan) {
     }
 
     @Override
@@ -3612,7 +3612,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
             }
             for (final Network network : networks) {
                 if (network.getNameLabel(conn).startsWith("VLAN")) {
-                    disableVlanNetwork(conn, network);
+                    disableVlanNetwork(conn, network, true);
                 }
             }
         } catch (final Exception e) {

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServer56Resource.java

@@ -37,12 +37,13 @@ public class XenServer56Resource extends CitrixResourceBase {
     }
 
     @Override
-    public void disableVlanNetwork(final Connection conn, final Network network) {
+    public void disableVlanNetwork(final Connection conn, final Network network, boolean deleteVlan) {
         try {
             final Network.Record networkr = network.getRecord(conn);
             if (!networkr.nameLabel.startsWith("VLAN")) {
                 return;
             }
+            if (deleteVlan) {
                 final String bridge = networkr.bridge.trim();
                 for (final PIF pif : networkr.PIFs) {
                     final PIF.Record pifr = pif.getRecord(conn);
@@ -65,11 +66,12 @@ public class XenServer56Resource extends CitrixResourceBase {
                             host.forgetDataSourceArchives(conn, "pif_" + device + "." + vlannum + "_tx");
                             host.forgetDataSourceArchives(conn, "pif_" + device + "." + vlannum + "_rx");
                         } catch (final XenAPIException e) {
-                        s_logger.trace("Catch " + e.getClass().getName() + ": failed to destory VLAN " + device + " on host " + _host.getUuid() + " due to " + e.toString());
+                            s_logger.trace("Catch " + e.getClass().getName() + ": failed to destroy VLAN " + device + " on host " + _host.getUuid() + " due to " + e.toString());
                         }
                     }
                     return;
                 }
+            }
         } catch (final XenAPIException e) {
             final String msg = "Unable to disable VLAN network due to " + e.toString();
             s_logger.warn(msg, e);

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixCleanupPersistentNetworkResourceCommandWrapper.java

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.xenserver.resource.wrapper.xenbase;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CleanupPersistentNetworkResourceAnswer;
+import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand;
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase;
+import com.cloud.hypervisor.xenserver.resource.XsHost;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.xensource.xenapi.Connection;
+import com.xensource.xenapi.Network;
+
+@ResourceWrapper(handles = CleanupPersistentNetworkResourceCommand.class)
+public class CitrixCleanupPersistentNetworkResourceCommandWrapper extends CommandWrapper<CleanupPersistentNetworkResourceCommand, Answer, CitrixResourceBase> {
+    private static final Logger s_logger = Logger.getLogger(CitrixCleanupPersistentNetworkResourceCommandWrapper.class);
+
+    @Override
+    public Answer execute(CleanupPersistentNetworkResourceCommand command, CitrixResourceBase citrixResourceBase) {
+        final Connection conn = citrixResourceBase.getConnection();
+        final XsHost host = citrixResourceBase.getHost();
+        NicTO nic = command.getNicTO();
+        try {
+            Network network = citrixResourceBase.getNetwork(conn, nic);
+            if (network == null) {
+                return new CleanupPersistentNetworkResourceAnswer(command, false, "Failed to find network on host " + host.getIp() + " to cleanup");
+            }
+            citrixResourceBase.disableVlanNetwork(conn, network, true);
+            return new CleanupPersistentNetworkResourceAnswer(command, true, "Successfully deleted network VLAN on host: "+ host.getIp());
+        } catch (final Exception e) {
+            final String msg = " Failed to cleanup network VLAN on host: " + host.getIp() + " due to: " + e.toString();
+            s_logger.error(msg, e);
+            return new CleanupPersistentNetworkResourceAnswer(command, false, msg);
+        }
+    }
+}

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixSetupPersistentNetworkCommandWrapper.java

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.xenserver.resource.wrapper.xenbase;
+
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.SetupPersistentNetworkAnswer;
+import com.cloud.agent.api.SetupPersistentNetworkCommand;
+import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase;
+import com.cloud.hypervisor.xenserver.resource.XsHost;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.xensource.xenapi.Connection;
+import com.xensource.xenapi.Network;
+
+@ResourceWrapper(handles = SetupPersistentNetworkCommand.class)
+public class CitrixSetupPersistentNetworkCommandWrapper extends CommandWrapper<SetupPersistentNetworkCommand, Answer, CitrixResourceBase> {
+
+    private static final Logger s_logger = Logger.getLogger(CitrixSetupPersistentNetworkCommandWrapper.class);
+
+    @Override
+    public Answer execute(SetupPersistentNetworkCommand command, CitrixResourceBase citrixResourceBase) {
+        final Connection conn = citrixResourceBase.getConnection();
+        final XsHost host = citrixResourceBase.getHost();
+        try {
+            Network network = citrixResourceBase.getNetwork(conn, command.getNic());
+            if (network == null) {
+                return new SetupPersistentNetworkAnswer(command, false, "Failed to setup network on host: "+ host.getIp());
+            }
+            return new SetupPersistentNetworkAnswer(command, true, "Successfully setup network on host: "+ host.getIp());
+        } catch (final Exception e) {
+            final String msg = " Failed to setup network on host: " + host.getIp() + " due to: " + e.toString();
+            s_logger.error(msg, e);
+            return new SetupPersistentNetworkAnswer(command, false, msg);
+        }
+    }
+}

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixStopCommandWrapper.java

@@ -23,8 +23,10 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.collections.MapUtils;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
@@ -53,6 +55,7 @@ public final class CitrixStopCommandWrapper extends CommandWrapper<StopCommand,
     @Override
     public Answer execute(final StopCommand command, final CitrixResourceBase citrixResourceBase) {
         final String vmName = command.getVmName();
+        final Map<String, Boolean> vlanToPersistenceMap = command.getVlanToPersistenceMap();
         String platformstring = null;
         try {
             final Connection conn = citrixResourceBase.getConnection();
@@ -148,7 +151,8 @@ public final class CitrixStopCommandWrapper extends CommandWrapper<StopCommand,
                             for (final Network network : networks) {
                                 try {
                                     if (network.getNameLabel(conn).startsWith("VLAN")) {
-                                        citrixResourceBase.disableVlanNetwork(conn, network);
+                                        String networkLabel = network.getNameLabel(conn);
+                                        citrixResourceBase.disableVlanNetwork(conn, network, shouldDeleteVlan(networkLabel, vlanToPersistenceMap));
                                     }
                                 } catch (final Exception e) {
                                     // network might be destroyed by other host
@@ -172,4 +176,18 @@ public final class CitrixStopCommandWrapper extends CommandWrapper<StopCommand,
         }
         return new StopAnswer(command, "Stop VM failed", platformstring, false);
     }
+
+    private boolean shouldDeleteVlan(String networkLabel, Map<String, Boolean> vlanToPersistenceMap) {
+        String[] networkNameParts = null;
+        if (networkLabel.contains("-")) {
+            networkNameParts = networkLabel.split("-");
+        } else {
+            networkNameParts = networkLabel.split("VLAN");
+        }
+        String networkVlan = networkNameParts.length > 0 ? networkNameParts[networkNameParts.length - 1] : null;
+        if (networkVlan != null && MapUtils.isNotEmpty(vlanToPersistenceMap) && vlanToPersistenceMap.containsKey(networkVlan)) {
+            return vlanToPersistenceMap.get(networkVlan);
+        }
+        return true;
+    }
 }

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUnPlugNicCommandWrapper.java

@@ -19,9 +19,12 @@
 
 package com.cloud.hypervisor.xenserver.resource.wrapper.xenbase;
 
+import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.collections.MapUtils;
 import org.apache.log4j.Logger;
+import org.apache.xmlrpc.XmlRpcException;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.UnPlugNicAnswer;
@@ -32,6 +35,7 @@ import com.cloud.resource.CommandWrapper;
 import com.cloud.resource.ResourceWrapper;
 import com.xensource.xenapi.Connection;
 import com.xensource.xenapi.Network;
+import com.xensource.xenapi.Types;
 import com.xensource.xenapi.VIF;
 import com.xensource.xenapi.VM;
 
@@ -44,6 +48,7 @@ public final class CitrixUnPlugNicCommandWrapper extends CommandWrapper<UnPlugNi
     public Answer execute(final UnPlugNicCommand command, final CitrixResourceBase citrixResourceBase) {
         final Connection conn = citrixResourceBase.getConnection();
         final String vmName = command.getVmName();
+        final Map<String, Boolean> vlanToPersistenceMap = command.getVlanToPersistenceMap();
         try {
             final Set<VM> vms = VM.getByNameLabel(conn, vmName);
             if (vms == null || vms.isEmpty()) {
@@ -59,7 +64,8 @@ public final class CitrixUnPlugNicCommandWrapper extends CommandWrapper<UnPlugNi
                 vif.destroy(conn);
                 try {
                     if (network.getNameLabel(conn).startsWith("VLAN")) {
-                        citrixResourceBase.disableVlanNetwork(conn, network);
+                        String networkLabel = network.getNameLabel(conn);
+                        citrixResourceBase.disableVlanNetwork(conn, network, shouldDeleteVlan(networkLabel, vlanToPersistenceMap));
                     }
                 } catch (final Exception e) {
                 }
@@ -71,4 +77,13 @@ public final class CitrixUnPlugNicCommandWrapper extends CommandWrapper<UnPlugNi
             return new UnPlugNicAnswer(command, false, msg);
         }
     }
+
+    private boolean shouldDeleteVlan(String networkLabel, Map<String, Boolean> vlanToPersistenceMap) throws XmlRpcException, Types.XenAPIException {
+        String[] networkNameParts = networkLabel.split("-");
+        String networkVlan = networkNameParts[networkNameParts.length -1];
+        if (MapUtils.isNotEmpty(vlanToPersistenceMap) && networkVlan != null && vlanToPersistenceMap.containsKey(networkVlan)) {
+            return vlanToPersistenceMap.get(networkVlan);
+        }
+        return true;
+    }
 }

scripts/vm/network/vnet/modifyvlan.sh

@@ -22,7 +22,7 @@
 # set -x
 
 usage() {
-  printf "Usage: %s: -o <op>(add | delete) -v <vlan id> -p <pif> -b <bridge name>\n" 
+  printf "Usage: %s: -o <op>(add | delete) -v <vlan id> -p <pif> -b <bridge name> -d <delete bridge>(true | false)\n"
 }
 
 addVlan() {
@@ -91,7 +91,10 @@ deleteVlan() {
 	local pif=$2
 	local vlanDev=$pif.$vlanId
   local vlanBr=$3
+  local deleteBr=$4
 
+  if [ $deleteBr == "true" ]
+  then
 	  ip link delete $vlanDev type vlan > /dev/null
 	
   	if [ $? -gt 0 ]
@@ -99,7 +102,6 @@ deleteVlan() {
 		  printf "Failed to del vlan: $vlanId"
 		  return 1
 	  fi
-
     ip link set $vlanBr down
 
     if [ $? -gt 0 ]
@@ -114,16 +116,17 @@ deleteVlan() {
       printf "Failed to del bridge $vlanBr"
       return 1
     fi
-
+  fi
 	return 0
 	
 }
 
 op=
 vlanId=
+deleteBr="true"
 option=$@
 
-while getopts 'o:v:p:b:' OPTION
+while getopts 'o:v:p:b:d:' OPTION
 do
   case $OPTION in
   o)	oflag=1
@@ -138,6 +141,9 @@ do
   b)    bflag=1
     brName="$OPTARG"
     ;;
+  d)    dflag=1
+    deleteBr="$OPTARG"
+    ;;
   ?)	usage
 		exit 2
 		;;
@@ -177,7 +183,7 @@ else
 	if [ "$op" == "delete" ]
 	then
 		# Delete the vlan
-		deleteVlan $vlanId $pif $brName
+		deleteVlan $vlanId $pif $brName $deleteBr
 	
 		# Always exit with success
 		exit 0

server/src/main/java/com/cloud/network/NetworkServiceImpl.java

@@ -1429,10 +1429,6 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
         // if the network offering has persistent set to true, implement the network
         if (ntwkOff.isPersistent()) {
             try {
-                if (network.getState() == Network.State.Setup) {
-                    s_logger.debug("Network id=" + network.getId() + " is already provisioned");
-                    return network;
-                }
                 DeployDestination dest = new DeployDestination(zone, null, null, null);
                 UserVO callerUser = _userDao.findById(CallContext.current().getCallingUserId());
                 Journal journal = new Journal.LogJournal("Implementing " + network, s_logger);

server/src/main/java/com/cloud/network/guru/GuestNetworkGuru.java

@@ -214,7 +214,9 @@ public abstract class GuestNetworkGuru extends AdapterBase implements NetworkGur
 
             if (offering.isSpecifyVlan()) {
                 network.setBroadcastUri(userSpecified.getBroadcastUri());
+                if (!offering.isPersistent()) {
                     network.setState(State.Setup);
+                }
                 if (userSpecified.getPvlanType() != null) {
                     network.setBroadcastDomainType(BroadcastDomainType.Pvlan);
                     network.setPvlanType(userSpecified.getPvlanType());

server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java

@@ -74,6 +74,11 @@ public class MockNetworkDaoImpl extends GenericDaoBase<NetworkVO, Long> implemen
         return null;
     }
 
+    @Override
+    public int getOtherPersistentNetworksCount(long id, String broadcastURI, boolean isPersistent) {
+        return 0;
+    }
+
     @Override
     public String getNextAvailableMacAddress(final long networkConfigId, Integer zoneMacIdentifier) {
         return null;

test/integration/smoke/test_persistent_network.py

@@ -0,0 +1,385 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.lib.utils import (cleanup_resources,
+                              validateList,
+                              get_hypervisor_type, get_process_status)
+from marvin.lib.base import (Account,
+                             Cluster,
+                             Configurations,
+                             Host,
+                             VPC,
+                             VirtualMachine,
+                             Network,
+                             Router,
+                             ServiceOffering,
+                             NetworkOffering)
+from marvin.lib.common import (get_zone,
+                               get_template,
+                               verifyNetworkState,
+                               wait_for_cleanup, list_routers, list_hosts)
+from nose.plugins.attrib import attr
+from marvin.sshClient import SshClient
+from distutils.util import strtobool
+from pyVmomi import vim, vmodl
+from marvin.lib.vcenter import Vcenter
+import logging
+
+logger = logging.getLogger('TestPesistentNetwork')
+stream_handler = logging.StreamHandler()
+logger.setLevel(logging.DEBUG)
+logger.addHandler(stream_handler)
+
+
+class TestL2PersistentNetworks(cloudstackTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.testClient = super(TestL2PersistentNetworks, cls).getClsTestClient()
+        cls.api_client = cls.testClient.getApiClient()
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+        # Fill services from the external config file
+        cls.services = cls.testClient.getParsedTestDataConfig()
+        cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][
+            0].__dict__
+        # Get Zone and templates
+        cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests())
+        cls.template = get_template(
+            cls.api_client,
+            cls.zone.id,
+            cls.services["ostype"]
+        )
+
+        cls.services["virtual_machine"]["zoneid"] = cls.zone.id
+        cls.services["virtual_machine"]["template"] = cls.template.id
+        cls.service_offering = ServiceOffering.create(
+            cls.api_client,
+            cls.services["service_offering"]
+        )
+        cls.l2_persistent_network_offering = cls.create_network_offering("nw_off_L2_persistent")
+        cls.isolated_persistent_network_offering = cls.create_network_offering("nw_off_isolated_persistent")
+
+        # network will be deleted as part of account cleanup
+        cls._cleanup = [
+            cls.service_offering,
+            cls.isolated_persistent_network_offering,
+            cls.l2_persistent_network_offering]
+        return
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            # Cleanup resources used
+            cleanup_resources(cls.api_client, cls._cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.dbclient = self.testClient.getDbConnection()
+        self.hypervisor = self.testClient.getHypervisorInfo()
+        self.cleanup = []
+        return
+
+    def tearDown(self):
+        try:
+            # Clean up, terminate the resources created
+            cleanup_resources(self.apiclient, self.cleanup)
+            self.cleanup[:] = []
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    @classmethod
+    def is_ssh_enabled(cls):
+        conf = Configurations.list(cls.apiclient, name="kvm.ssh.to.agent")
+        if not conf:
+            return False
+        else:
+            return bool(strtobool(conf[0].value)) if conf[0].value else False
+
+    @classmethod
+    def create_network_offering(cls, network_offering_type):
+        network_offering = NetworkOffering.create(
+            cls.api_client,
+            cls.services[network_offering_type],
+            conservemode=False
+        )
+
+        # Update network offering state from disabled to enabled.
+        NetworkOffering.update(
+            network_offering,
+            cls.api_client,
+            id=network_offering.id,
+            state="enabled")
+        return network_offering
+
+    def get_ssh_client(self, ip, username, password, retries=10):
+        """ Setup ssh client connection and return connection """
+        try:
+            ssh_client = SshClient(ip, 22, username, password, retries)
+        except Exception as e:
+            raise unittest.SkipTest("Unable to create ssh connection: " % e)
+
+        self.assertIsNotNone(
+            ssh_client, "Failed to setup ssh connection to ip=%s" % ip)
+
+        return ssh_client
+
+    def list_all_hosts_in_zone(self, zone_id):
+        hosts = Host.list(
+            self.apiclient,
+            type='Routing',
+            resourcestate='Enabled',
+            state='Up',
+            zoneid=zone_id
+        )
+        return hosts
+
+    '''
+    Verifies creation of bridge on KVM host 
+    '''
+    def verify_bridge_creation(self, host, vlan_id):
+        username = self.hostConfig["username"]
+        password = self.hostConfig["password"]
+        try:
+            ssh_client = self.get_ssh_client(host.ipaddress, username, password)
+            res = ssh_client.execute("ip addr | grep breth1-" + str(vlan_id) + " > /dev/null 2>&1; echo $?")
+            return res[0]
+        except Exception as e:
+            self.fail(e)
+
+    '''
+    Gets all port groups on the host
+    '''
+    def capture_host_portgroups(self, host):
+        host_portgroups = []
+        for portgroup in host.config.network.portgroup:
+            host_portgroups.append(portgroup.spec.name)
+        return host_portgroups
+
+    '''
+    Fetches port group names based on VMware switch type - Distributed Virtual Switch(DVS) and
+    Standard Virtual Switch(SVS)
+    '''
+    def get_port_group_name(self, switch_type, vlan_id, network_rate):
+        if switch_type == 'DVS':
+            return 'cloud.guest.' + str(vlan_id) + '.' + str(network_rate) + '.1-dvSwitch1'
+        elif switch_type == 'SVS':
+            return 'cloud.guest.' + str(vlan_id) + '.' + str(network_rate) + '.1-vSwitch1'
+        else:
+            return None
+
+    '''
+    Verifies creation of port group on the Distributed vSwitch or a host in a cluster connected to 
+    a Standard vSwitch
+    '''
+    def verify_port_group_creation(self, vlan_id):
+        config = self.get_vmware_dc_config(self.zone.id)
+        vc_object = Vcenter(config[0][0], config[0][1], 'P@ssword123')
+        dvs = vc_object.get_dvswitches()
+        port_group_present = False
+        if dvs is not None:
+            port_group_name = self.get_port_group_name('DVS', vlan_id,
+                                                       self.isolated_persistent_network_offering.networkrate)
+            port_group_present = port_group_name in dvs[0]['dvswitch']['portgroupNameList']
+
+        else:
+            port_group_name = self.get_port_group_name('SVS', vlan_id,
+                                                       self.isolated_persistent_network_offering.networkrate)
+            hosts = vc_object._get_obj([vim.HostSystem])
+            host = hosts[0]['host']
+            host_pg = self.capture_host_portgroups(host)
+            port_group_present = port_group_name in host_pg
+        return port_group_present
+
+    '''
+    Fetch vmware datacenter login details
+    '''
+    def get_vmware_dc_config(self, zone_id):
+        zid = self.dbclient.execute("select id from data_center where uuid='%s';" %
+                                    zone_id)
+        vmware_dc_id = self.dbclient.execute(
+            "select vmware_data_center_id from vmware_data_center_zone_map where zone_id='%s';" %
+            zid[0])
+        vmware_dc_config = self.dbclient.execute(
+            "select vcenter_host, username, password from vmware_data_center where id = '%s';" % vmware_dc_id[0])
+
+        return vmware_dc_config
+
+    '''
+    Verify VLAN creation on specific host in a cluster
+    '''
+    def verify_vlan_network_creation(self, host, vlan_id):
+        username = self.hostConfig["username"]
+        password = self.hostConfig["password"]
+        try:
+            ssh_client = self.get_ssh_client(host.ipaddress, username, password)
+            res = ssh_client.execute(
+                "xe vlan-list | grep -x  \"^\s*tag ( RO): \"" + str(vlan_id) + "> /dev/null 2>&1; echo $?")
+            return res[0]
+        except Exception as e:
+            self.fail(e)
+
+    def verify_network_setup_on_host_per_cluster(self, hypervisor, vlan_id):
+        clusters = Cluster.list(
+            self.apiclient,
+            zoneid=self.zone.id,
+            allocationstate="Enabled",
+            listall=True
+        )
+        for cluster in clusters:
+            hosts = Host.list(self.apiclient,
+                              clusterid=cluster.id,
+                              type="Routing",
+                              state="Up",
+                              resourcestate="Enabled")
+            host = hosts[0]
+            if hypervisor == "xenserver":
+                result = self.verify_vlan_network_creation(host, vlan_id)
+                self.assertEqual(
+                    int(result),
+                    0,
+                    "Failed to find vlan on host: " + host.name + " in cluster: " + cluster.name)
+            if hypervisor == "vmware":
+                result = self.verify_port_group_creation(vlan_id)
+                self.assertEqual(
+                        result,
+                        True,
+                        "Failed to find port group on hosts of cluster: " + cluster.name)
+
+    '''
+    This test verifies that on creation of an Isolated network with network offering with isPersistent flag 
+    set to true the corresponding network resources are created without having to deploy a VM - VR created
+    '''
+    @attr(tags=["advanced", "isolated", "persistent", "network"], required_hardware="false")
+    def test_01_isolated_persistent_network(self):
+        network = Network.create(
+            self.apiclient,
+            self.services["isolated_network"],
+            networkofferingid=self.isolated_persistent_network_offering.id,
+            zoneid=self.zone.id)
+        self.cleanup.append(network)
+        networkVlan = network.vlan
+        response = verifyNetworkState(
+            self.apiclient,
+            network.id,
+            "implemented")
+        exceptionOccured = response[0]
+        isNetworkInDesiredState = response[1]
+        exceptionMessage = response[2]
+
+        if (exceptionOccured or (not isNetworkInDesiredState)):
+            self.fail(exceptionMessage)
+        self.assertIsNotNone(
+            networkVlan,
+            "vlan must not be null for persistent network")
+
+        router = Router.list(self.apiclient, networkid=network.id)[0]
+        router_host_id = router.hostid
+        host = Host.list(self.apiclient, id=router_host_id)[0]
+        if host.hypervisor.lower() in "kvm":
+            result = self.verify_bridge_creation(host, networkVlan)
+            self.assertEqual(
+                int(result),
+                0,
+                "Failed to find bridge on the breth1$-{networkVlan}")
+        elif host.hypervisor.lower() in ["xenserver", "vmware"]:
+            self.verify_network_setup_on_host_per_cluster(host.hypervisor.lower(), networkVlan)
+
+    '''
+    This test verifies that on creation of an L2 network with network offering with isPersistent flag 
+    set to true the corresponding network resources are created without having to deploy a VM - VR created
+    '''
+    @attr(tags=["advanced", "l2", "persistent", "network"], required_hardware="false")
+    def test_02_L2_persistent_network(self):
+        network_vlan = 90
+        network = Network.create(
+            self.apiclient,
+            self.services["l2_network"],
+            networkofferingid=self.l2_persistent_network_offering.id,
+            zoneid=self.zone.id,
+            vlan=network_vlan)
+        self.cleanup.append(network)
+        response = verifyNetworkState(
+            self.apiclient,
+            network.id,
+            "implemented")
+        exceptionOccured = response[0]
+        isNetworkInDesiredState = response[1]
+        exceptionMessage = response[2]
+
+        if (exceptionOccured or (not isNetworkInDesiredState)):
+            self.fail(exceptionMessage)
+        self.assertIsNotNone(
+            network_vlan,
+            "vlan must not be null for persistent network")
+
+        self.validate_persistent_network_resources_created_on_host(network_vlan)
+
+    @attr(tags=["advanced", "l2", "persistent", "network"], required_hardware="false")
+    def test_03_deploy_and_destroy_VM_and_verify_network_resources_persist(self):
+        network_vlan = 99
+        network = Network.create(
+            self.apiclient,
+            self.services["l2_network"],
+            networkofferingid=self.l2_persistent_network_offering.id,
+            zoneid=self.zone.id,
+            vlan=network_vlan)
+        self.cleanup.append(network)
+        response = verifyNetworkState(
+            self.apiclient,
+            network.id,
+            "implemented")
+        logger.debug(response)
+        exceptionOccured = response[0]
+        isNetworkInDesiredState = response[1]
+        exceptionMessage = response[2]
+
+        if (exceptionOccured or (not isNetworkInDesiredState)):
+            self.fail(exceptionMessage)
+        self.assertIsNotNone(
+            network_vlan,
+            "vlan must not be null for persistent network")
+        try:
+            virtual_machine = VirtualMachine.create(
+                self.apiclient,
+                self.services["virtual_machine"],
+                networkids=[
+                    network.id],
+                serviceofferingid=self.service_offering.id)
+
+            VirtualMachine.delete(virtual_machine, self.apiclient, expunge=True)
+
+            self.validate_persistent_network_resources_created_on_host(network_vlan)
+        except Exception as e:
+            self.fail("Exception occurred: %s" % e)
+        return
+
+
+    def validate_persistent_network_resources_created_on_host(self, network_vlan):
+        hosts = self.list_all_hosts_in_zone(self.zone.id)
+        if self.hypervisor.lower() in "kvm":
+            for host in hosts:
+                result = self.verify_bridge_creation(host, network_vlan)
+                self.assertEqual(
+                    int(result),
+                    0,
+                    "Failed to find bridge on the breth1-" + str(network_vlan))
+        elif self.hypervisor.lower() in ["xenserver", "vmware"]:
+            self.verify_network_setup_on_host_per_cluster(self.hypervisor.lower(), network_vlan)

tools/marvin/marvin/config/test_data.py

@@ -306,13 +306,21 @@ test_data = {
             "StaticNat": "VirtualRouter"
         }
     },
+    "nw_off_L2_persistent": {
+        "name": 'Test L2 Network Offering persistent',
+        "displaytext": 'Test L2 Network Offering persistent',
+        "guestiptype": 'L2',
+        "traffictype": 'GUEST',
+        "ispersistent": 'True',
+        "specifyVlan": 'True'
+    },
     "network_offering_vlan": {
         "name": 'Test Network offering',
         "displaytext": 'Test Network offering',
         "guestiptype": 'Isolated',
         "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding',
         "traffictype": 'GUEST',
-        "specifyvlan": 'False',
+        "specifyVlan": 'False',
         "availability": 'Optional',
         "serviceProviderList": {
             "Dhcp": 'VirtualRouter',
@@ -338,6 +346,10 @@ test_data = {
         "name": "Isolated Network",
         "displaytext": "Isolated Network"
     },
+    "l2_network": {
+        "name": "L2 Network",
+        "displaytext": "L2 Network"
+    },
     "netscaler_VPX": {
         "ipaddress": "10.223.240.174",
         "username": "nsroot",

tools/marvin/marvin/lib/vcenter.py

@@ -137,6 +137,8 @@ class Vcenter():
             parsedObject['dvportgroup'] = Vcenter._parse_dvportgroup(obj)
         elif vim.VirtualMachine in vimtype:
             parsedObject['vm'] = Vcenter._parse_vm(obj)
+        elif vim.HostSystem in vimtype:
+            parsedObject['host'] = obj
         else:
             parsedObject['name'] = obj.name
         return parsedObject

ui/public/locales/en.json

@@ -630,6 +630,7 @@
 "label.create.ssh.key.pair": "Create a SSH Key Pair",
 "label.create.template": "Create template",
 "label.create.user": "Create user",
+"label.create.vpc.tier": "Create VPC tier",
 "label.create.vpn.connection": "Create VPN Connection",
 "label.created": "Created",
 "label.created.by.system": "Created by system",

ui/src/views/network/VpcTiersTab.vue

@@ -169,12 +169,26 @@
           </a-form-item>
           <a-form-item :label="$t('label.networkofferingid')">
             <a-select
-              v-decorator="['networkOffering',{rules: [{ required: true, message: `${$t('label.required')}` }]}]">
+              v-decorator="['networkOffering',{rules: [{ required: true, message: `${$t('label.required')}` }]}]"
+              @change="val => { this.handleNetworkOfferingChange(val) }">
               <a-select-option v-for="item in networkOfferings" :key="item.id" :value="item.id">
                 {{ item.displaytext || item.name || item.description }}
               </a-select-option>
             </a-select>
           </a-form-item>
+          <a-form-item v-if="!this.isObjectEmpty(this.selectedNetworkOffering) && this.selectedNetworkOffering.specifyvlan">
+            <span slot="label">
+              {{ $t('label.vlan') }}
+              <a-tooltip :title="$t('label.vlan')">
+                <a-icon type="info-circle" style="color: rgba(0,0,0,.45)" />
+              </a-tooltip>
+            </span>
+            <a-input
+              v-decorator="['vlan', {
+                rules: [{ required: true, message: $t('message.please.enter.value') }]
+              }]"
+              :placeholder="this.$t('label.vlan')"/>
+          </a-form-item>
           <a-form-item :label="$t('label.gateway')">
             <a-input
               :placeholder="$t('label.create.network.gateway.description')"
@@ -284,6 +298,7 @@ export default {
       LBPublicIPs: {},
       staticNats: {},
       vms: {},
+      selectedNetworkOffering: {},
       algorithms: {
         Source: 'source',
         'Round-robin': 'roundrobin',
@@ -387,6 +402,9 @@ export default {
     this.form = this.$form.createForm(this)
   },
   methods: {
+    isObjectEmpty (obj) {
+      return !(obj !== null && obj !== undefined && Object.keys(obj).length > 0 && obj.constructor === Object)
+    },
     showIlb (network) {
       return network.service.filter(s => (s.name === 'Lb') && (s.capability.filter(c => c.name === 'LbSchemes' && c.value === 'Internal').length > 0)).length > 0 || false
     },
@@ -432,6 +450,7 @@ export default {
             networkOffering: this.networkOfferings[0].id
           })
         })
+        this.selectedNetworkOffering = this.networkOfferings[0]
       }).catch(error => {
         this.$notifyError(error)
       }).finally(() => {
@@ -469,6 +488,9 @@ export default {
         this.fetchLoading = false
       })
     },
+    handleNetworkOfferingChange (networkOfferingId) {
+      this.selectedNetworkOffering = this.networkOfferings.filter(offering => offering.id === networkOfferingId)[0]
+    },
     closeModal () {
       this.$emit('close-action')
     },
@@ -493,7 +515,7 @@ export default {
         }
 
         this.showCreateNetworkModal = false
-        api('createNetwork', {
+        var params = {
           vpcid: this.resource.id,
           domainid: this.resource.domainid,
           account: this.resource.account,
@@ -505,7 +527,13 @@ export default {
           zoneId: this.resource.zoneid,
           externalid: values.externalId,
           aclid: values.acl
-        }).then(() => {
+        }
+
+        if (values.vlan) {
+          params.vlan = values.vlan
+        }
+
+        api('createNetwork', params).then(() => {
           this.$notification.success({
             message: this.$t('message.success.add.vpc.network')
           })