Synchronization of network devices on newly added hosts for Persistent Networks (#5977)

* Persistent Network feature & Marvin component tests

* Cleaned up comments and imports

* fixed small error

* add support to add setup persistent networks' resources when a disabled host is enabled

* small fix

* use wildcard instead of hard-coding the bridge name

* allow clean up of resources when removing a host in maintenance mode

* skip test for simulator hypervisor

Co-authored-by: shatoboar <sang-woo.bae@campus.tu-berlin.de>

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/HypervisorHostListener.java

@@ -30,4 +30,6 @@ public interface HypervisorHostListener {
     boolean hostAboutToBeRemoved(long hostId);
 
     boolean hostRemoved(long hostId, long clusterId);
+
+    boolean hostEnabled(long hostId);
 }

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -280,6 +280,8 @@ public interface StorageManager extends StorageService {
 
     void disconnectHostFromSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException;
 
+    void enableHost(long hostId) throws StorageUnavailableException, StorageConflictException;
+
     void createCapacityEntry(long poolId);
 
     DataStore createLocalStorage(Host host, StoragePoolInfo poolInfo) throws ConnectionException;

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java

@@ -31,6 +31,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
+import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand;
 import org.apache.cloudstack.agent.lb.SetupMSListCommand;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.log4j.Logger;
@@ -118,7 +119,8 @@ public abstract class AgentAttache {
         StopCommand.class.toString(), CheckVirtualMachineCommand.class.toString(), PingTestCommand.class.toString(), CheckHealthCommand.class.toString(),
         ReadyCommand.class.toString(), ShutdownCommand.class.toString(), SetupCommand.class.toString(),
         CleanupNetworkRulesCmd.class.toString(), CheckNetworkCommand.class.toString(), PvlanSetupCommand.class.toString(), CheckOnHostCommand.class.toString(),
-        ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString()};
+        ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(),
+            CleanupPersistentNetworkResourceCommand.class.toString()};
     protected final static String[] s_commandsNotAllowedInConnectingMode = new String[] { StartCommand.class.toString(), CreateCommand.class.toString() };
     static {
         Arrays.sort(s_commandsAllowedInMaintenanceMode);

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -699,7 +699,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
                 if (network.getId() != -1) {
                     if (network instanceof NetworkVO) {
-                        networks.add((NetworkVO)network);
+                        networks.add((NetworkVO) network);
                     } else {
                         networks.add(_networksDao.findById(network.getId()));
                     }
@@ -731,9 +731,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
                         updateRouterIpInNetworkDetails(networkPersisted.getId(), network.getRouterIp(), network.getRouterIpv6());
 
-                        if (predefined instanceof NetworkVO && guru instanceof NetworkGuruAdditionalFunctions){
+                        if (predefined instanceof NetworkVO && guru instanceof NetworkGuruAdditionalFunctions) {
                             final NetworkGuruAdditionalFunctions functions = (NetworkGuruAdditionalFunctions) guru;
-                            functions.finalizeNetworkDesign(networkPersisted.getId(), ((NetworkVO)predefined).getVlanIdAsUUID());
+                            functions.finalizeNetworkDesign(networkPersisted.getId(), ((NetworkVO) predefined).getVlanIdAsUUID());
                         }
 
                         if (domainId != null && aclType == ACLType.Domain) {
@@ -766,7 +766,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             @Override
             public void doInTransactionWithoutResult(final TransactionStatus status) throws InsufficientCapacityException {
                 if (s_logger.isTraceEnabled()) {
-                    s_logger.trace(String.format("allocating networks for %s(template %s); %d networks",vm.getInstanceName(), vm.getTemplate().getUuid(), networks.size()));
+                    s_logger.trace(String.format("allocating networks for %s(template %s); %d networks", vm.getInstanceName(), vm.getTemplate().getUuid(), networks.size()));
                 }
                 int deviceId = 0;
                 int size;
@@ -779,7 +779,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 final List<NicProfile> nics = new ArrayList<NicProfile>(size);
                 NicProfile defaultNic = null;
                 Network nextNetwork = null;
-                for (Pair <Network, NicProfile> networkNicPair : profilesList) {
+                for (Pair<Network, NicProfile> networkNicPair : profilesList) {
                     nextNetwork = networkNicPair.first();
                     Pair<NicProfile, Integer> newDeviceInfo = addRequestedNicToNicListWithDeviceNumberAndRetrieveDefaultDevice(networkNicPair.second(), deviceIds, deviceId, nextNetwork, nics, defaultNic);
                     defaultNic = newDeviceInfo.first();
@@ -795,9 +795,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             /**
              * private transaction method to check and add devices to the nic list and update the info
              */
-            Pair<NicProfile,Integer> addRequestedNicToNicListWithDeviceNumberAndRetrieveDefaultDevice(NicProfile requested, boolean[] deviceIds, int deviceId, Network nextNetwork, List<NicProfile> nics, NicProfile defaultNic)
+            Pair<NicProfile, Integer> addRequestedNicToNicListWithDeviceNumberAndRetrieveDefaultDevice(NicProfile requested, boolean[] deviceIds, int deviceId, Network nextNetwork, List<NicProfile> nics, NicProfile defaultNic)
                     throws InsufficientAddressCapacityException, InsufficientVirtualNetworkCapacityException {
-                Pair<NicProfile, Integer> rc = new Pair<>(null,null);
+                Pair<NicProfile, Integer> rc = new Pair<>(null, null);
                 Boolean isDefaultNic = false;
                 if (vm != null && requested != null && requested.isDefaultNic()) {
                     isDefaultNic = true;
@@ -918,7 +918,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                             throw new CloudRuntimeException(String.format("can not assign network to %d remaining required NICs", size - nics.size()));
                         }
                         // create extra
-                        for ( int extraNicNum = nics.size() ; extraNicNum < size; extraNicNum ++) {
+                        for (int extraNicNum = nics.size(); extraNicNum < size; extraNicNum++) {
                             final Pair<NicProfile, Integer> vmNicPair = allocateNic(new NicProfile(), finalNetwork, false, extraNicNum, vm);
                         }
                     }
@@ -930,9 +930,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     @Override
     public void saveExtraDhcpOptions(final String networkUuid, final Long nicId, final Map<String, Map<Integer, String>> extraDhcpOptionMap) {
 
-        if(extraDhcpOptionMap != null) {
+        if (extraDhcpOptionMap != null) {
             Map<Integer, String> extraDhcpOption = extraDhcpOptionMap.get(networkUuid);
-            if(extraDhcpOption != null) {
+            if (extraDhcpOption != null) {
                 List<NicExtraDhcpOptionVO> nicExtraDhcpOptionList = new LinkedList<>();
 
                 for (Integer code : extraDhcpOption.keySet()) {
@@ -1038,7 +1038,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     }
 
     /**
-     *  Acquires lock in "user_ip_address" and checks if the requested IPv4 address is Free.
+     * Acquires lock in "user_ip_address" and checks if the requested IPv4 address is Free.
      */
     protected void acquireLockAndCheckIfIpv4IsFree(Network network, String requestedIpv4Address) {
         IPAddressVO ipVO = _ipAddressDao.findByIpAndSourceNetworkId(network.getId(), requestedIpv4Address);
@@ -1198,7 +1198,6 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     }
 
     /**
-     *
      * Creates a dummy NicTO object which is used by the respective hypervisors to setup network elements / resources
      * - bridges(KVM), VLANs(Xen) and portgroups(VMWare) for L2 network
      */
@@ -1220,7 +1219,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         if (hosts == null) {
             hosts = new ArrayList<>();
         }
-        if (hostVO.getHypervisorType() == HypervisorType.KVM || hostVO.getHypervisorType() == HypervisorType.XenServer ) {
+        if (hostVO.getHypervisorType() == HypervisorType.KVM || hostVO.getHypervisorType() == HypervisorType.XenServer) {
             hosts.add(hostVO.getId());
             clusterToHostsMap.put(clusterId, hosts);
             return new Pair<>(false, createNicTOFromNetworkAndOffering(networkVO, networkOfferingVO, hostVO));
@@ -1259,7 +1258,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                     clusterToHostsMap.get(host.getClusterId()).remove(host.getId());
                 }
             } catch (Exception e) {
-                s_logger.warn("Failed to connect to host: "+ host.getName());
+                s_logger.warn("Failed to connect to host: " + host.getName());
             }
         }
         if (clusterToHostsMap.keySet().size() != clusterVOs.size()) {
@@ -1276,6 +1275,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             return criteriaMet && (network.getGuestType() == GuestType.L2 || network.getGuestType() == GuestType.Isolated);
         }
     }
+
     @Override
     @DB
     public Pair<NetworkGuru, NetworkVO> implementNetwork(final long networkId, final DeployDestination dest, final ReservationContext context) throws ConcurrentOperationException,
@@ -1335,7 +1335,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             implementNetworkElementsAndResources(dest, context, network, offering);
 
             long dcId = dest.getDataCenter().getId();
-            if (networkMeetsPersistenceCriteria(network,offering, false)) {
+            if (networkMeetsPersistenceCriteria(network, offering, false)) {
                 setupPersistentNetwork(network, offering, dcId);
             }
             if (isSharedNetworkWithServices(network)) {
@@ -1421,15 +1421,15 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
         //Reset the extra DHCP option that may have been cleared per nic.
         List<NicVO> nicVOs = _nicDao.listByNetworkId(network.getId());
-        for(NicVO nicVO : nicVOs) {
+        for (NicVO nicVO : nicVOs) {
-            if(nicVO.getState() == Nic.State.Reserved) {
+            if (nicVO.getState() == Nic.State.Reserved) {
                 configureExtraDhcpOptions(network, nicVO.getId());
             }
         }
 
         for (final NetworkElement element : networkElements) {
             if (element instanceof AggregatedCommandExecutor && providersToImplement.contains(element.getProvider())) {
-                ((AggregatedCommandExecutor)element).prepareAggregatedExecution(network, dest);
+                ((AggregatedCommandExecutor) element).prepareAggregatedExecution(network, dest);
             }
         }
 
@@ -1446,7 +1446,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             }
             for (final NetworkElement element : networkElements) {
                 if (element instanceof AggregatedCommandExecutor && providersToImplement.contains(element.getProvider())) {
-                    if (!((AggregatedCommandExecutor)element).completeAggregatedExecution(network, dest)) {
+                    if (!((AggregatedCommandExecutor) element).completeAggregatedExecution(network, dest)) {
                         s_logger.warn("Failed to re-program the network as a part of network " + network + " implement due to aggregated commands execution failure!");
                         // see DataCenterVO.java
                         final ResourceUnavailableException ex = new ResourceUnavailableException("Unable to apply network rules as a part of network " + network + " implement", DataCenter.class,
@@ -1459,7 +1459,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         } finally {
             for (final NetworkElement element : networkElements) {
                 if (element instanceof AggregatedCommandExecutor && providersToImplement.contains(element.getProvider())) {
-                    ((AggregatedCommandExecutor)element).cleanupAggregatedExecution(network, dest);
+                    ((AggregatedCommandExecutor) element).cleanupAggregatedExecution(network, dest);
                 }
             }
         }
@@ -1579,32 +1579,32 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         if (vmProfile.getType() == Type.User && element.getProvider() != null) {
             if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)
                     && _networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.Dhcp, element.getProvider()) && element instanceof DhcpServiceProvider) {
-                final DhcpServiceProvider sp = (DhcpServiceProvider)element;
+                final DhcpServiceProvider sp = (DhcpServiceProvider) element;
                 if (isDhcpAccrossMultipleSubnetsSupported(sp)) {
                     if (!sp.configDhcpSupportForSubnet(network, profile, vmProfile, dest, context)) {
                         return false;
                     }
                 }
-                if(!sp.addDhcpEntry(network, profile, vmProfile, dest, context)) {
+                if (!sp.addDhcpEntry(network, profile, vmProfile, dest, context)) {
                     return false;
                 }
             }
             if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dns)
                     && _networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.Dns, element.getProvider()) && element instanceof DnsServiceProvider) {
-                final DnsServiceProvider sp = (DnsServiceProvider)element;
+                final DnsServiceProvider sp = (DnsServiceProvider) element;
                 if (profile.getIPv6Address() == null) {
                     if (!sp.configDnsSupportForSubnet(network, profile, vmProfile, dest, context)) {
                         return false;
                     }
                 }
-                if(!sp.addDnsEntry(network, profile, vmProfile, dest, context)) {
+                if (!sp.addDnsEntry(network, profile, vmProfile, dest, context)) {
                     return false;
                 }
             }
             if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData)
                     && _networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.UserData, element.getProvider()) && element instanceof UserDataServiceProvider) {
-                final UserDataServiceProvider sp = (UserDataServiceProvider)element;
+                final UserDataServiceProvider sp = (UserDataServiceProvider) element;
-                if(!sp.addPasswordAndUserdata(network, profile, vmProfile, dest, context)){
+                if (!sp.addPasswordAndUserdata(network, profile, vmProfile, dest, context)) {
                     return false;
                 }
             }
@@ -1613,20 +1613,20 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     }
 
     @Override
-    public boolean canUpdateInSequence(Network network, boolean forced){
+    public boolean canUpdateInSequence(Network network, boolean forced) {
         List<Provider> providers = getNetworkProviders(network.getId());
 
         //check if the there are no service provider other than virtualrouter.
-        for(Provider provider : providers) {
+        for (Provider provider : providers) {
             if (provider != Provider.VirtualRouter)
                 throw new UnsupportedOperationException("Cannot update the network resources in sequence when providers other than virtualrouter are used");
         }
         //check if routers are in correct state before proceeding with the update
         List<DomainRouterVO> routers = _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER);
-        for (DomainRouterVO router : routers){
+        for (DomainRouterVO router : routers) {
             if (router.getRedundantState() == VirtualRouter.RedundantState.UNKNOWN) {
                 if (!forced) {
-                    throw new CloudRuntimeException("Domain router: "+router.getInstanceName()+" is in unknown state, Cannot update network. set parameter forced to true for forcing an update");
+                    throw new CloudRuntimeException("Domain router: " + router.getInstanceName() + " is in unknown state, Cannot update network. set parameter forced to true for forcing an update");
                 }
             }
         }
@@ -1634,23 +1634,23 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     }
 
     @Override
-    public List<String> getServicesNotSupportedInNewOffering(Network network,long newNetworkOfferingId){
+    public List<String> getServicesNotSupportedInNewOffering(Network network, long newNetworkOfferingId) {
-        NetworkOffering offering =_networkOfferingDao.findById(newNetworkOfferingId);
+        NetworkOffering offering = _networkOfferingDao.findById(newNetworkOfferingId);
-        List<String> services=_ntwkOfferingSrvcDao.listServicesForNetworkOffering(offering.getId());
+        List<String> services = _ntwkOfferingSrvcDao.listServicesForNetworkOffering(offering.getId());
-        List<NetworkServiceMapVO> serviceMap= _ntwkSrvcDao.getServicesInNetwork(network.getId());
+        List<NetworkServiceMapVO> serviceMap = _ntwkSrvcDao.getServicesInNetwork(network.getId());
-        List<String> servicesNotInNewOffering=new ArrayList<>();
+        List<String> servicesNotInNewOffering = new ArrayList<>();
-        for(NetworkServiceMapVO serviceVO :serviceMap){
+        for (NetworkServiceMapVO serviceVO : serviceMap) {
-            boolean inlist=false;
+            boolean inlist = false;
-            for(String service: services){
+            for (String service : services) {
-                if(serviceVO.getService().equalsIgnoreCase(service)){
+                if (serviceVO.getService().equalsIgnoreCase(service)) {
-                    inlist=true;
+                    inlist = true;
                     break;
                 }
             }
-            if(!inlist){
+            if (!inlist) {
                 //ignore Gateway service as this has no effect on the
                 //behaviour of network.
-                if(!serviceVO.getService().equalsIgnoreCase(Service.Gateway.getName()))
+                if (!serviceVO.getService().equalsIgnoreCase(Service.Gateway.getName()))
                     servicesNotInNewOffering.add(serviceVO.getService());
             }
         }
@@ -1658,21 +1658,21 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     }
 
     @Override
-    public void cleanupConfigForServicesInNetwork(List<String> services, final Network network){
+    public void cleanupConfigForServicesInNetwork(List<String> services, final Network network) {
-        long networkId=network.getId();
+        long networkId = network.getId();
-        Account caller=_accountDao.findById(Account.ACCOUNT_ID_SYSTEM);
+        Account caller = _accountDao.findById(Account.ACCOUNT_ID_SYSTEM);
-        long userId=User.UID_SYSTEM;
+        long userId = User.UID_SYSTEM;
         //remove all PF/Static Nat rules for the network
-        s_logger.info("Services:"+services+" are no longer supported in network:"+network.getUuid()+
+        s_logger.info("Services:" + services + " are no longer supported in network:" + network.getUuid() +
-                " after applying new network offering:"+network.getNetworkOfferingId()+" removing the related configuration");
+                " after applying new network offering:" + network.getNetworkOfferingId() + " removing the related configuration");
-        if(services.contains(Service.StaticNat.getName())|| services.contains(Service.PortForwarding.getName())) {
+        if (services.contains(Service.StaticNat.getName()) || services.contains(Service.PortForwarding.getName())) {
             try {
                 if (_rulesMgr.revokeAllPFStaticNatRulesForNetwork(networkId, userId, caller)) {
                     s_logger.debug("Successfully cleaned up portForwarding/staticNat rules for network id=" + networkId);
                 } else {
                     s_logger.warn("Failed to release portForwarding/StaticNat rules as a part of network id=" + networkId + " cleanup");
                 }
-                if(services.contains(Service.StaticNat.getName())){
+                if (services.contains(Service.StaticNat.getName())) {
                     //removing static nat configured on ips.
                     //optimizing the db operations using transaction.
                     Transaction.execute(new TransactionCallbackNoReturn() {
@@ -1683,7 +1683,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                                 ip.setOneToOneNat(false);
                                 ip.setAssociatedWithVmId(null);
                                 ip.setVmIp(null);
-                                _ipAddressDao.update(ip.getId(),ip);
+                                _ipAddressDao.update(ip.getId(), ip);
                             }
                         }
                     });
@@ -1692,20 +1692,20 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 s_logger.warn("Failed to release portForwarding/StaticNat rules as a part of network id=" + networkId + " cleanup due to resourceUnavailable ", ex);
             }
         }
-        if(services.contains(Service.SourceNat.getName())){
+        if (services.contains(Service.SourceNat.getName())) {
             Transaction.execute(new TransactionCallbackNoReturn() {
                 @Override
                 public void doInTransactionWithoutResult(TransactionStatus status) {
-                    List<IPAddressVO> ips = _ipAddressDao.listByAssociatedNetwork(network.getId(),true);
+                    List<IPAddressVO> ips = _ipAddressDao.listByAssociatedNetwork(network.getId(), true);
                     //removing static nat configured on ips.
                     for (IPAddressVO ip : ips) {
                         ip.setSourceNat(false);
-                        _ipAddressDao.update(ip.getId(),ip);
+                        _ipAddressDao.update(ip.getId(), ip);
                     }
                 }
             });
         }
-        if(services.contains(Service.Lb.getName())){
+        if (services.contains(Service.Lb.getName())) {
             //remove all LB rules for the network
             if (_lbMgr.removeAllLoadBalanacersForNetwork(networkId, caller, userId)) {
                 s_logger.debug("Successfully cleaned up load balancing rules for network id=" + networkId);
@@ -1714,7 +1714,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             }
         }
 
-        if(services.contains(Service.Firewall.getName())){
+        if (services.contains(Service.Firewall.getName())) {
             //revoke all firewall rules for the network
             try {
                 if (_firewallMgr.revokeAllFirewallRulesForNetwork(networkId, userId, caller)) {
@@ -1728,12 +1728,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         }
 
         //do not remove vpn service for vpc networks.
-        if(services.contains(Service.Vpn.getName()) && network.getVpcId()==null){
+        if (services.contains(Service.Vpn.getName()) && network.getVpcId() == null) {
-            RemoteAccessVpnVO vpn = _remoteAccessVpnDao.findByAccountAndNetwork(network.getAccountId(),networkId);
+            RemoteAccessVpnVO vpn = _remoteAccessVpnDao.findByAccountAndNetwork(network.getAccountId(), networkId);
             try {
                 _vpnMgr.destroyRemoteAccessVpnForIp(vpn.getServerAddressId(), caller, true);
             } catch (ResourceUnavailableException ex) {
-                s_logger.warn("Failed to cleanup remote access vpn resources of network:"+network.getUuid() + " due to Exception: ", ex);
+                s_logger.warn("Failed to cleanup remote access vpn resources of network:" + network.getUuid() + " due to Exception: ", ex);
             }
         }
     }
@@ -1751,14 +1751,14 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     }
 
     @Override
-    public int getResourceCount(Network network){
+    public int getResourceCount(Network network) {
         List<Provider> providers = getNetworkProviders(network.getId());
-        int resourceCount=0;
+        int resourceCount = 0;
         for (NetworkElement element : networkElements) {
             if (providers.contains(element.getProvider())) {
                 //currently only one element implements the redundant resource interface
                 if (element instanceof RedundantResource) {
-                    resourceCount= ((RedundantResource) element).getResourceCount(network);
+                    resourceCount = ((RedundantResource) element).getResourceCount(network);
                     break;
                 }
             }
@@ -1768,7 +1768,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
     @Override
     public void configureExtraDhcpOptions(Network network, long nicId, Map<Integer, String> extraDhcpOptions) {
-        if(_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) {
+        if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) {
             if (_networkModel.getNetworkServiceCapabilities(network.getId(), Service.Dhcp).containsKey(Capability.ExtraDhcpOptions)) {
                 DhcpServiceProvider sp = getDhcpServiceProvider(network);
                 sp.setExtraDhcpOptions(network, nicId, extraDhcpOptions);
@@ -1789,7 +1789,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             if (providers.contains(element.getProvider())) {
                 //currently only one element implements the redundant resource interface
                 if (element instanceof RedundantResource) {
-                    ((RedundantResource) element).finalize(network,success);
+                    ((RedundantResource) element).finalize(network, success);
                     break;
                 }
             }
@@ -1953,7 +1953,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
     @Override
     public void prepareNicForMigration(final VirtualMachineProfile vm, final DeployDestination dest) {
-        if(vm.getType().equals(VirtualMachine.Type.DomainRouter) && (vm.getHypervisorType().equals(HypervisorType.KVM) || vm.getHypervisorType().equals(HypervisorType.VMware))) {
+        if (vm.getType().equals(VirtualMachine.Type.DomainRouter) && (vm.getHypervisorType().equals(HypervisorType.KVM) || vm.getHypervisorType().equals(HypervisorType.VMware))) {
             //Include nics hot plugged and not stored in DB
             prepareAllNicsForMigration(vm, dest);
             return;
@@ -1968,7 +1968,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             final NicProfile profile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network),
                     _networkModel.getNetworkTag(vm.getHypervisorType(), network));
             if (guru instanceof NetworkMigrationResponder) {
-                if (!((NetworkMigrationResponder)guru).prepareMigration(profile, network, vm, dest, context)) {
+                if (!((NetworkMigrationResponder) guru).prepareMigration(profile, network, vm, dest, context)) {
                     s_logger.error("NetworkGuru " + guru + " prepareForMigration failed."); // XXX: Transaction error
                 }
             }
@@ -1985,7 +1985,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                                 + network.getPhysicalNetworkId());
                     }
                     if (element instanceof NetworkMigrationResponder) {
-                        if (!((NetworkMigrationResponder)element).prepareMigration(profile, network, vm, dest, context)) {
+                        if (!((NetworkMigrationResponder) element).prepareMigration(profile, network, vm, dest, context)) {
                             s_logger.error("NetworkElement " + element + " prepareForMigration failed."); // XXX: Transaction error
                         }
                     }
@@ -2008,7 +2008,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         Long guestNetworkId = null;
         for (final NicVO nic : nics) {
             final NetworkVO network = _networksDao.findById(nic.getNetworkId());
-            if(network.getTrafficType().equals(TrafficType.Guest) && network.getGuestType().equals(GuestType.Isolated)){
+            if (network.getTrafficType().equals(TrafficType.Guest) && network.getGuestType().equals(GuestType.Isolated)) {
                 guestNetworkId = network.getId();
             }
             final Integer networkRate = _networkModel.getNetworkRate(network.getId(), vm.getId());
@@ -2016,9 +2016,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, network.getGuruName());
             final NicProfile profile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), networkRate,
                     _networkModel.isSecurityGroupSupportedInNetwork(network), _networkModel.getNetworkTag(vm.getHypervisorType(), network));
-            if(guru instanceof NetworkMigrationResponder){
+            if (guru instanceof NetworkMigrationResponder) {
-                if(!((NetworkMigrationResponder) guru).prepareMigration(profile, network, vm, dest, context)){
+                if (!((NetworkMigrationResponder) guru).prepareMigration(profile, network, vm, dest, context)) {
-                    s_logger.error("NetworkGuru "+guru+" prepareForMigration failed."); // XXX: Transaction error
+                    s_logger.error("NetworkGuru " + guru + " prepareForMigration failed."); // XXX: Transaction error
                 }
             }
             final List<Provider> providersToImplement = getNetworkProviders(network.getId());
@@ -2027,9 +2027,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                     if (!_networkModel.isProviderEnabledInPhysicalNetwork(_networkModel.getPhysicalNetworkId(network), element.getProvider().getName())) {
                         throw new CloudRuntimeException("Service provider " + element.getProvider().getName() + " either doesn't exist or is not enabled in physical network id: " + network.getPhysicalNetworkId());
                     }
-                    if(element instanceof NetworkMigrationResponder){
+                    if (element instanceof NetworkMigrationResponder) {
-                        if(!((NetworkMigrationResponder) element).prepareMigration(profile, network, vm, dest, context)){
+                        if (!((NetworkMigrationResponder) element).prepareMigration(profile, network, vm, dest, context)) {
-                            s_logger.error("NetworkElement "+element+" prepareForMigration failed."); // XXX: Transaction error
+                            s_logger.error("NetworkElement " + element + " prepareForMigration failed."); // XXX: Transaction error
                         }
                     }
                 }
@@ -2039,18 +2039,18 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         }
 
         final List<String> addedURIs = new ArrayList<String>();
-        if(guestNetworkId != null){
+        if (guestNetworkId != null) {
             final List<IPAddressVO> publicIps = _ipAddressDao.listByAssociatedNetwork(guestNetworkId, null);
-            for (final IPAddressVO userIp : publicIps){
+            for (final IPAddressVO userIp : publicIps) {
                 final PublicIp publicIp = PublicIp.createFromAddrAndVlan(userIp, _vlanDao.findById(userIp.getVlanId()));
                 final URI broadcastUri = BroadcastDomainType.Vlan.toUri(publicIp.getVlanTag());
                 final long ntwkId = publicIp.getNetworkId();
                 final Nic nic = _nicDao.findByNetworkIdInstanceIdAndBroadcastUri(ntwkId, vm.getId(),
                         broadcastUri.toString());
-                if(nic == null && !addedURIs.contains(broadcastUri.toString())){
+                if (nic == null && !addedURIs.contains(broadcastUri.toString())) {
                     //Nic details are not available in DB
                     //Create nic profile for migration
-                    s_logger.debug("Creating nic profile for migration. BroadcastUri: "+broadcastUri.toString()+" NetworkId: "+ntwkId+" Vm: "+vm.getId());
+                    s_logger.debug("Creating nic profile for migration. BroadcastUri: " + broadcastUri.toString() + " NetworkId: " + ntwkId + " Vm: " + vm.getId());
                     final NetworkVO network = _networksDao.findById(ntwkId);
                     final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, network.getGuruName());
                     final NicProfile profile = new NicProfile();
@@ -2095,7 +2095,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             final ReservationContext dst_context = new ReservationContextImpl(nicDst.getReservationId(), null, null);
 
             if (guru instanceof NetworkMigrationResponder) {
-                ((NetworkMigrationResponder)guru).commitMigration(nicSrc, network, src, src_context, dst_context);
+                ((NetworkMigrationResponder) guru).commitMigration(nicSrc, network, src, src_context, dst_context);
             }
 
             if (network.getGuestType() == Network.GuestType.L2 && src.getType() == VirtualMachine.Type.User) {
@@ -2110,7 +2110,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                                 + network.getPhysicalNetworkId());
                     }
                     if (element instanceof NetworkMigrationResponder) {
-                        ((NetworkMigrationResponder)element).commitMigration(nicSrc, network, src, src_context, dst_context);
+                        ((NetworkMigrationResponder) element).commitMigration(nicSrc, network, src, src_context, dst_context);
                     }
                 }
             }
@@ -2131,7 +2131,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             final ReservationContext dst_context = new ReservationContextImpl(nicDst.getReservationId(), null, null);
 
             if (guru instanceof NetworkMigrationResponder) {
-                ((NetworkMigrationResponder)guru).rollbackMigration(nicDst, network, dst, src_context, dst_context);
+                ((NetworkMigrationResponder) guru).rollbackMigration(nicDst, network, dst, src_context, dst_context);
             }
 
             if (network.getGuestType() == Network.GuestType.L2 && src.getType() == VirtualMachine.Type.User) {
@@ -2146,7 +2146,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                                 + network.getPhysicalNetworkId());
                     }
                     if (element instanceof NetworkMigrationResponder) {
-                        ((NetworkMigrationResponder)element).rollbackMigration(nicDst, network, dst, src_context, dst_context);
+                        ((NetworkMigrationResponder) element).rollbackMigration(nicDst, network, dst, src_context, dst_context);
                     }
                 }
             }
@@ -2210,12 +2210,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         });
 
         // cleanup the entry in vm_network_map
-        if(vmProfile.getType().equals(VirtualMachine.Type.User)) {
+        if (vmProfile.getType().equals(VirtualMachine.Type.User)) {
             final NicVO nic = _nicDao.findById(nicId);
-            if(nic != null) {
+            if (nic != null) {
                 final NetworkVO vmNetwork = _networksDao.findById(nic.getNetworkId());
                 final VMNetworkMapVO vno = _vmNetworkMapDao.findByVmAndNetworkId(vmProfile.getVirtualMachine().getId(), vmNetwork.getId());
-                if(vno != null) {
+                if (vno != null) {
                     _vmNetworkMapDao.remove(vno.getId());
                 }
             }
@@ -2320,11 +2320,11 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                     removeDhcpServiceInSubnet(nic);
                 }
             }
-            if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dns)){
+            if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dns)) {
                 final DnsServiceProvider dnsServiceProvider = getDnsServiceProvider(network);
                 if (dnsServiceProvider != null) {
                     try {
-                        if(!dnsServiceProvider.removeDnsSupportForSubnet(network)) {
+                        if (!dnsServiceProvider.removeDnsSupportForSubnet(network)) {
                             s_logger.warn("Failed to remove the ip alias on the dns server");
                         }
                     } catch (final ResourceUnavailableException e) {
@@ -2550,7 +2550,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 throw new InvalidParameterValueException("The VLAN tag for isolated PVLAN " + isolatedPvlan + " is already being used for dynamic vlan allocation for the guest network in zone "
                         + zone.getName());
             }
-            if (! UuidUtils.validateUUID(vlanId)){
+            if (!UuidUtils.validateUUID(vlanId)) {
                 // For Isolated and L2 networks, don't allow to create network with vlan that already exists in the zone
                 if (!hasGuestBypassVlanOverlapCheck(bypassVlanOverlapCheck, ntwkOff, isPrivateNetwork)) {
                     if (_networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), null).size() > 0) {
@@ -2587,7 +2587,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 } else {
                     // don't allow to creating shared network with given Vlan ID, if there already exists a isolated network or
                     // shared network with same Vlan ID in the zone
-                    if (!bypassVlanOverlapCheck && _networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), GuestType.Isolated).size() > 0 ) {
+                    if (!bypassVlanOverlapCheck && _networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), GuestType.Isolated).size() > 0) {
                         throw new InvalidParameterValueException("There is an existing isolated/shared network that overlaps with vlan id:" + vlanId + " in zone " + zoneId);
                     }
                 }
@@ -2698,7 +2698,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 if (vlanIdFinal != null) {
                     if (isolatedPvlan == null) {
                         URI uri = null;
-                        if (UuidUtils.validateUUID(vlanIdFinal)){
+                        if (UuidUtils.validateUUID(vlanIdFinal)) {
                             //Logical router's UUID provided as VLAN_ID
                             userNetwork.setVlanIdAsUUID(vlanIdFinal); //Set transient field
                         } else {
@@ -2707,7 +2707,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
                         if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) {
                             throw new InvalidParameterValueException("Network with vlan " + vlanIdFinal +
-                                " already exists or overlaps with other network pvlans in zone " + zoneId);
+                                    " already exists or overlaps with other network pvlans in zone " + zoneId);
                         }
 
                         userNetwork.setBroadcastUri(uri);
@@ -2766,6 +2766,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
     /**
      * Encodes VLAN/VXLAN ID into a Broadcast URI according to the isolation method from the Physical Network.
+     *
      * @return Broadcast URI, e.g. 'vlan://vlan_ID' or 'vxlan://vlxan_ID'
      */
     protected URI encodeVlanIdIntoBroadcastUri(String vlanId, PhysicalNetwork pNtwk) {
@@ -2773,29 +2774,31 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             throw new InvalidParameterValueException(String.format("Failed to encode VLAN/VXLAN %s into a Broadcast URI. Physical Network cannot be null.", vlanId));
         }
 
-        if(!pNtwk.getIsolationMethods().isEmpty() && StringUtils.isNotBlank(pNtwk.getIsolationMethods().get(0))) {
+        if (!pNtwk.getIsolationMethods().isEmpty() && StringUtils.isNotBlank(pNtwk.getIsolationMethods().get(0))) {
             String isolationMethod = pNtwk.getIsolationMethods().get(0).toLowerCase();
             String vxlan = BroadcastDomainType.Vxlan.toString().toLowerCase();
-            if(isolationMethod.equals(vxlan)) {
+            if (isolationMethod.equals(vxlan)) {
                 return BroadcastDomainType.encodeStringIntoBroadcastUri(vlanId, BroadcastDomainType.Vxlan);
             }
         }
         return BroadcastDomainType.fromString(vlanId);
     }
 
-  /**
+    /**
-   * Checks bypass VLAN id/range overlap check during network creation for guest networks
+     * Checks bypass VLAN id/range overlap check during network creation for guest networks
-   * @param bypassVlanOverlapCheck bypass VLAN id/range overlap check
+     *
-   * @param ntwkOff network offering
+     * @param bypassVlanOverlapCheck bypass VLAN id/range overlap check
-   */
+     * @param ntwkOff                network offering
-  private boolean hasGuestBypassVlanOverlapCheck(final boolean bypassVlanOverlapCheck, final NetworkOfferingVO ntwkOff, final boolean isPrivateNetwork) {
+     */
-    return bypassVlanOverlapCheck && (ntwkOff.getGuestType() != GuestType.Isolated || isPrivateNetwork);
+    private boolean hasGuestBypassVlanOverlapCheck(final boolean bypassVlanOverlapCheck, final NetworkOfferingVO ntwkOff, final boolean isPrivateNetwork) {
-  }
+        return bypassVlanOverlapCheck && (ntwkOff.getGuestType() != GuestType.Isolated || isPrivateNetwork);
+    }
 
     /**
      * Checks for L2 network offering services. Only 2 cases allowed:
      * - No services
      * - User Data service only, provided by ConfigDrive
+     *
      * @param ntwkOff network offering
      */
     protected void checkL2OfferingServices(NetworkOfferingVO ntwkOff) {
@@ -2921,7 +2924,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         boolean cleanupResult = true;
         boolean cleanupNeeded = false;
         try {
-            for (final Provider provider: providersToShutdown) {
+            for (final Provider provider : providersToShutdown) {
                 if (provider.cleanupNeededOnShutdown()) {
                     cleanupNeeded = true;
                     break;
@@ -2987,7 +2990,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                             s_logger.warn("Unable to setup agent " + host.getId() + " due to " + answer.getDetails());
                         }
                     } catch (Exception e) {
-                        s_logger.warn("Failed to cleanup network resources on host: "+ host.getName());
+                        s_logger.warn("Failed to cleanup network resources on host: " + host.getName());
                     }
                 }
             }
@@ -3264,7 +3267,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         // implement the network
         s_logger.debug("Starting network " + network + "...");
         final Pair<NetworkGuru, NetworkVO> implementedNetwork = implementNetwork(networkId, dest, context);
-        if (implementedNetwork== null || implementedNetwork.first() == null) {
+        if (implementedNetwork == null || implementedNetwork.first() == null) {
             s_logger.warn("Failed to start the network " + network);
             return false;
         } else {
@@ -3419,7 +3422,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         if (network.isRedundant() || (oldRouters.size() == 1 && oldRouters.get(0).getIsRedundantRouter())) {
             try {
                 Thread.sleep(NetworkOrchestrationService.RVRHandoverTime);
-            } catch (final InterruptedException ignored) {}
+            } catch (final InterruptedException ignored) {
+            }
         }
 
         // Destroy old routers
@@ -3468,7 +3472,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             return null;
         }
 
-        return (UserDataServiceProvider)_networkModel.getElementImplementingProvider(passwordProvider);
+        return (UserDataServiceProvider) _networkModel.getElementImplementingProvider(passwordProvider);
     }
 
     @Override
@@ -3480,7 +3484,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             return null;
         }
 
-        return (UserDataServiceProvider)_networkModel.getElementImplementingProvider(SSHKeyProvider);
+        return (UserDataServiceProvider) _networkModel.getElementImplementingProvider(SSHKeyProvider);
     }
 
     @Override
@@ -3493,8 +3497,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         }
 
         final NetworkElement element = _networkModel.getElementImplementingProvider(DhcpProvider);
-        if ( element instanceof DhcpServiceProvider ) {
+        if (element instanceof DhcpServiceProvider) {
-            return (DhcpServiceProvider)element;
+            return (DhcpServiceProvider) element;
         } else {
             return null;
         }
@@ -3509,7 +3513,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             return null;
         }
 
-        return  (DnsServiceProvider) _networkModel.getElementImplementingProvider(dnsProvider);
+        return (DnsServiceProvider) _networkModel.getElementImplementingProvider(dnsProvider);
     }
 
     protected boolean isSharedNetworkWithServices(final Network network) {
@@ -3558,7 +3562,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 final NetworkGuruAdditionalFunctions guruFunctions = (NetworkGuruAdditionalFunctions) guru;
 
                 final Map<String, ? extends Object> nsxParams = guruFunctions.listAdditionalNicParams(nic.getUuid());
-                if (nsxParams != null){
+                if (nsxParams != null) {
                     final String lswitchUuuid = nsxParams.containsKey(NetworkGuruAdditionalFunctions.NSX_LSWITCH_UUID)
                             ? (String) nsxParams.get(NetworkGuruAdditionalFunctions.NSX_LSWITCH_UUID) : null;
                     final String lswitchPortUuuid = nsxParams.containsKey(NetworkGuruAdditionalFunctions.NSX_LSWITCHPORT_UUID)
@@ -3884,7 +3888,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             return;
         }
         final long hostId = host.getId();
-        final StartupRoutingCommand startup = (StartupRoutingCommand)cmd;
+        final StartupRoutingCommand startup = (StartupRoutingCommand) cmd;
 
         final String dataCenter = startup.getDataCenter();
 
@@ -3937,7 +3941,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         }
         final CheckNetworkCommand nwCmd = new CheckNetworkCommand(networkInfoList);
 
-        final CheckNetworkAnswer answer = (CheckNetworkAnswer)_agentMgr.easySend(hostId, nwCmd);
+        final CheckNetworkAnswer answer = (CheckNetworkAnswer) _agentMgr.easySend(hostId, nwCmd);
 
         if (answer == null) {
             s_logger.warn("Unable to get an answer to the CheckNetworkCommand from agent:" + host.getId());
@@ -3945,7 +3949,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         }
 
         if (!answer.getResult()) {
-            s_logger.warn("Unable to setup agent " + hostId + " due to " + answer.getDetails() );
+            s_logger.warn("Unable to setup agent " + hostId + " due to " + answer.getDetails());
             final String msg = "Incorrect Network setup on agent, Reinitialize agent after network names are setup, details : " + answer.getDetails();
             _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, dcId, host.getPodId(), msg, msg);
             throw new ConnectionException(true, msg);
@@ -4090,7 +4094,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             }
 
             //Update vm_network_map table
-            if(vmProfile.getType() == VirtualMachine.Type.User) {
+            if (vmProfile.getType() == VirtualMachine.Type.User) {
                 final VMNetworkMapVO vno = new VMNetworkMapVO(vm.getId(), network.getId());
                 _vmNetworkMapDao.persist(vno);
             }
@@ -4158,6 +4162,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 accessDetails.put(network.getTrafficType().name(), address);
             }
         }
+
         if (privateIpAddress != null && StringUtils.isEmpty(accessDetails.get(NetworkElementCommand.ROUTER_IP))) {
             accessDetails.put(NetworkElementCommand.ROUTER_IP,  privateIpAddress);
         }
@@ -4223,7 +4228,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         //only one provider per Static nat service is supoprted
         final NetworkElement element = getElementForServiceInNetwork(network, Service.StaticNat).get(0);
         assert element instanceof StaticNatServiceProvider;
-        return (StaticNatServiceProvider)element;
+        return (StaticNatServiceProvider) element;
     }
 
     @Override
@@ -4249,7 +4254,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
         assert lbElement != null;
         assert lbElement instanceof LoadBalancingServiceProvider;
-        return (LoadBalancingServiceProvider)lbElement;
+        return (LoadBalancingServiceProvider) lbElement;
     }
 
     @Override
@@ -4398,7 +4403,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {NetworkGcWait, NetworkGcInterval, NetworkLockTimeout,
+        return new ConfigKey<?>[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout,
                 GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion,
                 PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled};
     }

engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java

@@ -130,4 +130,6 @@ public interface NetworkDao extends GenericDao<NetworkVO, Long>, StateDao<State,
     List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri, Network.PVlanType pVlanType);
 
     List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri);
+
+    List<NetworkVO> getAllPersistentNetworksFromZone(long dataCenterId);
 }

engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java

@@ -419,6 +419,16 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
         return persistentNetworks.size();
     }
 
+    @Override
+    public List<NetworkVO> getAllPersistentNetworksFromZone(long dataCenterId) {
+        Object[] guestTypes = {"Isolated", "L2"};
+        final SearchCriteria<NetworkVO> sc = PersistentNetworkSearch.create();
+        sc.setParameters("guestType", guestTypes);
+        sc.setParameters("dc", dataCenterId);
+        sc.setJoinParameters("persistent", "persistent", true);
+        return search(sc, null);
+    }
+
     @Override
     public String getNextAvailableMacAddress(final long networkConfigId, Integer zoneMacIdentifier) {
         final SequenceFetcher fetch = SequenceFetcher.getInstance();

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java

@@ -20,10 +20,21 @@ package org.apache.cloudstack.storage.datastore.provider;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand;
 import com.cloud.agent.api.ModifyStoragePoolAnswer;
 import com.cloud.agent.api.ModifyStoragePoolCommand;
+import com.cloud.agent.api.SetupPersistentNetworkCommand;
+import com.cloud.agent.api.to.NicTO;
 import com.cloud.alert.AlertManager;
+import com.cloud.configuration.ConfigurationManager;
 import com.cloud.exception.StorageConflictException;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.network.NetworkModel;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
+import com.cloud.offerings.NetworkOfferingVO;
+import com.cloud.offerings.dao.NetworkOfferingDao;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Storage;
 import com.cloud.storage.StorageManager;
@@ -62,12 +73,50 @@ public class DefaultHostListener implements HypervisorHostListener {
     StorageManager storageManager;
     @Inject
     StorageService storageService;
+    @Inject
+    NetworkOfferingDao networkOfferingDao;
+    @Inject
+    HostDao hostDao;
+    @Inject
+    NetworkModel networkModel;
+    @Inject
+    ConfigurationManager configManager;
+    @Inject
+    NetworkDao networkDao;
+
 
     @Override
     public boolean hostAdded(long hostId) {
         return true;
     }
 
+    private boolean createPersistentNetworkResourcesOnHost(long hostId) {
+        HostVO host = hostDao.findById(hostId);
+        if (host == null) {
+            s_logger.warn(String.format("Host with id %ld can't be found", hostId));
+            return false;
+        }
+        setupPersistentNetwork(host);
+        return true;
+    }
+
+    /**
+     * Creates a dummy NicTO object which is used by the respective hypervisors to setup network elements / resources
+     * - bridges(KVM), VLANs(Xen) and portgroups(VMWare) for L2 network
+     */
+    private NicTO createNicTOFromNetworkAndOffering(NetworkVO networkVO, NetworkOfferingVO networkOfferingVO, HostVO hostVO) {
+        NicTO to = new NicTO();
+        to.setName(networkModel.getNetworkTag(hostVO.getHypervisorType(), networkVO));
+        to.setBroadcastType(networkVO.getBroadcastDomainType());
+        to.setType(networkVO.getTrafficType());
+        to.setBroadcastUri(networkVO.getBroadcastUri());
+        to.setIsolationuri(networkVO.getBroadcastUri());
+        to.setNetworkRateMbps(configManager.getNetworkOfferingNetworkRate(networkOfferingVO.getId(), networkVO.getDataCenterId()));
+        to.setSecurityGroupEnabled(networkModel.isSecurityGroupSupportedInNetwork(networkVO));
+        return to;
+    }
+
+
     @Override
     public boolean hostConnect(long hostId, long poolId) throws StorageConflictException {
         StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
@@ -109,7 +158,8 @@ public class DefaultHostListener implements HypervisorHostListener {
         storageService.updateStorageCapabilities(poolId, false);
 
         s_logger.info("Connection established between storage pool " + pool + " and host " + hostId);
-        return true;
+
+        return createPersistentNetworkResourcesOnHost(hostId);
     }
 
     private void updateStoragePoolHostVOAndDetails(StoragePool pool, long hostId, ModifyStoragePoolAnswer mspAnswer) {
@@ -124,7 +174,7 @@ public class DefaultHostListener implements HypervisorHostListener {
         StoragePoolVO poolVO = this.primaryStoreDao.findById(pool.getId());
         poolVO.setUsedBytes(mspAnswer.getPoolInfo().getCapacityBytes() - mspAnswer.getPoolInfo().getAvailableBytes());
         poolVO.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes());
-        if(StringUtils.isNotEmpty(mspAnswer.getPoolType())) {
+        if (StringUtils.isNotEmpty(mspAnswer.getPoolType())) {
             StoragePoolDetailVO poolType = storagePoolDetailsDao.findDetail(pool.getId(), "pool_type");
             if (poolType == null) {
                 StoragePoolDetailVO storagePoolDetailVO = new StoragePoolDetailVO(pool.getId(), "pool_type", mspAnswer.getPoolType(), false);
@@ -142,6 +192,28 @@ public class DefaultHostListener implements HypervisorHostListener {
 
     @Override
     public boolean hostAboutToBeRemoved(long hostId) {
+        // send host the cleanup persistent network resources
+        HostVO host = hostDao.findById(hostId);
+        if (host == null) {
+            s_logger.warn("Host with id " + hostId + " can't be found");
+            return false;
+        }
+
+        List<NetworkVO> allPersistentNetworks = networkDao.getAllPersistentNetworksFromZone(host.getDataCenterId()); // find zoneId of host
+        for (NetworkVO persistentNetworkVO : allPersistentNetworks) {
+            NetworkOfferingVO networkOfferingVO = networkOfferingDao.findById(persistentNetworkVO.getNetworkOfferingId());
+            CleanupPersistentNetworkResourceCommand cleanupCmd =
+                    new CleanupPersistentNetworkResourceCommand(createNicTOFromNetworkAndOffering(persistentNetworkVO, networkOfferingVO, host));
+            Answer answer = agentMgr.easySend(hostId, cleanupCmd);
+            if (answer == null) {
+                s_logger.error("Unable to get answer to the cleanup persistent network command " + persistentNetworkVO.getId());
+                continue;
+            }
+            if (!answer.getResult()) {
+                String msg = String.format("Unable to cleanup persistent network resources from network %d on the host %d", persistentNetworkVO.getId(), hostId);
+                s_logger.error(msg);
+            }
+        }
         return true;
     }
 
@@ -149,4 +221,33 @@ public class DefaultHostListener implements HypervisorHostListener {
     public boolean hostRemoved(long hostId, long clusterId) {
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        HostVO host = hostDao.findById(hostId);
+        if (host == null) {
+            s_logger.warn(String.format("Host with id %d can't be found", hostId));
+            return false;
+        }
+        setupPersistentNetwork(host);
+        return true;
+    }
+
+    private void setupPersistentNetwork(HostVO host) {
+        List<NetworkVO> allPersistentNetworks = networkDao.getAllPersistentNetworksFromZone(host.getDataCenterId());
+        for (NetworkVO networkVO : allPersistentNetworks) {
+            NetworkOfferingVO networkOfferingVO = networkOfferingDao.findById(networkVO.getNetworkOfferingId());
+
+            SetupPersistentNetworkCommand persistentNetworkCommand =
+                    new SetupPersistentNetworkCommand(createNicTOFromNetworkAndOffering(networkVO, networkOfferingVO, host));
+            Answer answer = agentMgr.easySend(host.getId(), persistentNetworkCommand);
+            if (answer == null) {
+                throw new CloudRuntimeException("Unable to get answer to the setup persistent network command " + networkVO.getId());
+            }
+            if (!answer.getResult()) {
+                String msg = String.format("Unable to create persistent network resources for network %d on the host %d in zone %d", networkVO.getId(), host.getId(), networkVO.getDataCenterId());
+                s_logger.error(msg);
+            }
+        }
+    }
 }

plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/provider/ElastistorHostListener.java

@@ -140,4 +140,9 @@ public class ElastistorHostListener implements HypervisorHostListener {
     public boolean hostRemoved(long hostId, long clusterId) {
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
 }

plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java

@@ -178,6 +178,11 @@ public class DateraHostListener implements HypervisorHostListener {
         return true;
     }
 
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
+
     private void handleXenServer(long clusterId, long hostId, long storagePoolId) {
         List<String> storagePaths = getStoragePaths(clusterId, storagePoolId);
 

plugins/storage/volume/nexenta/src/main/java/org/apache/cloudstack/storage/datastore/provider/NexentaHostListener.java

@@ -59,4 +59,9 @@ public class NexentaHostListener implements HypervisorHostListener {
 
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
 }

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java

@@ -138,4 +138,9 @@ public class ScaleIOHostListener implements HypervisorHostListener {
     public boolean hostRemoved(long hostId, long clusterId) {
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
 }

plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java

@@ -141,6 +141,11 @@ public class SolidFireHostListener implements HypervisorHostListener {
         return true;
     }
 
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
+
     private void handleXenServer(long clusterId, long hostId, long storagePoolId) {
         List<String> storagePaths = getStoragePaths(clusterId, storagePoolId);
 

plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireSharedHostListener.java

@@ -140,6 +140,11 @@ public class SolidFireSharedHostListener implements HypervisorHostListener {
         return true;
     }
 
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
+
     private void handleVMware(HostVO host, boolean add, ModifyTargetsCommand.TargetTypeToRemove targetTypeToRemove) {
         if (HypervisorType.VMware.equals(host.getHypervisorType())) {
             List<StoragePoolVO> storagePools = storagePoolDao.findPoolsByProvider(SolidFireUtil.SHARED_PROVIDER_NAME);

server/src/main/java/com/cloud/resource/ResourceManagerImpl.java

@@ -36,6 +36,8 @@ import java.util.Random;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.exception.StorageConflictException;
+import com.cloud.exception.StorageUnavailableException;
 import org.apache.cloudstack.annotation.AnnotationService;
 import org.apache.cloudstack.annotation.dao.AnnotationDao;
 import org.apache.cloudstack.api.ApiConstants;
@@ -1836,6 +1838,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
         if (url != null) {
             _storageMgr.updateSecondaryStorage(cmd.getId(), cmd.getUrl());
         }
+        try {
+            _storageMgr.enableHost(hostId);
+        } catch (StorageUnavailableException | StorageConflictException e) {
+            s_logger.error(String.format("Failed to setup host %s when enabled", host));
+        }
 
         final HostVO updatedHost = _hostDao.findById(hostId);
         return updatedHost;

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -79,6 +79,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreState
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle;
+import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreProvider;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
@@ -1118,6 +1119,26 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         listener.hostDisconnected(hostId, pool.getId());
     }
 
+    @Override
+    public void enableHost(long hostId) {
+        List<DataStoreProvider> providers = _dataStoreProviderMgr.getProviders();
+        if (providers != null) {
+            for (DataStoreProvider provider : providers) {
+                if (provider instanceof PrimaryDataStoreProvider) {
+                    try {
+                        HypervisorHostListener hypervisorHostListener = provider.getHostListener();
+                        if (hypervisorHostListener != null) {
+                            hypervisorHostListener.hostEnabled(hostId);
+                        }
+                    }
+                    catch (Exception ex) {
+                        s_logger.error("hostEnabled(long) failed for storage provider " + provider.getName(), ex);
+                    }
+                }
+            }
+        }
+    }
+
     @Override
     public BigDecimal getStorageOverProvisioningFactor(Long poolId) {
         return new BigDecimal(CapacityManager.StorageOverprovisioningFactor.valueIn(poolId));

server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java

@@ -260,4 +260,9 @@ public class MockNetworkDaoImpl extends GenericDaoBase<NetworkVO, Long> implemen
     public List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri) {
         return null;
     }
+
+    @Override
+    public List<NetworkVO> getAllPersistentNetworksFromZone(long dataCenterId) {
+        return null;
+    }
 }

test/integration/component/test_persistent_networks.py

@@ -40,7 +40,7 @@ from marvin.lib.common import (get_domain,
                                get_template,
                                verifyNetworkState,
                                add_netscaler,
-                               wait_for_cleanup,list_routers,list_hosts)
+                               wait_for_cleanup, list_routers, list_hosts, list_clusters)
 from nose.plugins.attrib import attr
 from marvin.codes import PASS, FAIL, FAILED
 from marvin.sshClient import SshClient
@@ -49,13 +49,12 @@ import unittest
 from ddt import ddt, data
 import time
 
-
 @ddt
 class TestPersistentNetworks(cloudstackTestCase):
-
     '''
     Test Persistent Networks without running VMs
     '''
+
     @classmethod
     def setUpClass(cls):
         cls.testClient = super(TestPersistentNetworks, cls).getClsTestClient()
@@ -66,8 +65,10 @@ class TestPersistentNetworks(cloudstackTestCase):
         cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][
             0].__dict__
         # Get Zone, Domain and templates
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
         cls.domain = get_domain(cls.api_client)
         cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests())
+        cls.cluster = list_clusters(cls.api_client)[0]
         cls.template = get_template(
             cls.api_client,
             cls.zone.id,
@@ -86,11 +87,11 @@ class TestPersistentNetworks(cloudstackTestCase):
         )
         cls.isolated_persistent_network_offering = cls.createNetworkOffering(
             "nw_off_isolated_persistent")
-        cls.isolated_persistent_network_offering_netscaler =\
+        cls.isolated_persistent_network_offering_netscaler = \
             cls.createNetworkOffering(
                 "nw_off_isolated_persistent_netscaler"
             )
-        cls.isolated_persistent_network_offering_RVR =\
+        cls.isolated_persistent_network_offering_RVR = \
             cls.createNetworkOffering(
                 "nw_off_persistent_RVR"
             )
@@ -99,7 +100,7 @@ class TestPersistentNetworks(cloudstackTestCase):
         cls.isolated_network_offering_netscaler = cls.createNetworkOffering(
             "nw_off_isolated_netscaler")
 
-        cls.services["configurableData"]["netscaler"]["lbdevicededicated"] =\
+        cls.services["configurableData"]["netscaler"]["lbdevicededicated"] = \
             False
 
         # Configure Netscaler device
@@ -230,6 +231,120 @@ class TestPersistentNetworks(cloudstackTestCase):
             raise Exception("Warning: Exception during cleanup : %s" % e)
         return
 
+    '''
+    Verifies creation of bridge on KVM host
+    '''
+    def verify_bridge_creation(self, host, vlan_id):
+        username = self.hostConfig["username"]
+        password = self.hostConfig["password"]
+        try:
+            ssh_client = self.get_ssh_client(host.ipaddress, username, password)
+            res = ssh_client.execute("ip addr | grep breth.*-" + str(vlan_id) + " > /dev/null 2>&1; echo $?")
+            return res[0]
+        except Exception as e:
+            self.fail(e)
+
+    def validate_persistent_network_resources_created_on_host(self, network_vlan):
+        hosts = self.list_all_hosts_in_zone(self.zone.id)
+        for host in hosts:
+            result = self.verify_bridge_creation(host, network_vlan)
+            self.assertEqual(
+                int(result),
+                0,
+                "Failed to find bridge on the breth0-" + str(network_vlan))
+
+    def list_all_hosts_in_zone(self, zone_id):
+        hosts = Host.list(
+            self.apiclient,
+            type='Routing',
+            resourcestate='Enabled',
+            state='Up',
+            zoneid=zone_id
+        )
+        return hosts
+
+    def get_ssh_client(self, ip, username, password, retries=10):
+        """ Setup ssh client connection and return connection """
+        try:
+            ssh_client = SshClient(ip, 22, username, password, retries)
+        except Exception as e:
+            raise unittest.SkipTest("Unable to create ssh connection: " % e)
+
+        self.assertIsNotNone(
+            ssh_client, "Failed to setup ssh connection to ip=%s" % ip)
+
+        return ssh_client
+
+    @attr(tags=["advanced", "xx"], required_hardware="false")
+    def test_newly_added_host_for_persistent_network_resources(self):
+        # steps
+        # 1. identify hosts in the zone, and remove the first
+        # 2. create a L2 persistent network
+        # 3. add the host back to the zone
+        #
+        # validation
+        # 1. Persistent network state should be implemented before adding the host
+        # 2. Host should be added back in successfully
+        # 3. Host should have the persistent networks resources after being added
+
+        if self.hypervisor.lower() == 'simulator':
+            raise self.skipTest("Skipping test case for Simulator hypervisor")
+
+        l2_persistent_network_offering = self.createNetworkOffering("nw_off_L2_persistent")
+        hosts = list_hosts(self.apiclient, clusterid=self.cluster.id)
+        host = hosts[0]
+        vlan_id = 991
+
+        Host(hosts[0].__dict__).delete(self.apiclient)  # remove host from zone before creating network
+
+        network = Network.create(
+            self.apiclient,
+            self.services["l2_network"],
+            networkofferingid=l2_persistent_network_offering.id,
+            accountid=self.account.name,
+            domainid=self.domain.id,
+            zoneid=self.zone.id,
+            vlan=vlan_id
+        )
+
+        self.cleanup.append(network)
+        self.cleanup.append(l2_persistent_network_offering)
+
+        response = verifyNetworkState(
+            self.apiclient,
+            network.id,
+            "implemented")
+        exceptionOccurred = response[0]
+        isNetworkInDesiredState = response[1]
+        exceptionMessage = response[2]
+
+        if exceptionOccurred or (not isNetworkInDesiredState):
+            self.fail(exceptionMessage)
+        self.assertIsNotNone(
+            network.vlan,
+            "vlan must not be null for persistent network")
+
+        newHost = {
+            "username": "root",
+            "password": "P@ssword123",
+            "url": "http://" + host.ipaddress,
+            "podid": host.podid,
+            "zoneid": host.zoneid
+        }
+
+        # add host back to the zone after creating network
+        try:
+            Host.create(
+                self.apiclient,
+                self.cluster,
+                newHost,
+                hypervisor=host.hypervisor
+            )
+        except Exception as e:
+            self.fail("Host creation failed: %s" % e)
+
+        self.validate_persistent_network_resources_created_on_host(network.vlan)
+
     @attr(tags=["advanced"], required_hardware="false")
     def test_network_state_after_destroying_vms(self):
         # steps
@@ -1540,14 +1655,14 @@ class TestPersistentNetworks(cloudstackTestCase):
                         state='Up',
                         id=router.hostid
                     )
-                    self.assertEqual(validateList(hosts)[0],PASS,"Check list host returns a valid list")
+                    self.assertEqual(validateList(hosts)[0], PASS, "Check list host returns a valid list")
                     host = hosts[0]
                     result = get_process_status(
-                        host.ipaddress,22,
+                        host.ipaddress, 22,
                         self.hostConfig["username"],
                         self.hostConfig["password"],
                         router.linklocalip,
-                    "iptables -I INPUT 1 -j DROP"
+                        "iptables -I INPUT 1 -j DROP"
                     )
                 except Exception as e:
                     raise Exception("Exception raised in accessing/running the command on hosts  : %s " % e)
@@ -1563,23 +1678,23 @@ class TestPersistentNetworks(cloudstackTestCase):
                 serviceofferingid=self.service_offering.id,
                 accountid=self.account.name,
                 domainid=self.domain.id)
-        #self.assertTrue('This is broken' in context.exception)
+        # self.assertTrue('This is broken' in context.exception)
         try:
             account.delete(self.api_client)
         except Exception as e:
             self.cleanup.append(account)
         qresultset = self.dbclient.execute(
-             "select id from usage_event where type = '%s' ORDER BY id DESC LIMIT 1;" %
+            "select id from usage_event where type = '%s' ORDER BY id DESC LIMIT 1;" %
-             str("VOLUME.DELETE"))
+            str("VOLUME.DELETE"))
         self.assertNotEqual(
-             len(qresultset),
+            len(qresultset),
-             0,
+            0,
-             "Check DB Query result set")
+            "Check DB Query result set")
         return
 
+
 @ddt
 class TestAssignVirtualMachine(cloudstackTestCase):
-
     """Test Persistent Network creation with
     assigning VM to different account/domain
     """
@@ -1613,14 +1728,14 @@ class TestAssignVirtualMachine(cloudstackTestCase):
         )
         cls.isolated_persistent_network_offering = cls.createNetworkOffering(
             "nw_off_isolated_persistent")
-        cls.isolated_persistent_network_offering_RVR =\
+        cls.isolated_persistent_network_offering_RVR = \
             cls.createNetworkOffering(
                 "nw_off_persistent_RVR"
             )
         cls.persistent_network_offering_netscaler = cls.createNetworkOffering(
             "nw_off_isolated_persistent_netscaler")
 
-        cls.services["configurableData"]["netscaler"]["lbdevicededicated"] =\
+        cls.services["configurableData"]["netscaler"]["lbdevicededicated"] = \
             False
 
         # Configure Netscaler device
@@ -1790,7 +1905,6 @@ class TestAssignVirtualMachine(cloudstackTestCase):
 
 @ddt
 class TestProjectAccountOperations(cloudstackTestCase):
-
     """Test suspend/disable/lock account/project operations
     when they have persistent network
     """
@@ -1913,7 +2027,7 @@ class TestProjectAccountOperations(cloudstackTestCase):
             accounts)
         self.assertEqual(str(accounts[0].state).lower(
         ), value, "account state should be %s, it is %s"
-            % (value, accounts[0].state))
+                  % (value, accounts[0].state))
 
         # Wait for network cleanup interval
         wait_for_cleanup(
@@ -2037,7 +2151,6 @@ class TestProjectAccountOperations(cloudstackTestCase):
 
 @ddt
 class TestRestartPersistentNetwork(cloudstackTestCase):
-
     """Test restart persistent network with cleanup parameter true and false
     """
 
@@ -2074,7 +2187,7 @@ class TestRestartPersistentNetwork(cloudstackTestCase):
             cls.services["nw_off_isolated_persistent_lb"],
             conservemode=False)
 
-        cls.isolated_persistent_network_offering_netscaler =\
+        cls.isolated_persistent_network_offering_netscaler = \
             NetworkOffering.create(
                 cls.api_client,
                 cls.services["nw_off_isolated_persistent_netscaler"],
@@ -2088,7 +2201,7 @@ class TestRestartPersistentNetwork(cloudstackTestCase):
             cls.api_client,
             state="enabled")
 
-        cls.services["configurableData"]["netscaler"]["lbdevicededicated"] =\
+        cls.services["configurableData"]["netscaler"]["lbdevicededicated"] = \
             False
 
         # Configure Netscaler device
@@ -2446,7 +2559,6 @@ class TestRestartPersistentNetwork(cloudstackTestCase):
 
 @ddt
 class TestVPCNetworkOperations(cloudstackTestCase):
-
     """Test VPC network operations consisting persistent networks
     """
 
@@ -3095,11 +3207,11 @@ class TestVPCNetworkOperations(cloudstackTestCase):
 
             # Create network ACL for both ther persistent networks (tiers of
             # VPC)
-            ingressAclNetwork1, egressAclNetwork1 =\
+            ingressAclNetwork1, egressAclNetwork1 = \
                 self.CreateIngressEgressNetworkACLForNetwork(
                     persistent_network_1.id
                 )
-            ingressAclNetwork2, egressAclNetwork2 =\
+            ingressAclNetwork2, egressAclNetwork2 = \
                 self.CreateIngressEgressNetworkACLForNetwork(
                     persistent_network_2.id
                 )
@@ -3310,11 +3422,11 @@ class TestVPCNetworkOperations(cloudstackTestCase):
         lb_rule.assign(self.api_client, [virtual_machine_3, virtual_machine_4])
 
         # Create network ACL for both ther persistent networks (tiers of VPC)
-        ingressAclNetwork1, egressAclNetwork1 =\
+        ingressAclNetwork1, egressAclNetwork1 = \
             self.CreateIngressEgressNetworkACLForNetwork(
                 persistent_network_1.id
             )
-        ingressAclNetwork2, egressAclNetwork2 =\
+        ingressAclNetwork2, egressAclNetwork2 = \
             self.CreateIngressEgressNetworkACLForNetwork(
                 persistent_network_2.id
             )