Synchronization of network devices on newly added hosts for Persistent Networks (#5977)

* Persistent Network feature & Marvin component tests

* Cleaned up comments and imports

* fixed small error

* add support to add setup persistent networks' resources when a disabled host is enabled

* small fix

* use wildcard instead of hard-coding the bridge name

* allow clean up of resources when removing a host in maintenance mode

* skip test for simulator hypervisor

Co-authored-by: shatoboar <sang-woo.bae@campus.tu-berlin.de>

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/HypervisorHostListener.java

@@ -30,4 +30,6 @@ public interface HypervisorHostListener {
     boolean hostAboutToBeRemoved(long hostId);
 
     boolean hostRemoved(long hostId, long clusterId);
+
+    boolean hostEnabled(long hostId);
 }

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -280,6 +280,8 @@ public interface StorageManager extends StorageService {
 
     void disconnectHostFromSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException;
 
+    void enableHost(long hostId) throws StorageUnavailableException, StorageConflictException;
+
     void createCapacityEntry(long poolId);
 
     DataStore createLocalStorage(Host host, StoragePoolInfo poolInfo) throws ConnectionException;

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java

@@ -31,6 +31,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
+import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand;
 import org.apache.cloudstack.agent.lb.SetupMSListCommand;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.log4j.Logger;
@@ -118,7 +119,8 @@ public abstract class AgentAttache {
         StopCommand.class.toString(), CheckVirtualMachineCommand.class.toString(), PingTestCommand.class.toString(), CheckHealthCommand.class.toString(),
         ReadyCommand.class.toString(), ShutdownCommand.class.toString(), SetupCommand.class.toString(),
         CleanupNetworkRulesCmd.class.toString(), CheckNetworkCommand.class.toString(), PvlanSetupCommand.class.toString(), CheckOnHostCommand.class.toString(),
-        ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString()};
+        ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(),
+            CleanupPersistentNetworkResourceCommand.class.toString()};
     protected final static String[] s_commandsNotAllowedInConnectingMode = new String[] { StartCommand.class.toString(), CreateCommand.class.toString() };
     static {
         Arrays.sort(s_commandsAllowedInMaintenanceMode);

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -1198,7 +1198,6 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
     }
 
     /**
-     *
      * Creates a dummy NicTO object which is used by the respective hypervisors to setup network elements / resources
      * - bridges(KVM), VLANs(Xen) and portgroups(VMWare) for L2 network
      */
@@ -1276,6 +1275,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             return criteriaMet && (network.getGuestType() == GuestType.L2 || network.getGuestType() == GuestType.Isolated);
         }
     }
+
     @Override
     @DB
     public Pair<NetworkGuru, NetworkVO> implementNetwork(final long networkId, final DeployDestination dest, final ReservationContext context) throws ConcurrentOperationException,
@@ -2766,6 +2766,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
     /**
      * Encodes VLAN/VXLAN ID into a Broadcast URI according to the isolation method from the Physical Network.
+     *
      * @return Broadcast URI, e.g. 'vlan://vlan_ID' or 'vxlan://vlxan_ID'
      */
     protected URI encodeVlanIdIntoBroadcastUri(String vlanId, PhysicalNetwork pNtwk) {
@@ -2785,6 +2786,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
     /**
      * Checks bypass VLAN id/range overlap check during network creation for guest networks
+     *
      * @param bypassVlanOverlapCheck bypass VLAN id/range overlap check
      * @param ntwkOff                network offering
      */
@@ -2796,6 +2798,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
      * Checks for L2 network offering services. Only 2 cases allowed:
      * - No services
      * - User Data service only, provided by ConfigDrive
+     *
      * @param ntwkOff network offering
      */
     protected void checkL2OfferingServices(NetworkOfferingVO ntwkOff) {
@@ -3419,7 +3422,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         if (network.isRedundant() || (oldRouters.size() == 1 && oldRouters.get(0).getIsRedundantRouter())) {
             try {
                 Thread.sleep(NetworkOrchestrationService.RVRHandoverTime);
-            } catch (final InterruptedException ignored) {}
+            } catch (final InterruptedException ignored) {
+            }
         }
 
         // Destroy old routers
@@ -4158,6 +4162,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 accessDetails.put(network.getTrafficType().name(), address);
             }
         }
+
         if (privateIpAddress != null && StringUtils.isEmpty(accessDetails.get(NetworkElementCommand.ROUTER_IP))) {
             accessDetails.put(NetworkElementCommand.ROUTER_IP,  privateIpAddress);
         }

engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java

@@ -130,4 +130,6 @@ public interface NetworkDao extends GenericDao<NetworkVO, Long>, StateDao<State,
     List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri, Network.PVlanType pVlanType);
 
     List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri);
+
+    List<NetworkVO> getAllPersistentNetworksFromZone(long dataCenterId);
 }

engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java

@@ -419,6 +419,16 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
         return persistentNetworks.size();
     }
 
+    @Override
+    public List<NetworkVO> getAllPersistentNetworksFromZone(long dataCenterId) {
+        Object[] guestTypes = {"Isolated", "L2"};
+        final SearchCriteria<NetworkVO> sc = PersistentNetworkSearch.create();
+        sc.setParameters("guestType", guestTypes);
+        sc.setParameters("dc", dataCenterId);
+        sc.setJoinParameters("persistent", "persistent", true);
+        return search(sc, null);
+    }
+
     @Override
     public String getNextAvailableMacAddress(final long networkConfigId, Integer zoneMacIdentifier) {
         final SequenceFetcher fetch = SequenceFetcher.getInstance();

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java

@@ -20,10 +20,21 @@ package org.apache.cloudstack.storage.datastore.provider;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand;
 import com.cloud.agent.api.ModifyStoragePoolAnswer;
 import com.cloud.agent.api.ModifyStoragePoolCommand;
+import com.cloud.agent.api.SetupPersistentNetworkCommand;
+import com.cloud.agent.api.to.NicTO;
 import com.cloud.alert.AlertManager;
+import com.cloud.configuration.ConfigurationManager;
 import com.cloud.exception.StorageConflictException;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.network.NetworkModel;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
+import com.cloud.offerings.NetworkOfferingVO;
+import com.cloud.offerings.dao.NetworkOfferingDao;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Storage;
 import com.cloud.storage.StorageManager;
@@ -62,12 +73,50 @@ public class DefaultHostListener implements HypervisorHostListener {
     StorageManager storageManager;
     @Inject
     StorageService storageService;
+    @Inject
+    NetworkOfferingDao networkOfferingDao;
+    @Inject
+    HostDao hostDao;
+    @Inject
+    NetworkModel networkModel;
+    @Inject
+    ConfigurationManager configManager;
+    @Inject
+    NetworkDao networkDao;
+
 
     @Override
     public boolean hostAdded(long hostId) {
         return true;
     }
 
+    private boolean createPersistentNetworkResourcesOnHost(long hostId) {
+        HostVO host = hostDao.findById(hostId);
+        if (host == null) {
+            s_logger.warn(String.format("Host with id %ld can't be found", hostId));
+            return false;
+        }
+        setupPersistentNetwork(host);
+        return true;
+    }
+
+    /**
+     * Creates a dummy NicTO object which is used by the respective hypervisors to setup network elements / resources
+     * - bridges(KVM), VLANs(Xen) and portgroups(VMWare) for L2 network
+     */
+    private NicTO createNicTOFromNetworkAndOffering(NetworkVO networkVO, NetworkOfferingVO networkOfferingVO, HostVO hostVO) {
+        NicTO to = new NicTO();
+        to.setName(networkModel.getNetworkTag(hostVO.getHypervisorType(), networkVO));
+        to.setBroadcastType(networkVO.getBroadcastDomainType());
+        to.setType(networkVO.getTrafficType());
+        to.setBroadcastUri(networkVO.getBroadcastUri());
+        to.setIsolationuri(networkVO.getBroadcastUri());
+        to.setNetworkRateMbps(configManager.getNetworkOfferingNetworkRate(networkOfferingVO.getId(), networkVO.getDataCenterId()));
+        to.setSecurityGroupEnabled(networkModel.isSecurityGroupSupportedInNetwork(networkVO));
+        return to;
+    }
+
+
     @Override
     public boolean hostConnect(long hostId, long poolId) throws StorageConflictException {
         StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
@@ -109,7 +158,8 @@ public class DefaultHostListener implements HypervisorHostListener {
         storageService.updateStorageCapabilities(poolId, false);
 
         s_logger.info("Connection established between storage pool " + pool + " and host " + hostId);
-        return true;
+
+        return createPersistentNetworkResourcesOnHost(hostId);
     }
 
     private void updateStoragePoolHostVOAndDetails(StoragePool pool, long hostId, ModifyStoragePoolAnswer mspAnswer) {
@@ -142,6 +192,28 @@ public class DefaultHostListener implements HypervisorHostListener {
 
     @Override
     public boolean hostAboutToBeRemoved(long hostId) {
+        // send host the cleanup persistent network resources
+        HostVO host = hostDao.findById(hostId);
+        if (host == null) {
+            s_logger.warn("Host with id " + hostId + " can't be found");
+            return false;
+        }
+
+        List<NetworkVO> allPersistentNetworks = networkDao.getAllPersistentNetworksFromZone(host.getDataCenterId()); // find zoneId of host
+        for (NetworkVO persistentNetworkVO : allPersistentNetworks) {
+            NetworkOfferingVO networkOfferingVO = networkOfferingDao.findById(persistentNetworkVO.getNetworkOfferingId());
+            CleanupPersistentNetworkResourceCommand cleanupCmd =
+                    new CleanupPersistentNetworkResourceCommand(createNicTOFromNetworkAndOffering(persistentNetworkVO, networkOfferingVO, host));
+            Answer answer = agentMgr.easySend(hostId, cleanupCmd);
+            if (answer == null) {
+                s_logger.error("Unable to get answer to the cleanup persistent network command " + persistentNetworkVO.getId());
+                continue;
+            }
+            if (!answer.getResult()) {
+                String msg = String.format("Unable to cleanup persistent network resources from network %d on the host %d", persistentNetworkVO.getId(), hostId);
+                s_logger.error(msg);
+            }
+        }
         return true;
     }
 
@@ -149,4 +221,33 @@ public class DefaultHostListener implements HypervisorHostListener {
     public boolean hostRemoved(long hostId, long clusterId) {
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        HostVO host = hostDao.findById(hostId);
+        if (host == null) {
+            s_logger.warn(String.format("Host with id %d can't be found", hostId));
+            return false;
+        }
+        setupPersistentNetwork(host);
+        return true;
+    }
+
+    private void setupPersistentNetwork(HostVO host) {
+        List<NetworkVO> allPersistentNetworks = networkDao.getAllPersistentNetworksFromZone(host.getDataCenterId());
+        for (NetworkVO networkVO : allPersistentNetworks) {
+            NetworkOfferingVO networkOfferingVO = networkOfferingDao.findById(networkVO.getNetworkOfferingId());
+
+            SetupPersistentNetworkCommand persistentNetworkCommand =
+                    new SetupPersistentNetworkCommand(createNicTOFromNetworkAndOffering(networkVO, networkOfferingVO, host));
+            Answer answer = agentMgr.easySend(host.getId(), persistentNetworkCommand);
+            if (answer == null) {
+                throw new CloudRuntimeException("Unable to get answer to the setup persistent network command " + networkVO.getId());
+            }
+            if (!answer.getResult()) {
+                String msg = String.format("Unable to create persistent network resources for network %d on the host %d in zone %d", networkVO.getId(), host.getId(), networkVO.getDataCenterId());
+                s_logger.error(msg);
+            }
+        }
+    }
 }

plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/provider/ElastistorHostListener.java

@@ -140,4 +140,9 @@ public class ElastistorHostListener implements HypervisorHostListener {
     public boolean hostRemoved(long hostId, long clusterId) {
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
 }

plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java

@@ -178,6 +178,11 @@ public class DateraHostListener implements HypervisorHostListener {
         return true;
     }
 
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
+
     private void handleXenServer(long clusterId, long hostId, long storagePoolId) {
         List<String> storagePaths = getStoragePaths(clusterId, storagePoolId);
 

plugins/storage/volume/nexenta/src/main/java/org/apache/cloudstack/storage/datastore/provider/NexentaHostListener.java

@@ -59,4 +59,9 @@ public class NexentaHostListener implements HypervisorHostListener {
 
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
 }

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java

@@ -138,4 +138,9 @@ public class ScaleIOHostListener implements HypervisorHostListener {
     public boolean hostRemoved(long hostId, long clusterId) {
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
 }

plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java

@@ -141,6 +141,11 @@ public class SolidFireHostListener implements HypervisorHostListener {
         return true;
     }
 
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
+
     private void handleXenServer(long clusterId, long hostId, long storagePoolId) {
         List<String> storagePaths = getStoragePaths(clusterId, storagePoolId);
 

plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireSharedHostListener.java

@@ -140,6 +140,11 @@ public class SolidFireSharedHostListener implements HypervisorHostListener {
         return true;
     }
 
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
+
     private void handleVMware(HostVO host, boolean add, ModifyTargetsCommand.TargetTypeToRemove targetTypeToRemove) {
         if (HypervisorType.VMware.equals(host.getHypervisorType())) {
             List<StoragePoolVO> storagePools = storagePoolDao.findPoolsByProvider(SolidFireUtil.SHARED_PROVIDER_NAME);

server/src/main/java/com/cloud/resource/ResourceManagerImpl.java

@@ -36,6 +36,8 @@ import java.util.Random;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.exception.StorageConflictException;
+import com.cloud.exception.StorageUnavailableException;
 import org.apache.cloudstack.annotation.AnnotationService;
 import org.apache.cloudstack.annotation.dao.AnnotationDao;
 import org.apache.cloudstack.api.ApiConstants;
@@ -1836,6 +1838,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
         if (url != null) {
             _storageMgr.updateSecondaryStorage(cmd.getId(), cmd.getUrl());
         }
+        try {
+            _storageMgr.enableHost(hostId);
+        } catch (StorageUnavailableException | StorageConflictException e) {
+            s_logger.error(String.format("Failed to setup host %s when enabled", host));
+        }
 
         final HostVO updatedHost = _hostDao.findById(hostId);
         return updatedHost;

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -79,6 +79,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreState
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle;
+import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreProvider;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
@@ -1118,6 +1119,26 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         listener.hostDisconnected(hostId, pool.getId());
     }
 
+    @Override
+    public void enableHost(long hostId) {
+        List<DataStoreProvider> providers = _dataStoreProviderMgr.getProviders();
+        if (providers != null) {
+            for (DataStoreProvider provider : providers) {
+                if (provider instanceof PrimaryDataStoreProvider) {
+                    try {
+                        HypervisorHostListener hypervisorHostListener = provider.getHostListener();
+                        if (hypervisorHostListener != null) {
+                            hypervisorHostListener.hostEnabled(hostId);
+                        }
+                    }
+                    catch (Exception ex) {
+                        s_logger.error("hostEnabled(long) failed for storage provider " + provider.getName(), ex);
+                    }
+                }
+            }
+        }
+    }
+
     @Override
     public BigDecimal getStorageOverProvisioningFactor(Long poolId) {
         return new BigDecimal(CapacityManager.StorageOverprovisioningFactor.valueIn(poolId));

server/src/test/java/com/cloud/vpc/dao/MockNetworkDaoImpl.java

@@ -260,4 +260,9 @@ public class MockNetworkDaoImpl extends GenericDaoBase<NetworkVO, Long> implemen
     public List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri) {
         return null;
     }
+
+    @Override
+    public List<NetworkVO> getAllPersistentNetworksFromZone(long dataCenterId) {
+        return null;
+    }
 }

test/integration/component/test_persistent_networks.py

@@ -40,7 +40,7 @@ from marvin.lib.common import (get_domain,
                                get_template,
                                verifyNetworkState,
                                add_netscaler,
-                               wait_for_cleanup,list_routers,list_hosts)
+                               wait_for_cleanup, list_routers, list_hosts, list_clusters)
 from nose.plugins.attrib import attr
 from marvin.codes import PASS, FAIL, FAILED
 from marvin.sshClient import SshClient
@@ -49,13 +49,12 @@ import unittest
 from ddt import ddt, data
 import time
 
-
 @ddt
 class TestPersistentNetworks(cloudstackTestCase):
-
     '''
     Test Persistent Networks without running VMs
     '''
+
     @classmethod
     def setUpClass(cls):
         cls.testClient = super(TestPersistentNetworks, cls).getClsTestClient()
@@ -66,8 +65,10 @@ class TestPersistentNetworks(cloudstackTestCase):
         cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][
             0].__dict__
         # Get Zone, Domain and templates
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
         cls.domain = get_domain(cls.api_client)
         cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests())
+        cls.cluster = list_clusters(cls.api_client)[0]
         cls.template = get_template(
             cls.api_client,
             cls.zone.id,
@@ -230,6 +231,120 @@ class TestPersistentNetworks(cloudstackTestCase):
             raise Exception("Warning: Exception during cleanup : %s" % e)
         return
 
+    '''
+    Verifies creation of bridge on KVM host
+    '''
+    def verify_bridge_creation(self, host, vlan_id):
+        username = self.hostConfig["username"]
+        password = self.hostConfig["password"]
+        try:
+            ssh_client = self.get_ssh_client(host.ipaddress, username, password)
+            res = ssh_client.execute("ip addr | grep breth.*-" + str(vlan_id) + " > /dev/null 2>&1; echo $?")
+            return res[0]
+        except Exception as e:
+            self.fail(e)
+
+    def validate_persistent_network_resources_created_on_host(self, network_vlan):
+        hosts = self.list_all_hosts_in_zone(self.zone.id)
+        for host in hosts:
+            result = self.verify_bridge_creation(host, network_vlan)
+            self.assertEqual(
+                int(result),
+                0,
+                "Failed to find bridge on the breth0-" + str(network_vlan))
+
+    def list_all_hosts_in_zone(self, zone_id):
+        hosts = Host.list(
+            self.apiclient,
+            type='Routing',
+            resourcestate='Enabled',
+            state='Up',
+            zoneid=zone_id
+        )
+        return hosts
+
+    def get_ssh_client(self, ip, username, password, retries=10):
+        """ Setup ssh client connection and return connection """
+        try:
+            ssh_client = SshClient(ip, 22, username, password, retries)
+        except Exception as e:
+            raise unittest.SkipTest("Unable to create ssh connection: " % e)
+
+        self.assertIsNotNone(
+            ssh_client, "Failed to setup ssh connection to ip=%s" % ip)
+
+        return ssh_client
+
+    @attr(tags=["advanced", "xx"], required_hardware="false")
+    def test_newly_added_host_for_persistent_network_resources(self):
+        # steps
+        # 1. identify hosts in the zone, and remove the first
+        # 2. create a L2 persistent network
+        # 3. add the host back to the zone
+        #
+        # validation
+        # 1. Persistent network state should be implemented before adding the host
+        # 2. Host should be added back in successfully
+        # 3. Host should have the persistent networks resources after being added
+
+        if self.hypervisor.lower() == 'simulator':
+            raise self.skipTest("Skipping test case for Simulator hypervisor")
+
+        l2_persistent_network_offering = self.createNetworkOffering("nw_off_L2_persistent")
+        hosts = list_hosts(self.apiclient, clusterid=self.cluster.id)
+        host = hosts[0]
+        vlan_id = 991
+
+        Host(hosts[0].__dict__).delete(self.apiclient)  # remove host from zone before creating network
+
+        network = Network.create(
+            self.apiclient,
+            self.services["l2_network"],
+            networkofferingid=l2_persistent_network_offering.id,
+            accountid=self.account.name,
+            domainid=self.domain.id,
+            zoneid=self.zone.id,
+            vlan=vlan_id
+        )
+
+        self.cleanup.append(network)
+        self.cleanup.append(l2_persistent_network_offering)
+
+        response = verifyNetworkState(
+            self.apiclient,
+            network.id,
+            "implemented")
+        exceptionOccurred = response[0]
+        isNetworkInDesiredState = response[1]
+        exceptionMessage = response[2]
+
+        if exceptionOccurred or (not isNetworkInDesiredState):
+            self.fail(exceptionMessage)
+        self.assertIsNotNone(
+            network.vlan,
+            "vlan must not be null for persistent network")
+
+        newHost = {
+            "username": "root",
+            "password": "P@ssword123",
+            "url": "http://" + host.ipaddress,
+            "podid": host.podid,
+            "zoneid": host.zoneid
+        }
+
+        # add host back to the zone after creating network
+        try:
+            Host.create(
+                self.apiclient,
+                self.cluster,
+                newHost,
+                hypervisor=host.hypervisor
+            )
+        except Exception as e:
+            self.fail("Host creation failed: %s" % e)
+
+        self.validate_persistent_network_resources_created_on_host(network.vlan)
+
     @attr(tags=["advanced"], required_hardware="false")
     def test_network_state_after_destroying_vms(self):
         # steps
@@ -1577,9 +1692,9 @@ class TestPersistentNetworks(cloudstackTestCase):
             "Check DB Query result set")
         return
 
+
 @ddt
 class TestAssignVirtualMachine(cloudstackTestCase):
-
     """Test Persistent Network creation with
     assigning VM to different account/domain
     """
@@ -1790,7 +1905,6 @@ class TestAssignVirtualMachine(cloudstackTestCase):
 
 @ddt
 class TestProjectAccountOperations(cloudstackTestCase):
-
     """Test suspend/disable/lock account/project operations
     when they have persistent network
     """
@@ -2037,7 +2151,6 @@ class TestProjectAccountOperations(cloudstackTestCase):
 
 @ddt
 class TestRestartPersistentNetwork(cloudstackTestCase):
-
     """Test restart persistent network with cleanup parameter true and false
     """
 
@@ -2446,7 +2559,6 @@ class TestRestartPersistentNetwork(cloudstackTestCase):
 
 @ddt
 class TestVPCNetworkOperations(cloudstackTestCase):
-
     """Test VPC network operations consisting persistent networks
     """