Merge release branch 4.13 to master

* 4.13:
  VR: Fix Redundant VRouter guest network on wrong interface (#3847)

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -1793,17 +1793,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
         try {
             conn = getLibvirtUtilitiesHelper().getConnectionByVmName(routerName);
+            Pair<Map<String, Integer>, Integer> macAddressToNicNumPair = getMacAddressToNicNumPair(conn, routerName);
+            final Map<String, Integer> macAddressToNicNum = macAddressToNicNumPair.first();
+            Integer devNum = macAddressToNicNumPair.second();
+
             final IpAddressTO[] ips = cmd.getIpAddresses();
-            Integer devNum = 0;
-            final List<InterfaceDef> pluggedNics = getInterfaces(conn, routerName);
-            final Map<String, Integer> macAddressToNicNum = new HashMap<>(pluggedNics.size());
-
-            for (final InterfaceDef pluggedNic : pluggedNics) {
-                final String pluggedVlan = pluggedNic.getBrName();
-                macAddressToNicNum.put(pluggedNic.getMacAddress(), devNum);
-                devNum++;
-            }
-
             for (final IpAddressTO ip : ips) {
                 ip.setNicDevId(macAddressToNicNum.get(ip.getVifMacAddress()));
             }
@@ -1820,35 +1814,22 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         final String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
         Connect conn;
         try {
-            conn = LibvirtConnection.getConnectionByVmName(routerName);
+            conn = getLibvirtUtilitiesHelper().getConnectionByVmName(routerName);
-            final List<InterfaceDef> nics = getInterfaces(conn, routerName);
+            Pair<Map<String, Integer>, Integer> macAddressToNicNumPair = getMacAddressToNicNumPair(conn, routerName);
-            final Map<String, Integer> broadcastUriAllocatedToVM = new HashMap<String, Integer>();
+            final Map<String, Integer> macAddressToNicNum = macAddressToNicNumPair.first();
-            Integer nicPos = 0;
+            Integer devNum = macAddressToNicNumPair.second();
-            for (final InterfaceDef nic : nics) {
+
-                if (nic.getBrName().equalsIgnoreCase(_linkLocalBridgeName)) {
-                    broadcastUriAllocatedToVM.put("LinkLocal", nicPos);
-                } else {
-                    if (nic.getBrName().equalsIgnoreCase(_publicBridgeName) || nic.getBrName().equalsIgnoreCase(_privBridgeName) ||
-                            nic.getBrName().equalsIgnoreCase(_guestBridgeName)) {
-                        broadcastUriAllocatedToVM.put(BroadcastDomainType.Vlan.toUri(Vlan.UNTAGGED).toString(), nicPos);
-                    } else {
-                        final String broadcastUri = getBroadcastUriFromBridge(nic.getBrName());
-                        broadcastUriAllocatedToVM.put(broadcastUri, nicPos);
-                    }
-                }
-                nicPos++;
-            }
             final IpAddressTO[] ips = cmd.getIpAddresses();
             int nicNum = 0;
             for (final IpAddressTO ip : ips) {
                 boolean newNic = false;
-                if (!broadcastUriAllocatedToVM.containsKey(ip.getBroadcastUri())) {
+                if (!macAddressToNicNum.containsKey(ip.getVifMacAddress())) {
                     /* plug a vif into router */
                     VifHotPlug(conn, routerName, ip.getBroadcastUri(), ip.getVifMacAddress());
-                    broadcastUriAllocatedToVM.put(ip.getBroadcastUri(), nicPos++);
+                    macAddressToNicNum.put(ip.getVifMacAddress(), devNum++);
                     newNic = true;
                 }
-                nicNum = broadcastUriAllocatedToVM.get(ip.getBroadcastUri());
+                nicNum = macAddressToNicNum.get(ip.getVifMacAddress());
                 networkUsage(routerIp, "addVif", "eth" + nicNum);
 
                 ip.setNicDevId(nicNum);
@@ -1870,39 +1851,21 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         final String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
         final String lastIp = cmd.getAccessDetail(NetworkElementCommand.NETWORK_PUB_LAST_IP);
         Connect conn;
-
+        try {
-
+            conn = getLibvirtUtilitiesHelper().getConnectionByVmName(routerName);
-        try{
+            Pair<Map<String, Integer>, Integer> macAddressToNicNumPair = getMacAddressToNicNumPair(conn, routerName);
-            conn = LibvirtConnection.getConnectionByVmName(routerName);
+            final Map<String, Integer> macAddressToNicNum = macAddressToNicNumPair.first();
-            final List<InterfaceDef> nics = getInterfaces(conn, routerName);
+            Integer devNum = macAddressToNicNumPair.second();
-            final Map<String, Integer> broadcastUriAllocatedToVM = new HashMap<String, Integer>();
-
-            Integer nicPos = 0;
-            for (final InterfaceDef nic : nics) {
-                if (nic.getBrName().equalsIgnoreCase(_linkLocalBridgeName)) {
-                    broadcastUriAllocatedToVM.put("LinkLocal", nicPos);
-                } else {
-                    if (nic.getBrName().equalsIgnoreCase(_publicBridgeName) || nic.getBrName().equalsIgnoreCase(_privBridgeName) ||
-                            nic.getBrName().equalsIgnoreCase(_guestBridgeName)) {
-                        broadcastUriAllocatedToVM.put(BroadcastDomainType.Vlan.toUri(Vlan.UNTAGGED).toString(), nicPos);
-                    } else {
-                        final String broadcastUri = getBroadcastUriFromBridge(nic.getBrName());
-                        broadcastUriAllocatedToVM.put(broadcastUri, nicPos);
-                    }
-                }
-                nicPos++;
-            }
 
             final IpAddressTO[] ips = cmd.getIpAddresses();
             int nicNum = 0;
             for (final IpAddressTO ip : ips) {
-
+                if (!macAddressToNicNum.containsKey(ip.getVifMacAddress())) {
-                if (!broadcastUriAllocatedToVM.containsKey(ip.getBroadcastUri())) {
                     /* plug a vif into router */
                     VifHotPlug(conn, routerName, ip.getBroadcastUri(), ip.getVifMacAddress());
-                    broadcastUriAllocatedToVM.put(ip.getBroadcastUri(), nicPos++);
+                    macAddressToNicNum.put(ip.getVifMacAddress(), devNum++);
                 }
-                nicNum = broadcastUriAllocatedToVM.get(ip.getBroadcastUri());
+                nicNum = macAddressToNicNum.get(ip.getVifMacAddress());
 
                 if (org.apache.commons.lang.StringUtils.equalsIgnoreCase(lastIp, "true") && !ip.isAdd()) {
                     // in isolated network eth2 is the default public interface. We don't want to delete it.
@@ -1924,6 +1887,19 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         return new ExecutionResult(true, null);
     }
 
+
+    private Pair<Map<String, Integer>, Integer> getMacAddressToNicNumPair(Connect conn, String routerName) {
+        Integer devNum = 0;
+        final List<InterfaceDef> pluggedNics = getInterfaces(conn, routerName);
+        final Map<String, Integer> macAddressToNicNum = new HashMap<>(pluggedNics.size());
+        for (final InterfaceDef pluggedNic : pluggedNics) {
+            final String pluggedVlan = pluggedNic.getBrName();
+            macAddressToNicNum.put(pluggedNic.getMacAddress(), devNum);
+            devNum++;
+        }
+        return new Pair<Map<String, Integer>, Integer>(macAddressToNicNum, devNum);
+    }
+
     protected PowerState convertToPowerState(final DomainState ps) {
         final PowerState state = s_powerStatesTable.get(ps);
         return state == null ? PowerState.PowerUnknown : state;

server/src/main/java/com/cloud/network/router/CommandSetupHelper.java

@@ -674,7 +674,20 @@ public class CommandSetupHelper {
             vlanIpMap.put(vlanTag, ipList);
         }
 
+        Long guestNetworkId = null;
+        final List<NicVO> nics = _nicDao.listByVmId(router.getId());
+        for (final NicVO nic : nics) {
+            final NetworkVO nw = _networkDao.findById(nic.getNetworkId());
+            if (nw.getTrafficType() == TrafficType.Guest) {
+                guestNetworkId = nw.getId();
+                break;
+            }
+        }
+
+        Map<String, Boolean> vlanLastIpMap = getVlanLastIpMap(router.getVpcId(), guestNetworkId);
+
         for (final Map.Entry<String, ArrayList<PublicIpAddress>> vlanAndIp : vlanIpMap.entrySet()) {
+            final String vlanTagKey = vlanAndIp.getKey();
             final List<PublicIpAddress> ipAddrList = vlanAndIp.getValue();
 
             // Source nat ip address should always be sent first
@@ -732,6 +745,8 @@ public class CommandSetupHelper {
             final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId());
             cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString());
 
+            setAccessDetailNetworkLastPublicIp(vlanLastIpMap, vlanTagKey, cmd);
+
             cmds.addCommand(ipAssocCommand, cmd);
         }
 
@@ -769,15 +784,25 @@ public class CommandSetupHelper {
 
         final List<NicVO> nics = _nicDao.listByVmId(router.getId());
         String baseMac = null;
+        Map<String, String> vlanMacAddress = new HashMap<String, String>();;
+        Long guestNetworkId = null;
         for (final NicVO nic : nics) {
             final NetworkVO nw = _networkDao.findById(nic.getNetworkId());
             if (nw.getTrafficType() == TrafficType.Public) {
-                baseMac = nic.getMacAddress();
+                if (baseMac == null) {
-                break;
+                    baseMac = nic.getMacAddress();
+                }
+                final String vlanTag = BroadcastDomainType.getValue(nic.getBroadcastUri());
+                vlanMacAddress.put(vlanTag, nic.getMacAddress());
+            } else if (nw.getTrafficType() == TrafficType.Guest && guestNetworkId == null) {
+                guestNetworkId = nw.getId();
             }
         }
 
+        Map<String, Boolean> vlanLastIpMap = getVlanLastIpMap(router.getVpcId(), guestNetworkId);
+
         for (final Map.Entry<String, ArrayList<PublicIpAddress>> vlanAndIp : vlanIpMap.entrySet()) {
+            final String vlanTagKey = vlanAndIp.getKey();
             final List<PublicIpAddress> ipAddrList = vlanAndIp.getValue();
             // Source nat ip address should always be sent first
             Collections.sort(ipAddrList, new Comparator<PublicIpAddress>() {
@@ -809,19 +834,16 @@ public class CommandSetupHelper {
                 final String vlanGateway = ipAddr.getGateway();
                 final String vlanNetmask = ipAddr.getNetmask();
                 String vifMacAddress = null;
-                // For non-source nat IP, set the mac to be something based on
+                final String vlanTag = BroadcastDomainType.getValue(BroadcastDomainType.fromString(ipAddr.getVlanTag()));
-                // first public nic's MAC
+                if (vlanMacAddress.containsKey(vlanTag)) {
-                // We cannot depend on first ip because we need to deal with
+                    vifMacAddress = vlanMacAddress.get(vlanTag);
-                // first ip of other nics
-                if (router.getVpcId() != null) {
-                    //vifMacAddress = NetUtils.generateMacOnIncrease(baseMac, ipAddr.getVlanId());
-                    vifMacAddress = ipAddr.getMacAddress();
                 } else {
-                    if (!sourceNat && ipAddr.getVlanId() != 0) {
+                    if (ipAddr.getVlanId() != 0) {
                         vifMacAddress = NetUtils.generateMacOnIncrease(baseMac, ipAddr.getVlanId());
                     } else {
                         vifMacAddress = ipAddr.getMacAddress();
                     }
+                    vlanMacAddress.put(vlanTag, vifMacAddress);
                 }
 
                 final IpAddressTO ip = new IpAddressTO(ipAddr.getAccountId(), ipAddr.getAddress().addr(), add, firstIP, sourceNat, vlanId, vlanGateway, vlanNetmask,
@@ -839,56 +861,59 @@ public class CommandSetupHelper {
                 }
             }
 
-            Long associatedWithNetworkId = ipAddrList.get(0).getAssociatedWithNetworkId();
+            final IpAssocCommand cmd;
-            if (associatedWithNetworkId == null || associatedWithNetworkId == 0) {
+            if (router.getVpcId() != null) {
-                associatedWithNetworkId = ipAddrList.get(0).getNetworkId();
+                cmd = new IpAssocVpcCommand(ipsToSend);
+            } else {
+                cmd = new IpAssocCommand(ipsToSend);
             }
-
-            // for network if the ips does not have any rules, then only last ip
-            final List<IPAddressVO> userIps = _ipAddressDao.listByAssociatedNetwork(associatedWithNetworkId, null);
-            boolean hasSourceNat = false;
-            if (isVPC && userIps.size() > 0 && userIps.get(0) != null) {
-                // All ips should belong to a VPC
-                final Long vpcId = userIps.get(0).getVpcId();
-                final List<IPAddressVO> sourceNatIps = _ipAddressDao.listByAssociatedVpc(vpcId, true);
-                if (sourceNatIps != null && sourceNatIps.size() > 0) {
-                    hasSourceNat = true;
-                }
-            }
-
-            int ipsWithrules = 0;
-            int ipsStaticNat = 0;
-            for (IPAddressVO ip : userIps) {
-                if ( _rulesDao.countRulesByIpIdAndState(ip.getId(), FirewallRule.State.Active) > 0){
-                    ipsWithrules++;
-                }
-
-                // check onetoonenat and also check if the ip "add":false. If there are 2 PF rules remove and
-                // 1 static nat rule add
-                if (ip.isOneToOneNat() && ip.getRuleState() == null) {
-                    ipsStaticNat++;
-                }
-            }
-
-            final IpAssocCommand cmd = new IpAssocCommand(ipsToSend);
             cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId()));
-            cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, _routerControlHelper.getRouterIpInNetwork(associatedWithNetworkId, router.getId()));
+            cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, _routerControlHelper.getRouterIpInNetwork(ipAddrList.get(0).getNetworkId(), router.getId()));
             cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
             final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId());
             cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString());
 
-            // if there is 1 static nat then it will be checked for remove at the resource
+            setAccessDetailNetworkLastPublicIp(vlanLastIpMap, vlanTagKey, cmd);
-            if (ipsWithrules == 0 && ipsStaticNat == 0 && !hasSourceNat) {
-                // there is only one ip address for the network.
-                cmd.setAccessDetail(NetworkElementCommand.NETWORK_PUB_LAST_IP, "true");
-            } else {
-                cmd.setAccessDetail(NetworkElementCommand.NETWORK_PUB_LAST_IP, "false");
-            }
 
             cmds.addCommand(ipAssocCommand, cmd);
         }
     }
 
+    private void setAccessDetailNetworkLastPublicIp(Map<String, Boolean> vlanLastIpMap, String vlanTagKey, IpAssocCommand cmd) {
+        // if public ip is the last ip in the vlan which is used for static nat or has active rules,
+        // it will be checked for remove at the resource
+        Boolean lastIp = vlanLastIpMap.get(vlanTagKey);
+        if (lastIp == null) {
+            cmd.setAccessDetail(NetworkElementCommand.NETWORK_PUB_LAST_IP, "true");
+        } else {
+            cmd.setAccessDetail(NetworkElementCommand.NETWORK_PUB_LAST_IP, "false");
+        }
+    }
+
+    private Map<String, Boolean> getVlanLastIpMap(Long vpcId, Long guestNetworkId) {
+        // for network if the ips does not have any rules, then only last ip
+        final Map<String, Boolean> vlanLastIpMap = new HashMap<String, Boolean>();
+        final List<IPAddressVO> userIps;
+        if (vpcId != null) {
+            userIps = _ipAddressDao.listByAssociatedVpc(vpcId, null);
+        } else {
+            userIps = _ipAddressDao.listByAssociatedNetwork(guestNetworkId, null);
+        }
+        for (IPAddressVO ip : userIps) {
+            String vlanTag = _vlanDao.findById(ip.getVlanId()).getVlanTag();
+            Boolean lastIp = vlanLastIpMap.get(vlanTag);
+            if (lastIp != null && !lastIp) {
+                continue;
+            }
+            if (ip.isSourceNat()
+                    || _rulesDao.countRulesByIpIdAndState(ip.getId(), FirewallRule.State.Active) > 0
+                    || (ip.isOneToOneNat() && ip.getRuleState() == null)) {
+                vlanLastIpMap.put(vlanTag, false);
+            }
+        }
+        return vlanLastIpMap;
+    }
+
     public void createStaticRouteCommands(final List<StaticRouteProfile> staticRoutes, final DomainRouterVO router, final Commands cmds) {
         final SetStaticRouteCommand cmd = new SetStaticRouteCommand(staticRoutes);
         cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId()));

systemvm/debian/etc/sysctl.conf

@@ -27,6 +27,10 @@ net.ipv4.conf.default.send_redirects = 0
 net.ipv4.conf.all.secure_redirects = 0
 net.ipv4.conf.default.secure_redirects = 0
 
+# Promote secondary ip to be primary if primary IP is removed
+net.ipv4.conf.all.promote_secondaries = 1
+net.ipv4.conf.default.promote_secondaries = 1
+
 # For smooth transition of the vip address in case of a keepalived failover
 net.ipv4.ip_nonlocal_bind = 1
 

test/integration/component/test_multiple_subnets_in_isolated_network.py

@@ -0,0 +1,701 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Tests of acquiring IPs in multiple subnets for isolated network or vpc
+"""
+
+from nose.plugins.attrib import attr
+from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.lib.utils import (validateList,
+                              get_host_credentials,
+                              get_process_status,
+                              cleanup_resources)
+from marvin.lib.base import (Account,
+                             Domain,
+                             VirtualMachine,
+                             ServiceOffering,
+                             Zone,
+                             Network,
+                             NetworkOffering,
+                             VPC,
+                             VpcOffering,
+                             NATRule,
+                             PublicIPAddress,
+                             PublicIpRange)
+from marvin.lib.common import (get_domain,
+                               get_zone,
+                               get_free_vlan,
+                               get_template,
+                               list_hosts,
+                               list_routers)
+import logging
+import random
+class TestMultiplePublicIpSubnets(cloudstackTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.testClient = super(
+            TestMultiplePublicIpSubnets,
+            cls).getClsTestClient()
+        cls.apiclient = cls.testClient.getApiClient()
+        cls.services = cls.testClient.getParsedTestDataConfig()
+
+        zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests())
+        cls.zone = Zone(zone.__dict__)
+        cls.template = get_template(cls.apiclient, cls.zone.id)
+        cls._cleanup = []
+        cls.skip = False
+
+        if str(cls.zone.securitygroupsenabled) == "True":
+            cls.skip = True
+            return
+
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+        if cls.hypervisor.lower() not in ['kvm']:
+            cls.skip = True
+            return
+
+        cls.logger = logging.getLogger("TestMultiplePublicIpSubnets")
+        cls.stream_handler = logging.StreamHandler()
+        cls.logger.setLevel(logging.DEBUG)
+        cls.logger.addHandler(cls.stream_handler)
+
+        # Get Zone, Domain and templates
+        cls.domain = get_domain(cls.apiclient)
+
+        # Create small service offering
+        cls.service_offering = ServiceOffering.create(
+            cls.apiclient,
+            cls.services["service_offerings"]["small"]
+        )
+        cls._cleanup.append(cls.service_offering)
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls._cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def setUp(cls):
+        if cls.skip:
+            cls.skipTest("Test can be run only on advanced zone and KVM hypervisor")
+        cls.apiclient = cls.testClient.getApiClient()
+        cls.cleanup = []
+        return
+
+    def tearDown(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls.cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def get_routers(self, network_id):
+        routers = list_routers(
+            self.apiclient,
+            networkid=network_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers
+
+    def get_vpc_routers(self, vpc_id):
+        routers = list_routers(
+            self.apiclient,
+            vpcid=vpc_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers
+
+    def get_router_host(self, router):
+        self.assertEqual(
+            router.state,
+            'Running',
+            "Check list router response for router state"
+        )
+        hosts = list_hosts(
+            self.apiclient,
+            id=router.hostid)
+        self.assertEqual(
+            isinstance(hosts, list),
+            True,
+            "Check for list hosts response return valid data")
+        host = hosts[0]
+        if host.hypervisor.lower() not in "kvm":
+            return
+        host.user, host.password = get_host_credentials(self.config, host.ipaddress)
+        host.port=22
+        return host
+
+    def get_router_ips(self, router):
+        guestIp = None
+        controlIp = None
+        sourcenatIp = None
+        for nic in router.nic:
+            if guestIp is None and nic.traffictype == "Guest":
+                guestIp = nic.ipaddress
+            elif nic.traffictype == "Control":
+                controlIp = nic.ipaddress
+            elif sourcenatIp is None and nic.traffictype == "Public":
+                sourcenatIp = nic.ipaddress
+        return guestIp, controlIp, sourcenatIp
+
+    def verify_network_interfaces_in_router(self, router, host, expectedNics):
+        command = 'ip link show |grep BROADCAST | cut -d ":" -f2 |tr -d " "|tr "\n" ","'
+        self.logger.debug("Executing command '%s'" % command)
+        result = get_process_status(
+            host.ipaddress,
+            host.port,
+            host.user,
+            host.password,
+            router.linklocalip,
+            command)
+        self.assertTrue(len(result) > 0 and result[0] == expectedNics, "Expected nics are %s but actual nics are %s" %(expectedNics, result))
+
+    def verify_ip_address_in_router(self, router, host, ipaddress, device, isExist=True):
+        command = 'ip addr show %s |grep "inet "|cut -d " " -f6 |cut -d "/" -f1 |grep -w %s' % (device,ipaddress)
+        self.logger.debug("Executing command '%s'" % command)
+        result = get_process_status(
+            host.ipaddress,
+            host.port,
+            host.user,
+            host.password,
+            router.linklocalip,
+            command)
+        self.assertEqual(len(result) > 0 and result[0] == ipaddress, isExist, "ip %s verification failed" % ipaddress)
+
+    def get_free_ipaddress(self, vlanId):
+        ipaddresses = PublicIPAddress.list(
+            self.apiclient,
+            vlanid=vlanId,
+            state='Free'
+        )
+        self.assertEqual(
+            isinstance(ipaddresses, list),
+            True,
+            "List ipaddresses should return a valid response for Free ipaddresses"
+             )
+        random.shuffle(ipaddresses)
+        return ipaddresses[0].ipaddress
+
+    @attr(tags=["advanced"], required_hardware="false")
+    def test_01_acquire_public_ips_in_isolated_network_with_single_vr(self):
+        """ Acquire IPs in multiple subnets in isolated networks with single VR
+
+        # Steps
+        # 1. Create network offering with single VR, and enable it
+        # 2. create isolated network with the network offering
+        # 3. create a vm in the network.
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP
+        # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP and new ip
+        # 5. remove the port forwarding rule, and release the new ip
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic IP, eth2 -> source nat IP
+
+        # 6. create new public ip range 1
+        # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1
+        # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2,
+        # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2, new ip 3
+        # 10. release new ip 2
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 3
+        # 11. release new ip 1
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3
+        # 12. create new public ip range 2
+        # 13. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4
+        # 14. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/5
+        # 15. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/5/6
+        # 16. release new ip 5
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/6
+        # 17. release new ip 4
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 6
+        """
+
+        # Create new domain1
+        self.domain1 = Domain.create(
+            self.apiclient,
+            services=self.services["acl"]["domain1"],
+            parentdomainid=self.domain.id)
+        # Create account1
+        self.account1 = Account.create(
+            self.apiclient,
+            self.services["acl"]["accountD1"],
+            domainid=self.domain1.id
+        )
+        self.cleanup.append(self.account1)
+        self.cleanup.append(self.domain1)
+
+        # 1. Create network offering with single VR, and enable it
+        self.network_offering = NetworkOffering.create(
+            self.apiclient,
+            self.services["isolated_network_offering"],
+        )
+        self.network_offering.update(self.apiclient, state='Enabled')
+        self.cleanup.append(self.network_offering)
+
+        # 2. create isolated network with the network offering
+        self.services["network"]["zoneid"] = self.zone.id
+        self.services["network"]["networkoffering"] = self.network_offering.id
+        self.network1 = Network.create(
+            self.apiclient,
+            self.services["network"],
+            self.account1.name,
+            self.account1.domainid
+        )
+
+        # 3. create a vm in the network.
+        try:
+            self.virtual_machine1 = VirtualMachine.create(
+                self.apiclient,
+                self.services["virtual_machine"],
+                accountid=self.account1.name,
+                domainid=self.account1.domainid,
+                serviceofferingid=self.service_offering.id,
+                templateid=self.template.id,
+                zoneid=self.zone.id,
+                networkids=self.network1.id
+            )
+        except Exception as e:
+            self.fail("Exception while deploying virtual machine: %s" % e)
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+
+        # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm
+        ipaddress = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+        )
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress.ipaddress.id,
+            openfirewall=True
+        )
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP/new ip
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth2", True)
+
+        # 5. release the new ip
+        ipaddress.delete(self.apiclient)
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth2", False)
+
+        # 6. create new public ip range 1
+        self.services["publiciprange"]["zoneid"] = self.zone.id
+        self.services["publiciprange"]["forvirtualnetwork"] = "true"
+        random_subnet_number = random.randrange(10,20)
+        self.services["publiciprange"]["vlan"] = get_free_vlan(
+            self.apiclient,
+            self.zone.id)[1]
+        self.services["publiciprange"]["gateway"] = "172.16." + str(random_subnet_number) + ".1"
+        self.services["publiciprange"]["startip"] = "172.16." + str(random_subnet_number) + ".2"
+        self.services["publiciprange"]["endip"] = "172.16." + str(random_subnet_number) + ".10"
+        self.services["publiciprange"]["netmask"] = "255.255.255.0"
+        self.public_ip_range1 = PublicIpRange.create(
+            self.apiclient,
+            self.services["publiciprange"]
+        )
+        self.cleanup.append(self.public_ip_range1)
+
+        # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        ip_address_1 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_1 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_1
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_1.ipaddress.id,
+            openfirewall=True
+        )
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+
+        # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2,
+        ip_address_2 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_2 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_2
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_2.ipaddress.id,
+            openfirewall=True
+        )
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True)
+
+        # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2, new ip 3
+        ip_address_3 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_3 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_3
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_3.ipaddress.id,
+            openfirewall=True
+        )
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 10. release new ip 2
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 3
+        ipaddress_2.delete(self.apiclient)
+
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 11. release new ip 1
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3
+        ipaddress_1.delete(self.apiclient)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 12. create new public ip range 2
+        self.services["publiciprange"]["zoneid"] = self.zone.id
+        self.services["publiciprange"]["forvirtualnetwork"] = "true"
+        self.services["publiciprange"]["vlan"] = get_free_vlan(
+            self.apiclient,
+            self.zone.id)[1]
+        self.services["publiciprange"]["gateway"] = "172.16." + str(random_subnet_number + 1) + ".1"
+        self.services["publiciprange"]["startip"] = "172.16." + str(random_subnet_number + 1) + ".2"
+        self.services["publiciprange"]["endip"] = "172.16." + str(random_subnet_number + 1) + ".10"
+        self.services["publiciprange"]["netmask"] = "255.255.255.0"
+        self.public_ip_range2 = PublicIpRange.create(
+            self.apiclient,
+            self.services["publiciprange"]
+        )
+        self.cleanup.append(self.public_ip_range2)
+
+        # 13. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4
+
+        ip_address_4 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_4 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_4
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_4.ipaddress.id,
+            openfirewall=True
+        )
+
+
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True)
+
+        # 14. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/5
+        ip_address_5 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_5 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_5
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_5.ipaddress.id,
+            openfirewall=True
+        )
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", True)
+
+        # 15. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/5/6
+        ip_address_6 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_6 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_6
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_6.ipaddress.id,
+            openfirewall=True
+        )
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
+
+        # 16. release new ip 5
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/6
+        ipaddress_5.delete(self.apiclient)
+
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
+
+        # 17. release new ip 4
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 6
+        ipaddress_4.delete(self.apiclient)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
+
+        # 18. release new ip 3
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth4 -> new ip 6
+        ipaddress_3.delete(self.apiclient)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
+
+        # 19. restart network
+        self.network1.restart(self.apiclient)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
+
+        # 20. restart network with cleanup
+        self.network1.restart(self.apiclient, cleanup=True)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True)
+
+        # 21. restart network with cleanup, makeredundant=true
+        self.network1.restart(self.apiclient, cleanup=True, makeredundant=True)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True)

test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py

@@ -0,0 +1,687 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Tests of acquiring IPs in multiple subnets for isolated network or vpc
+"""
+
+from nose.plugins.attrib import attr
+from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.lib.utils import (validateList,
+                              get_host_credentials,
+                              get_process_status,
+                              cleanup_resources)
+from marvin.lib.base import (Account,
+                             Domain,
+                             VirtualMachine,
+                             ServiceOffering,
+                             Zone,
+                             Network,
+                             NetworkOffering,
+                             VPC,
+                             VpcOffering,
+                             NATRule,
+                             PublicIPAddress,
+                             PublicIpRange)
+from marvin.lib.common import (get_domain,
+                               get_zone,
+                               get_free_vlan,
+                               get_template,
+                               list_hosts,
+                               list_routers)
+import logging
+import random
+class TestMultiplePublicIpSubnets(cloudstackTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.testClient = super(
+            TestMultiplePublicIpSubnets,
+            cls).getClsTestClient()
+        cls.apiclient = cls.testClient.getApiClient()
+        cls.services = cls.testClient.getParsedTestDataConfig()
+
+        zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests())
+        cls.zone = Zone(zone.__dict__)
+        cls.template = get_template(cls.apiclient, cls.zone.id)
+        cls._cleanup = []
+        cls.skip = False
+
+        if str(cls.zone.securitygroupsenabled) == "True":
+            cls.skip = True
+            return
+
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+        if cls.hypervisor.lower() not in ['kvm']:
+            cls.skip = True
+            return
+
+        cls.logger = logging.getLogger("TestMultiplePublicIpSubnets")
+        cls.stream_handler = logging.StreamHandler()
+        cls.logger.setLevel(logging.DEBUG)
+        cls.logger.addHandler(cls.stream_handler)
+
+        # Get Zone, Domain and templates
+        cls.domain = get_domain(cls.apiclient)
+
+        # Create small service offering
+        cls.service_offering = ServiceOffering.create(
+            cls.apiclient,
+            cls.services["service_offerings"]["small"]
+        )
+        cls._cleanup.append(cls.service_offering)
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls._cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def setUp(cls):
+        if cls.skip:
+            cls.skipTest("Test can be run only on advanced zone and KVM hypervisor")
+        cls.apiclient = cls.testClient.getApiClient()
+        cls.cleanup = []
+        return
+
+    def tearDown(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls.cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def get_routers(self, network_id):
+        routers = list_routers(
+            self.apiclient,
+            networkid=network_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers
+
+    def get_vpc_routers(self, vpc_id):
+        routers = list_routers(
+            self.apiclient,
+            vpcid=vpc_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers
+
+    def get_router_host(self, router):
+        self.assertEqual(
+            router.state,
+            'Running',
+            "Check list router response for router state"
+        )
+        hosts = list_hosts(
+            self.apiclient,
+            id=router.hostid)
+        self.assertEqual(
+            isinstance(hosts, list),
+            True,
+            "Check for list hosts response return valid data")
+        host = hosts[0]
+        if host.hypervisor.lower() not in "kvm":
+            return
+        host.user, host.password = get_host_credentials(self.config, host.ipaddress)
+        host.port=22
+        return host
+
+    def get_router_ips(self, router):
+        guestIp = None
+        controlIp = None
+        sourcenatIp = None
+        for nic in router.nic:
+            if guestIp is None and nic.traffictype == "Guest":
+                guestIp = nic.ipaddress
+            elif nic.traffictype == "Control":
+                controlIp = nic.ipaddress
+            elif sourcenatIp is None and nic.traffictype == "Public":
+                sourcenatIp = nic.ipaddress
+        return guestIp, controlIp, sourcenatIp
+
+    def verify_network_interfaces_in_router(self, router, host, expectedNics):
+        command = 'ip link show |grep BROADCAST | cut -d ":" -f2 |tr -d " "|tr "\n" ","'
+        self.logger.debug("Executing command '%s'" % command)
+        result = get_process_status(
+            host.ipaddress,
+            host.port,
+            host.user,
+            host.password,
+            router.linklocalip,
+            command)
+        self.assertTrue(len(result) > 0 and result[0] == expectedNics, "Expected nics are %s but actual nics are %s" %(expectedNics, result))
+
+    def verify_ip_address_in_router(self, router, host, ipaddress, device, isExist=True):
+        command = 'ip addr show %s |grep "inet "|cut -d " " -f6 |cut -d "/" -f1 |grep -w %s' % (device,ipaddress)
+        self.logger.debug("Executing command '%s'" % command)
+        result = get_process_status(
+            host.ipaddress,
+            host.port,
+            host.user,
+            host.password,
+            router.linklocalip,
+            command)
+        self.assertEqual(len(result) > 0 and result[0] == ipaddress, isExist, "ip %s verification failed" % ipaddress)
+
+    def get_free_ipaddress(self, vlanId):
+        ipaddresses = PublicIPAddress.list(
+            self.apiclient,
+            vlanid=vlanId,
+            state='Free'
+        )
+        self.assertEqual(
+            isinstance(ipaddresses, list),
+            True,
+            "List ipaddresses should return a valid response for Free ipaddresses"
+             )
+        random.shuffle(ipaddresses)
+        return ipaddresses[0].ipaddress
+
+    @attr(tags=["advanced"], required_hardware="false")
+    def test_02_acquire_public_ips_in_isolated_network_with_redundant_vrs(self):
+        """ Acquire IPs in multiple subnets in isolated networks with redundant VRs
+
+        # Steps
+        # 1. Create network offering with single VR, and enable it
+        # 2. create isolated network with the network offering
+        # 3. create a vm in the network.
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP
+        # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP and new ip
+        # 5. remove the port forwarding rule, and release the new ip
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic IP, eth2 -> source nat IP
+
+        # 6. create new public ip range 1
+        # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1
+        # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2,
+        # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2, new ip 3
+        # 10. release new ip 2
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 3
+        # 11. release new ip 1
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3
+        # 12. create new public ip range 2
+        # 13. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4
+        # 14. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/5
+        # 15. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/5/6
+        # 16. release new ip 5
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/6
+        # 17. release new ip 4
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 6
+        """
+
+        # Create new domain1
+        self.domain1 = Domain.create(
+            self.apiclient,
+            services=self.services["acl"]["domain1"],
+            parentdomainid=self.domain.id)
+        # Create account1
+        self.account1 = Account.create(
+            self.apiclient,
+            self.services["acl"]["accountD1"],
+            domainid=self.domain1.id
+        )
+        self.cleanup.append(self.account1)
+        self.cleanup.append(self.domain1)
+
+        # 1. Create network offering with redundant VRs, and enable it
+        self.network_offering = NetworkOffering.create(
+            self.apiclient,
+            self.services["nw_off_isolated_RVR"],
+        )
+        self.network_offering.update(self.apiclient, state='Enabled')
+        self.cleanup.append(self.network_offering)
+
+        # 2. create isolated network with the network offering
+        self.services["network"]["zoneid"] = self.zone.id
+        self.services["network"]["networkoffering"] = self.network_offering.id
+        self.network1 = Network.create(
+            self.apiclient,
+            self.services["network"],
+            self.account1.name,
+            self.account1.domainid
+        )
+
+        # 3. create a vm in the network.
+        try:
+            self.virtual_machine1 = VirtualMachine.create(
+                self.apiclient,
+                self.services["virtual_machine"],
+                accountid=self.account1.name,
+                domainid=self.account1.domainid,
+                serviceofferingid=self.service_offering.id,
+                templateid=self.template.id,
+                zoneid=self.zone.id,
+                networkids=self.network1.id
+            )
+        except Exception as e:
+            self.fail("Exception while deploying virtual machine: %s" % e)
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+
+        # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm
+        ipaddress = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+        )
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress.ipaddress.id,
+            openfirewall=True
+        )
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP/new ip
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth2", True)
+
+        # 5. release the new ip
+        ipaddress.delete(self.apiclient)
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth2", False)
+
+        # 6. create new public ip range 1
+        self.services["publiciprange"]["zoneid"] = self.zone.id
+        self.services["publiciprange"]["forvirtualnetwork"] = "true"
+        random_subnet_number = random.randrange(10,20)
+        self.services["publiciprange"]["vlan"] = get_free_vlan(
+            self.apiclient,
+            self.zone.id)[1]
+        self.services["publiciprange"]["gateway"] = "172.16." + str(random_subnet_number) + ".1"
+        self.services["publiciprange"]["startip"] = "172.16." + str(random_subnet_number) + ".2"
+        self.services["publiciprange"]["endip"] = "172.16." + str(random_subnet_number) + ".10"
+        self.services["publiciprange"]["netmask"] = "255.255.255.0"
+        self.public_ip_range1 = PublicIpRange.create(
+            self.apiclient,
+            self.services["publiciprange"]
+        )
+        self.cleanup.append(self.public_ip_range1)
+
+        # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        ip_address_1 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_1 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_1
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_1.ipaddress.id,
+            openfirewall=True
+        )
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+
+        # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2,
+        ip_address_2 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_2 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_2
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_2.ipaddress.id,
+            openfirewall=True
+        )
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True)
+
+        # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2, new ip 3
+        ip_address_3 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_3 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_3
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_3.ipaddress.id,
+            openfirewall=True
+        )
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 10. release new ip 2
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 3
+        ipaddress_2.delete(self.apiclient)
+
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 11. release new ip 1
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3
+        ipaddress_1.delete(self.apiclient)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 12. create new public ip range 2
+        self.services["publiciprange"]["zoneid"] = self.zone.id
+        self.services["publiciprange"]["forvirtualnetwork"] = "true"
+        self.services["publiciprange"]["vlan"] = get_free_vlan(
+            self.apiclient,
+            self.zone.id)[1]
+        self.services["publiciprange"]["gateway"] = "172.16." + str(random_subnet_number + 1) + ".1"
+        self.services["publiciprange"]["startip"] = "172.16." + str(random_subnet_number + 1) + ".2"
+        self.services["publiciprange"]["endip"] = "172.16." + str(random_subnet_number + 1) + ".10"
+        self.services["publiciprange"]["netmask"] = "255.255.255.0"
+        self.public_ip_range2 = PublicIpRange.create(
+            self.apiclient,
+            self.services["publiciprange"]
+        )
+        self.cleanup.append(self.public_ip_range2)
+
+        # 13. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4
+
+        ip_address_4 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_4 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_4
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_4.ipaddress.id,
+            openfirewall=True
+        )
+
+
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True)
+
+        # 14. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/5
+        ip_address_5 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_5 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_5
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_5.ipaddress.id,
+            openfirewall=True
+        )
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", True)
+
+        # 15. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/5/6
+        ip_address_6 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_6 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.network1.id,
+            ipaddress=ip_address_6
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_6.ipaddress.id,
+            openfirewall=True
+        )
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
+
+        # 16. release new ip 5
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 4/6
+        ipaddress_5.delete(self.apiclient)
+
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
+
+        # 17. release new ip 4
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 6
+        ipaddress_4.delete(self.apiclient)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
+
+        # 18. release new ip 3
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth4,"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth4 -> new ip 6
+        ipaddress_3.delete(self.apiclient)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
+
+        # 19. restart network
+        self.network1.restart(self.apiclient)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
+
+        # 20. restart network with cleanup
+        self.network1.restart(self.apiclient, cleanup=True)
+        routers = self.get_routers(self.network1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True)

test/integration/component/test_multiple_subnets_in_vpc.py

@@ -0,0 +1,817 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Tests of acquiring IPs in multiple subnets for isolated network or vpc
+"""
+
+from nose.plugins.attrib import attr
+from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.lib.utils import (validateList,
+                              get_host_credentials,
+                              get_process_status,
+                              cleanup_resources)
+from marvin.lib.base import (Account,
+                             Domain,
+                             VirtualMachine,
+                             ServiceOffering,
+                             Zone,
+                             Network,
+                             NetworkOffering,
+                             VPC,
+                             VpcOffering,
+                             NATRule,
+                             PublicIPAddress,
+                             PublicIpRange)
+from marvin.lib.common import (get_domain,
+                               get_zone,
+                               get_free_vlan,
+                               get_template,
+                               list_hosts,
+                               list_routers)
+import logging
+import random
+class TestMultiplePublicIpSubnets(cloudstackTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.testClient = super(
+            TestMultiplePublicIpSubnets,
+            cls).getClsTestClient()
+        cls.apiclient = cls.testClient.getApiClient()
+        cls.services = cls.testClient.getParsedTestDataConfig()
+
+        zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests())
+        cls.zone = Zone(zone.__dict__)
+        cls.template = get_template(cls.apiclient, cls.zone.id)
+        cls._cleanup = []
+        cls.skip = False
+
+        if str(cls.zone.securitygroupsenabled) == "True":
+            cls.skip = True
+            return
+
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+        if cls.hypervisor.lower() not in ['kvm']:
+            cls.skip = True
+            return
+
+        cls.logger = logging.getLogger("TestMultiplePublicIpSubnets")
+        cls.stream_handler = logging.StreamHandler()
+        cls.logger.setLevel(logging.DEBUG)
+        cls.logger.addHandler(cls.stream_handler)
+
+        # Get Zone, Domain and templates
+        cls.domain = get_domain(cls.apiclient)
+
+        # Create small service offering
+        cls.service_offering = ServiceOffering.create(
+            cls.apiclient,
+            cls.services["service_offerings"]["small"]
+        )
+        cls._cleanup.append(cls.service_offering)
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls._cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def setUp(cls):
+        if cls.skip:
+            cls.skipTest("Test can be run only on advanced zone and KVM hypervisor")
+        cls.apiclient = cls.testClient.getApiClient()
+        cls.cleanup = []
+        return
+
+    def tearDown(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls.cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def get_routers(self, network_id):
+        routers = list_routers(
+            self.apiclient,
+            networkid=network_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers
+
+    def get_vpc_routers(self, vpc_id):
+        routers = list_routers(
+            self.apiclient,
+            vpcid=vpc_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers
+
+    def get_router_host(self, router):
+        self.assertEqual(
+            router.state,
+            'Running',
+            "Check list router response for router state"
+        )
+        hosts = list_hosts(
+            self.apiclient,
+            id=router.hostid)
+        self.assertEqual(
+            isinstance(hosts, list),
+            True,
+            "Check for list hosts response return valid data")
+        host = hosts[0]
+        if host.hypervisor.lower() not in "kvm":
+            return
+        host.user, host.password = get_host_credentials(self.config, host.ipaddress)
+        host.port=22
+        return host
+
+    def get_vpc_router_ips(self, router):
+        controlIp = None
+        sourcenatIp = None
+        tier1_Ip = None
+        tier2_Ip = None
+        for nic in router.nic:
+            if nic.traffictype == "Guest" and nic.ipaddress.startswith("10.250.1."):
+                tier1_Ip = nic.ipaddress
+            elif nic.traffictype == "Guest" and nic.ipaddress.startswith("10.250.2."):
+                tier2_Ip = nic.ipaddress
+            elif nic.traffictype == "Control":
+                controlIp = nic.ipaddress
+            elif sourcenatIp is None and nic.traffictype == "Public":
+                sourcenatIp = nic.ipaddress
+        return controlIp, sourcenatIp, tier1_Ip, tier2_Ip
+
+    def verify_network_interfaces_in_router(self, router, host, expectedNics):
+        command = 'ip link show |grep BROADCAST | cut -d ":" -f2 |tr -d " "|tr "\n" ","'
+        self.logger.debug("Executing command '%s'" % command)
+        result = get_process_status(
+            host.ipaddress,
+            host.port,
+            host.user,
+            host.password,
+            router.linklocalip,
+            command)[0]
+        self.assertEqual(result, expectedNics, "Expected nics are %s but actual nics are %s" %(expectedNics, result))
+
+    def verify_ip_address_in_router(self, router, host, ipaddress, device, isExist=True):
+        command = 'ip addr show %s |grep "inet "|cut -d " " -f6 |cut -d "/" -f1 |grep -w %s' % (device,ipaddress)
+        self.logger.debug("Executing command '%s'" % command)
+        result = get_process_status(
+            host.ipaddress,
+            host.port,
+            host.user,
+            host.password,
+            router.linklocalip,
+            command)
+        self.assertEqual(len(result) > 0 and result[0] == ipaddress, isExist, "ip %s verification failed" % ipaddress)
+
+    def get_free_ipaddress(self, vlanId):
+        ipaddresses = PublicIPAddress.list(
+            self.apiclient,
+            vlanid=vlanId,
+            state='Free'
+        )
+        self.assertEqual(
+            isinstance(ipaddresses, list),
+            True,
+            "List ipaddresses should return a valid response for Free ipaddresses"
+             )
+        random.shuffle(ipaddresses)
+        return ipaddresses[0].ipaddress
+
+    @attr(tags=["advanced"], required_hardware="false")
+    def test_03_acquire_public_ips_in_vpc_with_single_vr(self):
+        """ Acquire IPs in multiple subnets in vpc with single VR
+        # Steps
+        # 1. get vpc offering with single VR
+        # 2. create a vpc with the vpc offering
+        #   verify the available nics in VR should be "eth0,eth1"
+        #   verify the IPs in VR. eth0 -> control nic, eth1 -> source nat IP
+        # 3. create a tier in the vpc, and create a vm in the tier.
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> control nic, eth1 -> source nat IP, eth2 -> tier 1
+        # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> control nic, eth1 -> source nat IP, eth2 -> tier 1
+        # 5. remove the port forwarding rule, and release the new ip
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> control nic, eth1 -> source nat IP, eth2 -> tier 1
+
+        # 6. create new public ip range 1
+        # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 1
+        # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 1/2
+        # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 1/2/3
+        # 10. release new ip 2
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 1/3
+        # 11. release new ip 1
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3
+
+        # 12. create a tier 2 in the vpc, and create a vm 2 in the tier2.
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2
+
+        # 13. create new public ip range 2
+        # 14. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4
+        # 15. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/5
+        # 16. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/5/6
+        # 17. release new ip 5
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/6
+        # 18. release new ip 4
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 6
+        """
+
+        # Create new domain1
+        self.domain1 = Domain.create(
+            self.apiclient,
+            services=self.services["acl"]["domain1"],
+            parentdomainid=self.domain.id)
+        # Create account1
+        self.account1 = Account.create(
+            self.apiclient,
+            self.services["acl"]["accountD1"],
+            domainid=self.domain1.id
+        )
+        self.cleanup.append(self.account1)
+        self.cleanup.append(self.domain1)
+
+        # Create new domain1
+        self.domain1 = Domain.create(
+            self.apiclient,
+            services=self.services["acl"]["domain1"],
+            parentdomainid=self.domain.id)
+        # Create account1
+        self.account1 = Account.create(
+            self.apiclient,
+            self.services["acl"]["accountD1"],
+            domainid=self.domain1.id
+        )
+        self.cleanup.append(self.account1)
+        self.cleanup.append(self.domain1)
+
+        # 1. get vpc offering with single VR
+        vpc_offering = VpcOffering.list(self.apiclient, name="Default VPC offering")
+
+        # 2. create a vpc with the vpc offering
+        self.services["vpc"]["cidr"] = "10.250.0.0/16"
+        self.vpc1 = VPC.create(
+            apiclient=self.apiclient,
+            services=self.services["vpc"],
+            vpcofferingid=vpc_offering[0].id,
+            zoneid=self.zone.id,
+            account=self.account1.name,
+            domainid=self.account1.domainid
+        )
+
+        #   verify the available nics in VR should be "eth0,eth1"
+        #   verify the IPs in VR. eth0 -> control nic, eth1 -> source nat IP
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+
+        # 3. create a tier in the vpc, and create a vm in the tier.
+        vpc_tier_offerings = NetworkOffering.list(self.apiclient, name="DefaultIsolatedNetworkOfferingForVpcNetworks")
+        self.assertTrue(vpc_tier_offerings is not None and len(vpc_tier_offerings) > 0, "No VPC based network offering")
+        vpc_tier_offering = vpc_tier_offerings[0]
+        self.services["vpctier"] = {}
+        self.services["vpctier"]["name"] = "vpc-tier-1"
+        self.services["vpctier"]["displaytext"] = "vpc-tier-1"
+        vpc_tier_1 = Network.create(
+            apiclient=self.apiclient,
+            services=self.services["vpctier"],
+            accountid=self.account1.name,
+            domainid=self.domain1.id,
+            networkofferingid=vpc_tier_offering.id,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            gateway="10.250.1.1",
+            netmask="255.255.255.0"
+        )
+
+        try:
+            self.virtual_machine1 = VirtualMachine.create(
+                self.apiclient,
+                self.services["virtual_machine"],
+                accountid=self.account1.name,
+                domainid=self.account1.domainid,
+                serviceofferingid=self.service_offering.id,
+                templateid=self.template.id,
+                zoneid=self.zone.id,
+                networkids=vpc_tier_1.id
+            )
+        except Exception as e:
+            self.fail("Exception while deploying virtual machine: %s" % e)
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+
+        # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm
+        ipaddress = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+        )
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress.ipaddress.id,
+            networkid=vpc_tier_1.id
+        )
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP/new ip
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth1", True)
+
+        # 5. release the new ip
+        ipaddress.delete(self.apiclient)
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth1", False)
+
+        # 6. create new public ip range 1
+        self.services["publiciprange"]["zoneid"] = self.zone.id
+        self.services["publiciprange"]["forvirtualnetwork"] = "true"
+        random_subnet_number = random.randrange(10,20)
+        self.services["publiciprange"]["vlan"] = get_free_vlan(
+            self.apiclient,
+            self.zone.id)[1]
+        self.services["publiciprange"]["gateway"] = "172.16." + str(random_subnet_number) + ".1"
+        self.services["publiciprange"]["startip"] = "172.16." + str(random_subnet_number) + ".2"
+        self.services["publiciprange"]["endip"] = "172.16." + str(random_subnet_number) + ".10"
+        self.services["publiciprange"]["netmask"] = "255.255.255.0"
+        self.public_ip_range1 = PublicIpRange.create(
+            self.apiclient,
+            self.services["publiciprange"]
+        )
+        self.cleanup.append(self.public_ip_range1)
+
+        # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        ip_address_1 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_1 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_1
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_1.ipaddress.id,
+            networkid=vpc_tier_1.id
+        )
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+
+        # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2,
+        ip_address_2 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_2 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_2
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_2.ipaddress.id,
+            networkid=vpc_tier_1.id
+        )
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True)
+
+        # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2, new ip 3
+        ip_address_3 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_3 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_3
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_3.ipaddress.id,
+            networkid=vpc_tier_1.id
+        )
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 10. release new ip 2
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 3
+        ipaddress_2.delete(self.apiclient)
+
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 11. release new ip 1
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3
+        ipaddress_1.delete(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 12. create a tier in the vpc, and create a vm in the tier.
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2
+        self.services["vpctier"]["name"] = "vpc-tier-2"
+        self.services["vpctier"]["displaytext"] = "vpc-tier-2"
+        vpc_tier_2 = Network.create(
+            apiclient=self.apiclient,
+            services=self.services["vpctier"],
+            accountid=self.account1.name,
+            domainid=self.domain1.id,
+            networkofferingid=vpc_tier_offering.id,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            gateway="10.250.2.1",
+            netmask="255.255.255.0"
+        )
+
+        try:
+            self.virtual_machine1 = VirtualMachine.create(
+                self.apiclient,
+                self.services["virtual_machine"],
+                accountid=self.account1.name,
+                domainid=self.account1.domainid,
+                serviceofferingid=self.service_offering.id,
+                templateid=self.template.id,
+                zoneid=self.zone.id,
+                networkids=vpc_tier_2.id
+            )
+        except Exception as e:
+            self.fail("Exception while deploying virtual machine: %s" % e)
+
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+
+        # 13. create new public ip range 2
+        self.services["publiciprange"]["zoneid"] = self.zone.id
+        self.services["publiciprange"]["forvirtualnetwork"] = "true"
+        self.services["publiciprange"]["vlan"] = get_free_vlan(
+            self.apiclient,
+            self.zone.id)[1]
+        self.services["publiciprange"]["gateway"] = "172.16." + str(random_subnet_number + 1) + ".1"
+        self.services["publiciprange"]["startip"] = "172.16." + str(random_subnet_number + 1) + ".2"
+        self.services["publiciprange"]["endip"] = "172.16." + str(random_subnet_number + 1) + ".10"
+        self.services["publiciprange"]["netmask"] = "255.255.255.0"
+        self.public_ip_range2 = PublicIpRange.create(
+            self.apiclient,
+            self.services["publiciprange"]
+        )
+        self.cleanup.append(self.public_ip_range2)
+
+        # 14. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4
+        ip_address_4 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_4 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_4
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_4.ipaddress.id,
+            networkid=vpc_tier_2.id
+        )
+
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True)
+
+        # 15. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/5
+        ip_address_5 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_5 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_5
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_5.ipaddress.id,
+            networkid=vpc_tier_2.id
+        )
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", True)
+
+        # 16. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/5/6
+        ip_address_6 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_6 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_6
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_6.ipaddress.id,
+            networkid=vpc_tier_2.id
+        )
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 17. release new ip 5
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/6
+        ipaddress_5.delete(self.apiclient)
+
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 18. release new ip 4
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 6
+        ipaddress_4.delete(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 19. release new ip 3
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth4 -> tier 2, eth5 -> new ip 6
+        ipaddress_3.delete(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 20. restart tier1
+        vpc_tier_1.restart(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        #21. restart tier2
+        vpc_tier_2.restart(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 22. restart VPC
+        self.vpc1.restart(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 23. restart VPC with cleanup
+        self.vpc1.restart(self.apiclient, cleanup=True)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+
+        # 24. restart VPC with cleanup, makeredundant=true
+        self.vpc1.restart(self.apiclient, cleanup=True, makeredundant=True)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)

test/integration/component/test_multiple_subnets_in_vpc_rvr.py

@@ -0,0 +1,817 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Tests of acquiring IPs in multiple subnets for isolated network or vpc
+"""
+
+from nose.plugins.attrib import attr
+from marvin.cloudstackTestCase import cloudstackTestCase, unittest
+from marvin.lib.utils import (validateList,
+                              get_host_credentials,
+                              get_process_status,
+                              cleanup_resources)
+from marvin.lib.base import (Account,
+                             Domain,
+                             VirtualMachine,
+                             ServiceOffering,
+                             Zone,
+                             Network,
+                             NetworkOffering,
+                             VPC,
+                             VpcOffering,
+                             NATRule,
+                             PublicIPAddress,
+                             PublicIpRange)
+from marvin.lib.common import (get_domain,
+                               get_zone,
+                               get_free_vlan,
+                               get_template,
+                               list_hosts,
+                               list_routers)
+import logging
+import random
+class TestMultiplePublicIpSubnets(cloudstackTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.testClient = super(
+            TestMultiplePublicIpSubnets,
+            cls).getClsTestClient()
+        cls.apiclient = cls.testClient.getApiClient()
+        cls.services = cls.testClient.getParsedTestDataConfig()
+
+        zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests())
+        cls.zone = Zone(zone.__dict__)
+        cls.template = get_template(cls.apiclient, cls.zone.id)
+        cls._cleanup = []
+        cls.skip = False
+
+        if str(cls.zone.securitygroupsenabled) == "True":
+            cls.skip = True
+            return
+
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+        if cls.hypervisor.lower() not in ['kvm']:
+            cls.skip = True
+            return
+
+        cls.logger = logging.getLogger("TestMultiplePublicIpSubnets")
+        cls.stream_handler = logging.StreamHandler()
+        cls.logger.setLevel(logging.DEBUG)
+        cls.logger.addHandler(cls.stream_handler)
+
+        # Get Zone, Domain and templates
+        cls.domain = get_domain(cls.apiclient)
+
+        # Create small service offering
+        cls.service_offering = ServiceOffering.create(
+            cls.apiclient,
+            cls.services["service_offerings"]["small"]
+        )
+        cls._cleanup.append(cls.service_offering)
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls._cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def setUp(cls):
+        if cls.skip:
+            cls.skipTest("Test can be run only on advanced zone and KVM hypervisor")
+        cls.apiclient = cls.testClient.getApiClient()
+        cls.cleanup = []
+        return
+
+    def tearDown(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls.cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def get_routers(self, network_id):
+        routers = list_routers(
+            self.apiclient,
+            networkid=network_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers
+
+    def get_vpc_routers(self, vpc_id):
+        routers = list_routers(
+            self.apiclient,
+            vpcid=vpc_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers
+
+    def get_router_host(self, router):
+        self.assertEqual(
+            router.state,
+            'Running',
+            "Check list router response for router state"
+        )
+        hosts = list_hosts(
+            self.apiclient,
+            id=router.hostid)
+        self.assertEqual(
+            isinstance(hosts, list),
+            True,
+            "Check for list hosts response return valid data")
+        host = hosts[0]
+        if host.hypervisor.lower() not in "kvm":
+            return
+        host.user, host.password = get_host_credentials(self.config, host.ipaddress)
+        host.port=22
+        return host
+
+    def get_vpc_router_ips(self, router):
+        controlIp = None
+        sourcenatIp = None
+        tier1_Ip = None
+        tier2_Ip = None
+        for nic in router.nic:
+            if nic.traffictype == "Guest" and nic.ipaddress.startswith("10.250.1."):
+                tier1_Ip = nic.ipaddress
+            elif nic.traffictype == "Guest" and nic.ipaddress.startswith("10.250.2."):
+                tier2_Ip = nic.ipaddress
+            elif nic.traffictype == "Control":
+                controlIp = nic.ipaddress
+            elif sourcenatIp is None and nic.traffictype == "Public":
+                sourcenatIp = nic.ipaddress
+        return controlIp, sourcenatIp, tier1_Ip, tier2_Ip
+
+    def verify_network_interfaces_in_router(self, router, host, expectedNics):
+        command = 'ip link show |grep BROADCAST | cut -d ":" -f2 |tr -d " "|tr "\n" ","'
+        self.logger.debug("Executing command '%s'" % command)
+        result = get_process_status(
+            host.ipaddress,
+            host.port,
+            host.user,
+            host.password,
+            router.linklocalip,
+            command)[0]
+        self.assertEqual(result, expectedNics, "Expected nics are %s but actual nics are %s" %(expectedNics, result))
+
+    def verify_ip_address_in_router(self, router, host, ipaddress, device, isExist=True):
+        command = 'ip addr show %s |grep "inet "|cut -d " " -f6 |cut -d "/" -f1 |grep -w %s' % (device,ipaddress)
+        self.logger.debug("Executing command '%s'" % command)
+        result = get_process_status(
+            host.ipaddress,
+            host.port,
+            host.user,
+            host.password,
+            router.linklocalip,
+            command)
+        self.assertEqual(len(result) > 0 and result[0] == ipaddress, isExist, "ip %s verification failed" % ipaddress)
+
+    def get_free_ipaddress(self, vlanId):
+        ipaddresses = PublicIPAddress.list(
+            self.apiclient,
+            vlanid=vlanId,
+            state='Free'
+        )
+        self.assertEqual(
+            isinstance(ipaddresses, list),
+            True,
+            "List ipaddresses should return a valid response for Free ipaddresses"
+             )
+        random.shuffle(ipaddresses)
+        return ipaddresses[0].ipaddress
+
+    @attr(tags=["advanced"], required_hardware="false")
+    def test_04_acquire_public_ips_in_vpc_with_redundant_vrs(self):
+        """ Acquire IPs in multiple subnets in vpc with redundant VRs
+        # Steps
+        # 1. get vpc offering with redundant VRs
+        # 2. create a vpc with the vpc offering
+        #   verify the available nics in VR should be "eth0,eth1"
+        #   verify the IPs in VR. eth0 -> control nic, eth1 -> source nat IP
+        # 3. create a tier in the vpc, and create a vm in the tier.
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> control nic, eth1 -> source nat IP, eth2 -> tier 1
+        # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> control nic, eth1 -> source nat IP, eth2 -> tier 1
+        # 5. remove the port forwarding rule, and release the new ip
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> control nic, eth1 -> source nat IP, eth2 -> tier 1
+
+        # 6. create new public ip range 1
+        # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 1
+        # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 1/2
+        # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 1/2/3
+        # 10. release new ip 2
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 1/3
+        # 11. release new ip 1
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3
+
+        # 12. create a tier 2 in the vpc, and create a vm 2 in the tier2.
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2
+
+        # 13. create new public ip range 2
+        # 14. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4
+        # 15. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/5
+        # 16. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/5/6
+        # 17. release new ip 5
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/6
+        # 18. release new ip 4
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 6
+        """
+
+        # Create new domain1
+        self.domain1 = Domain.create(
+            self.apiclient,
+            services=self.services["acl"]["domain1"],
+            parentdomainid=self.domain.id)
+        # Create account1
+        self.account1 = Account.create(
+            self.apiclient,
+            self.services["acl"]["accountD1"],
+            domainid=self.domain1.id
+        )
+        self.cleanup.append(self.account1)
+        self.cleanup.append(self.domain1)
+
+        # Create new domain1
+        self.domain1 = Domain.create(
+            self.apiclient,
+            services=self.services["acl"]["domain1"],
+            parentdomainid=self.domain.id)
+        # Create account1
+        self.account1 = Account.create(
+            self.apiclient,
+            self.services["acl"]["accountD1"],
+            domainid=self.domain1.id
+        )
+        self.cleanup.append(self.account1)
+        self.cleanup.append(self.domain1)
+
+        # 1. get vpc offering with redundant VRs
+        vpc_offering = VpcOffering.list(self.apiclient, name="Redundant VPC offering")
+
+        # 2. create a vpc with the vpc offering
+        self.services["vpc"]["cidr"] = "10.250.0.0/16"
+        self.vpc1 = VPC.create(
+            apiclient=self.apiclient,
+            services=self.services["vpc"],
+            vpcofferingid=vpc_offering[0].id,
+            zoneid=self.zone.id,
+            account=self.account1.name,
+            domainid=self.account1.domainid
+        )
+
+        #   verify the available nics in VR should be "eth0,eth1"
+        #   verify the IPs in VR. eth0 -> control nic, eth1 -> source nat IP
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+
+        # 3. create a tier in the vpc, and create a vm in the tier.
+        vpc_tier_offerings = NetworkOffering.list(self.apiclient, name="DefaultIsolatedNetworkOfferingForVpcNetworks")
+        self.assertTrue(vpc_tier_offerings is not None and len(vpc_tier_offerings) > 0, "No VPC based network offering")
+        vpc_tier_offering = vpc_tier_offerings[0]
+        self.services["vpctier"] = {}
+        self.services["vpctier"]["name"] = "vpc-tier-1"
+        self.services["vpctier"]["displaytext"] = "vpc-tier-1"
+        vpc_tier_1 = Network.create(
+            apiclient=self.apiclient,
+            services=self.services["vpctier"],
+            accountid=self.account1.name,
+            domainid=self.domain1.id,
+            networkofferingid=vpc_tier_offering.id,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            gateway="10.250.1.1",
+            netmask="255.255.255.0"
+        )
+
+        try:
+            self.virtual_machine1 = VirtualMachine.create(
+                self.apiclient,
+                self.services["virtual_machine"],
+                accountid=self.account1.name,
+                domainid=self.account1.domainid,
+                serviceofferingid=self.service_offering.id,
+                templateid=self.template.id,
+                zoneid=self.zone.id,
+                networkids=vpc_tier_1.id
+            )
+        except Exception as e:
+            self.fail("Exception while deploying virtual machine: %s" % e)
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+
+        # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm
+        ipaddress = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+        )
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress.ipaddress.id,
+            networkid=vpc_tier_1.id
+        )
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP/new ip
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth1", True)
+
+        # 5. release the new ip
+        ipaddress.delete(self.apiclient)
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth1", False)
+
+        # 6. create new public ip range 1
+        self.services["publiciprange"]["zoneid"] = self.zone.id
+        self.services["publiciprange"]["forvirtualnetwork"] = "true"
+        random_subnet_number = random.randrange(10,20)
+        self.services["publiciprange"]["vlan"] = get_free_vlan(
+            self.apiclient,
+            self.zone.id)[1]
+        self.services["publiciprange"]["gateway"] = "172.16." + str(random_subnet_number) + ".1"
+        self.services["publiciprange"]["startip"] = "172.16." + str(random_subnet_number) + ".2"
+        self.services["publiciprange"]["endip"] = "172.16." + str(random_subnet_number) + ".10"
+        self.services["publiciprange"]["netmask"] = "255.255.255.0"
+        self.public_ip_range1 = PublicIpRange.create(
+            self.apiclient,
+            self.services["publiciprange"]
+        )
+        self.cleanup.append(self.public_ip_range1)
+
+        # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        ip_address_1 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_1 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_1
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_1.ipaddress.id,
+            networkid=vpc_tier_1.id
+        )
+
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+
+        # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2,
+        ip_address_2 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_2 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_2
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_2.ipaddress.id,
+            networkid=vpc_tier_1.id
+        )
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True)
+
+        # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 2, new ip 3
+        ip_address_3 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress_3 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_3
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_3.ipaddress.id,
+            networkid=vpc_tier_1.id
+        )
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 10. release new ip 2
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1, new ip 3
+        ipaddress_2.delete(self.apiclient)
+
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 11. release new ip 1
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3"
+        #   verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3
+        ipaddress_1.delete(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+
+        # 12. create a tier in the vpc, and create a vm in the tier.
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2
+        self.services["vpctier"]["name"] = "vpc-tier-2"
+        self.services["vpctier"]["displaytext"] = "vpc-tier-2"
+        vpc_tier_2 = Network.create(
+            apiclient=self.apiclient,
+            services=self.services["vpctier"],
+            accountid=self.account1.name,
+            domainid=self.domain1.id,
+            networkofferingid=vpc_tier_offering.id,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            gateway="10.250.2.1",
+            netmask="255.255.255.0"
+        )
+
+        try:
+            self.virtual_machine1 = VirtualMachine.create(
+                self.apiclient,
+                self.services["virtual_machine"],
+                accountid=self.account1.name,
+                domainid=self.account1.domainid,
+                serviceofferingid=self.service_offering.id,
+                templateid=self.template.id,
+                zoneid=self.zone.id,
+                networkids=vpc_tier_2.id
+            )
+        except Exception as e:
+            self.fail("Exception while deploying virtual machine: %s" % e)
+
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+
+        # 13. create new public ip range 2
+        self.services["publiciprange"]["zoneid"] = self.zone.id
+        self.services["publiciprange"]["forvirtualnetwork"] = "true"
+        self.services["publiciprange"]["vlan"] = get_free_vlan(
+            self.apiclient,
+            self.zone.id)[1]
+        self.services["publiciprange"]["gateway"] = "172.16." + str(random_subnet_number + 1) + ".1"
+        self.services["publiciprange"]["startip"] = "172.16." + str(random_subnet_number + 1) + ".2"
+        self.services["publiciprange"]["endip"] = "172.16." + str(random_subnet_number + 1) + ".10"
+        self.services["publiciprange"]["netmask"] = "255.255.255.0"
+        self.public_ip_range2 = PublicIpRange.create(
+            self.apiclient,
+            self.services["publiciprange"]
+        )
+        self.cleanup.append(self.public_ip_range2)
+
+        # 14. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4
+        ip_address_4 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_4 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_4
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_4.ipaddress.id,
+            networkid=vpc_tier_2.id
+        )
+
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True)
+
+        # 15. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/5
+        ip_address_5 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_5 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_5
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_5.ipaddress.id,
+            networkid=vpc_tier_2.id
+        )
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", True)
+
+        # 16. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/5/6
+        ip_address_6 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress_6 = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            vpcid=self.vpc1.id,
+            ipaddress=ip_address_6
+        )
+
+        nat_rule = NATRule.create(
+            self.apiclient,
+            self.virtual_machine1,
+            self.services["natrule"],
+            ipaddressid=ipaddress_6.ipaddress.id,
+            networkid=vpc_tier_2.id
+        )
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 17. release new ip 5
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4/6
+        ipaddress_5.delete(self.apiclient)
+
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 18. release new ip 4
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 6
+        ipaddress_4.delete(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 19. release new ip 3
+        #   verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5,"
+        #   verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth4 -> tier 2, eth5 -> new ip 6
+        ipaddress_3.delete(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 20. restart tier1
+        vpc_tier_1.restart(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        #21. restart tier2
+        vpc_tier_2.restart(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 22. restart VPC
+        self.vpc1.restart(self.apiclient)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,eth5,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
+
+        # 23. restart VPC with cleanup
+        self.vpc1.restart(self.apiclient, cleanup=True)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+
+        # 24. restart VPC with cleanup, makeredundant=true
+        self.vpc1.restart(self.apiclient, cleanup=True, makeredundant=True)
+        routers = self.get_vpc_routers(self.vpc1.id)
+        for router in routers:
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)

tools/marvin/marvin/lib/base.py

@@ -3103,13 +3103,15 @@ class Network:
         [setattr(cmd, k, v) for k, v in kwargs.items()]
         return (apiclient.updateNetwork(cmd))
 
-    def restart(self, apiclient, cleanup=None):
+    def restart(self, apiclient, cleanup=None, makeredundant=None):
         """Restarts the network"""
 
         cmd = restartNetwork.restartNetworkCmd()
         cmd.id = self.id
         if cleanup:
             cmd.cleanup = cleanup
+        if makeredundant:
+            cmd.makeredundant = makeredundant
         return (apiclient.restartNetwork(cmd))
 
     def migrate(self, apiclient, network_offering_id, resume=False):
@@ -4516,11 +4518,15 @@ class VPC:
         cmd.id = self.id
         return apiclient.deleteVPC(cmd)
 
-    def restart(self, apiclient):
+    def restart(self, apiclient, cleanup=None, makeredundant=None):
         """Restarts the VPC connections"""
 
         cmd = restartVPC.restartVPCCmd()
         cmd.id = self.id
+        if cleanup:
+            cmd.cleanup = cleanup
+        if makeredundant:
+            cmd.makeredundant = makeredundant
         return apiclient.restartVPC(cmd)
 
     @classmethod