vpc: fix ips on wrong interfaces after rebooting vpc vrs (#4467)

* vpc: fix ips on wrong interfaces after rebooting vpc vrs

* #4467: Rename to updateNicWithDeviceId

* CLSTACK-8923 vr: Force a restart of keepalived if conntrackd is not running or configuration has changed

server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java

@@ -339,6 +339,7 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
                     vlanMacAddress.put(vlanTag, routerNic.getMacAddress());
                 }
             }
+            int deviceId = 1; //Public and Guest networks start from device_id = 1
 
             final List<Command> usageCmds = new ArrayList<Command>();
 
@@ -347,7 +348,8 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
                 // add VPC router to public networks
                 final List<PublicIp> sourceNat = new ArrayList<PublicIp>(1);
                 for (final Pair<Nic, Network> nicNtwk : publicNics) {
-                    final Nic publicNic = nicNtwk.first();
+                    final Nic publicNic = updateNicWithDeviceId(nicNtwk.first().getId(), deviceId);
+                    deviceId ++;
                     final Network publicNtwk = nicNtwk.second();
                     final IPAddressVO userIp = _ipAddressDao.findByIpAndSourceNetworkId(publicNtwk.getId(), publicNic.getIPv4Address());
 
@@ -385,7 +387,8 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
 
                 // add VPC router to guest networks
                 for (final Pair<Nic, Network> nicNtwk : guestNics) {
-                    final Nic guestNic = nicNtwk.first();
+                    final Nic guestNic = updateNicWithDeviceId(nicNtwk.first().getId(), deviceId);
+                    deviceId ++;
                     // plug guest nic
                     final PlugNicCommand plugNicCmd = new PlugNicCommand(_nwHelper.getNicTO(domainRouterVO, guestNic.getNetworkId(), null), domainRouterVO.getInstanceName(), domainRouterVO.getType(), details);
                     cmds.addCommand(plugNicCmd);
@@ -834,4 +837,11 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
         // once from VpcVirtualNetworkApplianceManagerImpl and once from VirtualNetworkApplianceManagerImpl itself
         return true;
     }
+
+    private Nic updateNicWithDeviceId(final long nicId, int deviceId) {
+        NicVO nic = _nicDao.findById(nicId);
+        nic.setDeviceId(deviceId);
+        _nicDao.update(nic.getId(), nic);
+        return nic;
+    }
 }

systemvm/debian/opt/cloud/bin/cs/CsRedundant.py

@@ -173,7 +173,7 @@ class CsRedundant(object):
         force_keepalived_restart = False
         proc = CsProcess(['/etc/conntrackd/conntrackd.conf'])
 
-        if not proc.find() and not is_equals:
+        if not proc.find() or not is_equals:
             CsHelper.copy(conntrackd_template_conf, self.CONNTRACKD_CONF)
             CsHelper.service("conntrackd", "restart")
             force_keepalived_restart = True

systemvm/debian/opt/cloud/bin/setup/bootstrap.sh

@@ -104,6 +104,9 @@ config_guest() {
           for i in {1..60}; do
             if [ -s $CMDLINE ]; then
               log_it "Received a new non-empty cmdline file from qemu-guest-agent"
+              # Remove old configuration files in /etc/cloudstack if VR is booted from cloudstack
+              rm -rf /etc/cloudstack/*.json
+              log_it "Booting from cloudstack, remove old configuration files in /etc/cloudstack/"
               break
             fi
             sleep 1

test/integration/component/test_multiple_subnets_in_isolated_network.py

@@ -20,6 +20,7 @@ Tests of acquiring IPs in multiple subnets for isolated network or vpc
 """
 
 from nose.plugins.attrib import attr
+from marvin.cloudstackAPI import rebootRouter
 from marvin.cloudstackTestCase import cloudstackTestCase, unittest
 from marvin.lib.utils import (validateList,
                               get_host_credentials,
@@ -106,6 +107,23 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase):
             raise Exception("Warning: Exception during cleanup : %s" % e)
         return
 
+    def get_router(self, router_id):
+        routers = list_routers(
+            self.apiclient,
+            id=router_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers[0]
+
     def get_routers(self, network_id):
         routers = list_routers(
             self.apiclient,
@@ -672,6 +690,22 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase):
             self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
             self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
 
+        # reboot router
+        for router in routers:
+            cmd = rebootRouter.rebootRouterCmd()
+            cmd.id = router.id
+            self.apiclient.rebootRouter(cmd)
+            router = self.get_router(router.id)
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True)
+
         # 20. restart network with cleanup
         self.network1.restart(self.apiclient, cleanup=True)
         routers = self.get_routers(self.network1.id)

test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py

@@ -20,6 +20,7 @@ Tests of acquiring IPs in multiple subnets for isolated network or vpc
 """
 
 from nose.plugins.attrib import attr
+from marvin.cloudstackAPI import rebootRouter
 from marvin.cloudstackTestCase import cloudstackTestCase, unittest
 from marvin.lib.utils import (validateList,
                               get_host_credentials,
@@ -106,6 +107,23 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase):
             raise Exception("Warning: Exception during cleanup : %s" % e)
         return
 
+    def get_router(self, router_id):
+        routers = list_routers(
+            self.apiclient,
+            id=router_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers[0]
+
     def get_routers(self, network_id):
         routers = list_routers(
             self.apiclient,
@@ -672,6 +690,22 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase):
             self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
             self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
 
+        # reboot router
+        for router in routers:
+            cmd = rebootRouter.rebootRouterCmd()
+            cmd.id = router.id
+            self.apiclient.rebootRouter(cmd)
+            router = self.get_router(router.id)
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True)
+
         # 20. restart network with cleanup
         self.network1.restart(self.apiclient, cleanup=True)
         routers = self.get_routers(self.network1.id)

test/integration/component/test_multiple_subnets_in_vpc.py

@@ -20,6 +20,7 @@ Tests of acquiring IPs in multiple subnets for isolated network or vpc
 """
 
 from nose.plugins.attrib import attr
+from marvin.cloudstackAPI import rebootRouter
 from marvin.cloudstackTestCase import cloudstackTestCase, unittest
 from marvin.lib.utils import (validateList,
                               get_host_credentials,
@@ -106,6 +107,23 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase):
             raise Exception("Warning: Exception during cleanup : %s" % e)
         return
 
+    def get_router(self, router_id):
+        routers = list_routers(
+            self.apiclient,
+            id=router_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers[0]
+
     def get_routers(self, network_id):
         routers = list_routers(
             self.apiclient,
@@ -790,6 +808,21 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase):
             self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
             self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
 
+        # reboot router
+        for router in routers:
+            cmd = rebootRouter.rebootRouterCmd()
+            cmd.id = router.id
+            self.apiclient.rebootRouter(cmd)
+            router = self.get_router(router.id)
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+
         # 23. restart VPC with cleanup
         self.vpc1.restart(self.apiclient, cleanup=True)
         routers = self.get_vpc_routers(self.vpc1.id)

test/integration/component/test_multiple_subnets_in_vpc_rvr.py

@@ -20,6 +20,7 @@ Tests of acquiring IPs in multiple subnets for isolated network or vpc
 """
 
 from nose.plugins.attrib import attr
+from marvin.cloudstackAPI import rebootRouter
 from marvin.cloudstackTestCase import cloudstackTestCase, unittest
 from marvin.lib.utils import (validateList,
                               get_host_credentials,
@@ -106,6 +107,23 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase):
             raise Exception("Warning: Exception during cleanup : %s" % e)
         return
 
+    def get_router(self, router_id):
+        routers = list_routers(
+            self.apiclient,
+            id=router_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers[0]
+
     def get_routers(self, network_id):
         routers = list_routers(
             self.apiclient,
@@ -790,6 +808,21 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase):
             self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
             self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
 
+        # reboot router
+        for router in routers:
+            cmd = rebootRouter.rebootRouterCmd()
+            cmd.id = router.id
+            self.apiclient.rebootRouter(cmd)
+            router = self.get_router(router.id)
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+
         # 23. restart VPC with cleanup
         self.vpc1.restart(self.apiclient, cleanup=True)
         routers = self.get_vpc_routers(self.vpc1.id)