From 75fdb07387eaaf07365a612f9035cbd98d98cb1b Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 20 Nov 2020 16:32:53 +0100 Subject: [PATCH] vpc: fix ips on wrong interfaces after rebooting vpc vrs (#4467) * vpc: fix ips on wrong interfaces after rebooting vpc vrs * #4467: Rename to updateNicWithDeviceId * CLSTACK-8923 vr: Force a restart of keepalived if conntrackd is not running or configuration has changed --- ...VpcVirtualNetworkApplianceManagerImpl.java | 14 ++++++-- .../debian/opt/cloud/bin/cs/CsRedundant.py | 2 +- .../debian/opt/cloud/bin/setup/bootstrap.sh | 3 ++ ...st_multiple_subnets_in_isolated_network.py | 34 +++++++++++++++++++ ...ultiple_subnets_in_isolated_network_rvr.py | 34 +++++++++++++++++++ .../component/test_multiple_subnets_in_vpc.py | 33 ++++++++++++++++++ .../test_multiple_subnets_in_vpc_rvr.py | 33 ++++++++++++++++++ 7 files changed, 150 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java index e9d32a2152b..556fe18388f 100644 --- a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java @@ -339,6 +339,7 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian vlanMacAddress.put(vlanTag, routerNic.getMacAddress()); } } + int deviceId = 1; //Public and Guest networks start from device_id = 1 final List usageCmds = new ArrayList(); @@ -347,7 +348,8 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian // add VPC router to public networks final List sourceNat = new ArrayList(1); for (final Pair nicNtwk : publicNics) { - final Nic publicNic = nicNtwk.first(); + final Nic publicNic = updateNicWithDeviceId(nicNtwk.first().getId(), deviceId); + deviceId ++; final Network publicNtwk = nicNtwk.second(); final IPAddressVO userIp = _ipAddressDao.findByIpAndSourceNetworkId(publicNtwk.getId(), publicNic.getIPv4Address()); @@ -385,7 +387,8 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian // add VPC router to guest networks for (final Pair nicNtwk : guestNics) { - final Nic guestNic = nicNtwk.first(); + final Nic guestNic = updateNicWithDeviceId(nicNtwk.first().getId(), deviceId); + deviceId ++; // plug guest nic final PlugNicCommand plugNicCmd = new PlugNicCommand(_nwHelper.getNicTO(domainRouterVO, guestNic.getNetworkId(), null), domainRouterVO.getInstanceName(), domainRouterVO.getType(), details); cmds.addCommand(plugNicCmd); @@ -834,4 +837,11 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian // once from VpcVirtualNetworkApplianceManagerImpl and once from VirtualNetworkApplianceManagerImpl itself return true; } + + private Nic updateNicWithDeviceId(final long nicId, int deviceId) { + NicVO nic = _nicDao.findById(nicId); + nic.setDeviceId(deviceId); + _nicDao.update(nic.getId(), nic); + return nic; + } } diff --git a/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py b/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py index cefedbfe84f..190de1ab82e 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py @@ -173,7 +173,7 @@ class CsRedundant(object): force_keepalived_restart = False proc = CsProcess(['/etc/conntrackd/conntrackd.conf']) - if not proc.find() and not is_equals: + if not proc.find() or not is_equals: CsHelper.copy(conntrackd_template_conf, self.CONNTRACKD_CONF) CsHelper.service("conntrackd", "restart") force_keepalived_restart = True diff --git a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh index 0fb317b3203..769078ea8f0 100755 --- a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh +++ b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh @@ -104,6 +104,9 @@ config_guest() { for i in {1..60}; do if [ -s $CMDLINE ]; then log_it "Received a new non-empty cmdline file from qemu-guest-agent" + # Remove old configuration files in /etc/cloudstack if VR is booted from cloudstack + rm -rf /etc/cloudstack/*.json + log_it "Booting from cloudstack, remove old configuration files in /etc/cloudstack/" break fi sleep 1 diff --git a/test/integration/component/test_multiple_subnets_in_isolated_network.py b/test/integration/component/test_multiple_subnets_in_isolated_network.py index 0e5054732ee..9892a3bc6e9 100644 --- a/test/integration/component/test_multiple_subnets_in_isolated_network.py +++ b/test/integration/component/test_multiple_subnets_in_isolated_network.py @@ -20,6 +20,7 @@ Tests of acquiring IPs in multiple subnets for isolated network or vpc """ from nose.plugins.attrib import attr +from marvin.cloudstackAPI import rebootRouter from marvin.cloudstackTestCase import cloudstackTestCase, unittest from marvin.lib.utils import (validateList, get_host_credentials, @@ -106,6 +107,23 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return + def get_router(self, router_id): + routers = list_routers( + self.apiclient, + id=router_id, + listall=True) + self.assertEqual( + isinstance(routers, list), + True, + "Check for list routers response return valid data" + ) + self.assertNotEqual( + len(routers), + 0, + "Check list router response" + ) + return routers[0] + def get_routers(self, network_id): routers = list_routers( self.apiclient, @@ -672,6 +690,22 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + # reboot router + for router in routers: + cmd = rebootRouter.rebootRouterCmd() + cmd.id = router.id + self.apiclient.rebootRouter(cmd) + router = self.get_router(router.id) + host = self.get_router_host(router) + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,") + guestIp, controlIp, sourcenatIp = self.get_router_ips(router) + self.verify_ip_address_in_router(router, host, guestIp, "eth0", True) + self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) + self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) + self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False) + self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False) + self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) + # 20. restart network with cleanup self.network1.restart(self.apiclient, cleanup=True) routers = self.get_routers(self.network1.id) diff --git a/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py b/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py index b3dd864738a..2abd076be01 100644 --- a/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py +++ b/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py @@ -20,6 +20,7 @@ Tests of acquiring IPs in multiple subnets for isolated network or vpc """ from nose.plugins.attrib import attr +from marvin.cloudstackAPI import rebootRouter from marvin.cloudstackTestCase import cloudstackTestCase, unittest from marvin.lib.utils import (validateList, get_host_credentials, @@ -106,6 +107,23 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return + def get_router(self, router_id): + routers = list_routers( + self.apiclient, + id=router_id, + listall=True) + self.assertEqual( + isinstance(routers, list), + True, + "Check for list routers response return valid data" + ) + self.assertNotEqual( + len(routers), + 0, + "Check list router response" + ) + return routers[0] + def get_routers(self, network_id): routers = list_routers( self.apiclient, @@ -672,6 +690,22 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + # reboot router + for router in routers: + cmd = rebootRouter.rebootRouterCmd() + cmd.id = router.id + self.apiclient.rebootRouter(cmd) + router = self.get_router(router.id) + host = self.get_router_host(router) + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,") + guestIp, controlIp, sourcenatIp = self.get_router_ips(router) + self.verify_ip_address_in_router(router, host, guestIp, "eth0", True) + self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) + self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) + self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False) + self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False) + self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) + # 20. restart network with cleanup self.network1.restart(self.apiclient, cleanup=True) routers = self.get_routers(self.network1.id) diff --git a/test/integration/component/test_multiple_subnets_in_vpc.py b/test/integration/component/test_multiple_subnets_in_vpc.py index 09f6ec64f89..a9e36966b72 100644 --- a/test/integration/component/test_multiple_subnets_in_vpc.py +++ b/test/integration/component/test_multiple_subnets_in_vpc.py @@ -20,6 +20,7 @@ Tests of acquiring IPs in multiple subnets for isolated network or vpc """ from nose.plugins.attrib import attr +from marvin.cloudstackAPI import rebootRouter from marvin.cloudstackTestCase import cloudstackTestCase, unittest from marvin.lib.utils import (validateList, get_host_credentials, @@ -106,6 +107,23 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return + def get_router(self, router_id): + routers = list_routers( + self.apiclient, + id=router_id, + listall=True) + self.assertEqual( + isinstance(routers, list), + True, + "Check for list routers response return valid data" + ) + self.assertNotEqual( + len(routers), + 0, + "Check list router response" + ) + return routers[0] + def get_routers(self, network_id): routers = list_routers( self.apiclient, @@ -790,6 +808,21 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + # reboot router + for router in routers: + cmd = rebootRouter.rebootRouterCmd() + cmd.id = router.id + self.apiclient.rebootRouter(cmd) + router = self.get_router(router.id) + host = self.get_router_host(router) + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,") + controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router) + self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) + self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) + self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) + self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) + self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + # 23. restart VPC with cleanup self.vpc1.restart(self.apiclient, cleanup=True) routers = self.get_vpc_routers(self.vpc1.id) diff --git a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py index a03a91b2057..392620f88cb 100644 --- a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py +++ b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py @@ -20,6 +20,7 @@ Tests of acquiring IPs in multiple subnets for isolated network or vpc """ from nose.plugins.attrib import attr +from marvin.cloudstackAPI import rebootRouter from marvin.cloudstackTestCase import cloudstackTestCase, unittest from marvin.lib.utils import (validateList, get_host_credentials, @@ -106,6 +107,23 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return + def get_router(self, router_id): + routers = list_routers( + self.apiclient, + id=router_id, + listall=True) + self.assertEqual( + isinstance(routers, list), + True, + "Check for list routers response return valid data" + ) + self.assertNotEqual( + len(routers), + 0, + "Check list router response" + ) + return routers[0] + def get_routers(self, network_id): routers = list_routers( self.apiclient, @@ -790,6 +808,21 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + # reboot router + for router in routers: + cmd = rebootRouter.rebootRouterCmd() + cmd.id = router.id + self.apiclient.rebootRouter(cmd) + router = self.get_router(router.id) + host = self.get_router_host(router) + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,") + controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router) + self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) + self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) + self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) + self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) + self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + # 23. restart VPC with cleanup self.vpc1.restart(self.apiclient, cleanup=True) routers = self.get_vpc_routers(self.vpc1.id)