From 6491a69d63d7aff460aac93d525384b324087c37 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 17 Nov 2020 20:25:38 +0000 Subject: [PATCH 01/14] bugfix #2 vpc: Fix remove first public ip will remove all ips on the nic --- .../network/rules/NicPlugInOutRules.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java b/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java index 40cf72c77aa..996d1ccb05f 100644 --- a/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java +++ b/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java @@ -38,6 +38,8 @@ import com.cloud.network.NetworkModel; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.IsolationType; import com.cloud.network.PublicIpAddress; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; import com.cloud.network.router.VirtualRouter; import com.cloud.network.vpc.VpcManager; import com.cloud.network.vpc.VpcVO; @@ -159,6 +161,8 @@ public class NicPlugInOutRules extends RuleApplier { VpcManager vpcMgr = visitor.getVirtualNetworkApplianceFactory().getVpcMgr(); NicDao nicDao = visitor.getVirtualNetworkApplianceFactory().getNicDao(); + IPAddressDao ipAddressDao = visitor.getVirtualNetworkApplianceFactory().getIpAddressDao(); + // find out nics to unplug for (PublicIpAddress ip : _ipAddresses) { long publicNtwkId = ip.getNetworkId(); @@ -170,10 +174,23 @@ public class NicPlugInOutRules extends RuleApplier { } if (ip.getState() == IpAddress.State.Releasing) { - Nic nic = nicDao.findByIp4AddressAndNetworkIdAndInstanceId(publicNtwkId, _router.getId(), ip.getAddress().addr()); + NicVO nic = nicDao.findByIp4AddressAndNetworkIdAndInstanceId(publicNtwkId, _router.getId(), ip.getAddress().addr()); if (nic != null) { - nicsToUnplug.put(ip.getVlanTag(), ip); - s_logger.debug("Need to unplug the nic for ip=" + ip + "; vlan=" + ip.getVlanTag() + " in public network id =" + publicNtwkId); + final List allIps = ipAddressDao.listByAssociatedVpc(ip.getVpcId(), null); + boolean ipUpdated = false; + for (IPAddressVO allIp : allIps) { + if (allIp.getId() != ip.getId() && allIp.getVlanId() == ip.getVlanId() && allIp.getVmIp() != null) { + s_logger.debug("Updating the nic " + nic + " with new ip address " + allIp.getAddress().addr()); + nic.setIPv4Address(allIp.getAddress().addr()); + nicDao.update(nic.getId(), nic); + ipUpdated = true; + break; + } + } + if (!ipUpdated) { + nicsToUnplug.put(ip.getVlanTag(), ip); + s_logger.debug("Need to unplug the nic for ip=" + ip + "; vlan=" + ip.getVlanTag() + " in public network id =" + publicNtwkId); + } } } } @@ -215,4 +232,4 @@ public class NicPlugInOutRules extends RuleApplier { return nicsToChange; } -} \ No newline at end of file +} From f5ab87c153bd5b320701cb92f73c7a3c1791215e Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 17 Nov 2020 20:42:18 +0000 Subject: [PATCH 02/14] bugfix #3 apply ip dessociation before unplugging a nic so ip is marked as add:false in ips.json --- .../network/rules/NicPlugInOutRules.java | 27 +++++++++++++++++++ .../rules/VirtualNetworkApplianceFactory.java | 11 +++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java b/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java index 996d1ccb05f..a90264e164a 100644 --- a/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java +++ b/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java @@ -29,6 +29,8 @@ import org.apache.log4j.Logger; import com.cloud.agent.api.Command; import com.cloud.agent.api.NetworkUsageCommand; import com.cloud.agent.manager.Commands; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.ResourceUnavailableException; @@ -53,6 +55,9 @@ import com.cloud.vm.NicVO; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.NicDao; +import org.apache.cloudstack.network.topology.NetworkTopology; +import org.apache.cloudstack.network.topology.NetworkTopologyContext; + public class NicPlugInOutRules extends RuleApplier { private static final Logger s_logger = Logger.getLogger(NicPlugInOutRules.class); @@ -77,6 +82,28 @@ public class NicPlugInOutRules extends RuleApplier { NetworkModel networkModel = visitor.getVirtualNetworkApplianceFactory().getNetworkModel(); VirtualMachineManager itMgr = visitor.getVirtualNetworkApplianceFactory().getItMgr(); + NicDao nicDao = visitor.getVirtualNetworkApplianceFactory().getNicDao(); + + // de-associate IPs before unplugging nics + if (!nicsToUnplug.isEmpty()) { + NetworkTopologyContext networkTopologyContext = visitor.getVirtualNetworkApplianceFactory().getNetworkTopologyContext(); + final DataCenterDao dcDao = visitor.getVirtualNetworkApplianceFactory().getDcDao(); + final DataCenterVO dcVO = dcDao.findById(router.getDataCenterId()); + final NetworkTopology networkTopology = networkTopologyContext.retrieveNetworkTopology(dcVO); + + final String typeString = "vpc ip association before unplugging nics"; + final boolean isPodLevelException = false; + final boolean failWhenDisconnect = false; + final Long podId = null; + final VpcIpAssociationRules ipAssociationRules = new VpcIpAssociationRules(_network, _ipAddresses); + final boolean result = networkTopology.applyRules(_network, router, typeString, isPodLevelException, podId, failWhenDisconnect, + new RuleApplierWrapper(ipAssociationRules)); + if (!result) { + s_logger.warn("Failed to de-associate IPs before unplugging nics"); + return false; + } + } + // 1) Unplug the nics for (Entry entry : nicsToUnplug.entrySet()) { Network publicNtwk = null; diff --git a/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java b/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java index 34400ea02ab..b3edc3b736d 100644 --- a/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java +++ b/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java @@ -43,6 +43,8 @@ import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.NicIpAliasDao; import com.cloud.vm.dao.UserVmDao; +import org.apache.cloudstack.network.topology.NetworkTopologyContext; + public class VirtualNetworkApplianceFactory { @Inject @@ -91,6 +93,9 @@ public class VirtualNetworkApplianceFactory { @Inject private NicProfileHelper _nicProfileHelper; + @Inject + private NetworkTopologyContext _networkTopologyContext; + public NetworkModel getNetworkModel() { return _networkModel; } @@ -174,4 +179,8 @@ public class VirtualNetworkApplianceFactory { public NicProfileHelper getNicProfileHelper() { return _nicProfileHelper; } -} \ No newline at end of file + + public NetworkTopologyContext getNetworkTopologyContext() { + return _networkTopologyContext; + } +} From 655ed10655c0848471d43627a85b1433f38da9d5 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 18 Nov 2020 15:56:03 +0000 Subject: [PATCH 03/14] bugfix #2 vpc vr: fix issue if static nat is disabled but still other IP used by lb/pf --- .../network/rules/NicPlugInOutRules.java | 7 ++++++- .../rules/VirtualNetworkApplianceFactory.java | 7 +++++++ ...st_multiple_subnets_in_isolated_network.py | 19 ++++++++--------- ...ultiple_subnets_in_isolated_network_rvr.py | 19 ++++++++--------- .../component/test_multiple_subnets_in_vpc.py | 21 +++++++++---------- .../test_multiple_subnets_in_vpc_rvr.py | 21 +++++++++---------- 6 files changed, 51 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java b/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java index a90264e164a..6ee5e85f271 100644 --- a/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java +++ b/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java @@ -40,6 +40,7 @@ import com.cloud.network.NetworkModel; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.IsolationType; import com.cloud.network.PublicIpAddress; +import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.router.VirtualRouter; @@ -189,6 +190,7 @@ public class NicPlugInOutRules extends RuleApplier { VpcManager vpcMgr = visitor.getVirtualNetworkApplianceFactory().getVpcMgr(); NicDao nicDao = visitor.getVirtualNetworkApplianceFactory().getNicDao(); IPAddressDao ipAddressDao = visitor.getVirtualNetworkApplianceFactory().getIpAddressDao(); + FirewallRulesDao rulesDao = visitor.getVirtualNetworkApplianceFactory().getFirewallRulesDao(); // find out nics to unplug for (PublicIpAddress ip : _ipAddresses) { @@ -206,7 +208,10 @@ public class NicPlugInOutRules extends RuleApplier { final List allIps = ipAddressDao.listByAssociatedVpc(ip.getVpcId(), null); boolean ipUpdated = false; for (IPAddressVO allIp : allIps) { - if (allIp.getId() != ip.getId() && allIp.getVlanId() == ip.getVlanId() && allIp.getVmIp() != null) { + if (allIp.getId() != ip.getId() && allIp.getVlanId() == ip.getVlanId() + && (allIp.isSourceNat() + || rulesDao.countRulesByIpIdAndState(allIp.getId(), FirewallRule.State.Active) > 0 + || (allIp.isOneToOneNat() && allIp.getRuleState() == null))) { s_logger.debug("Updating the nic " + nic + " with new ip address " + allIp.getAddress().addr()); nic.setIPv4Address(allIp.getAddress().addr()); nicDao.update(nic.getId(), nic); diff --git a/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java b/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java index b3edc3b736d..9d4660da557 100644 --- a/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java +++ b/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java @@ -26,6 +26,7 @@ import com.cloud.dc.dao.HostPodDao; import com.cloud.dc.dao.VlanDao; import com.cloud.network.IpAddressManager; import com.cloud.network.NetworkModel; +import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.NetworkDao; @@ -85,6 +86,8 @@ public class VirtualNetworkApplianceFactory { private IpAddressManager _ipAddrMgr; @Inject private NetworkACLManager _networkACLMgr; + @Inject + private FirewallRulesDao _rulesDao; @Autowired @Qualifier("networkHelper") @@ -183,4 +186,8 @@ public class VirtualNetworkApplianceFactory { public NetworkTopologyContext getNetworkTopologyContext() { return _networkTopologyContext; } + + public FirewallRulesDao getFirewallRulesDao() { + return _rulesDao; + } } diff --git a/test/integration/component/test_multiple_subnets_in_isolated_network.py b/test/integration/component/test_multiple_subnets_in_isolated_network.py index 9892a3bc6e9..dee90663a78 100644 --- a/test/integration/component/test_multiple_subnets_in_isolated_network.py +++ b/test/integration/component/test_multiple_subnets_in_isolated_network.py @@ -35,6 +35,7 @@ from marvin.lib.base import (Account, NetworkOffering, VPC, VpcOffering, + StaticNATRule, NATRule, PublicIPAddress, PublicIpRange) @@ -247,7 +248,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): # verify the IPs in VR. eth0 -> guest nic IP, eth2 -> source nat IP # 6. create new public ip range 1 - # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm + # 7. get a free ip 4 in new ip range 2, assign to network, and enable static nat to vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3" # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1 # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm @@ -395,7 +396,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ) self.cleanup.append(self.public_ip_range1) - # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm + # 7. get a free ip 4 in new ip range 2, assign to network, and enable static nat to vm ip_address_1 = self.get_free_ipaddress(self.public_ip_range1.vlan.id) ipaddress_1 = PublicIPAddress.create( self.apiclient, @@ -404,12 +405,11 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ipaddress=ip_address_1 ) - nat_rule = NATRule.create( + StaticNATRule.enable( self.apiclient, - self.virtual_machine1, - self.services["natrule"], + virtualmachineid=self.virtual_machine1.id, ipaddressid=ipaddress_1.ipaddress.id, - openfirewall=True + networkid=self.network1.id ) # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -544,12 +544,11 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ipaddress=ip_address_4 ) - nat_rule = NATRule.create( + StaticNATRule.enable( self.apiclient, - self.virtual_machine1, - self.services["natrule"], + virtualmachineid=self.virtual_machine1.id, ipaddressid=ipaddress_4.ipaddress.id, - openfirewall=True + networkid=self.network1.id ) diff --git a/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py b/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py index 2abd076be01..778e04fcee6 100644 --- a/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py +++ b/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py @@ -35,6 +35,7 @@ from marvin.lib.base import (Account, NetworkOffering, VPC, VpcOffering, + StaticNATRule, NATRule, PublicIPAddress, PublicIpRange) @@ -247,7 +248,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): # verify the IPs in VR. eth0 -> guest nic IP, eth2 -> source nat IP # 6. create new public ip range 1 - # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm + # 7. get a free ip 4 in new ip range 2, assign to network, and enable static nat to vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3" # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 1 # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm @@ -395,7 +396,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ) self.cleanup.append(self.public_ip_range1) - # 7. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm + # 7. get a free ip 4 in new ip range 2, assign to network, and enable static nat to vm ip_address_1 = self.get_free_ipaddress(self.public_ip_range1.vlan.id) ipaddress_1 = PublicIPAddress.create( self.apiclient, @@ -404,12 +405,11 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ipaddress=ip_address_1 ) - nat_rule = NATRule.create( + StaticNATRule.enable( self.apiclient, - self.virtual_machine1, - self.services["natrule"], + virtualmachineid=self.virtual_machine1.id, ipaddressid=ipaddress_1.ipaddress.id, - openfirewall=True + networkid=self.network1.id ) # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -544,12 +544,11 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ipaddress=ip_address_4 ) - nat_rule = NATRule.create( + StaticNATRule.enable( self.apiclient, - self.virtual_machine1, - self.services["natrule"], + virtualmachineid=self.virtual_machine1.id, ipaddressid=ipaddress_4.ipaddress.id, - openfirewall=True + networkid=self.network1.id ) diff --git a/test/integration/component/test_multiple_subnets_in_vpc.py b/test/integration/component/test_multiple_subnets_in_vpc.py index a9e36966b72..5366bf4dd7d 100644 --- a/test/integration/component/test_multiple_subnets_in_vpc.py +++ b/test/integration/component/test_multiple_subnets_in_vpc.py @@ -35,6 +35,7 @@ from marvin.lib.base import (Account, NetworkOffering, VPC, VpcOffering, + StaticNATRule, NATRule, PublicIPAddress, PublicIpRange) @@ -272,7 +273,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2 # 13. create new public ip range 2 - # 14. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm + # 14. get a free ip 4 in new ip range 2, assign to network, and enable static nat to vm 2 in tier 2 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4 # 15. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm @@ -451,10 +452,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ipaddress=ip_address_1 ) - nat_rule = NATRule.create( + StaticNATRule.enable( self.apiclient, - self.virtual_machine1, - self.services["natrule"], + virtualmachineid=self.virtual_machine1.id, ipaddressid=ipaddress_1.ipaddress.id, networkid=vpc_tier_1.id ) @@ -581,7 +581,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ) try: - self.virtual_machine1 = VirtualMachine.create( + self.virtual_machine2 = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], accountid=self.account1.name, @@ -621,7 +621,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ) self.cleanup.append(self.public_ip_range2) - # 14. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm + # 14. get a free ip 4 in new ip range 2, assign to network, and enable static nat to vm 2 in tier 2 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4 ip_address_4 = self.get_free_ipaddress(self.public_ip_range2.vlan.id) @@ -632,10 +632,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ipaddress=ip_address_4 ) - nat_rule = NATRule.create( + StaticNATRule.enable( self.apiclient, - self.virtual_machine1, - self.services["natrule"], + virtualmachineid=self.virtual_machine2.id, ipaddressid=ipaddress_4.ipaddress.id, networkid=vpc_tier_2.id ) @@ -665,7 +664,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): nat_rule = NATRule.create( self.apiclient, - self.virtual_machine1, + self.virtual_machine2, self.services["natrule"], ipaddressid=ipaddress_5.ipaddress.id, networkid=vpc_tier_2.id @@ -696,7 +695,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): nat_rule = NATRule.create( self.apiclient, - self.virtual_machine1, + self.virtual_machine2, self.services["natrule"], ipaddressid=ipaddress_6.ipaddress.id, networkid=vpc_tier_2.id diff --git a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py index 392620f88cb..18a4ac78eec 100644 --- a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py +++ b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py @@ -35,6 +35,7 @@ from marvin.lib.base import (Account, NetworkOffering, VPC, VpcOffering, + StaticNATRule, NATRule, PublicIPAddress, PublicIpRange) @@ -272,7 +273,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2 # 13. create new public ip range 2 - # 14. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm + # 14. get a free ip 4 in new ip range 2, assign to network, and enable static nat to vm 2 in tier 2 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4 # 15. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm @@ -451,10 +452,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ipaddress=ip_address_1 ) - nat_rule = NATRule.create( + StaticNATRule.enable( self.apiclient, - self.virtual_machine1, - self.services["natrule"], + virtualmachineid=self.virtual_machine1.id, ipaddressid=ipaddress_1.ipaddress.id, networkid=vpc_tier_1.id ) @@ -581,7 +581,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ) try: - self.virtual_machine1 = VirtualMachine.create( + self.virtual_machine2 = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], accountid=self.account1.name, @@ -621,7 +621,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ) self.cleanup.append(self.public_ip_range2) - # 14. get a free ip 4 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm + # 14. get a free ip 4 in new ip range 2, assign to network, and enable static nat to vm 2 in tier 2 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 4 ip_address_4 = self.get_free_ipaddress(self.public_ip_range2.vlan.id) @@ -632,10 +632,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): ipaddress=ip_address_4 ) - nat_rule = NATRule.create( + StaticNATRule.enable( self.apiclient, - self.virtual_machine1, - self.services["natrule"], + virtualmachineid=self.virtual_machine2.id, ipaddressid=ipaddress_4.ipaddress.id, networkid=vpc_tier_2.id ) @@ -665,7 +664,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): nat_rule = NATRule.create( self.apiclient, - self.virtual_machine1, + self.virtual_machine2, self.services["natrule"], ipaddressid=ipaddress_5.ipaddress.id, networkid=vpc_tier_2.id @@ -696,7 +695,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): nat_rule = NATRule.create( self.apiclient, - self.virtual_machine1, + self.virtual_machine2, self.services["natrule"], ipaddressid=ipaddress_6.ipaddress.id, networkid=vpc_tier_2.id From 51f3756030f9b5a0c8d745da26602ff8d72c072a Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 18 Nov 2020 15:59:05 +0000 Subject: [PATCH 04/14] bugfix #4 vpc vr: Do NOT send Nic plug in/out command to Stopped/Stopping VR --- .../network/topology/AdvancedNetworkTopology.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/network/topology/AdvancedNetworkTopology.java b/server/src/main/java/org/apache/cloudstack/network/topology/AdvancedNetworkTopology.java index f35f1425f81..fc29fcca998 100644 --- a/server/src/main/java/org/apache/cloudstack/network/topology/AdvancedNetworkTopology.java +++ b/server/src/main/java/org/apache/cloudstack/network/topology/AdvancedNetworkTopology.java @@ -214,7 +214,11 @@ public class AdvancedNetworkTopology extends BasicNetworkTopology { final boolean result = applyRules(network, router, typeString, isPodLevelException, podId, failWhenDisconnect, new RuleApplierWrapper(ipAssociationRules)); if (result) { - _advancedVisitor.visit(nicPlugInOutRules); + if (router.getState() == State.Stopped || router.getState() == State.Stopping) { + s_logger.debug("Router " + router.getInstanceName() + " is in " + router.getState() + ", so not sending NicPlugInOutRules command to the backend"); + } else { + _advancedVisitor.visit(nicPlugInOutRules); + } } return result; @@ -241,4 +245,4 @@ public class AdvancedNetworkTopology extends BasicNetworkTopology { final boolean result = applyRules(network, router, typeString, isPodLevelException, podId, failWhenDisconnect, new RuleApplierWrapper(aclsRules)); return result; } -} \ No newline at end of file +} From 5cc6fedb1f2aae97a1d69e63c45b62adba2110cc Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 18 Nov 2020 16:04:31 +0000 Subject: [PATCH 05/14] Revert "Handle private gateways more reliably" This reverts commit f4f9b3ab4ef2ef34e4d8a04c6ebfbf0784497227. --- systemvm/debian/opt/cloud/bin/merge.py | 42 -------------------------- 1 file changed, 42 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/merge.py b/systemvm/debian/opt/cloud/bin/merge.py index b988b7a3e37..4ab9911824d 100755 --- a/systemvm/debian/opt/cloud/bin/merge.py +++ b/systemvm/debian/opt/cloud/bin/merge.py @@ -153,13 +153,10 @@ class updateDataBag: dp['gateway'] = d['router_guest_gateway'] dp['nic_dev_id'] = d['device'][3:] dp['nw_type'] = 'guest' - dp = PrivateGatewayHack.update_network_type_for_privategateway(dbag, dp) qf = QueueFile() qf.load({'ip_address': [dp], 'type': 'ips'}) if 'domain_name' not in d.keys() or d['domain_name'] == '': d['domain_name'] = "cloudnine.internal" - - d = PrivateGatewayHack.update_network_type_for_privategateway(dbag, d) return cs_guestnetwork.merge(dbag, d) def process_dhcp_entry(self, dbag): @@ -329,42 +326,3 @@ class QueueFile: os.remove(origPath) logging.debug("Processed file written to %s", zipped_file_name) - - -class PrivateGatewayHack: - - @classmethod - def update_network_type_for_privategateway(cls, dbag, data): - ip = data['router_guest_ip'] if 'router_guest_ip' in data.keys() else data['public_ip'] - - initial_data = cls.load_inital_data() - has_private_gw_ip = cls.if_config_has_privategateway(initial_data) - private_gw_matches = 'privategateway' in initial_data['config'] and cls.ip_matches_private_gateway_ip(ip, initial_data['config']['privategateway']) - - if has_private_gw_ip and private_gw_matches: - data['nw_type'] = "public" - logging.debug("Updating nw_type for ip %s" % ip) - else: - logging.debug("Not updating nw_type for ip %s because has_private_gw_ip = %s and private_gw_matches = %s " % (ip, has_private_gw_ip, private_gw_matches)) - return data - - @classmethod - def if_config_has_privategateway(cls, dbag): - return 'privategateway' in dbag['config'].keys() and dbag['config']['privategateway'] != "None" - - @classmethod - def ip_matches_private_gateway_ip(cls, ip, private_gateway_ip): - new_ip_matches_private_gateway_ip = False - if ip == private_gateway_ip: - new_ip_matches_private_gateway_ip = True - return new_ip_matches_private_gateway_ip - - @classmethod - def load_inital_data(cls): - initial_data_bag = DataBag() - initial_data_bag.setKey('cmdline') - initial_data_bag.load() - initial_data = initial_data_bag.getDataBag() - logging.debug("Initial data = %s" % initial_data) - - return initial_data From 66d3e1f6d45b07dde7c973a89b5413ab53b3521f Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 18 Nov 2020 16:05:02 +0000 Subject: [PATCH 06/14] Revert "Add private gateway IP to router initialization config" This reverts commit 65cb22216aa1d5d6257f99d4ad84f80c319cdea9. --- .../router/VpcVirtualNetworkApplianceManagerImpl.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java index 556fe18388f..7c957614ddd 100644 --- a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java @@ -77,7 +77,6 @@ import com.cloud.network.vpc.StaticRoute; import com.cloud.network.vpc.StaticRouteProfile; import com.cloud.network.vpc.Vpc; import com.cloud.network.vpc.VpcGateway; -import com.cloud.network.vpc.VpcGatewayVO; import com.cloud.network.vpc.VpcManager; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.PrivateIpDao; @@ -277,15 +276,6 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian if (defaultDns2 != null) { buf.append(" dns2=").append(defaultDns2); } - - VpcGatewayVO privateGatewayForVpc = _vpcGatewayDao.getPrivateGatewayForVpc(domainRouterVO.getVpcId()); - if (privateGatewayForVpc != null) { - String ip4Address = privateGatewayForVpc.getIp4Address(); - buf.append(" privategateway=").append(ip4Address); - s_logger.debug("Set privategateway field in cmd_line.json to " + ip4Address); - } else { - buf.append(" privategateway=None"); - } } } From 7e6f484332ed00a94511209a6af05ddf734e157d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 18 Nov 2020 16:05:14 +0000 Subject: [PATCH 07/14] Revert "Fix Policy Based Routing for private gateway static routes (#3604)" This reverts commit 82d94a87c56e1414d155271cff2481175640bc20. --- systemvm/debian/opt/cloud/bin/configure.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/configure.py b/systemvm/debian/opt/cloud/bin/configure.py index 01e580fce2f..0f9d6eadf76 100755 --- a/systemvm/debian/opt/cloud/bin/configure.py +++ b/systemvm/debian/opt/cloud/bin/configure.py @@ -949,11 +949,11 @@ class CsForwardingRules(CsDataBag): raise Exception("Ip address %s has no device in the ips databag" % rule["public_ip"]) self.fw.append(["mangle", "front", - "-A PREROUTING -d %s/32 -m state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" % - rule["public_ip"]]) + "-A PREROUTING -s %s/32 -m state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" % + rule["internal_ip"]]) self.fw.append(["mangle", "front", - "-A PREROUTING -d %s/32 -m state --state NEW -j MARK --set-xmark %s/0xffffffff" % - (rule["public_ip"], hex(100 + int(device[len("eth"):])))]) + "-A PREROUTING -s %s/32 -m state --state NEW -j MARK --set-xmark %s/0xffffffff" % + (rule["internal_ip"], hex(100 + int(device[len("eth"):])))]) self.fw.append(["nat", "front", "-A PREROUTING -d %s/32 -j DNAT --to-destination %s" % (rule["public_ip"], rule["internal_ip"])]) self.fw.append(["nat", "front", From 8fb2efee1c2021ee57150248775ef3cbfa54bc7f Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 19 Nov 2020 08:33:15 +0000 Subject: [PATCH 08/14] bugfix #6 vpc vr: Add iptables rules for ACL of private gateway --- .../com/cloud/agent/api/to/IpAddressTO.java | 9 +++++++ .../facade/IpAssociationConfigItem.java | 1 + .../virtualnetwork/model/IpAddress.java | 9 +++++++ .../network/router/CommandSetupHelper.java | 26 ++++++++----------- systemvm/debian/opt/cloud/bin/cs/CsAddress.py | 17 ++++++++++++ 5 files changed, 47 insertions(+), 15 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/to/IpAddressTO.java b/api/src/main/java/com/cloud/agent/api/to/IpAddressTO.java index 1169820e19a..5be71bad0ed 100644 --- a/api/src/main/java/com/cloud/agent/api/to/IpAddressTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/IpAddressTO.java @@ -35,6 +35,7 @@ public class IpAddressTO { private String networkName; private Integer nicDevId; private boolean newNic; + private boolean isPrivateGateway; public IpAddressTO(long accountId, String ipAddress, boolean add, boolean firstIP, boolean sourceNat, String broadcastUri, String vlanGateway, String vlanNetmask, String vifMacAddress, Integer networkRate, boolean isOneToOneNat) { @@ -133,4 +134,12 @@ public class IpAddressTO { public void setNewNic(boolean newNic) { this.newNic = newNic; } + + public boolean isPrivateGateway() { + return isPrivateGateway; + } + + public void setPrivateGateway(boolean isPrivateGateway) { + this.isPrivateGateway = isPrivateGateway; + } } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/IpAssociationConfigItem.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/IpAssociationConfigItem.java index 64f953dc726..36908536659 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/IpAssociationConfigItem.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/IpAssociationConfigItem.java @@ -42,6 +42,7 @@ public class IpAssociationConfigItem extends AbstractConfigItemFacade { for (final IpAddressTO ip : command.getIpAddresses()) { final IpAddress ipAddress = new IpAddress(ip.getPublicIp(), ip.isSourceNat(), ip.isAdd(), ip.isOneToOneNat(), ip.isFirstIP(), ip.getVlanGateway(), ip.getVlanNetmask(), ip.getVifMacAddress(), ip.getNicDevId(), ip.isNewNic(), ip.getTrafficType().toString()); + ipAddress.setPrivateGateway(ip.isPrivateGateway()); ips.add(ipAddress); } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/IpAddress.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/IpAddress.java index 70aeb4cfc89..627bcf0dace 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/IpAddress.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/IpAddress.java @@ -32,6 +32,7 @@ public class IpAddress { private Integer nicDevId; private boolean newNic; private String nwType; + private boolean isPrivateGateway; public IpAddress() { // Empty constructor for (de)serialization @@ -133,4 +134,12 @@ public class IpAddress { this.newNic = newNic; } + public boolean isPrivateGateway() { + return isPrivateGateway; + } + + public void setPrivateGateway(boolean isPrivateGateway) { + this.isPrivateGateway = isPrivateGateway; + } + } diff --git a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java index 63e9d8025f3..167fba9d261 100644 --- a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java +++ b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java @@ -104,9 +104,7 @@ import com.cloud.network.vpc.PrivateIpAddress; import com.cloud.network.vpc.StaticRouteProfile; import com.cloud.network.vpc.Vpc; import com.cloud.network.vpc.VpcGateway; -import com.cloud.network.vpc.VpcGatewayVO; import com.cloud.network.vpc.dao.VpcDao; -import com.cloud.network.vpc.dao.VpcGatewayDao; import com.cloud.offering.NetworkOffering; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; @@ -172,8 +170,6 @@ public class CommandSetupHelper { @Inject private VpcDao _vpcDao; @Inject - private VpcGatewayDao _vpcGatewayDao; - @Inject private VlanDao _vlanDao; @Inject private IPAddressDao _ipAddressDao; @@ -726,8 +722,7 @@ public class CommandSetupHelper { final IpAddressTO ip = new IpAddressTO(ipAddr.getAccountId(), ipAddr.getAddress().addr(), add, firstIP, sourceNat, BroadcastDomainType.fromString(ipAddr.getVlanTag()).toString(), ipAddr.getGateway(), ipAddr.getNetmask(), macAddress, networkRate, ipAddr.isOneToOneNat()); - ip.setTrafficType(getNetworkTrafficType(network)); - ip.setNetworkName(_networkModel.getNetworkTag(router.getHypervisorType(), network)); + setIpAddressNetworkParams(ip, network, router); ipsToSend[i++] = ip; if (ipAddr.isSourceNat()) { sourceNatIpAdd = new Pair(ip, ipAddr.getNetworkId()); @@ -851,8 +846,7 @@ public class CommandSetupHelper { final IpAddressTO ip = new IpAddressTO(ipAddr.getAccountId(), ipAddr.getAddress().addr(), add, firstIP, sourceNat, vlanId, vlanGateway, vlanNetmask, vifMacAddress, networkRate, ipAddr.isOneToOneNat()); - ip.setTrafficType(getNetworkTrafficType(network)); - ip.setNetworkName(_networkModel.getNetworkTag(router.getHypervisorType(), network)); + setIpAddressNetworkParams(ip, network, router); ipsToSend[i++] = ip; /* * send the firstIP = true for the first Add, this is to create @@ -979,8 +973,7 @@ public class CommandSetupHelper { final IpAddressTO ip = new IpAddressTO(Account.ACCOUNT_ID_SYSTEM, ipAddr.getIpAddress(), add, false, ipAddr.getSourceNat(), ipAddr.getBroadcastUri(), ipAddr.getGateway(), ipAddr.getNetmask(), ipAddr.getMacAddress(), null, false); - ip.setTrafficType(getNetworkTrafficType(network)); - ip.setNetworkName(_networkModel.getNetworkTag(router.getHypervisorType(), network)); + setIpAddressNetworkParams(ip, network, router); ipsToSend[i++] = ip; } @@ -1136,13 +1129,16 @@ public class CommandSetupHelper { return dhcpRange; } - private TrafficType getNetworkTrafficType(Network network) { - final VpcGatewayVO gateway = _vpcGatewayDao.getVpcGatewayByNetworkId(network.getId()); - if (gateway != null) { + private void setIpAddressNetworkParams(IpAddressTO ipAddress, final Network network, final VirtualRouter router) { + if (_networkModel.isPrivateGateway(network.getId())) { s_logger.debug("network " + network.getId() + " (name: " + network.getName() + " ) is a vpc private gateway, set traffic type to Public"); - return TrafficType.Public; + ipAddress.setTrafficType(TrafficType.Public); + ipAddress.setPrivateGateway(true); } else { - return network.getTrafficType(); + ipAddress.setTrafficType(network.getTrafficType()); + ipAddress.setPrivateGateway(false); } + ipAddress.setNetworkName(_networkModel.getNetworkTag(router.getHypervisorType(), network)); } + } diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index 3340a5527cd..ff0d855766b 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -197,6 +197,11 @@ class CsInterface: return True return False + def is_private_gateway(self): + if "is_private_gateway" in self.address: + return self.address['is_private_gateway'] + return False + def is_added(self): return self.get_attr("add") @@ -476,6 +481,13 @@ class CsIP: self.fw.append(["", "front", "-A NETWORK_STATS_%s -o %s -s %s" % ("eth1", "eth1", guestNetworkCidr)]) + if self.is_private_gateway(): + self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % + (self.address['network'], self.dev, self.dev)]) + self.fw.append(["filter", "", "-A ACL_INBOUND_%s -j DROP" % self.dev]) + self.fw.append(["mangle", "", + "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" % + (self.dev, self.address['network'], self.address['gateway'], self.dev)]) if self.address["source_nat"]: self.fw.append(["nat", "front", "-A POSTROUTING -o %s -j SNAT --to-source %s" % @@ -625,6 +637,11 @@ class CsIP: return True return False + def is_private_gateway(self): + if "is_private_gateway" in self.address: + return self.address['is_private_gateway'] + return False + def ip(self): return str(self.address['cidr']) From a8c9b4531b754c3395496abe096365d775cb8fa0 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 19 Nov 2020 08:35:59 +0000 Subject: [PATCH 09/14] bugfix #7 vpc vr: allow servers in private gateway to reach internet via the VPC VR if it is gateway --- systemvm/debian/opt/cloud/bin/cs/CsAddress.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index ff0d855766b..7cdca8f41c3 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -492,6 +492,10 @@ class CsIP: self.fw.append(["nat", "front", "-A POSTROUTING -o %s -j SNAT --to-source %s" % (self.dev, self.address['public_ip'])]) + if self.get_gateway() == self.get_ip_address(): + # Accept packet from private gateway if VPC VR is used as gateway + self.fw.append(["filter", "", "-A FORWARD -s %s ! -d %s -j ACCEPT" % + (self.address['network'], self.address['network'])]) if self.get_type() in ["public"]: self.fw.append( From 69c0f71cf7d890d20cfbcab69274f6924209a95a Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 19 Nov 2020 15:53:28 +0000 Subject: [PATCH 10/14] bugfix #8 vpc: add rule for traffic between vm and private gateway --- systemvm/debian/opt/cloud/bin/cs/CsAddress.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index 7cdca8f41c3..489840028e8 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -493,6 +493,15 @@ class CsIP: "-A POSTROUTING -o %s -j SNAT --to-source %s" % (self.dev, self.address['public_ip'])]) if self.get_gateway() == self.get_ip_address(): + for inf, addresses in self.config.address().dbag.iteritems(): + if not inf.startswith("eth"): + continue + for address in addresses: + if "nw_type" in address and address["nw_type"] == "guest": + self.fw.append(["filter", "front", "-A FORWARD -s %s -d %s -j ACL_INBOUND_%s" % + (address["network"], self.address["network"], self.dev)]) + self.fw.append(["filter", "front", "-A FORWARD -s %s -d %s -j ACL_INBOUND_%s" % + (self.address["network"], address["network"], address["device"])]) # Accept packet from private gateway if VPC VR is used as gateway self.fw.append(["filter", "", "-A FORWARD -s %s ! -d %s -j ACCEPT" % (self.address['network'], self.address['network'])]) From 8a68617eee1a8a78f8d5c06a67aee40b2e19baac Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 19 Nov 2020 17:23:53 +0000 Subject: [PATCH 11/14] bugfix #9 vpc vr: Add PREROUTING rule for vm with static nat to multiple private gateways --- systemvm/debian/opt/cloud/bin/configure.py | 26 +++++++++++++++---- systemvm/debian/opt/cloud/bin/cs/CsAddress.py | 3 +++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/configure.py b/systemvm/debian/opt/cloud/bin/configure.py index 0f9d6eadf76..be67f403c8b 100755 --- a/systemvm/debian/opt/cloud/bin/configure.py +++ b/systemvm/debian/opt/cloud/bin/configure.py @@ -817,6 +817,13 @@ class CsForwardingRules(CsDataBag): return interface.get_gateway() return None + def getPrivateGatewayNetworks(self): + interfaces = [] + for interface in self.config.address().get_interfaces(): + if interface.is_private_gateway(): + interfaces.append(interface) + return interfaces + def portsToString(self, ports, delimiter): ports_parts = ports.split(":", 2) if ports_parts[0] == ports_parts[1]: @@ -948,12 +955,21 @@ class CsForwardingRules(CsDataBag): if device is None: raise Exception("Ip address %s has no device in the ips databag" % rule["public_ip"]) + chain_name = "PREROUTING-%s-def" % device self.fw.append(["mangle", "front", - "-A PREROUTING -s %s/32 -m state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" % - rule["internal_ip"]]) - self.fw.append(["mangle", "front", - "-A PREROUTING -s %s/32 -m state --state NEW -j MARK --set-xmark %s/0xffffffff" % - (rule["internal_ip"], hex(100 + int(device[len("eth"):])))]) + "-A PREROUTING -s %s/32 -m state --state NEW -j %s" % + (rule["internal_ip"], chain_name)]) + self.fw.append(["mangle", "", + "-A %s -j MARK --set-xmark %s/0xffffffff" % + (chain_name, hex(100 + int(device[len("eth"):])))]) + self.fw.append(["mangle", "", + "-A %s -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" % + chain_name]) + private_gateways = self.getPrivateGatewayNetworks() + for private_gw in private_gateways: + self.fw.append(["mangle", "front", "-A %s -d %s -j RETURN" % + (chain_name, private_gw.get_network())]) + self.fw.append(["nat", "front", "-A PREROUTING -d %s/32 -j DNAT --to-destination %s" % (rule["public_ip"], rule["internal_ip"])]) self.fw.append(["nat", "front", diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index 489840028e8..44b69500b4c 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -488,6 +488,9 @@ class CsIP: self.fw.append(["mangle", "", "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" % (self.dev, self.address['network'], self.address['gateway'], self.dev)]) + self.fw.append(["mangle", "front", + "-A PREROUTING -s %s -d %s -m state --state NEW -j MARK --set-xmark %s/0xffffffff" % + (self.cl.get_vpccidr(), self.address['network'], hex(100 + int(self.dev[3:])))]) if self.address["source_nat"]: self.fw.append(["nat", "front", "-A POSTROUTING -o %s -j SNAT --to-source %s" % From b2d8fffeea312de9b2190149fa1ab27bd6e7ac39 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 20 Nov 2020 21:46:17 +0000 Subject: [PATCH 12/14] integration test: verify public nics state --- ...st_multiple_subnets_in_isolated_network.py | 46 +++++++++++++++++ ...ultiple_subnets_in_isolated_network_rvr.py | 46 +++++++++++++++++ .../component/test_multiple_subnets_in_vpc.py | 50 +++++++++++++++++++ .../test_multiple_subnets_in_vpc_rvr.py | 50 +++++++++++++++++++ 4 files changed, 192 insertions(+) diff --git a/test/integration/component/test_multiple_subnets_in_isolated_network.py b/test/integration/component/test_multiple_subnets_in_isolated_network.py index dee90663a78..df0171c2330 100644 --- a/test/integration/component/test_multiple_subnets_in_isolated_network.py +++ b/test/integration/component/test_multiple_subnets_in_isolated_network.py @@ -192,6 +192,35 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): sourcenatIp = nic.ipaddress return guestIp, controlIp, sourcenatIp + def verify_router_publicnic_state(self, router, host, publicNics): + command = '/opt/cloud/bin/checkrouter.sh | cut -d ":" -f2 |tr -d " "' + self.logger.debug("Executing command '%s'" % command) + result = get_process_status( + host.ipaddress, + host.port, + host.user, + host.password, + router.linklocalip, + command) + self.assertTrue(len(result) > 0, "Cannot get router %s redundant state" % router.name) + redundant_state = result[0] + self.logger.debug("router %s redudnant state is %s" % (router.name, redundant_state)) + if redundant_state == "FAULT": + self.logger.debug("Skip as redundant_state is %s" % redundant_state) + return + elif redundant_state == "MASTER": + command = 'ip link show |grep BROADCAST | egrep "%s" |grep "state DOWN" |wc -l' % publicNics + elif redundant_state == "BACKUP": + command = 'ip link show |grep BROADCAST | egrep "%s" |grep "state UP" |wc -l' % publicNics + result = get_process_status( + host.ipaddress, + host.port, + host.user, + host.password, + router.linklocalip, + command) + self.assertTrue(len(result) > 0 and result[0] == "0", "Expected result is 0 but actual result is %s" % result[0]) + def verify_network_interfaces_in_router(self, router, host, expectedNics): command = 'ip link show |grep BROADCAST | cut -d ":" -f2 |tr -d " "|tr "\n" ","' self.logger.debug("Executing command '%s'" % command) @@ -338,6 +367,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, guestIp, "eth0", True) self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) + self.verify_router_publicnic_state(router, host, "eth2") # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm ipaddress = PublicIPAddress.create( @@ -363,6 +393,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth2", True) + self.verify_router_publicnic_state(router, host, "eth2") # 5. release the new ip ipaddress.delete(self.apiclient) @@ -378,6 +409,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth2", False) + self.verify_router_publicnic_state(router, host, "eth2") # 6. create new public ip range 1 self.services["publiciprange"]["zoneid"] = self.zone.id @@ -423,6 +455,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -452,6 +485,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -482,6 +516,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 10. release new ip 2 # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -499,6 +534,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 11. release new ip 1 # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -515,6 +551,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 12. create new public ip range 2 self.services["publiciprange"]["zoneid"] = self.zone.id @@ -562,6 +599,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3|eth4") # 14. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," @@ -592,6 +630,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3|eth4") # 15. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," @@ -623,6 +662,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3|eth4") # 16. release new ip 5 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," @@ -641,6 +681,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3|eth4") # 17. release new ip 4 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," @@ -658,6 +699,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3|eth4") # 18. release new ip 3 # verify the available nics in VR should be "eth0,eth1,eth2,eth4," @@ -674,6 +716,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth4") # 19. restart network self.network1.restart(self.apiclient) @@ -688,6 +731,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth4") # reboot router for router in routers: @@ -704,6 +748,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 20. restart network with cleanup self.network1.restart(self.apiclient, cleanup=True) @@ -732,3 +777,4 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") diff --git a/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py b/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py index 778e04fcee6..d798c2b2d9d 100644 --- a/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py +++ b/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py @@ -192,6 +192,35 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): sourcenatIp = nic.ipaddress return guestIp, controlIp, sourcenatIp + def verify_router_publicnic_state(self, router, host, publicNics): + command = '/opt/cloud/bin/checkrouter.sh | cut -d ":" -f2 |tr -d " "' + self.logger.debug("Executing command '%s'" % command) + result = get_process_status( + host.ipaddress, + host.port, + host.user, + host.password, + router.linklocalip, + command) + self.assertTrue(len(result) > 0, "Cannot get router %s redundant state" % router.name) + redundant_state = result[0] + self.logger.debug("router %s redudnant state is %s" % (router.name, redundant_state)) + if redundant_state == "FAULT": + self.logger.debug("Skip as redundant_state is %s" % redundant_state) + return + elif redundant_state == "MASTER": + command = 'ip link show |grep BROADCAST | egrep "%s" |grep "state DOWN" |wc -l' % publicNics + elif redundant_state == "BACKUP": + command = 'ip link show |grep BROADCAST | egrep "%s" |grep "state UP" |wc -l' % publicNics + result = get_process_status( + host.ipaddress, + host.port, + host.user, + host.password, + router.linklocalip, + command) + self.assertTrue(len(result) > 0 and result[0] == "0", "Expected result is 0 but actual result is %s" % result[0]) + def verify_network_interfaces_in_router(self, router, host, expectedNics): command = 'ip link show |grep BROADCAST | cut -d ":" -f2 |tr -d " "|tr "\n" ","' self.logger.debug("Executing command '%s'" % command) @@ -338,6 +367,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, guestIp, "eth0", True) self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) + self.verify_router_publicnic_state(router, host, "eth2") # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm ipaddress = PublicIPAddress.create( @@ -363,6 +393,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth2", True) + self.verify_router_publicnic_state(router, host, "eth2") # 5. release the new ip ipaddress.delete(self.apiclient) @@ -378,6 +409,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth2", False) + self.verify_router_publicnic_state(router, host, "eth2") # 6. create new public ip range 1 self.services["publiciprange"]["zoneid"] = self.zone.id @@ -423,6 +455,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -452,6 +485,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -482,6 +516,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 10. release new ip 2 # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -499,6 +534,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 11. release new ip 1 # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -515,6 +551,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 12. create new public ip range 2 self.services["publiciprange"]["zoneid"] = self.zone.id @@ -562,6 +599,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3|eth4") # 14. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," @@ -592,6 +630,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3|eth4") # 15. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," @@ -623,6 +662,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3|eth4") # 16. release new ip 5 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," @@ -641,6 +681,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3|eth4") # 17. release new ip 4 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," @@ -658,6 +699,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3|eth4") # 18. release new ip 3 # verify the available nics in VR should be "eth0,eth1,eth2,eth4," @@ -674,6 +716,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth4") # 19. restart network self.network1.restart(self.apiclient) @@ -688,6 +731,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth2|eth4") # reboot router for router in routers: @@ -704,6 +748,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") # 20. restart network with cleanup self.network1.restart(self.apiclient, cleanup=True) @@ -718,3 +763,4 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") diff --git a/test/integration/component/test_multiple_subnets_in_vpc.py b/test/integration/component/test_multiple_subnets_in_vpc.py index 5366bf4dd7d..454a294d548 100644 --- a/test/integration/component/test_multiple_subnets_in_vpc.py +++ b/test/integration/component/test_multiple_subnets_in_vpc.py @@ -195,6 +195,35 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): sourcenatIp = nic.ipaddress return controlIp, sourcenatIp, tier1_Ip, tier2_Ip + def verify_router_publicnic_state(self, router, host, publicNics): + command = '/opt/cloud/bin/checkrouter.sh | cut -d ":" -f2 |tr -d " "' + self.logger.debug("Executing command '%s'" % command) + result = get_process_status( + host.ipaddress, + host.port, + host.user, + host.password, + router.linklocalip, + command) + self.assertTrue(len(result) > 0, "Cannot get router %s redundant state" % router.name) + redundant_state = result[0] + self.logger.debug("router %s redudnant state is %s" % (router.name, redundant_state)) + if redundant_state == "FAULT": + self.logger.debug("Skip as redundant_state is %s" % redundant_state) + return + elif redundant_state == "MASTER": + command = 'ip link show |grep BROADCAST | egrep "%s" |grep "state DOWN" |wc -l' % publicNics + elif redundant_state == "BACKUP": + command = 'ip link show |grep BROADCAST | egrep "%s" |grep "state UP" |wc -l' % publicNics + result = get_process_status( + host.ipaddress, + host.port, + host.user, + host.password, + router.linklocalip, + command) + self.assertTrue(len(result) > 0 and result[0] == "0", "Expected result is 0 but actual result is %s" % result[0]) + def verify_network_interfaces_in_router(self, router, host, expectedNics): command = 'ip link show |grep BROADCAST | cut -d ":" -f2 |tr -d " "|tr "\n" ","' self.logger.debug("Executing command '%s'" % command) @@ -385,6 +414,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) + self.verify_router_publicnic_state(router, host, "eth1") # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm ipaddress = PublicIPAddress.create( @@ -410,6 +440,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth1", True) + self.verify_router_publicnic_state(router, host, "eth1") # 5. release the new ip ipaddress.delete(self.apiclient) @@ -425,6 +456,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth1", False) + self.verify_router_publicnic_state(router, host, "eth1") # 6. create new public ip range 1 self.services["publiciprange"]["zoneid"] = self.zone.id @@ -470,6 +502,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -499,6 +532,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -529,6 +563,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 10. release new ip 2 # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -546,6 +581,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 11. release new ip 1 # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -562,6 +598,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 12. create a tier in the vpc, and create a vm in the tier. # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," @@ -604,6 +641,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 13. create new public ip range 2 self.services["publiciprange"]["zoneid"] = self.zone.id @@ -650,6 +688,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 15. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," @@ -681,6 +720,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 16. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," @@ -713,6 +753,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 17. release new ip 5 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," @@ -732,6 +773,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 18. release new ip 4 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," @@ -750,6 +792,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 19. release new ip 3 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," @@ -767,6 +810,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth5") # 20. restart tier1 vpc_tier_1.restart(self.apiclient) @@ -780,6 +824,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth5") #21. restart tier2 vpc_tier_2.restart(self.apiclient) @@ -793,6 +838,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth5") # 22. restart VPC self.vpc1.restart(self.apiclient) @@ -806,6 +852,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth5") # reboot router for router in routers: @@ -821,6 +868,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2") # 23. restart VPC with cleanup self.vpc1.restart(self.apiclient, cleanup=True) @@ -834,6 +882,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2") # 24. restart VPC with cleanup, makeredundant=true self.vpc1.restart(self.apiclient, cleanup=True, makeredundant=True) @@ -847,3 +896,4 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2") diff --git a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py index 18a4ac78eec..a9cf3f0d892 100644 --- a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py +++ b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py @@ -195,6 +195,35 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): sourcenatIp = nic.ipaddress return controlIp, sourcenatIp, tier1_Ip, tier2_Ip + def verify_router_publicnic_state(self, router, host, publicNics): + command = '/opt/cloud/bin/checkrouter.sh | cut -d ":" -f2 |tr -d " "' + self.logger.debug("Executing command '%s'" % command) + result = get_process_status( + host.ipaddress, + host.port, + host.user, + host.password, + router.linklocalip, + command) + self.assertTrue(len(result) > 0, "Cannot get router %s redundant state" % router.name) + redundant_state = result[0] + self.logger.debug("router %s redudnant state is %s" % (router.name, redundant_state)) + if redundant_state == "FAULT": + self.logger.debug("Skip as redundant_state is %s" % redundant_state) + return + elif redundant_state == "MASTER": + command = 'ip link show |grep BROADCAST | egrep "%s" |grep "state DOWN" |wc -l' % publicNics + elif redundant_state == "BACKUP": + command = 'ip link show |grep BROADCAST | egrep "%s" |grep "state UP" |wc -l' % publicNics + result = get_process_status( + host.ipaddress, + host.port, + host.user, + host.password, + router.linklocalip, + command) + self.assertTrue(len(result) > 0 and result[0] == "0", "Expected result is 0 but actual result is %s" % result[0]) + def verify_network_interfaces_in_router(self, router, host, expectedNics): command = 'ip link show |grep BROADCAST | cut -d ":" -f2 |tr -d " "|tr "\n" ","' self.logger.debug("Executing command '%s'" % command) @@ -385,6 +414,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) + self.verify_router_publicnic_state(router, host, "eth1") # 4. get a free public ip, assign to network, and create port forwarding rules (ssh) to the vm ipaddress = PublicIPAddress.create( @@ -410,6 +440,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth1", True) + self.verify_router_publicnic_state(router, host, "eth1") # 5. release the new ip ipaddress.delete(self.apiclient) @@ -425,6 +456,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress.ipaddress.ipaddress, "eth1", False) + self.verify_router_publicnic_state(router, host, "eth1") # 6. create new public ip range 1 self.services["publiciprange"]["zoneid"] = self.zone.id @@ -470,6 +502,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 8. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -499,6 +532,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 9. get a free ip in new ip range, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -529,6 +563,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 10. release new ip 2 # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -546,6 +581,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 11. release new ip 1 # verify the available nics in VR should be "eth0,eth1,eth2,eth3" @@ -562,6 +598,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_1.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_2.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 12. create a tier in the vpc, and create a vm in the tier. # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," @@ -604,6 +641,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3") # 13. create new public ip range 2 self.services["publiciprange"]["zoneid"] = self.zone.id @@ -650,6 +688,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 15. get a free ip 5 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," @@ -681,6 +720,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 16. get a free ip 6 in new ip range 2, assign to network, and create port forwarding rules (ssh) to the vm # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," @@ -713,6 +753,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 17. release new ip 5 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," @@ -732,6 +773,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", True) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 18. release new ip 4 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," @@ -750,6 +792,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 19. release new ip 3 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," @@ -767,6 +810,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth5", False) self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth5", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth5") # 20. restart tier1 vpc_tier_1.restart(self.apiclient) @@ -780,6 +824,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth5") #21. restart tier2 vpc_tier_2.restart(self.apiclient) @@ -793,6 +838,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth5") # 22. restart VPC self.vpc1.restart(self.apiclient) @@ -806,6 +852,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth5") # reboot router for router in routers: @@ -821,6 +868,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2") # 23. restart VPC with cleanup self.vpc1.restart(self.apiclient, cleanup=True) @@ -834,6 +882,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2") # 24. restart VPC with cleanup, makeredundant=true self.vpc1.restart(self.apiclient, cleanup=True, makeredundant=True) @@ -847,3 +896,4 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2") From 6a91b8ace718e3208263cf8623517f7286c66256 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 20 Nov 2020 21:48:13 +0000 Subject: [PATCH 13/14] integration test: add private gateway in test --- .../component/test_multiple_subnets_in_vpc.py | 47 +++++++++++++++---- .../test_multiple_subnets_in_vpc_rvr.py | 47 +++++++++++++++---- 2 files changed, 76 insertions(+), 18 deletions(-) diff --git a/test/integration/component/test_multiple_subnets_in_vpc.py b/test/integration/component/test_multiple_subnets_in_vpc.py index 454a294d548..f2b72af36c3 100644 --- a/test/integration/component/test_multiple_subnets_in_vpc.py +++ b/test/integration/component/test_multiple_subnets_in_vpc.py @@ -35,6 +35,7 @@ from marvin.lib.base import (Account, NetworkOffering, VPC, VpcOffering, + PrivateGateway, StaticNATRule, NATRule, PublicIPAddress, @@ -854,46 +855,74 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) self.verify_router_publicnic_state(router, host, "eth1|eth5") + # Add private gateway + private_gateway_ip = "172.16." + str(random_subnet_number + 2) + ".1" + private_gateway = PrivateGateway.create( + self.apiclient, + gateway=private_gateway_ip, + ipaddress=private_gateway_ip, + netmask='255.255.255.0', + vlan=get_free_vlan(self.apiclient, self.zone.id)[1], + vpcid=self.vpc1.id + ) + routers = self.get_vpc_routers(self.vpc1.id) + for router in routers: + host = self.get_router_host(router) + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,eth5,eth3,") + controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router) + self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) + self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) + self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) + self.verify_ip_address_in_router(router, host, private_gateway_ip, "eth3", True) + self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", False) + self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") + # reboot router + routers = self.get_vpc_routers(self.vpc1.id) for router in routers: cmd = rebootRouter.rebootRouterCmd() cmd.id = router.id self.apiclient.rebootRouter(cmd) router = self.get_router(router.id) host = self.get_router_host(router) - self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,") + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,") controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router) self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) - self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) - self.verify_router_publicnic_state(router, host, "eth1|eth2") + self.verify_ip_address_in_router(router, host, private_gateway_ip, "eth4", True) + self.verify_ip_address_in_router(router, host, tier2_Ip, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2|eth4") # 23. restart VPC with cleanup self.vpc1.restart(self.apiclient, cleanup=True) routers = self.get_vpc_routers(self.vpc1.id) for router in routers: host = self.get_router_host(router) - self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,") + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,") controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router) self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) - self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) - self.verify_router_publicnic_state(router, host, "eth1|eth2") + self.verify_ip_address_in_router(router, host, private_gateway_ip, "eth4", True) + self.verify_ip_address_in_router(router, host, tier2_Ip, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2|eth4") # 24. restart VPC with cleanup, makeredundant=true self.vpc1.restart(self.apiclient, cleanup=True, makeredundant=True) routers = self.get_vpc_routers(self.vpc1.id) for router in routers: host = self.get_router_host(router) - self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,") + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,") controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router) self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) - self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) - self.verify_router_publicnic_state(router, host, "eth1|eth2") + self.verify_ip_address_in_router(router, host, private_gateway_ip, "eth4", True) + self.verify_ip_address_in_router(router, host, tier2_Ip, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2|eth4") diff --git a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py index a9cf3f0d892..84e8c824cd2 100644 --- a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py +++ b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py @@ -35,6 +35,7 @@ from marvin.lib.base import (Account, NetworkOffering, VPC, VpcOffering, + PrivateGateway, StaticNATRule, NATRule, PublicIPAddress, @@ -854,46 +855,74 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) self.verify_router_publicnic_state(router, host, "eth1|eth5") + # Add private gateway + private_gateway_ip = "172.16." + str(random_subnet_number + 2) + ".1" + private_gateway = PrivateGateway.create( + self.apiclient, + gateway=private_gateway_ip, + ipaddress=private_gateway_ip, + netmask='255.255.255.0', + vlan=get_free_vlan(self.apiclient, self.zone.id)[1], + vpcid=self.vpc1.id + ) + routers = self.get_vpc_routers(self.vpc1.id) + for router in routers: + host = self.get_router_host(router) + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth4,eth5,eth3,") + controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router) + self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) + self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) + self.verify_ip_address_in_router(router, host, tier1_Ip, "eth2", True) + self.verify_ip_address_in_router(router, host, private_gateway_ip, "eth3", True) + self.verify_ip_address_in_router(router, host, ipaddress_3.ipaddress.ipaddress, "eth3", False) + self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) + self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") + # reboot router + routers = self.get_vpc_routers(self.vpc1.id) for router in routers: cmd = rebootRouter.rebootRouterCmd() cmd.id = router.id self.apiclient.rebootRouter(cmd) router = self.get_router(router.id) host = self.get_router_host(router) - self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,") + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,") controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router) self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) - self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) - self.verify_router_publicnic_state(router, host, "eth1|eth2") + self.verify_ip_address_in_router(router, host, private_gateway_ip, "eth4", True) + self.verify_ip_address_in_router(router, host, tier2_Ip, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2|eth4") # 23. restart VPC with cleanup self.vpc1.restart(self.apiclient, cleanup=True) routers = self.get_vpc_routers(self.vpc1.id) for router in routers: host = self.get_router_host(router) - self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,") + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,") controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router) self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) - self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) - self.verify_router_publicnic_state(router, host, "eth1|eth2") + self.verify_ip_address_in_router(router, host, private_gateway_ip, "eth4", True) + self.verify_ip_address_in_router(router, host, tier2_Ip, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2|eth4") # 24. restart VPC with cleanup, makeredundant=true self.vpc1.restart(self.apiclient, cleanup=True, makeredundant=True) routers = self.get_vpc_routers(self.vpc1.id) for router in routers: host = self.get_router_host(router) - self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,") + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,eth5,") controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router) self.verify_ip_address_in_router(router, host, controlIp, "eth0", True) self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True) self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True) - self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True) - self.verify_router_publicnic_state(router, host, "eth1|eth2") + self.verify_ip_address_in_router(router, host, private_gateway_ip, "eth4", True) + self.verify_ip_address_in_router(router, host, tier2_Ip, "eth5", True) + self.verify_router_publicnic_state(router, host, "eth1|eth2|eth4") From a979ab9050fb78dea6ba4359fdf93ec09b231c99 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 23 Nov 2020 09:06:42 +0000 Subject: [PATCH 14/14] integration test: update steps --- ...st_multiple_subnets_in_isolated_network.py | 29 +++++++++++-- ...ultiple_subnets_in_isolated_network_rvr.py | 42 ++++++++++++++++++- .../component/test_multiple_subnets_in_vpc.py | 35 +++++++++++++--- .../test_multiple_subnets_in_vpc_rvr.py | 35 +++++++++++++--- 4 files changed, 126 insertions(+), 15 deletions(-) diff --git a/test/integration/component/test_multiple_subnets_in_isolated_network.py b/test/integration/component/test_multiple_subnets_in_isolated_network.py index df0171c2330..9cbd3bb2fba 100644 --- a/test/integration/component/test_multiple_subnets_in_isolated_network.py +++ b/test/integration/component/test_multiple_subnets_in_isolated_network.py @@ -308,6 +308,21 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): # 17. release new ip 4 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 6 + # 18. release new ip 3 + # verify the available nics in VR should be "eth0,eth1,eth2,eth4," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth4 -> new ip 6 + # 19. restart network + # verify the available nics in VR should be "eth0,eth1,eth2,eth4," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth4 -> new ip 6 + # 20. reboot router + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 + # 21. restart network with cleanup + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 + # 22. restart network with cleanup, makeredundant=true + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 """ # Create new domain1 @@ -719,6 +734,8 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_router_publicnic_state(router, host, "eth2|eth4") # 19. restart network + # verify the available nics in VR should be "eth0,eth1,eth2,eth4," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth4 -> new ip 6 self.network1.restart(self.apiclient) routers = self.get_routers(self.network1.id) for router in routers: @@ -733,7 +750,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) self.verify_router_publicnic_state(router, host, "eth2|eth4") - # reboot router + # 20. reboot router + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 for router in routers: cmd = rebootRouter.rebootRouterCmd() cmd.id = router.id @@ -750,7 +769,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) self.verify_router_publicnic_state(router, host, "eth2|eth3") - # 20. restart network with cleanup + # 21. restart network with cleanup + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 self.network1.restart(self.apiclient, cleanup=True) routers = self.get_routers(self.network1.id) for router in routers: @@ -764,7 +785,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) - # 21. restart network with cleanup, makeredundant=true + # 22. restart network with cleanup, makeredundant=true + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 self.network1.restart(self.apiclient, cleanup=True, makeredundant=True) routers = self.get_routers(self.network1.id) for router in routers: diff --git a/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py b/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py index d798c2b2d9d..7b58e003e4a 100644 --- a/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py +++ b/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py @@ -308,6 +308,21 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): # 17. release new ip 4 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4," # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 3, eth4 -> new ip 6 + # 18. release new ip 3 + # verify the available nics in VR should be "eth0,eth1,eth2,eth4," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth4 -> new ip 6 + # 19. restart network + # verify the available nics in VR should be "eth0,eth1,eth2,eth4," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth4 -> new ip 6 + # 20. reboot router + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 + # 21. restart network with cleanup + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 + # 22. restart network with cleanup, makeredundant=true + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 """ # Create new domain1 @@ -719,6 +734,8 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_router_publicnic_state(router, host, "eth2|eth4") # 19. restart network + # verify the available nics in VR should be "eth0,eth1,eth2,eth4," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth4 -> new ip 6 self.network1.restart(self.apiclient) routers = self.get_routers(self.network1.id) for router in routers: @@ -733,7 +750,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True) self.verify_router_publicnic_state(router, host, "eth2|eth4") - # reboot router + # 20. reboot router + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 for router in routers: cmd = rebootRouter.rebootRouterCmd() cmd.id = router.id @@ -750,7 +769,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) self.verify_router_publicnic_state(router, host, "eth2|eth3") - # 20. restart network with cleanup + # 21. restart network with cleanup + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 self.network1.restart(self.apiclient, cleanup=True) routers = self.get_routers(self.network1.id) for router in routers: @@ -764,3 +785,20 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False) self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) self.verify_router_publicnic_state(router, host, "eth2|eth3") + + # 22. restart network with cleanup, makeredundant=true + # verify the available nics in VR should be "eth0,eth1,eth2,eth3," + # verify the IPs in VR. eth0 -> guest nic, eth2 -> source nat IP, eth3 -> new ip 6 + self.network1.restart(self.apiclient, cleanup=True, makeredundant=True) + routers = self.get_routers(self.network1.id) + for router in routers: + host = self.get_router_host(router) + self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,") + guestIp, controlIp, sourcenatIp = self.get_router_ips(router) + self.verify_ip_address_in_router(router, host, guestIp, "eth0", True) + self.verify_ip_address_in_router(router, host, controlIp, "eth1", True) + self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True) + self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False) + self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False) + self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True) + self.verify_router_publicnic_state(router, host, "eth2|eth3") diff --git a/test/integration/component/test_multiple_subnets_in_vpc.py b/test/integration/component/test_multiple_subnets_in_vpc.py index f2b72af36c3..eaa42169771 100644 --- a/test/integration/component/test_multiple_subnets_in_vpc.py +++ b/test/integration/component/test_multiple_subnets_in_vpc.py @@ -318,6 +318,23 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): # 18. release new ip 4 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 6 + # 19. release new ip 3 + # verify the available nics in VR should be "eth0,eth1,eth2,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth4 -> tier 2, eth5 -> new ip 6 + # 20. restart tier1 + # 22. restart VPC + # 23. Add private gateway + # verify the available nics in VR should be "eth0,eth1,eth2,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth4 -> tier 2, eth5 -> new ip 6, eth3-> private gateway + # 24. reboot router + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 + # 25. restart VPC with cleanup + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 + # 26. restart VPC with cleanup, makeredundant=true + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 """ # Create new domain1 @@ -796,7 +813,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 19. release new ip 3 - # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the available nics in VR should be "eth0,eth1,eth2,eth4,eth5," # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth4 -> tier 2, eth5 -> new ip 6 ipaddress_3.delete(self.apiclient) routers = self.get_vpc_routers(self.vpc1.id) @@ -855,7 +872,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) self.verify_router_publicnic_state(router, host, "eth1|eth5") - # Add private gateway + # 23. Add private gateway + # verify the available nics in VR should be "eth0,eth1,eth2,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth4 -> tier 2, eth5 -> new ip 6, eth3-> private gateway private_gateway_ip = "172.16." + str(random_subnet_number + 2) + ".1" private_gateway = PrivateGateway.create( self.apiclient, @@ -879,7 +898,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") - # reboot router + # 24. reboot router + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 routers = self.get_vpc_routers(self.vpc1.id) for router in routers: cmd = rebootRouter.rebootRouterCmd() @@ -897,7 +918,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier2_Ip, "eth5", True) self.verify_router_publicnic_state(router, host, "eth1|eth2|eth4") - # 23. restart VPC with cleanup + # 25. restart VPC with cleanup + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 self.vpc1.restart(self.apiclient, cleanup=True) routers = self.get_vpc_routers(self.vpc1.id) for router in routers: @@ -912,7 +935,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier2_Ip, "eth5", True) self.verify_router_publicnic_state(router, host, "eth1|eth2|eth4") - # 24. restart VPC with cleanup, makeredundant=true + # 26. restart VPC with cleanup, makeredundant=true + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 self.vpc1.restart(self.apiclient, cleanup=True, makeredundant=True) routers = self.get_vpc_routers(self.vpc1.id) for router in routers: diff --git a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py index 84e8c824cd2..f6c0f5242d0 100644 --- a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py +++ b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py @@ -318,6 +318,23 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): # 18. release new ip 4 # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth3 -> new ip 3, eth4 -> tier 2, eth5 -> new ip 6 + # 19. release new ip 3 + # verify the available nics in VR should be "eth0,eth1,eth2,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth4 -> tier 2, eth5 -> new ip 6 + # 20. restart tier1 + # 22. restart VPC + # 23. Add private gateway + # verify the available nics in VR should be "eth0,eth1,eth2,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth4 -> tier 2, eth5 -> new ip 6, eth3-> private gateway + # 24. reboot router + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 + # 25. restart VPC with cleanup + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 + # 26. restart VPC with cleanup, makeredundant=true + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 """ # Create new domain1 @@ -796,7 +813,7 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") # 19. release new ip 3 - # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the available nics in VR should be "eth0,eth1,eth2,eth4,eth5," # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth4 -> tier 2, eth5 -> new ip 6 ipaddress_3.delete(self.apiclient) routers = self.get_vpc_routers(self.vpc1.id) @@ -855,7 +872,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) self.verify_router_publicnic_state(router, host, "eth1|eth5") - # Add private gateway + # 23. Add private gateway + # verify the available nics in VR should be "eth0,eth1,eth2,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> tier 1, eth4 -> tier 2, eth5 -> new ip 6, eth3-> private gateway private_gateway_ip = "172.16." + str(random_subnet_number + 2) + ".1" private_gateway = PrivateGateway.create( self.apiclient, @@ -879,7 +898,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True) self.verify_router_publicnic_state(router, host, "eth1|eth3|eth5") - # reboot router + # 24. reboot router + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 routers = self.get_vpc_routers(self.vpc1.id) for router in routers: cmd = rebootRouter.rebootRouterCmd() @@ -897,7 +918,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier2_Ip, "eth5", True) self.verify_router_publicnic_state(router, host, "eth1|eth2|eth4") - # 23. restart VPC with cleanup + # 25. restart VPC with cleanup + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 self.vpc1.restart(self.apiclient, cleanup=True) routers = self.get_vpc_routers(self.vpc1.id) for router in routers: @@ -912,7 +935,9 @@ class TestMultiplePublicIpSubnets(cloudstackTestCase): self.verify_ip_address_in_router(router, host, tier2_Ip, "eth5", True) self.verify_router_publicnic_state(router, host, "eth1|eth2|eth4") - # 24. restart VPC with cleanup, makeredundant=true + # 26. restart VPC with cleanup, makeredundant=true + # verify the available nics in VR should be "eth0,eth1,eth2,eth3,eth4,eth5," + # verify the IPs in VR. eth1 -> source nat IP, eth2 -> new ip 6, eth3 -> tier 1, eth4 -> private gateway, eth5 -> tier 2 self.vpc1.restart(self.apiclient, cleanup=True, makeredundant=True) routers = self.get_vpc_routers(self.vpc1.id) for router in routers: