Merge pull request #2162 from nuagenetworks/vrouter_preformance_fix

CLOUDSTACK-9980: Fix for DNS after network restart with clean up.

systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh

@@ -81,7 +81,12 @@ mv $cfg /var/cache/cloud/processed/
 
 unset DEFER_CONFIG
 # trigger finish_config()
-/opt/cloud/bin/configure.py
+if [ -f /etc/cloudstack/dhcpentry.json ]; then
+    /opt/cloud/bin/configure.py vm_dhcp_entry.json
+fi
+if [ -f /etc/cloudstack/vmdata.json ]; then
+    /opt/cloud/bin/configure.py vm_metadata.json
+fi
 
 # Flush kernel conntrack table
 log_it "VR config: Flushing conntrack table"

test/integration/plugins/nuagevsp/test_nuage_internal_dns.py

@@ -220,8 +220,177 @@ class TestNuageInternalDns(nuageTestCase):
             else:
                 self.fail("excepted value not found in vm: " + item)
 
+    @attr(tags=["advanced", "nuagevsp"], required_hardware="true")
+    def test_03_Isolated_Network_restarts(self):
+        """ Verify InternalDns on Isolated Network with restart networks and
+        ping by hostname
+        """
+
+        # Validate the following
+        # 1. Create an Isolated network - network1 (10.1.1.1/24) by using DNS
+        #    network offering.
+        # 2. Deploy vm1 in network1.
+        # 3. Verify dhcp option 06 and 0f for subnet
+        # 4. Verify dhcp option 06,15 and 0f for vm Interface.
+        # 5. Deploy VM2 in network1.
+        # 6. Verify end to end by pinging with hostname while restarting
+        #    network1 without and with cleanup.
+
+        cmd = updateZone.updateZoneCmd()
+        cmd.id = self.zone.id
+        cmd.domain = "isolated.com"
+        self.apiclient.updateZone(cmd)
+
+        self.debug("Creating and enabling Nuage Vsp Isolated Network "
+                   "offering...")
+        network_offering = self.create_NetworkOffering(
+            self.dnsdata["isolated_network_offering"])
+        self.validate_NetworkOffering(network_offering, state="Enabled")
+
+        network_1 = self.create_Network(network_offering)
+        vm_1 = self.create_VM(network_1)
+
+        # VSD verification
+        self.verify_vsd_network(self.domain.id, network_1)
+        self.verify_vsd_vm(vm_1)
+
+        # Internal DNS check point on VSD
+        self.verify_vsd_dhcp_option(self.DNS, "10.1.1.2", network_1)
+        self.verify_vsd_dhcp_option(self.DOMAINNAME, "isolated.com", network_1)
+        for nic in vm_1.nic:
+            self.verify_vsd_dhcp_option(self.DNS, "10.1.1.2", nic, True)
+            self.verify_vsd_dhcp_option(
+                self.DOMAINNAME, "isolated.com", nic, True)
+            self.verify_vsd_dhcp_option(self.HOSTNAME, "vm1", nic, True)
+
+        self.test_data["virtual_machine"]["displayname"] = "vm2"
+        self.test_data["virtual_machine"]["name"] = "vm2"
+        vm_2 = self.create_VM(network_1)
+        self.test_data["virtual_machine"]["displayname"] = "vm1"
+        self.test_data["virtual_machine"]["name"] = "vm1"
+        self.verify_vsd_vm(vm_2)
+        for nic in vm_2.nic:
+            self.verify_vsd_dhcp_option(self.DNS, "10.1.1.2", nic, True)
+            self.verify_vsd_dhcp_option(
+                self.DOMAINNAME, "isolated.com", nic, True)
+            self.verify_vsd_dhcp_option(self.HOSTNAME, "vm2", nic, True)
+
+        public_ip_1 = self.acquire_PublicIPAddress(network_1)
+        self.create_and_verify_fw(vm_1, public_ip_1, network_1)
+
+        vm_public_ip = public_ip_1.ipaddress.ipaddress
+
+        try:
+            vm_1.ssh_ip = vm_public_ip
+            vm_1.ssh_port = self.test_data["virtual_machine"]["ssh_port"]
+            vm_1.username = self.test_data["virtual_machine"]["username"]
+            vm_1.password = self.test_data["virtual_machine"]["password"]
+            self.debug("SSHing into VM: %s with %s" %
+                       (vm_1.ssh_ip, vm_1.password))
+
+            ssh = vm_1.get_ssh_client(ipaddress=vm_public_ip)
+
+        except Exception as e:
+            self.fail("SSH into VM failed with exception %s" % e)
+
+        cmd = 'ping -c 2 vm2'
+        self.debug("ping vm2 by hostname with command: " + cmd)
+        outputlist = ssh.execute(cmd)
+        self.debug("command is executed properly " + cmd)
+        completeoutput = str(outputlist).strip('[]')
+        self.debug("complete output is " + completeoutput)
+        expectedlist = ['2 received', 'vm2.isolated.com', vm_2.ipaddress]
+        for item in expectedlist:
+            if item in completeoutput:
+                self.debug("excepted value found in vm: " + item)
+            else:
+                self.fail("excepted value not found in vm: " + item)
+
+        # Restarting Isolated network (cleanup = false)
+        self.debug("Restarting the created Isolated network without "
+                   "cleanup...")
+        Network.restart(network_1, self.api_client, cleanup=False)
+        self.validate_Network(network_1, state="Implemented")
+        vr = self.get_Router(network_1)
+        self.check_Router_state(vr, state="Running")
+        self.check_VM_state(vm_1, state="Running")
+        self.check_VM_state(vm_2, state="Running")
+
+        # VSD verification
+        self.verify_vsd_network(self.domain.id, network_1)
+        self.verify_vsd_router(vr)
+        self.verify_vsd_vm(vm_1)
+        self.verify_vsd_vm(vm_2)
+
+        try:
+            vm_1.ssh_ip = vm_public_ip
+            vm_1.ssh_port = self.test_data["virtual_machine"]["ssh_port"]
+            vm_1.username = self.test_data["virtual_machine"]["username"]
+            vm_1.password = self.test_data["virtual_machine"]["password"]
+            self.debug("SSHing into VM: %s with %s" %
+                       (vm_1.ssh_ip, vm_1.password))
+
+            ssh = vm_1.get_ssh_client(ipaddress=vm_public_ip)
+
+        except Exception as e:
+            self.fail("SSH into VM failed with exception %s" % e)
+
+        cmd = 'ping -c 2 vm2'
+        self.debug("ping vm2 by hostname with command: " + cmd)
+        outputlist = ssh.execute(cmd)
+        self.debug("command is executed properly " + cmd)
+        completeoutput = str(outputlist).strip('[]')
+        self.debug("complete output is " + completeoutput)
+        expectedlist = ['2 received', 'vm2.isolated.com', vm_2.ipaddress]
+        for item in expectedlist:
+            if item in completeoutput:
+                self.debug("excepted value found in vm: " + item)
+            else:
+                self.fail("excepted value not found in vm: " + item)
+
+        # Restarting Isolated network (cleanup = true)
+        self.debug("Restarting the created Isolated network with cleanup...")
+        Network.restart(network_1, self.api_client, cleanup=True)
+        self.validate_Network(network_1, state="Implemented")
+        vr = self.get_Router(network_1)
+        self.check_Router_state(vr, state="Running")
+        self.check_VM_state(vm_1, state="Running")
+        self.check_VM_state(vm_2, state="Running")
+
+        # VSD verification
+        self.verify_vsd_network(self.domain.id, network_1)
+        self.verify_vsd_router(vr)
+        self.verify_vsd_vm(vm_1)
+        self.verify_vsd_vm(vm_2)
+
+        try:
+            vm_1.ssh_ip = vm_public_ip
+            vm_1.ssh_port = self.test_data["virtual_machine"]["ssh_port"]
+            vm_1.username = self.test_data["virtual_machine"]["username"]
+            vm_1.password = self.test_data["virtual_machine"]["password"]
+            self.debug("SSHing into VM: %s with %s" %
+                       (vm_1.ssh_ip, vm_1.password))
+
+            ssh = vm_1.get_ssh_client(ipaddress=vm_public_ip)
+
+        except Exception as e:
+            self.fail("SSH into VM failed with exception %s" % e)
+
+        cmd = 'ping -c 2 vm2'
+        self.debug("ping vm2 by hostname with command: " + cmd)
+        outputlist = ssh.execute(cmd)
+        self.debug("command is executed properly " + cmd)
+        completeoutput = str(outputlist).strip('[]')
+        self.debug("complete output is " + completeoutput)
+        expectedlist = ['2 received', 'vm2.isolated.com', vm_2.ipaddress]
+        for item in expectedlist:
+            if item in completeoutput:
+                self.debug("excepted value found in vm: " + item)
+            else:
+                self.fail("excepted value not found in vm: " + item)
+
     @attr(tags=["advanced", "nuagevsp"], required_hardware="false")
-    def test_03_Update_Network_with_Domain(self):
+    def test_04_Update_Network_with_Domain(self):
         """ Verify update NetworkDomain for InternalDns on Isolated Network
         """
 
@@ -278,7 +447,7 @@ class TestNuageInternalDns(nuageTestCase):
             self.verify_vsd_dhcp_option(self.HOSTNAME, "vm1", nic, True)
 
     @attr(tags=["advanced", "nuagevsp"], required_hardware="true")
-    def test_04_Update_Network_with_Domain(self):
+    def test_05_Update_Network_with_Domain(self):
         """ Verify update NetworkDomain for InternalDns on Isolated Network
         with ping VM
         """
@@ -391,7 +560,7 @@ class TestNuageInternalDns(nuageTestCase):
                 self.fail("excepted value not found in vm: " + item)
 
     @attr(tags=["advanced", "nuagevsp"], required_hardware="false")
-    def test_05_VPC_Network_With_InternalDns(self):
+    def test_06_VPC_Network_With_InternalDns(self):
         """ Verify InternalDns on VPC Network
         """
 
@@ -432,7 +601,7 @@ class TestNuageInternalDns(nuageTestCase):
             self.verify_vsd_dhcp_option(self.HOSTNAME, "vm1", nic, True)
 
     @attr(tags=["advanced", "nuagevsp"], required_hardware="true")
-    def test_06_VPC_Network_With_InternalDns(self):
+    def test_07_VPC_Network_With_InternalDns(self):
         """ Verify InternalDns on VPC Network by ping with hostname
         """
 
@@ -521,3 +690,258 @@ class TestNuageInternalDns(nuageTestCase):
                 self.debug("excepted value found in vm: " + item)
             else:
                 self.fail("excepted value not found in vm: " + item)
+
+    @attr(tags=["advanced", "nuagevsp"], required_hardware="true")
+    def test_08_VPC_Network_Restarts_With_InternalDns(self):
+        """ Verify InternalDns on VPC Network with restarts and ping by
+        hostname
+        """
+
+        # Validate the following
+        # 1. Create a VPC and Tier network by using DNS network offering.
+        # 2. Deploy vm1 in Tier network network1.
+        # 3. Verify dhcp option 06 and 0f for subnet
+        # 4. Verify dhcp option 06,15 and 0f for vm Interface.
+        # 5. Deploy Vm2.
+        # 6. Verify end to end by pinging with hostname while restarting
+        #    VPC and Tier without and with cleanup.
+
+        cmd = updateZone.updateZoneCmd()
+        cmd.id = self.zone.id
+        cmd.domain = "vpc.com"
+        self.apiclient.updateZone(cmd)
+
+        vpc_off = self.create_VpcOffering(self.dnsdata["vpc_offering"])
+        self.validate_VpcOffering(vpc_off, state="Enabled")
+        vpc = self.create_Vpc(vpc_off, cidr='10.1.0.0/16', cleanup=False)
+
+        self.debug("Creating Nuage Vsp VPC Network offering...")
+        network_offering = self.create_NetworkOffering(
+            self.dnsdata["vpc_network_offering"])
+        self.validate_NetworkOffering(network_offering, state="Enabled")
+        network_1 = self.create_Network(
+            network_offering, gateway='10.1.1.1', vpc=vpc)
+
+        vm_1 = self.create_VM(network_1)
+
+        # VSD verification
+        self.verify_vsd_network(self.domain.id, network_1, vpc)
+        self.verify_vsd_vm(vm_1)
+        # Internal DNS check point on VSD
+        self.verify_vsd_dhcp_option(self.DNS, "10.1.1.2", network_1)
+        self.verify_vsd_dhcp_option(self.DOMAINNAME, "vpc.com", network_1)
+        for nic in vm_1.nic:
+            self.verify_vsd_dhcp_option(self.DNS, "10.1.1.2", nic, True)
+            self.verify_vsd_dhcp_option(self.DOMAINNAME, "vpc.com", nic, True)
+            self.verify_vsd_dhcp_option(self.HOSTNAME, "vm1", nic, True)
+
+        self.test_data["virtual_machine"]["displayname"] = "vm2"
+        self.test_data["virtual_machine"]["name"] = "vm2"
+        vm_2 = self.create_VM(network_1)
+        self.test_data["virtual_machine"]["displayname"] = "vm1"
+        self.test_data["virtual_machine"]["name"] = "vm1"
+        self.verify_vsd_vm(vm_2)
+        for nic in vm_2.nic:
+            self.verify_vsd_dhcp_option(self.DNS, "10.1.1.2", nic, True)
+            self.verify_vsd_dhcp_option(self.DOMAINNAME, "vpc.com", nic, True)
+            self.verify_vsd_dhcp_option(self.HOSTNAME, "vm2", nic, True)
+
+        public_ip_1 = self.acquire_PublicIPAddress(network_1, vpc)
+        self.create_StaticNatRule_For_VM(vm_1, public_ip_1, network_1)
+        # Adding Network ACL rule in the Public tier
+        self.debug("Adding Network ACL rule to make the created NAT rule "
+                   "(SSH) accessible...")
+        public_ssh_rule = self.create_NetworkAclRule(
+            self.test_data["ingress_rule"], network=network_1)
+
+        # VSD verification
+        self.verify_vsd_firewall_rule(public_ssh_rule)
+        vm_public_ip = public_ip_1.ipaddress.ipaddress
+
+        try:
+            vm_1.ssh_ip = vm_public_ip
+            vm_1.ssh_port = self.test_data["virtual_machine"]["ssh_port"]
+            vm_1.username = self.test_data["virtual_machine"]["username"]
+            vm_1.password = self.test_data["virtual_machine"]["password"]
+            self.debug("SSHing into VM: %s with %s" %
+                       (vm_1.ssh_ip, vm_1.password))
+
+            ssh = vm_1.get_ssh_client(ipaddress=vm_public_ip)
+
+        except Exception as e:
+            self.fail("SSH into VM failed with exception %s" % e)
+
+        cmd = 'ping -c 2 vm2'
+        self.debug("ping vm2 by hostname with command: " + cmd)
+        outputlist = ssh.execute(cmd)
+        self.debug("command is executed properly " + cmd)
+        completeoutput = str(outputlist).strip('[]')
+        self.debug("complete output is " + completeoutput)
+        expectedlist = ['2 received', 'vm2.vpc.com', vm_2.ipaddress]
+        for item in expectedlist:
+            if item in completeoutput:
+                self.debug("excepted value found in vm: " + item)
+            else:
+                self.fail("excepted value not found in vm: " + item)
+
+        # Restarting VPC network (cleanup = false)
+        self.debug("Restarting the created VPC network without cleanup...")
+        Network.restart(network_1, self.api_client, cleanup=False)
+        self.validate_Network(network_1, state="Implemented")
+        vr = self.get_Router(network_1)
+        self.check_Router_state(vr, state="Running")
+        self.check_VM_state(vm_1, state="Running")
+        self.check_VM_state(vm_2, state="Running")
+
+        # VSD verification
+        self.verify_vsd_network(self.domain.id, network_1, vpc)
+        self.verify_vsd_router(vr)
+        self.verify_vsd_vm(vm_1)
+        self.verify_vsd_vm(vm_2)
+
+        try:
+            vm_1.ssh_ip = vm_public_ip
+            vm_1.ssh_port = self.test_data["virtual_machine"]["ssh_port"]
+            vm_1.username = self.test_data["virtual_machine"]["username"]
+            vm_1.password = self.test_data["virtual_machine"]["password"]
+            self.debug("SSHing into VM: %s with %s" %
+                       (vm_1.ssh_ip, vm_1.password))
+
+            ssh = vm_1.get_ssh_client(ipaddress=vm_public_ip)
+
+        except Exception as e:
+            self.fail("SSH into VM failed with exception %s" % e)
+
+        cmd = 'ping -c 2 vm2'
+        self.debug("ping vm2 by hostname with command: " + cmd)
+        outputlist = ssh.execute(cmd)
+        self.debug("command is executed properly " + cmd)
+        completeoutput = str(outputlist).strip('[]')
+        self.debug("complete output is " + completeoutput)
+        expectedlist = ['2 received', 'vm2.vpc.com', vm_2.ipaddress]
+        for item in expectedlist:
+            if item in completeoutput:
+                self.debug("excepted value found in vm: " + item)
+            else:
+                self.fail("excepted value not found in vm: " + item)
+
+        # Restarting VPC network (cleanup = true)
+        self.debug("Restarting the created VPC network with cleanup...")
+        Network.restart(network_1, self.api_client, cleanup=True)
+        self.validate_Network(network_1, state="Implemented")
+        vr = self.get_Router(network_1)
+        self.check_Router_state(vr, state="Running")
+        self.check_VM_state(vm_1, state="Running")
+        self.check_VM_state(vm_2, state="Running")
+
+        # VSD verification
+        self.verify_vsd_network(self.domain.id, network_1, vpc)
+        self.verify_vsd_router(vr)
+        self.verify_vsd_vm(vm_1)
+        self.verify_vsd_vm(vm_2)
+
+        try:
+            vm_1.ssh_ip = vm_public_ip
+            vm_1.ssh_port = self.test_data["virtual_machine"]["ssh_port"]
+            vm_1.username = self.test_data["virtual_machine"]["username"]
+            vm_1.password = self.test_data["virtual_machine"]["password"]
+            self.debug("SSHing into VM: %s with %s" %
+                       (vm_1.ssh_ip, vm_1.password))
+
+            ssh = vm_1.get_ssh_client(ipaddress=vm_public_ip)
+
+        except Exception as e:
+            self.fail("SSH into VM failed with exception %s" % e)
+
+        cmd = 'ping -c 2 vm2'
+        self.debug("ping vm2 by hostname with command: " + cmd)
+        outputlist = ssh.execute(cmd)
+        self.debug("command is executed properly " + cmd)
+        completeoutput = str(outputlist).strip('[]')
+        self.debug("complete output is " + completeoutput)
+        expectedlist = ['2 received', 'vm2.vpc.com', vm_2.ipaddress]
+        for item in expectedlist:
+            if item in completeoutput:
+                self.debug("excepted value found in vm: " + item)
+            else:
+                self.fail("excepted value not found in vm: " + item)
+
+        # Restarting VPC (cleanup = false)
+        self.debug("Restarting the VPC without cleanup...")
+        self.restart_Vpc(vpc, cleanup=False)
+        self.validate_Network(network_1, state="Implemented")
+        vr = self.get_Router(network_1)
+        self.check_Router_state(vr, state="Running")
+        self.check_VM_state(vm_1, state="Running")
+        self.check_VM_state(vm_2, state="Running")
+
+        # VSD verification
+        self.verify_vsd_network(self.domain.id, network_1, vpc)
+        self.verify_vsd_router(vr)
+        self.verify_vsd_vm(vm_1)
+
+        try:
+            vm_1.ssh_ip = vm_public_ip
+            vm_1.ssh_port = self.test_data["virtual_machine"]["ssh_port"]
+            vm_1.username = self.test_data["virtual_machine"]["username"]
+            vm_1.password = self.test_data["virtual_machine"]["password"]
+            self.debug("SSHing into VM: %s with %s" %
+                       (vm_1.ssh_ip, vm_1.password))
+
+            ssh = vm_1.get_ssh_client(ipaddress=vm_public_ip)
+
+        except Exception as e:
+            self.fail("SSH into VM failed with exception %s" % e)
+
+        cmd = 'ping -c 2 vm2'
+        self.debug("ping vm2 by hostname with command: " + cmd)
+        outputlist = ssh.execute(cmd)
+        self.debug("command is executed properly " + cmd)
+        completeoutput = str(outputlist).strip('[]')
+        self.debug("complete output is " + completeoutput)
+        expectedlist = ['2 received', 'vm2.vpc.com', vm_2.ipaddress]
+        for item in expectedlist:
+            if item in completeoutput:
+                self.debug("excepted value found in vm: " + item)
+            else:
+                self.fail("excepted value not found in vm: " + item)
+
+        # Restarting VPC (cleanup = true)
+        self.debug("Restarting the VPC with cleanup...")
+        self.restart_Vpc(vpc, cleanup=True)
+        self.validate_Network(network_1, state="Implemented")
+        vr = self.get_Router(network_1)
+        self.check_Router_state(vr, state="Running")
+        self.check_VM_state(vm_1, state="Running")
+        self.check_VM_state(vm_2, state="Running")
+
+        # VSD verification
+        self.verify_vsd_network(self.domain.id, network_1, vpc)
+        self.verify_vsd_router(vr)
+        self.verify_vsd_vm(vm_1)
+
+        try:
+            vm_1.ssh_ip = vm_public_ip
+            vm_1.ssh_port = self.test_data["virtual_machine"]["ssh_port"]
+            vm_1.username = self.test_data["virtual_machine"]["username"]
+            vm_1.password = self.test_data["virtual_machine"]["password"]
+            self.debug("SSHing into VM: %s with %s" %
+                       (vm_1.ssh_ip, vm_1.password))
+
+            ssh = vm_1.get_ssh_client(ipaddress=vm_public_ip)
+
+        except Exception as e:
+            self.fail("SSH into VM failed with exception %s" % e)
+
+        cmd = 'ping -c 2 vm2'
+        self.debug("ping vm2 by hostname with command: " + cmd)
+        outputlist = ssh.execute(cmd)
+        self.debug("command is executed properly " + cmd)
+        completeoutput = str(outputlist).strip('[]')
+        self.debug("complete output is " + completeoutput)
+        expectedlist = ['2 received', 'vm2.vpc.com', vm_2.ipaddress]
+        for item in expectedlist:
+            if item in completeoutput:
+                self.debug("excepted value found in vm: " + item)
+            else:
+                self.fail("excepted value not found in vm: " + item)