Merge pull request #1346 from borisroman/4.7-vr-performance

Enhance VR performanceWill post integration test results soon!

Ping @wilderrodrigues @remibergsma

* pr/1346:
  Enhance VR performance by selectively executing tasks instead of brute-forcing

Signed-off-by: Remi Bergsma <github@remi.nl>

systemvm/patches/debian/config/opt/cloud/bin/configure.py

@@ -288,7 +288,7 @@ class CsAcl(CsDataBag):
             if item == "id":
                 continue
             if self.config.is_vpc():
-                dev_obj = self.AclDevice(self.dbag[item], self.config).create()
+                self.AclDevice(self.dbag[item], self.config).create()
             else:
                 self.AclIP(self.dbag[item], self.config).create()
 
@@ -901,79 +901,123 @@ class CsForwardingRules(CsDataBag):
 
 
 def main(argv):
+    # The file we are currently processing, if it is "cmd_line.json" everything will be processed.
+    process_file = argv[1]
+
+    # process_file can be None, if so assume cmd_line.json
+    if process_file is None:
+        process_file = "cmd_line.json"
+
+    # Track if changes need to be committed to NetFilter
+    iptables_change = False
+
+    # The "GLOBAL" Configuration object
     config = CsConfig()
+
     logging.basicConfig(filename=config.get_logger(),
                         level=config.get_level(),
                         format=config.get_format())
+
+    # Load stored ip adresses from disk to CsConfig()
     config.set_address()
 
     logging.debug("Configuring ip addresses")
-    # IP configuration
     config.address().compare()
     config.address().process()
 
-    logging.debug("Configuring vmpassword")
+    if process_file in ["cmd_line.json", "guest_network.json"]:
-    password = CsPassword("vmpassword", config)
+        logging.debug("Configuring Guest Network")
-    password.process()
+        iptables_change = True
 
-    logging.debug("Configuring vmdata")
+    if process_file in ["cmd_line.json", "vm_password.json"]:
-    metadata = CsVmMetadata('vmdata', config)
+        logging.debug("Configuring vmpassword")
-    metadata.process()
+        password = CsPassword("vmpassword", config)
+        password.process()
 
-    logging.debug("Configuring networkacl")
+    if process_file in ["cmd_line.json", "vm_metadata.json"]:
+        logging.debug("Configuring vmdata")
+        metadata = CsVmMetadata('vmdata', config)
+        metadata.process()
+
+    # Always run both CsAcl().process() methods
+    # They fill the base rules in config.fw[]
     acls = CsAcl('networkacl', config)
     acls.process()
 
-    logging.debug("Configuring firewall rules")
     acls = CsAcl('firewallrules', config)
     acls.process()
 
-    logging.debug("Configuring PF rules")
     fwd = CsForwardingRules("forwardingrules", config)
     fwd.process()
 
-    logging.debug("Configuring s2s vpn")
     vpns = CsSite2SiteVpn("site2sitevpn", config)
     vpns.process()
 
-    logging.debug("Configuring remote access vpn")
-    #remote access vpn
     rvpn = CsRemoteAccessVpn("remoteaccessvpn", config)
     rvpn.process()
 
-    logging.debug("Configuring vpn users list")
-    #remote access vpn users
-    vpnuser = CsVpnUser("vpnuserlist", config)
-    vpnuser.process()
-
-    logging.debug("Configuring dhcp entry")
-    dhcp = CsDhcp("dhcpentry", config)
-    dhcp.process()
-
-    logging.debug("Configuring load balancer")
     lb = CsLoadBalancer("loadbalancer", config)
     lb.process()
 
-    logging.debug("Configuring monitor service")
+    if process_file in ["cmd_line.json", "network_acl.json"]:
-    mon = CsMonitor("monitorservice", config)
+        logging.debug("Configuring networkacl")
-    mon.process()
+        iptables_change = True
 
-    logging.debug("Configuring iptables rules")
+    if process_file in ["cmd_line.json", "firewall_rules.json"]:
-    nf = CsNetfilters()
+        logging.debug("Configuring firewall rules")
-    nf.compare(config.get_fw())
+        iptables_change = True
+
+    if process_file in ["cmd_line.json", "forwarding_rules.json", "staticnat_rules.json"]:
+        logging.debug("Configuring PF rules")
+        iptables_change = True
+
+    if process_file in ["cmd_line.json", "site_2_site_vpn.json"]:
+        logging.debug("Configuring s2s vpn")
+        iptables_change = True
+
+    if process_file in ["cmd_line.json", "remote_access_vpn.json"]:
+        logging.debug("Configuring remote access vpn")
+        iptables_change = True
+
+    if process_file in ["cmd_line.json", "vpn_user_list.json"]:
+        logging.debug("Configuring vpn users list")
+        vpnuser = CsVpnUser("vpnuserlist", config)
+        vpnuser.process()
+
+    if process_file in ["cmd_line.json", "vm_dhcp_entry.json", "dhcp.json"]:
+        logging.debug("Configuring dhcp entry")
+        dhcp = CsDhcp("dhcpentry", config)
+        dhcp.process()
+
+    if process_file in ["cmd_line.json", "load_balancer.json"]:
+        logging.debug("Configuring load balancer")
+        iptables_change = True
+
+    if process_file in ["cmd_line.json", "monitor_service.json"]:
+        logging.debug("Configuring monitor service")
+        mon = CsMonitor("monitorservice", config)
+        mon.process()
+
+    # If iptable rules have changed, apply them.
+    if iptables_change:
+        logging.debug("Configuring iptables rules")
+        nf = CsNetfilters()
+        nf.compare(config.get_fw())
 
     red = CsRedundant(config)
     red.set()
 
-    logging.debug("Configuring static routes")
+    if process_file in ["cmd_line.json", "static_routes.json"]:
-    static_routes = CsStaticRoutes("staticroutes", config)
+        logging.debug("Configuring static routes")
-    static_routes.process()
+        static_routes = CsStaticRoutes("staticroutes", config)
+        static_routes.process()
 
-    logging.debug("Configuring iptables rules done ...saving rules")
+    if iptables_change:
+        logging.debug("Configuring iptables rules done ...saving rules")
 
-    # Save iptables configuration - will be loaded on reboot by the iptables-restore that is configured on /etc/rc.local
+        # Save iptables configuration - will be loaded on reboot by the iptables-restore that is configured on /etc/rc.local
-    CsHelper.save_iptables("iptables-save", "/etc/iptables/router_rules.v4")
+        CsHelper.save_iptables("iptables-save", "/etc/iptables/router_rules.v4")
-    CsHelper.save_iptables("ip6tables-save", "/etc/iptables/router_rules.v6")
+        CsHelper.save_iptables("ip6tables-save", "/etc/iptables/router_rules.v6")
 
 if __name__ == "__main__":
     main(sys.argv)

systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py

@@ -50,12 +50,12 @@ class CsDhcp(CsDataBag):
 
         self.configure_server()
 
-        # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems.
-        CsHelper.service("dnsmasq", "restart")
-
         self.conf.commit()
         self.cloud.commit()
 
+        # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems.
+        CsHelper.service("dnsmasq", "restart")
+
     def configure_server(self):
         # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS)
         for i in self.devinfo:
@@ -96,29 +96,8 @@ class CsDhcp(CsDataBag):
             self.conf.search(sline, line)
 
     def delete_leases(self):
-        changed = []
-        leases = []
         try:
-            for line in open(LEASES):
+            open(LEASES, 'w').close()
-                bits = line.strip().split(' ')
-                to = {"device": bits[0],
-                      "mac": bits[1],
-                      "ip": bits[2],
-                      "host": bits[3:],
-                      "del": False
-                      }
-                changed.append(to)
-
-                for v in changed:
-                    if v['mac'] == to['mac'] or v['ip'] == to['ip'] or v['host'] == to['host']:
-                        to['del'] = True
-                leases.append(to)
-
-            for o in leases:
-                if o['del']:
-                    cmd = "dhcp_release eth%s %s %s" % (o['device'], o['ip'], o['mac'])
-                    logging.info(cmd)
-                    CsHelper.execute(cmd)
         except IOError:
             return
 

systemvm/patches/debian/config/opt/cloud/bin/update_config.py

@@ -41,7 +41,7 @@ currentGuestNetConfig = "/etc/cloudstack/guestnetwork.json"
 
 def finish_config():
     # Converge
-    returncode = configure.main([])
+    returncode = configure.main(sys.argv)
     sys.exit(returncode)