Countless bug fixes, mostly do do with VR redundancy

Also added some new unit tests and adjusted the code to make them work

systemvm/patches/debian/config/opt/cloud/bin/configure.py

@@ -530,13 +530,11 @@ class CsForwardingRules(CsDataBag):
 
 
 def main(argv):
-    config = CsConfig(False)
+    config = CsConfig()
     logging.basicConfig(filename=config.get_logger(),
                         level=config.get_level(),
                         format=config.get_format())
-    config.set_cl()
     config.set_address()
-    cl = config.get_cmdline()
 
     # IP configuration
     config.address().compare()

systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py

@@ -26,7 +26,7 @@ import time
 from CsRoute import CsRoute
 from CsRule import CsRule
 
-VRRP_TYPES = ['guest', 'public']
+VRRP_TYPES = ['guest']
 
 
 class CsAddress(CsDataBag):
@@ -42,38 +42,37 @@ class CsAddress(CsDataBag):
             if dev == "id":
                 continue
             for ip in self.dbag[dev]:
-                ret.append(CsInterface(ip))
+                ret.append(CsInterface(ip, self.config))
         return ret
 
+    def get_guest_if(self):
+        """
+        Return CsIp object for the first guest interface
+        """
+        for ip in self.get_ips():
+            if ip.is_guest():
+                return ip
+        return None
+
     def get_guest_ip(self):
         """
         Return the ip of the first guest interface
         For use with routers not vpcrouters
         """
-        for ip in self.get_ips():
-            if ip.is_guest():
+        ip = self.get_guest_if()
+        if ip:
             return ip.get_ip()
         return None
 
-    def get_guest_gateway(self):
-        """
-        Return the gateway of the first guest interface
-        For use with routers not vpcrouters
-        """
-        for ip in self.get_ips():
-            if ip.is_guest():
-                return ip.get_gateway()
-        return None
-
     def get_guest_netmask(self):
         """
-        Return the gateway of the first guest interface
+        Return the netmask of the first guest interface
         For use with routers not vpcrouters
         """
-        for ip in self.get_ips():
-            if ip.is_guest():
+        ip = self.get_guest_if()
+        if ip:
             return ip.get_netmask()
-        return None
+        return "255.255.255.0"
 
     def needs_vrrp(self, o):
         """
@@ -144,8 +143,9 @@ class CsAddress(CsDataBag):
 
 class CsInterface:
     """ Hold one single ip """
-    def __init__(self, o):
+    def __init__(self, o, config):
         self.address = o
+        self.config = config
 
     def get_ip(self):
         return self.get_attr("public_ip")
@@ -154,7 +154,17 @@ class CsInterface:
         return self.get_attr("netmask")
 
     def get_gateway(self):
+        if self.config.is_vpc():
             return self.get_attr("gateway")
+        else:
+            return self.config.cmdline().get_guest_gw()
+
+    def get_gateway_cidr(self):
+        return "%s/%s" % (self.get_gateway(), self.get_size())
+
+    def get_size(self):
+        """ Return the network size in bits (24, 16, 8 etc) """
+        return self.get_attr("size")
 
     def get_device(self):
         return self.get_attr("device")
@@ -205,7 +215,7 @@ class CsDevice:
             self.tableNo = dev[3]
             self.table = "Table_%s" % dev
         self.fw = config.get_fw()
-        self.cl = config.get_cmdline()
+        self.cl = config.cmdline()
 
     def configure_rp(self):
         """
@@ -250,7 +260,7 @@ class CsIP:
         self.address = {}
         self.list()
         self.fw = config.get_fw()
-        self.cl = config.get_cmdline()
+        self.cl = config.cmdline()
         self.config = config
 
     def setAddress(self, address):
@@ -284,7 +294,7 @@ class CsIP:
             if " DOWN " in i:
                 cmd2 = "ip link set %s up" % self.getDevice()
                 # Do not change the state of ips on a redundant router that are managed by vrrp or CsRedundant
-                if self.cl.is_redundant() and self.needs_vrrp():
+                if self.config.cmdline().is_redundant() and self.needs_vrrp():
                     pass
                 else:
                     CsHelper.execute(cmd2)
@@ -412,7 +422,7 @@ class CsIP:
 
         if self.get_type() == "public" and self.config.is_vpc():
             if self.address["source_nat"]:
-                vpccidr = self.cl.get_vpccidr()
+                vpccidr = self.config.cmdline().get_vpccidr()
                 self.fw.append(["filter", "", "-A FORWARD -s %s ! -d %s -j ACCEPT" % (vpccidr, vpccidr)])
                 self.fw.append(["nat", "", "-A POSTROUTING -j SNAT -o %s --to-source %s" % (self.dev, self.address['public_ip'])])
         # route.flush()

systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py

@@ -28,24 +28,25 @@ class CsConfig(object):
     __LOG_FILE = "/var/log/cloud.log"
     __LOG_LEVEL = "DEBUG"
     __LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"
+    cl = None
 
-    def __init__(self, load=False):
-        if load:
-            self_set_cl()
-            self_set_address()
+    def __init__(self):
         self.fw = []
 
-    def set_cl(self):
-        self.cl = CsCmdLine("cmdline")
-
-    def address(self):
-        return self.ips
-
     def set_address(self):
         self.ips = CsAddress("ips", self)
 
-    def get_cmdline(self):
-        return self.cl
+    @classmethod
+    def get_cmdline_instance(cls):
+        if cls.cl is None:
+            cls.cl = CsCmdLine("cmdline")
+        return cls.cl
+
+    def cmdline(self):
+        return self.get_cmdline_instance()
+
+    def address(self):
+        return self.ips
 
     def get_fw(self):
         return self.fw
@@ -66,7 +67,7 @@ class CsConfig(object):
         return self.cl.get_domain()
 
     def get_dns(self):
-        return self.get_cmdline().get_dns()
+        return self.cmdline().get_dns()
 
     def get_format(self):
         return self.__LOG_FORMAT

systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py

@@ -28,7 +28,7 @@ class CsDataBag(object):
         self.dbag = self.db.getDataBag()
         if config:
             self.fw = config.get_fw()
-            self.cl = config.get_cmdline()
+            self.cl = config.cmdline()
             self.config = config
 
     def dump(self):
@@ -51,14 +51,32 @@ class CsDataBag(object):
 class CsCmdLine(CsDataBag):
     """ Get cmdline config parameters """
 
+    def idata(self):
+        if "config" in self.dbag:
+            return self.dbag['config']
+        else:
+            return {}
+
     def is_redundant(self):
-        if "redundant_router" in self.dbag['config']:
-            return self.dbag['config']['redundant_router'] == "true"
+        if "redundant_router" in self.idata():
+            return self.idata()['redundant_router'] == "true"
         return False
 
+    def get_guest_gw(self):
+        if "guestgw" in self.idata():
+            return self.idata()['guestgw']
+        else:
+            return "1.2.3.4"
+
+    def get_guest_gw_cidr(self):
+        if "guestgw" in self.idata():
+            return "%s/%s" % (self.idata()['guestgw'], self.idata()['guestcidrsize'])
+        else:
+            return "1.2.3.4/8"
+
     def get_name(self):
-        if "name" in self.dbag['config']:
-            return self.dbag['config']['name']
+        if "name" in self.idata():
+            return self.idata()['name']
         else:
             return "unloved-router"
 
@@ -66,31 +84,31 @@ class CsCmdLine(CsDataBag):
         dns = []
         names = "dns1 dns2"
         for name in names:
-            if name in self.dbag['config']:
-                dns.append(self.dbag['config'][name])
+            if name in self.idata():
+                dns.append(self.idata()[name])
         return dns
 
     def get_type(self):
-        if "type" in self.dbag['config']:
-            return self.dbag['config']['type']
+        if "type" in self.idata():
+            return self.idata()['type']
         else:
             return "unknown"
 
     def get_domain(self):
-        if "domain" in self.dbag['config']:
-            return self.dbag['config']['domain']
+        if "domain" in self.config:
+            return self.idata()['domain']
         else:
             return "cloudnine.internal"
 
     def get_vpccidr(self):
-        if "vpccidr" in self.dbag['config']:
-            return self.dbag['config']['vpccidr']
+        if "vpccidr" in self.idata():
+            return self.idata()['vpccidr']
         else:
             return "unknown"
 
     def is_master(self):
         if not self.is_redundant():
             return False
-        if "redundant_master" in self.dbag['config']:
-            return self.dbag['config']['redundant_master'] == "true"
+        if "redundant_master" in self.idata():
+            return self.idata()['redundant_master'] == "true"
         return False

systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py

@@ -101,6 +101,9 @@ class CsFile:
             if line.strip() == start:
                 sind = index + 1
                 found = True
+        if sind == -1:
+            content.insert(0, start + "\n")
+            content.append(end + "\n")
         self.new_config[sind:eind] = content
 
     def greplace(self, search, replace):

systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py

@@ -38,7 +38,6 @@ def is_mounted(name):
 
 def mount_tmpfs(name):
     if not is_mounted(name):
-        print "Mounting it"
         execute("mount tmpfs %s -t tmpfs" % name)
 
 

systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py

@@ -54,7 +54,7 @@ class CsRedundant(object):
     CONNTRACKD_CONFIG = "/etc/conntrackd/conntrackd.conf"
 
     def __init__(self, config):
-        self.cl = config.get_cmdline()
+        self.cl = config.cmdline()
         self.address = config.address()
 
     def set(self):
@@ -95,10 +95,16 @@ class CsRedundant(object):
         file.commit()
 
         # conntrackd configuration
-        control = self.address.get_control_if()
+        guest = self.address.get_guest_if()
         connt = CsFile("/etc/conntrackd/conntrackd.conf")
-        connt.search("[\s\t]IPv4_interface ", "\t\tIPv4_interface %s" % control.get_ip())
-        connt.search("[\s\t]Interface ", "\t\tInterface %s" % control.get_device())
+        connt.section("Multicast {", "}", [
+                      "IPv4_address 225.0.0.50\n",
+                      "Group 3780\n",
+                      "IPv4_interface %s\n" % guest.get_ip(),
+                      "Interface %s\n" % guest.get_device(),
+                      "SndSocketBuffer 1249280\n",
+                      "RcvSocketBuffer 1249280\n",
+                      "Checksum on\n"])
         connt.section("Address Ignore {", "}", self._collect_ignore_ips())
         connt.commit()
         if connt.is_changed():
@@ -209,6 +215,6 @@ class CsRedundant(object):
         lines = []
         for o in self.address.get_ips():
             if o.needs_vrrp():
-                str = "        %s brd %s dev %s\n" % (o.get_cidr(), o.get_broadcast(), o.get_ip())
+                str = "        %s brd %s dev %s\n" % (o.get_gateway_cidr(), o.get_broadcast(), o.get_device())
                 lines.append(str)
         return lines

systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py

@@ -32,6 +32,7 @@ def merge(dbag, ip):
         ip['device'] = 'eth' + str(ip['nic_dev_id'])
         ip['broadcast'] = str(ipo.broadcast)
         ip['cidr'] = str(ipo.ip) + '/' + str(ipo.prefixlen)
+        ip['size'] = str(ipo.prefixlen)
         ip['network'] = str(ipo.network) + '/' + str(ipo.prefixlen)
         if 'nw_type' not in ip.keys():
             ip['nw_type'] = 'public'

systemvm/test/python/TestCsAddress.py

@@ -7,10 +7,19 @@ class TestCsAddress(unittest.TestCase):
 
     def setUp(self):
         merge.DataBag.DPATH = "."
+        self.csaddress = CsAddress("ips", {})
 
     def test_needs_vrrp(self):
-        csaddress = CsAddress("ips", {})
-        self.assertTrue(csaddress.needs_vrrp({"nw_type": "public"}))
+        self.assertTrue(self.csaddress.needs_vrrp({"nw_type": "guest"}))
+
+    def test_get_guest_if(self):
+        self.assertTrue(self.csaddress.get_guest_if() is None)
+
+    def test_get_guest_ip(self):
+        self.assertTrue(self.csaddress.get_guest_ip() is None)
+
+    def test_get_guest_netmask(self):
+        self.assertTrue(self.csaddress.get_guest_netmask() == "255.255.255.0")
 
 if __name__ == '__main__':
     unittest.main()

systemvm/test/python/TestCsApp.py

@@ -12,7 +12,7 @@ class TestCsApp(unittest.TestCase):
 
     def test_init(self):
         csconfig = CsConfig()
-        csconfig.set_cl()
+        csconfig.cmdline()
         csip = CsIP("eth0", csconfig)
         csapp = CsApp(csip)
         self.assertTrue(csapp is not None)

systemvm/test/python/TestCsConfig.py

@@ -9,7 +9,7 @@ class TestCsConfig(unittest.TestCase):
         merge.DataBag.DPATH = "."
 
     def test_ini(self):
-        csconfig = CsConfig(False)
+        csconfig = CsConfig()
         self.assertTrue(csconfig is not None)
 
 if __name__ == '__main__':

systemvm/test/python/TestCsInterface.py

@@ -0,0 +1,21 @@
+import unittest
+from cs.CsAddress import CsInterface
+from cs.CsConfig import CsConfig
+from cs.CsDatabag import CsCmdLine
+import merge
+
+
+class TestCsInterface(unittest.TestCase):
+
+    def setUp(self):
+        merge.DataBag.DPATH = "."
+        csconfig = CsConfig()
+        self.cmdline = CsCmdLine("cmdline", csconfig)
+        csconfig.cl = self.cmdline
+        self.csinterface = CsInterface({}, csconfig)
+
+    def test_get_gateway(self):
+        self.assertTrue(self.csinterface.get_gateway() == "1.2.3.4")
+
+if __name__ == '__main__':
+    unittest.main()

systemvm/test/python/TestCsRedundant.py

@@ -1,6 +1,7 @@
 import unittest
 from cs.CsRedundant import CsRedundant
 from cs.CsConfig import CsConfig
+from cs.CsDatabag import CsCmdLine
 import merge
 
 
@@ -8,10 +9,11 @@ class TestCsRedundant(unittest.TestCase):
 
     def setUp(self):
         merge.DataBag.DPATH = "."
+        self.cmdline = CsCmdLine("cmdline", {})
 
     def test_init(self):
         csconfig = CsConfig()
-        csconfig.set_cl()
+        csconfig.cl = self.cmdline
         csconfig.set_address()
 
         csredundant = CsRedundant(csconfig)