CLOUDSTACK-9015 - Make sure the Backup router can talk to the Master router after a stop/start/reboot

- Stop KeepaliveD/ConntrackD if the eth2 (guest) interface is not configured and UP
   - Only setup the redundancy after all the router configuration is done
   - Open the FW for the VRRP communitation
     - 224.0.0.18 and 225.0.0.50
   - Set keepalived.conf.templ by default to use interface eth2 (guest)
     - It will be reconfigured anyway, but having eth2 there is more clear

systemvm/patches/debian/config/opt/cloud/bin/configure.py

@@ -906,9 +906,6 @@ def main(argv):
     fwd = CsForwardingRules("forwardingrules", config)
     fwd.process()
 
-    red = CsRedundant(config)
-    red.set()
-
     logging.debug("Configuring s2s vpn")
     vpns = CsSite2SiteVpn("site2sitevpn", config)
     vpns.process()
@@ -938,6 +935,9 @@ def main(argv):
     logging.debug("Configuring iptables rules .....")
     nf = CsNetfilters()
     nf.compare(config.get_fw())
+    
+    red = CsRedundant(config)
+    red.set()
 
     logging.debug("Configuring iptables rules done ...saving rules")
 

systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py

@@ -228,10 +228,10 @@ class CsDevice:
                 continue
             self.devlist.append(vals[0])
 
-    def waitfordevice(self):
+    def waitfordevice(self, timeout=15):
         """ Wait up to 15 seconds for a device to become available """
         count = 0
-        while count < 15:
+        while count < timeout:
             if self.dev in self.devlist:
                 return True
             time.sleep(1)
@@ -498,6 +498,9 @@ class CsIP:
         self.fw.append(["", "", "-A NETWORK_STATS ! -i eth0 -o eth2 -p tcp"])
         self.fw.append(["", "", "-A NETWORK_STATS -i eth2 ! -o eth0 -p tcp"])
 
+        self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"])
+        self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"])
+
         self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -i eth0 -p tcp -m tcp --dport 3922 -m state --state NEW,ESTABLISHED -j ACCEPT"])
 

systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py

@@ -86,6 +86,29 @@ class CsRedundant(object):
             self._redundant_off()
             return
 
+        interfaces = [interface for interface in self.address.get_ips() if interface.is_guest()]
+        isDeviceReady = False
+        dev = ''
+        for interface in interfaces:
+            if dev == interface.get_device():
+                continue
+            dev = interface.get_device()
+            logging.info("Wait for devices to be configured so we can start keepalived")
+            devConfigured = CsDevice(dev, self.config).waitfordevice()
+            if devConfigured:
+                command = "ip link show %s | grep 'state UP'" % dev
+                devUp = CsHelper.execute(command)
+                if devUp:
+                    logging.info("Device %s is present, let's start keepalive now." % dev)
+                    isDeviceReady = True
+        
+        if not isDeviceReady:
+            logging.info("Guest network not configured yet, let's stop router redundancy for now.")
+            CsHelper.service("conntrackd", "stop")
+            CsHelper.service("keepalived", "stop")
+            return
+
+
         CsHelper.mkdir(self.CS_RAMDISK_DIR, 0755, False)
         CsHelper.mount_tmpfs(self.CS_RAMDISK_DIR)
         CsHelper.mkdir(self.CS_ROUTER_DIR, 0755, False)
@@ -129,17 +152,16 @@ class CsRedundant(object):
         CsHelper.copy(conntrackd_template_conf, conntrackd_temp_bkp)
         
         conntrackd_tmpl = CsFile(conntrackd_template_conf)
-        if guest is not None:
+        conntrackd_tmpl.section("Multicast {", "}", [
-            conntrackd_tmpl.section("Multicast {", "}", [
+                      "IPv4_address 225.0.0.50\n",
-                          "IPv4_address 225.0.0.50\n",
+                      "Group 3780\n",
-                          "Group 3780\n",
+                      "IPv4_interface %s\n" % guest.get_ip(),
-                          "IPv4_interface %s\n" % guest.get_ip(),
+                      "Interface %s\n" % guest.get_device(),
-                          "Interface %s\n" % guest.get_device(),
+                      "SndSocketBuffer 1249280\n",
-                          "SndSocketBuffer 1249280\n",
+                      "RcvSocketBuffer 1249280\n",
-                          "RcvSocketBuffer 1249280\n",
+                      "Checksum on\n"])
-                          "Checksum on\n"])
+        conntrackd_tmpl.section("Address Ignore {", "}", self._collect_ignore_ips())
-            conntrackd_tmpl.section("Address Ignore {", "}", self._collect_ignore_ips())
+        conntrackd_tmpl.commit()
-            conntrackd_tmpl.commit()
 
         conntrackd_conf = CsFile(self.CONNTRACKD_CONF)
 
@@ -164,22 +186,6 @@ class CsRedundant(object):
             "* * * * * root sleep 30; $SHELL %s/check_heartbeat.sh 2>&1 > /dev/null" % self.CS_ROUTER_DIR, -1)
         heartbeat_cron.commit()
 
-        # Configure KeepaliveD cron job - runs at every reboot
-        keepalived_cron = CsFile("/etc/cron.d/keepalived")
-        keepalived_cron.add("SHELL=/bin/bash", 0)
-        keepalived_cron.add(
-            "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin", 1)
-        keepalived_cron.add("@reboot root service keepalived start", -1)
-        keepalived_cron.commit()
-
-        # Configure ConntrackD cron job - runs at every reboot
-        conntrackd_cron = CsFile("/etc/cron.d/conntrackd")
-        conntrackd_cron.add("SHELL=/bin/bash", 0)
-        conntrackd_cron.add(
-            "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin", 1)
-        conntrackd_cron.add("@reboot root service conntrackd start", -1)
-        conntrackd_cron.commit()
-
         proc = CsProcess(['/usr/sbin/keepalived'])
         if not proc.find() or keepalived_conf.is_changed():
             keepalived_conf.commit()

systemvm/patches/debian/config/opt/cloud/bin/master.py

@@ -42,7 +42,7 @@ logging.basicConfig(filename=config.get_logger(),
                     format=config.get_format())
 config.cmdline()
 cl = CsCmdLine("cmdline", config)
-#Update the configuration to set state as backup and let keepalived decide who is the real Master
+#Update the configuration to set state as backup and let keepalived decide who the real Master is!
 cl.set_master_state(False)
 cl.save()
 

systemvm/patches/debian/config/opt/cloud/templates/keepalived.conf.templ

@@ -26,7 +26,7 @@ vrrp_script heartbeat {
 
 vrrp_instance inside_network {
     state EQUAL
-    interface eth0
+    interface eth2
     virtual_router_id 51
     nopreempt
 
@@ -37,7 +37,7 @@ vrrp_instance inside_network {
     }
 
     virtual_ipaddress {
-        [ROUTER_IP] brd [BOARDCAST] dev eth0
+        [ROUTER_IP] brd [BOARDCAST] dev eth2
     }
 
     track_script {