cloudstack/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py

# -- coding: utf-8 --
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
# -------------------------------------------------------------------- #
# Notes
# -------------------------------------------------------------------- #
# Vrouter
#
# eth0 router gateway IP for isolated network
# eth1 Control IP for hypervisor
# eth2 public ip(s)
#
# VPC Router
#
# eth0 control interface
# eth1 public ip
# eth2+ Guest networks
# -------------------------------------------------------------------- #
import os
import logging
from . import CsHelper
from .CsFile import CsFile
from .CsProcess import CsProcess
from .CsApp import CsPasswdSvc
from .CsAddress import CsDevice
from .CsRoute import CsRoute
from .CsStaticRoutes import CsStaticRoutes
import socket
from time import sleep


class CsRedundant(object):

    CS_RAMDISK_DIR = "/ramdisk"
    CS_PRIO_UP = 1
    CS_PRIO_DOWN = -1
    CS_ROUTER_DIR = "%s/rrouter" % CS_RAMDISK_DIR
    CS_TEMPLATES = [
        "heartbeat.sh.templ", "check_heartbeat.sh.templ",
        "arping_gateways.sh.templ"
    ]
    CS_TEMPLATES_DIR = "/opt/cloud/templates"
    CONNTRACKD_BIN = "/usr/sbin/conntrackd"
    CONNTRACKD_KEEPALIVED_CONFLOCK = "/var/lock/conntrack.lock"
    CONNTRACKD_CONF = "/etc/conntrackd/conntrackd.conf"
    RROUTER_LOG = "/var/log/cloud.log"
    KEEPALIVED_CONF = "/etc/keepalived/keepalived.conf"

    def __init__(self, config):
        self.cl = config.cmdline()
        self.address = config.address()
        self.config = config

    def set(self):
        logging.debug("Router redundancy status is %s", self.cl.is_redundant())
        if self.cl.is_redundant():
            self._redundant_on()
        else:
            self._redundant_off()

    def _redundant_off(self):
        CsHelper.service("conntrackd", "stop")
        CsHelper.service("keepalived", "stop")
        CsHelper.umount_tmpfs(self.CS_RAMDISK_DIR)
        CsHelper.rmdir(self.CS_RAMDISK_DIR)
        CsHelper.rm(self.CONNTRACKD_CONF)
        CsHelper.rm(self.KEEPALIVED_CONF)

    def _redundant_on(self):
        guest = self.address.get_guest_if()

        # No redundancy if there is no guest network
        if guest is None:
            self.set_backup(restart_conntrackd=False)
            self._redundant_off()
            return

        interfaces = [interface for interface in self.address.get_interfaces() if interface.is_guest()]
        isDeviceReady = False
        dev = ''
        for interface in interfaces:
            if dev == interface.get_device():
                continue
            dev = interface.get_device()
            logging.info("Wait for devices to be configured so we can start keepalived")
            devConfigured = CsDevice(dev, self.config).waitfordevice()
            if devConfigured:
                command = "ip link show %s | grep 'state UP'" % dev
                devUp = CsHelper.execute(command)
                if devUp:
                    logging.info("Device %s is present, let's start keepalived now." % dev)
                    isDeviceReady = True

        if not isDeviceReady:
            logging.info("Guest network not configured yet, let's stop router redundancy for now.")
            CsHelper.service("conntrackd", "stop")
            CsHelper.service("keepalived", "stop")
            return

        CsHelper.mkdir(self.CS_RAMDISK_DIR, 0o755, False)
        CsHelper.mount_tmpfs(self.CS_RAMDISK_DIR)
        CsHelper.mkdir(self.CS_ROUTER_DIR, 0o755, False)
        for s in self.CS_TEMPLATES:
            d = s
            if s.endswith(".templ"):
                d = s.replace(".templ", "")
            CsHelper.copy_if_needed(
                "%s/%s" % (self.CS_TEMPLATES_DIR, s), "%s/%s" % (self.CS_ROUTER_DIR, d))

        CsHelper.copy_if_needed(
            "%s/%s" % (self.CS_TEMPLATES_DIR, "keepalived.conf.templ"), self.KEEPALIVED_CONF)
        CsHelper.copy_if_needed(
            "%s/%s" % (self.CS_TEMPLATES_DIR, "checkrouter.sh.templ"), "/opt/cloud/bin/checkrouter.sh")

        CsHelper.execute(
            'sed -i "s/--exec $DAEMON;/--exec $DAEMON -- --vrrp;/g" /etc/init.d/keepalived')
        # checkrouter.sh configuration
        check_router = CsFile("/opt/cloud/bin/checkrouter.sh")
        check_router.greplace("[RROUTER_LOG]", self.RROUTER_LOG)
        check_router.commit()

        # keepalived configuration
        keepalived_conf = CsFile(self.KEEPALIVED_CONF)
        keepalived_conf.search(
            " router_id ", "    router_id %s" % self.cl.get_name())
        keepalived_conf.search(
            " interface ", "    interface %s" % guest.get_device())
        keepalived_conf.search(
            " advert_int ", "    advert_int %s" % self.cl.get_advert_int())

        keepalived_conf.greplace("[RROUTER_BIN_PATH]", self.CS_ROUTER_DIR)
        keepalived_conf.section("authentication {", "}", [
                                "        auth_type AH \n", "        auth_pass %s\n" % self.cl.get_router_password()[:8]])
        keepalived_conf.section(
            "virtual_ipaddress {", "}", self._collect_ips())

        # conntrackd configuration
        conntrackd_template_conf = "%s/%s" % (self.CS_TEMPLATES_DIR, "conntrackd.conf.templ")
        conntrackd_temp_bkp = "%s/%s" % (self.CS_TEMPLATES_DIR, "conntrackd.conf.templ.bkp")

        CsHelper.copy(conntrackd_template_conf, conntrackd_temp_bkp)

        conntrackd_tmpl = CsFile(conntrackd_template_conf)
        conntrackd_tmpl.section("Multicast {", "}", [
                      "IPv4_address 225.0.0.50\n",
                      "Group 3780\n",
                      "IPv4_interface %s\n" % guest.get_ip(),
                      "Interface %s\n" % guest.get_device(),
                      "SndSocketBuffer 1249280\n",
                      "RcvSocketBuffer 1249280\n",
                      "Checksum on\n"])
        conntrackd_tmpl.section("Address Ignore {", "}", self._collect_ignore_ips())
        conntrackd_tmpl.commit()

        conntrackd_conf = CsFile(self.CONNTRACKD_CONF)

        is_equals = conntrackd_tmpl.compare(conntrackd_conf)

        force_keepalived_restart = False
        proc = CsProcess(['/etc/conntrackd/conntrackd.conf'])

        if not proc.find() or not is_equals:
            CsHelper.copy(conntrackd_template_conf, self.CONNTRACKD_CONF)
            CsHelper.service("conntrackd", "restart")
            force_keepalived_restart = True

        # Restore the template file and remove the backup.
        CsHelper.copy(conntrackd_temp_bkp, conntrackd_template_conf)
        CsHelper.execute("rm -rf %s" % conntrackd_temp_bkp)

        # Configure heartbeat cron job - runs every 30 seconds
        heartbeat_cron = CsFile("/etc/cron.d/heartbeat")
        heartbeat_cron.add("SHELL=/bin/bash", 0)
        heartbeat_cron.add(
            "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin", 1)
        heartbeat_cron.add(
            "* * * * * root $SHELL %s/check_heartbeat.sh 2>&1 > /dev/null" % self.CS_ROUTER_DIR, -1)
        heartbeat_cron.add(
            "* * * * * root sleep 30; $SHELL %s/check_heartbeat.sh 2>&1 > /dev/null" % self.CS_ROUTER_DIR, -1)
        heartbeat_cron.commit()

        proc = CsProcess(['/usr/sbin/keepalived'])
        if not proc.find():
            force_keepalived_restart = True
        if keepalived_conf.is_changed() or force_keepalived_restart:
            keepalived_conf.commit()
            os.chmod(self.KEEPALIVED_CONF, 0o644)
            if force_keepalived_restart or not self.cl.is_primary():
                CsHelper.service("keepalived", "restart")
            else:
                CsHelper.service("keepalived", "reload")

    def release_lock(self):
        try:
            os.remove("/tmp/primary_lock")
        except OSError:
            pass

    def set_lock(self):
        """
        Make sure that primary state changes happen sequentially
        """
        iterations = 10
        time_between = 1

        for iter in range(0, iterations):
            try:
                s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                s.bind('/tmp/primary_lock')
                return s
            except socket.error as e:
                error_code = e.args[0]
                error_string = e.args[1]
                logging.info("Primary is already running, waiting")
                sleep(time_between)

    def set_fault(self):
        """ Set fault mode on this router """
        if not self.cl.is_redundant():
            logging.error("Set fault called on non-redundant router")
            return

        self.set_lock()
        logging.info("Setting router to fault")

        interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()]
        for interface in interfaces:
            CsHelper.execute("ifconfig %s down" % interface.get_device())

        cmd = "%s -C %s" % (self.CONNTRACKD_BIN, self.CONNTRACKD_CONF)
        CsHelper.execute("%s -s" % cmd)
        CsHelper.service("ipsec", "stop")
        CsHelper.service("xl2tpd", "stop")
        CsHelper.service("dnsmasq", "stop")
        CsHelper.service("radvd", "stop")

        interfaces = [interface for interface in self.address.get_interfaces() if interface.needs_vrrp()]
        for interface in interfaces:
            CsPasswdSvc(interface.get_gateway() + "," + interface.get_ip()).stop()

        self.cl.set_fault_state()
        self.cl.save()
        self.release_lock()
        logging.info("Router switched to fault mode")

        interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()]
        CsHelper.reconfigure_interfaces(self.cl, interfaces)

    def set_backup(self, restart_conntrackd=True):
        """ Set the current router to backup """
        if not self.cl.is_redundant():
            logging.error("Set backup called on non-redundant router")
            return

        self.set_lock()
        logging.debug("Setting router to backup")

        dev = ''
        interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()]
        for interface in interfaces:
            if dev == interface.get_device():
                continue
            logging.info("Bringing public interface %s down" % interface.get_device())
            cmd2 = "ip link set %s down" % interface.get_device()
            CsHelper.execute(cmd2)
            dev = interface.get_device()

        self._remove_ipv6_guest_gateway()

        if restart_conntrackd:
            CsHelper.service("conntrackd", "restart")
        else:
            CsHelper.service("conntrackd", "stop")
        CsHelper.service("ipsec", "stop")
        CsHelper.service("xl2tpd", "stop")

        interfaces = [interface for interface in self.address.get_interfaces() if interface.needs_vrrp()]
        for interface in interfaces:
            CsPasswdSvc(interface.get_gateway() + "," + interface.get_ip()).stop()

        CsHelper.service("dnsmasq", "stop")

        self.cl.set_primary_state(False)
        self.cl.save()
        self.release_lock()

        interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()]
        CsHelper.reconfigure_interfaces(self.cl, interfaces)
        logging.info("Router switched to backup mode")

    def set_primary(self):
        """ Set the current router to primary """
        if not self.cl.is_redundant():
            logging.error("Set primary called on non-redundant router")
            return

        self.set_lock()
        logging.debug("Setting router to primary")

        dev = ''
        interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()]
        route = CsRoute()
        for interface in interfaces:
            if dev == interface.get_device():
                continue
            dev = interface.get_device()
            logging.info("Will proceed configuring device ==> %s" % dev)
            cmd = "ip link set %s up" % dev
            if CsDevice(dev, self.config).waitfordevice():
                CsHelper.execute(cmd)
                logging.info("Bringing public interface %s up" % dev)

                try:
                    gateway = interface.get_gateway()
                    logging.info("Adding gateway ==> %s to device ==> %s" % (gateway, dev))
                    if dev == CsHelper.PUBLIC_INTERFACES[self.cl.get_type()]:
                        route.add_defaultroute(gateway)
                except Exception:
                    logging.error("ERROR getting gateway from device %s" % dev)
                if dev == CsHelper.PUBLIC_INTERFACES[self.cl.get_type()]:
                    try:
                        self._add_ipv6_to_interface(interface, interface.get_ip6())
                        if interface.get_gateway6():
                            route.add_defaultroute_v6(interface.get_gateway6())
                    except Exception as e:
                        logging.error("ERROR adding IPv6, getting IPv6 gateway from device %s: %s" % (dev, e))
            else:
                logging.error("Device %s was not ready could not bring it up" % dev)

        self._add_ipv6_guest_gateway()

        logging.debug("Configuring static routes")
        static_routes = CsStaticRoutes("staticroutes", self.config)
        static_routes.process()

        cmd = "%s -C %s" % (self.CONNTRACKD_BIN, self.CONNTRACKD_CONF)
        CsHelper.execute("%s -c" % cmd)
        CsHelper.execute("%s -f" % cmd)
        CsHelper.execute("%s -R" % cmd)
        CsHelper.execute("%s -B" % cmd)
        CsHelper.service("ipsec", "restart")
        CsHelper.service("xl2tpd", "restart")

        interfaces = [interface for interface in self.address.get_interfaces() if interface.needs_vrrp()]
        for interface in interfaces:
            if interface.is_added():
                CsPasswdSvc(interface.get_gateway() + "," + interface.get_ip()).restart()

        CsHelper.service("dnsmasq", "restart")
        self.cl.set_primary_state(True)
        self.cl.save()
        self.release_lock()

        interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()]
        CsHelper.reconfigure_interfaces(self.cl, interfaces)

        public_devices = list(set([interface.get_device() for interface in interfaces]))
        if len(public_devices) > 1:
            # Handle specific failures when multiple public interfaces

            public_devices.sort()

            # Ensure the default route is added, or outgoing traffic from VMs with static NAT on
            # the subsequent interfaces will go from the wrong IP
            route = CsRoute()
            dev = ''
            for interface in interfaces:
                if dev == interface.get_device():
                    continue
                dev = interface.get_device()
                gateway = interface.get_gateway()
                if gateway:
                    route.add_route(dev, gateway)

            # The first public interface has a static MAC address between VRs.  Subsequent ones don't,
            # so an ARP announcement is needed on failover
            for device in public_devices[1:]:
                logging.info("Sending garp messages for IPs on %s" % device)
                for interface in interfaces:
                    if interface.get_device() == device:
                        CsHelper.execute("arping -I %s -U %s -c 1" % (device, interface.get_ip()))

        logging.info("Router switched to primary mode")

    def _collect_ignore_ips(self):
        """
        This returns a list of ip objects that should be ignored
        by conntrackd
        """
        lines = []
        lines.append("\t\t\tIPv4_address %s\n" % "127.0.0.1")
        lines.append("\t\t\tIPv4_address %s\n" %
                     self.address.get_control_if().get_ip())
        # FIXME - Do we need to also add any internal network gateways?
        return lines

    def _collect_ips(self):
        """
        Construct a list containing all the ips that need to be looked afer by vrrp
        This is based upon the address_needs_vrrp method in CsAddress which looks at
        the network type and decides if it is an internal address or an external one

        In a DomR there will only ever be one address in a VPC there can be many
        The new code also gives the possibility to cloudstack to have a hybrid device
        that could function as a router and VPC router at the same time
        """
        lines = []
        for interface in self.address.get_interfaces():
            if interface.needs_vrrp():
                cmdline = self.config.get_cmdline_instance()
                if not interface.is_added():
                    continue
                str = "        %s brd %s dev %s\n" % (interface.get_gateway_cidr(), interface.get_broadcast(), interface.get_device())
                lines.append(str)
        return lines

    def _add_ipv6_to_interface(self, interface, ipv6):
        """
        Add an IPv6 to an interface. This is useful for adding,
        - guest IPv6 gateway for primary VR guest NIC
        - public IPv6 for primary VR public NIC as its IPv6 gets lost on link down
        """
        dev = ''
        if dev == interface.get_device() or not ipv6:
            return
        dev = interface.get_device()
        command = "ip -6 address show %s | grep 'inet6 %s'" % (dev, ipv6)
        ipConfigured = CsHelper.execute(command)
        if ipConfigured:
            logging.info("IPv6 address %s already present for %s" % (ipv6, dev))
            return
        command = "ip link show %s | grep 'state UP'" % dev
        devUp = CsHelper.execute(command)
        if not devUp:
            logging.error("ERROR setting IPv6 address for device %s as it is not ready" % dev)
            return
        logging.info("Device %s is present, let's add IPv6 address  %s" % (dev, ipv6))
        cmd = "ip -6 addr add  %s dev %s" % (ipv6, dev)
        CsHelper.execute(cmd)

    def _remove_ipv6_to_interface(self, interface, ipv6):
        """
        Remove an IPv6 to an interface. This is useful for removing,
        - guest IPv6 gateway for primary VR guest NIC
        """
        dev = ''
        if dev == interface.get_device() or not ipv6:
            return
        dev = interface.get_device()
        command = "ip -6 address show %s | grep 'inet6 %s'" % (dev, ipv6)
        ipConfigured = CsHelper.execute(command)
        if ipConfigured:
            command = "ip link show %s | grep 'state UP'" % dev
            devUp = CsHelper.execute(command)
            if not devUp:
                logging.error("ERROR setting IPv6 address for device %s as it is not ready" % dev)
                return
            logging.info("Device %s is present, let's remove IPv6 address  %s" % (dev, ipv6))
            cmd = "ip -6 addr delete  %s dev %s" % (ipv6, dev)
            CsHelper.execute(cmd)
        else:
            logging.info("IPv6 address %s not present for %s" % (ipv6, dev))
            return

    def _enable_radvd(self, dev):
        """
        Setup radvd for primary VR
        """
        if dev == '':
            return
        CsHelper.service("radvd", "enable")
        CsHelper.start_if_stopped("radvd")

    def _disable_radvd(self, dev):
        """
        Disable radvd for non-primary VR
        """
        if dev == '':
            return
        CsHelper.service("radvd", "stop")
        CsHelper.service("radvd", "disable")
        logging.info(CsHelper.execute("systemctl status radvd"))

    def _add_ipv6_guest_gateway(self):
        """
        Configure guest network gateway as IPv6 address for guest interface
        for redundant primary VR
        """
        for interface in self.address.get_interfaces():
            if not interface.is_guest() or not interface.get_gateway6_cidr():
                continue
            self._add_ipv6_to_interface(interface, interface.get_gateway6_cidr())
            self._enable_radvd(interface.get_device())

    def _remove_ipv6_guest_gateway(self):
        """
        Remove guest network gateway as IPv6 address for guest interface
        for redundant backup VR
        """
        for interface in self.address.get_interfaces():
            if not interface.is_guest() or not interface.get_gateway6_cidr():
                continue
            self._remove_ipv6_to_interface(interface, interface.get_gateway6_cidr())
            self._disable_radvd(interface.get_device())