Revert "Merge pull request #1482 from remibergsma/iptables-fix"

Seems to have a license issue so reverting for now.

This reverts commit 9a20ab8bcbbd39aa012a0ec5a65e66bcc737ee0e, reversing
changes made to 7a0b37a29a8be14011427dcf61bf3ea86e47dbf4.

systemvm/patches/debian/config/opt/cloud/bin/configure.py

@@ -17,16 +17,27 @@
 # specific language governing permissions and limitations
 # under the License.
 import sys
+import os
 import base64
 
+from merge import DataBag
+from pprint import pprint
+import subprocess
+import logging
 import re
+import time
+import shutil
+import os.path
+import os
 from fcntl import flock, LOCK_EX, LOCK_UN
 
-from cs.CsDatabag import CsDataBag
+from cs.CsDatabag import CsDataBag, CsCmdLine
+import cs.CsHelper
 from cs.CsNetfilter import CsNetfilters
 from cs.CsDhcp import CsDhcp
 from cs.CsRedundant import *
 from cs.CsFile import CsFile
+from cs.CsApp import CsApache, CsDnsmasq
 from cs.CsMonitor import CsMonitor
 from cs.CsLoadBalancer import CsLoadBalancer
 from cs.CsConfig import CsConfig
@@ -197,23 +208,7 @@ class CsAcl(CsDataBag):
 
         def process(self, direction, rule_list, base):
             count = base
-            rule_list_splitted = []
+            for i in rule_list:
-            for rule in rule_list:
-                if ',' in rule['cidr']:
-                    cidrs = rule['cidr'].split(',')
-                    for cidr in cidrs:
-                        new_rule = {
-                            'cidr': cidr,
-                            'last_port': rule['last_port'],
-                            'type': rule['type'],
-                            'first_port': rule['first_port'],
-                            'allowed': rule['allowed']
-                        }
-                        rule_list_splitted.append(new_rule)
-                else:
-                    rule_list_splitted.append(rule)
-
-            for i in rule_list_splitted:
                 r = self.AclRule(direction, self, i, self.config, count)
                 r.create()
                 count += 1
@@ -266,7 +261,7 @@ class CsAcl(CsDataBag):
                     rstr = "%s -m icmp --icmp-type %s" % (rstr, self.icmp_type)
                 rstr = "%s %s -j %s" % (rstr, self.dport, self.action)
                 rstr = rstr.replace("  ", " ").lstrip()
-                self.fw.append([self.table, "", rstr])
+                self.fw.append([self.table, self.count, rstr])
 
     def process(self):
         for item in self.dbag:
@@ -480,7 +475,7 @@ class CsSite2SiteVpn(CsDataBag):
         self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])])
         self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 4500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])])
         self.fw.append(["", "front", "-A INPUT -i %s -p esp -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])])
-        self.fw.append(["nat", "front", "-A POSTROUTING -o %s -m mark --mark 0x525 -j ACCEPT" % dev])
+        self.fw.append(["nat", "front", "-A POSTROUTING -t nat -o %s -m mark --mark 0x525 -j ACCEPT" % dev])
         for net in obj['peer_guest_cidr_list'].lstrip().rstrip().split(','):
             self.fw.append(["mangle", "front",
                             "-A FORWARD -s %s -d %s -j MARK --set-xmark 0x525/0xffffffff" % (obj['local_guest_cidr'], net)])
@@ -796,7 +791,7 @@ class CsForwardingRules(CsDataBag):
                 rule['internal_ip'],
                 internal_fwports
               )
-        fw4 = "-A POSTROUTING -j SNAT --to-source %s -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \
+        fw4 = "-j SNAT --to-source %s -A POSTROUTING -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \
               (
                 self.getGuestIp(),
                 self.getNetworkByIp(rule['internal_ip']),
@@ -991,7 +986,7 @@ def main(argv):
         lb.process()
 
         logging.debug("Configuring iptables rules")
-        nf = CsNetfilters(False)
+        nf = CsNetfilters()
         nf.compare(config.get_fw())
 
         logging.debug("Configuring iptables rules done ...saving rules")

systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py

@@ -15,8 +15,9 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from CsDatabag import CsDataBag
+from CsDatabag import CsDataBag, CsCmdLine
 from CsApp import CsApache, CsDnsmasq, CsPasswdSvc
+import CsHelper
 import logging
 from netaddr import IPAddress, IPNetwork
 import CsHelper
@@ -197,7 +198,7 @@ class CsInterface:
         return self.get_attr("add")
 
     def to_str(self):
-        print(self.address)
+        pprint(self.address)
 
 
 class CsDevice:
@@ -370,6 +371,8 @@ class CsIP:
             self.fw.append(["mangle", "front",
                             "-A FIREWALL_%s " % self.address['public_ip'] +
                             "-m state --state RELATED,ESTABLISHED -j ACCEPT"])
+            self.fw.append(["mangle", "",
+                            "-A FIREWALL_%s DROP" % self.address['public_ip']])
             self.fw.append(["mangle", "",
                             "-A VPN_%s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.address['public_ip']])
             self.fw.append(["mangle", "",
@@ -387,7 +390,8 @@ class CsIP:
 
         self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"])
-        self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev])
+        self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" %
+                        self.dev])
         self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"])
 
@@ -430,13 +434,6 @@ class CsIP:
         self.fw.append(["mangle", "front", "-A PREROUTING " +
                         "-m state --state RELATED,ESTABLISHED " +
                         "-j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff"])
-
-        self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"])
-        self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"])
-        self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"])
-
-        self.fw.append(["filter", "", "-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT"])
-
         if self.get_type() in ["guest"]:
             self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" %
                             (self.address['network'], self.dev, self.dev)])
@@ -475,6 +472,10 @@ class CsIP:
                             ])
 
         if self.get_type() in ["public"]:
+            self.fw.append(["", "front",
+                            "-A FORWARD -o %s -d %s -j ACL_INBOUND_%s" % (
+                                self.dev, self.address['network'], self.dev)
+                            ])
             self.fw.append(
                 ["mangle", "", "-A FORWARD -j VPN_STATS_%s" % self.dev])
             self.fw.append(
@@ -482,7 +483,11 @@ class CsIP:
             self.fw.append(
                 ["mangle", "", "-A VPN_STATS_%s -i %s -m mark --mark 0x524/0xffffffff" % (self.dev, self.dev)])
             self.fw.append(
-                ["", "front", "-A FORWARD -j NETWORK_STATS_eth1"])
+                ["", "front", "-A FORWARD -j NETWORK_STATS_%s" % self.dev])
+
+        self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"])
+        self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"])
+        self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"])
 
         self.fw.append(["", "", "-A NETWORK_STATS -i eth0 -o eth2 -p tcp"])
         self.fw.append(["", "", "-A NETWORK_STATS -i eth2 -o eth0 -p tcp"])
@@ -491,11 +496,9 @@ class CsIP:
 
         self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"])
-        self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev])
+
-        self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -i eth0 -p tcp -m tcp --dport 3922 -m state --state NEW,ESTABLISHED -j ACCEPT"])
-        self.fw.append(["filter", "", "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT"])
 
         self.fw.append(["filter", "", "-P INPUT DROP"])
         self.fw.append(["filter", "", "-P FORWARD DROP"])

systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py

@@ -54,7 +54,7 @@ class CsDhcp(CsDataBag):
         self.cloud.commit()
 
         # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems.
-        CsHelper.execute2("service dnsmasq restart")
+        CsHelper.service("dnsmasq", "restart")
 
     def configure_server(self):
         # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS)

systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py

@@ -71,16 +71,14 @@ class CsLoadBalancer(CsDataBag):
             port = path[1]
             firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
 
+        for rules in remove_rules:
+            path = rules.split(':')
+            ip = path[0]
+            port = path[1]
+            firewall.append(["filter", "", "-D INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
+
         for rules in stat_rules:
             path = rules.split(':')
             ip = path[0]
             port = path[1]
             firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
-
-        for rules in remove_rules:
-            path = rules.split(':')
-            ip = path[0]
-            port = path[1]
-            if ["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)] in firewall:
-                firewall.remove(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
-

systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py

@@ -15,12 +15,10 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from __future__ import print_function
-
 import CsHelper
-from CsDatabag import CsCmdLine
+from pprint import pprint
+from CsDatabag import CsDataBag, CsCmdLine
 import logging
-from cs_iptables_save import Tables
 
 
 class CsChain(object):
@@ -83,7 +81,6 @@ class CsNetfilters(object):
 
     def __init__(self, load=True):
         self.rules = []
-        self.iptablerules = []
         self.table = CsTable()
         self.chain = CsChain()
         if load:
@@ -94,10 +91,7 @@ class CsNetfilters(object):
             if i.startswith('*'):  # Table
                 self.table.add(i[1:])
             if i.startswith(':'):  # Chain
-                string = i[1:].split(' ')[0]
+                self.chain.add(self.table.last(), i[1:].split(' ')[0])
-                cmd = "iptables -t %s -N %s" % (self.table.last(), string)
-                self.iptablerules.append(cmd)
-                self.chain.add(self.table.last(), string)
             if i.startswith('-A'):  # Rule
                 self.chain.add_rule(i.split()[1])
                 rule = CsNetfilter()
@@ -131,7 +125,10 @@ class CsNetfilters(object):
     def get_unseen(self):
         del_list = [x for x in self.rules if x.unseen()]
         for r in del_list:
-            self.delete(r)
+            cmd = "iptables -t %s %s" % (r.get_table(), r.to_str(True))
+            logging.debug("unseen cmd:  %s ", cmd)
+            CsHelper.execute(cmd)
+            # print "Delete rule %s from table %s" % (r.to_str(True), r.get_table())
             logging.info("Delete rule %s from table %s", r.to_str(True), r.get_table())
 
     def compare(self, list):
@@ -140,16 +137,12 @@ class CsNetfilters(object):
             # Ensure all inbound/outbound chains have a default drop rule
             if c.startswith("ACL_INBOUND") or c.startswith("ACL_OUTBOUND"):
                 list.append(["filter", "", "-A %s -j DROP" % c])
-        # PASS 1:  Ensure all chains are present and cleanup unused rules.
+        # PASS 1:  Ensure all chains are present
         for fw in list:
             new_rule = CsNetfilter()
             new_rule.parse(fw[2])
             new_rule.set_table(fw[0])
-            self.has_rule(new_rule)
+            self.add_chain(new_rule)
-
-        self.del_standard()
-        self.get_unseen()
-
         # PASS 2: Create rules
         for fw in list:
             new_rule = CsNetfilter()
@@ -158,6 +151,11 @@ class CsNetfilters(object):
             if isinstance(fw[1], int):
                 new_rule.set_count(fw[1])
 
+            logging.debug("Checking if the rule already exists: rule=%s table=%s chain=%s", new_rule.get_rule(), new_rule.get_table(), new_rule.get_chain())
+            if self.has_rule(new_rule):
+                logging.debug("Exists: rule=%s table=%s", fw[2], new_rule.get_table())
+            else:
+                # print "Add rule %s in table %s" % ( fw[2], new_rule.get_table())
                 logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table())
                 # front means insert instead of append
                 cpy = fw[2]
@@ -166,25 +164,15 @@ class CsNetfilters(object):
                 if isinstance(fw[1], int):
                     cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1]))
 
-            self.iptablerules.append("iptables -t %s %s" % (new_rule.get_table(), cpy))
+                CsHelper.execute("iptables -t %s %s" % (new_rule.get_table(), cpy))
-        self.apply_rules()
+        self.del_standard()
+        self.get_unseen()
 
-    def apply_rules(self):
+    def add_chain(self, rule):
-        s = []
+        """ Add the given chain if it is not already present """
-        for r in self.iptablerules:
+        if not self.has_chain(rule.get_table(), rule.get_chain()):
-            r.replace('  ', ' ')  # Remove duplicate spaces
+            CsHelper.execute("iptables -t %s -N %s" % (rule.get_table(), rule.get_chain()))
-            if r not in s:
+            self.chain.add(rule.get_table(), rule.get_chain())
-                s.append(r)
-
-        chains = Tables(s)
-        chains.table_printout()
-
-        # COMMIT all rules.
-        result = CsHelper.execute("iptables-restore < /tmp/rules.save")
-        if result:
-            logging.info("iptables-restore result: %s", result)
-        else:
-            logging.info("iptables-restore result: success!")
 
     def del_standard(self):
         """ Del rules that are there but should not be deleted

systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py

@@ -1,227 +0,0 @@
-#!/usr/bin/python
-#
-# -*- coding: utf-8 -*-
-#
-"""
-iptables_converter.py:
-    convert iptables commands within a script
-    into a correspondig iptables-save script
-
-    default filename to read is rules, to read some other
-        file, append: -s filename
-
-    output is written to stdout for maximum flexibilty
-
-Author:     Johannes Hubertz <johannes@hubertz.de>
-Date:       2015-03-17
-version:    0.9.8
-License:    GNU General Public License version 3 or later
-
-Have Fun!
-"""
-from __future__ import print_function
-
-try:
-    from collections import UserDict
-except ImportError:
-    from UserDict import UserDict
-import re
-import sys
-import logging
-
-
-class ConverterError():
-    """on accidential case of error show given reason"""
-
-    def __init__(self, message):
-        """message to stdout to compatible testings 2.7 and 3.4"""
-        print (message)
-        sys.exit(1)
-
-
-class Chains(UserDict):
-    """this is for one type of tables"""
-
-    def __init__(self, name, tables):
-        """init Chains object"""
-        UserDict.__init__(self)
-        self.name = name
-        self.tables = tables
-        self.predef = tables
-        self.reset()  # name, tables)
-
-    def put_into_fgr(self, content):
-        """fill this line into this tabular"""
-        self.length += 1
-        cha = "filter"
-        # act = ""
-        liste = content.split()
-        action = liste[0]
-        if "-t" in action:
-            liste.pop(0)  # remove 1st: -t
-            fname = liste.pop(0)
-            legals = ["filter", "nat", "raw", "mangle"]
-            if fname not in legals:
-                msg = "Valid is one of %s, got: %s" % (legals, fname)
-                raise ValueError(msg)
-            action = liste[0]
-            content = ""                       # rebuild content from here
-            for elem in liste:
-                content = content + elem + " "
-            if len(liste) > 1:
-                chain_name = liste[1]
-        if "-F" in action:
-            self.reset()
-            return
-        if "-P" in action:
-            liste.pop(0)
-            cha = liste.pop(0)
-            new = liste.pop(0)
-            if new not in ["ACCEPT", "DROP", "REJECT"]:
-                msg = "Illegal policy: % s" % (new)
-                raise ValueError(msg)
-            self.poli[cha] = new
-            return
-        if "-X" in action:
-            predef = ['INPUT', 'FORWARD', 'OUTPUT',
-                      'PREROUTING', 'POSTROUTING']
-            rem_chain_name = liste.pop(1)
-            if rem_chain_name in predef:
-                msg = "Cannot remove predefined chain"
-                raise ValueError(msg)
-            if rem_chain_name in self.data:
-                self.data[rem_chain_name] = []        # empty list
-                self.poli[rem_chain_name] = "-"       # empty policy, no need
-                self.data.pop(rem_chain_name)
-            return
-        if "-N" in action:
-            new_chain_name = liste.pop(1)
-            existing = self.data.keys()
-            if new_chain_name in existing:
-                logging.debug("Chain %s already exists" % new_chain_name)
-                return
-            self.data[new_chain_name] = []        # empty list
-            self.poli[new_chain_name] = "-"       # empty policy, no need
-            return
-        if "-I" in action:  # or "-A" in action:
-            chain_name = liste[1]
-            existing = self.data.keys()
-            if chain_name not in existing:
-                self.data[chain_name] = []
-                self.poli[chain_name] = "-"
-            kette = self.data[chain_name]
-            kette.insert(0, content.replace("-I", "-A"))
-            self.data[chain_name] = kette
-            return
-        if "-A" in action:  # or "-I" in action:
-            chain_name = liste[1]
-            existing = self.data.keys()
-            if chain_name not in existing:
-                self.data[chain_name] = []
-                self.poli[chain_name] = "-"
-            kette = self.data[chain_name]
-            kette.append(content)
-            self.data[chain_name] = kette
-            return
-        msg = "Unknown filter command in input:", content
-        raise ValueError(msg)
-
-    def reset(self):  # name, tables):
-        """
-        name is one of filter, nat, raw, mangle,
-        tables is a list of tables in that table-class
-        """
-        self.poli = {}               # empty dict
-        self.length = 0
-        self.policy = "-"
-        for tabular in self.tables:
-            self.data[tabular] = []
-            self.poli[tabular] = "ACCEPT"
-
-
-class Tables(UserDict):
-    """
-    some chaingroups in tables are predef: filter, nat, mangle, raw
-    """
-
-    def __init__(self, rules):
-        """init Tables Object is easy going"""
-        UserDict.__init__(self)
-        self.reset(rules)
-
-    def reset(self, rules):
-        """all predefined Chains aka lists are setup as new here"""
-        filter = Chains("filter", ["INPUT", "FORWARD", "OUTPUT"])
-
-        mang = ["PREROUTING", "INPUT", "FORWARD", "OUTPUT", "POSTROUTING", ]
-        mangle = Chains("mangle", mang)
-
-        # kernel 2.6.32 has no INPUT in NAT!
-        nat = Chains("nat", ["PREROUTING", "OUTPUT", "POSTROUTING"])
-
-        raw = Chains("raw", ["PREROUTING", "OUTPUT", ])
-
-        self.data["filter"] = filter
-        self.data["mangle"] = mangle
-        self.data["nat"] = nat
-        self.data["raw"] = raw
-        if rules is not None:
-            self.read_file(rules)
-
-    def table_printout(self):
-        """printout nonempty tabulars in fixed sequence"""
-        with open("/tmp/rules.save", 'w') as f:
-            for key in ["raw", "nat", "mangle", "filter"]:
-                len = self.data[key].length
-                if len > -1:
-                    print("*%s" % (self.data[key].name), file=f)
-                    for chain in self.data[key].keys():
-                        poli = self.data[key].poli[chain]
-                        print(":%s %s [0:0]" % (chain, poli), file=f)
-                    for chain in self.data[key].values():
-                        for elem in chain:
-                            print(elem, file=f)
-                    print("COMMIT", file=f)
-
-    def put_into_tables(self, line):
-        """put line into matching Chains-object"""
-        liste = line.split()
-        liste.pop(0)                        # we always know, it's iptables
-        rest = ""
-        for elem in liste:                  # remove redirects and the like
-            if ">" not in elem:
-                rest = rest + elem + " "    # string again with single blanks
-        action = liste.pop(0)               # action is one of {N,F,A,I, etc.}
-        fam = "filter"
-        if "-t nat" in line:                # nat filter group
-            fam = "nat"
-        elif "-t mangle" in line:           # mangle filter group
-            fam = "mangle"
-        elif "-t raw" in line:              # raw filter group
-            fam = "raw"
-        fam_dict = self.data[fam]           # select the group dictionary
-        fam_dict.put_into_fgr(rest)         # do action thers
-
-    def read_file(self, rules):
-        """read file into Tables-object"""
-        self.linecounter = 0
-        self.tblctr = 0
-        for zeile in rules:
-            line = str(zeile.strip())
-            self.linecounter += 1
-            if line.startswith('#'):
-                continue
-            for element in ['\$', '\(', '\)', ]:
-                if re.search(element, line):
-                    m1 = "Line %d:\n%s\nplain files only, " % \
-                         (self.linecounter, line)
-                    if element in ['\(', '\)', ]:
-                        m2 = "unable to convert shell functions, abort"
-                    else:
-                        m2 = "unable to resolve shell variables, abort"
-                    msg = m1 + m2
-                    raise ConverterError(msg)
-            for muster in ["^/sbin/iptables ", "^iptables "]:
-                if re.search(muster, line):
-                    self.tblctr += 1
-                    self.put_into_tables(line)