From 666dc16e58a1ad57483b3ba68efe92bd4ed9fadd Mon Sep 17 00:00:00 2001 From: Ian Southam Date: Mon, 4 Aug 2014 18:39:21 +0200 Subject: [PATCH] Moved bag location to /et/cloudstack Updated test script to also process command line Added connmark stuff to merge --- .../config/etc/init.d/cloud-early-config | 4 +- .../debian/config/opt/cloud/bin/configure.py | 282 +++++++++++------- .../debian/config/opt/cloud/bin/cs_cmdline.py | 5 + .../debian/config/opt/cloud/bin/cs_ip.py | 2 + .../debian/config/opt/cloud/bin/merge.py | 30 +- .../debian/config/opt/cloud/bin/test.sh | 1 + 6 files changed, 195 insertions(+), 129 deletions(-) create mode 100644 systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py diff --git a/systemvm/patches/debian/config/etc/init.d/cloud-early-config b/systemvm/patches/debian/config/etc/init.d/cloud-early-config index 76e6e94011a..06780aee699 100755 --- a/systemvm/patches/debian/config/etc/init.d/cloud-early-config +++ b/systemvm/patches/debian/config/etc/init.d/cloud-early-config @@ -1338,7 +1338,7 @@ VM_PASSWORD="" CHEF_TMP_FILE=/tmp/cmdline.json COMMA="\t" -echo -e "{\n\"id\": \"cmdline\"," > ${CHEF_TMP_FILE} +echo -e "{\n\"type\": \"cmdline\"," > ${CHEF_TMP_FILE} echo -e "\n\"cmd_line\": {" >> ${CHEF_TMP_FILE} for i in $CMDLINE @@ -1491,7 +1491,7 @@ done echo -e "\n\t}\n}" >> ${CHEF_TMP_FILE} if [ "$TYPE" != "unknown" ] then - mv ${CHEF_TMP_FILE} /var/chef/data_bags/vr/cmd_line.json + mv ${CHEF_TMP_FILE} /etc/cloudstack/cmd_line.json fi [ $ETH0_IP ] && LOCAL_ADDRS=$ETH0_IP diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index bac7b10fb67..714ba525f17 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -7,7 +7,7 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an @@ -23,89 +23,104 @@ import logging import re import time -class csHelper: +class CsHelper: + """ General helper functions + for use in the configuation process - def upFile(self, fn, val, mode): - for line in open(fn): + TODO - Convert it to a module + """ + def updatefile(self, filename, val, mode): + """ add val to file """ + for line in open(filename): if line.strip().lstrip("0") == val: - return + return # set the value - f = open(fn, mode) - f.write(val) - f.close + handle = open(filename, mode) + handle.write(val) + handle.close() - def definedInFile(self, fn, val): - for line in open(fn): + def definedinfile(self, filename, val): + """ Check if val is defined in the file """ + for line in open(filename): if re.search(val, line): - return True + return True return False - def addIfMissing(self, fn, val): - if not csHelper().definedInFile(fn, val): - csHelper().upFile(fn, val + "\n", "a") - logging.debug("Added %s to file %s" % (val, fn)) + def addifmissing(self, filename, val): + """ Add something to a file + if it is not already there """ + if not CsHelper().definedinfile(filename, val): + CsHelper().updatefile(filename, val + "\n", "a") + logging.debug("Added %s to file %s" % (val, filename)) def execute(self, command): + """ Execute command """ p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) result = p.communicate()[0] return result.splitlines() -# ----------------------------------------------------------- # -# Manage ip rules (such as fwmark) -# ----------------------------------------------------------- # -class csRule: -#sudo ip rule add fwmark $tableNo table $tableName + +class CsRule: + """ Manage iprules + Supported Types: + fwmark + """ + def __init__(self, dev): self.dev = dev self.tableNo = dev[3] - self.table = "Table_%s" % (dev) + self.table = "Table_%s" % (dev) def addMark(self): if not self.findMark(): - cmd = "ip rule add fwmark %s table %s" % (self.tableNo, self.table) - csHelper().execute(cmd) - logging.info("Added fwmark rule for %s" % (self.table)) - + cmd = "ip rule add fwmark %s table %s" % (self.tableNo, self.table) + CsHelper().execute(cmd) + logging.info("Added fwmark rule for %s" % (self.table)) + def findMark(self): srch = "from all fwmark 0x%s lookup %s" % (self.tableNo, self.table) - for i in csHelper().execute("ip rule show"): + for i in CsHelper().execute("ip rule show"): if srch in i.strip(): - return True + return True return False + class csRoute: + """ Manage routes """ def __init__(self, dev): self.dev = dev self.tableNo = dev[3] - self.table = "Table_%s" % (dev) + self.table = "Table_%s" % (dev) def routeTable(self): str = "%s %s" % (self.tableNo, self.table) - fn = "/etc/iproute2/rt_tables" - csHelper().addIfMissing(fn, str) + filename = "/etc/iproute2/rt_tables" + CsHelper().addifmissing(filename, str) def flush(self): - csHelper().execute("ip route flush table %s" % (self.table) ) - csHelper().execute("ip route flush cache") + CsHelper().execute("ip route flush table %s" % (self.table)) + CsHelper().execute("ip route flush cache") def add(self, address): - # ip route show dev eth1 table Table_eth1 10.0.2.0/24 - # sudo ip route add default via $defaultGwIP table $tableName proto static + # ip route show dev eth1 table Table_eth1 10.0.2.0/24 + # sudo ip route add default via $defaultGwIP table $tableName proto static cmd = "dev %s table %s %s" % (self.dev, self.table, address['network']) - self.addIfMissing(cmd) + self.addifmissing(cmd) - def addIfMissing(self, cmd): + def addifmissing(self, cmd): + """ Add a route is it is not already defined """ found = False - for i in csHelper().execute("ip route show " + cmd): + for i in CsHelper().execute("ip route show " + cmd): found = True if not found: - logging.info("Add " + cmd) - cmd = "ip route add " + cmd - csHelper().execute(cmd) - + logging.info("Add " + cmd) + cmd = "ip route add " + cmd + CsHelper().execute(cmd) + class csRpsrfs: + """ Configure rpsrfs if there is more than one cpu """ def __init__(self, dev): self.dev = dev @@ -115,11 +130,11 @@ class csRpsrfs: cpus = self.cpus() if cpus < 2: return val = format((1 << cpus) - 1, "x") - fn = "/sys/class/net/%s/queues/rx-0/rps_cpus" % (self.dev) - csHelper().upFile(fn, val, "w+") - csHelper().upFile("/proc/sys/net/core/rps_sock_flow_entries", "256", "w+") - fn = "/sys/class/net/%s/queues/rx-0/rps_flow_cnt" % (self.dev) - csHelper().upFile(fn, "256", "w+") + filename = "/sys/class/net/%s/queues/rx-0/rps_cpus" % (self.dev) + CsHelper().updatefile(filename, val, "w+") + CsHelper().updatefile("/proc/sys/net/core/rps_sock_flow_entries", "256", "w+") + filename = "/sys/class/net/%s/queues/rx-0/rps_flow_cnt" % (self.dev) + CsHelper().updatefile(filename, "256", "w+") logging.debug("rpsfr is configured for %s cpus" % (cpus)) def inKernel(self): @@ -140,57 +155,86 @@ class csRpsrfs: if count < 2: logging.debug("Single CPU machine") return count -class csDevice: +class CsDevice: + """ Configure Network Devices """ def __init__(self, dev): self.devlist = [] self.dev = dev self.buildlist() + self.table = '' + self.tableNo = '' + if dev != '': + self.tableNo = dev[3] + self.table = "Table_%s" % (dev) - # ------------------------------------------------------- # - # List all available network devices on the system - # ------------------------------------------------------- # def buildlist(self): + """ + List all available network devices on the system + """ self.devlist = [] for line in open('/proc/net/dev'): vals = line.lstrip().split(':') - if(not vals[0].startswith("eth")): - continue + if (not vals[0].startswith("eth")): + continue # Ignore control interface for now - if(vals[0] == 'eth0'): - continue + if (vals[0] == 'eth0'): + continue self.devlist.append(vals[0]) - # ------------------------------------------------------- # - # Wait up to 15 seconds for a device to become available - # ------------------------------------------------------- # + def set_connmark(self): + """ Set connmark for device """ + if not self.has_connmark(): + cmd="-A PREROUTING -i %s -m state --state NEW -j CONNMARK --set-mark %s" \ + % (self.dev, self.tableNo) + CsHelper().execute("iptables -t mangle %s" % (cmd)) + logging.error("Set connmark for device %s (Table %s)", self.dev, self.tableNo) + + def has_connmark(self): + cmd = "iptables-save -t mangle" + for line in CsHelper().execute(cmd): + if not "PREROUTING" in line: + continue + if not "state" in line: + continue + if not "CONNMARK" in line: + continue + if not "set-xmark" in line: + continue + if not self.dev in line: + continue + return True + return False + + def waitForDevice(self): + """ Wait up to 15 seconds for a device to become available """ count = 0 while count < 15: - if self.dev in self.devlist: - return True - time.sleep(1) - count += 1 - self.buildlist(); - logging.error("Address %s on device %s cannot be configured - device was not found", ip.ip(), dev) + if self.dev in self.devlist: + return True + time.sleep(1) + count += 1 + self.buildlist(); + logging.error("Address %s on device %s cannot be configured - device was not found", ip.ip(), self.dev) return False def list(self): return self.devlist - # ------------------------------------------------------- # - # Ensure device is up - # ------------------------------------------------------- # def setUp(self): + """ Ensure device is up """ cmd = "ip link show %s | grep 'state DOWN'" % (self.dev) - for i in csHelper().execute(cmd): + for i in CsHelper().execute(cmd): if " DOWN " in i: - cmd2 = "ip link set %s up" % (self.dev) - csHelper().execute(cmd2) + cmd2 = "ip link set %s up" % (self.dev) + CsHelper().execute(cmd2) + self.set_connmark() -class csIp: - def __init__(self,dev): +class CsIP: + + def __init__(self, dev): self.dev = dev self.iplist = {} self.address = {} @@ -199,14 +243,19 @@ class csIp: def setAddress(self, address): self.address = address + def configure(self): logging.info("Configuring address %s on device %s", self.ip(), self.dev) cmd = "ip addr add dev %s %s brd +" % (self.dev, self.ip()) subprocess.call(cmd, shell=True) + self.post_configure() + + def post_configure(self): + """ The steps that must be done after a device is configured """ route = csRoute(self.dev) route.routeTable() - csRule(self.dev).addMark() - csDevice(self.dev).setUp() + CsRule(self.dev).addMark() + CsDevice(self.dev).setUp() self.arpPing() route.add(self.address) csRpsrfs(self.dev).enable() @@ -215,46 +264,46 @@ class csIp: def list(self): self.iplist = {} cmd = ("ip addr show dev " + self.dev) - for i in csHelper().execute(cmd): + for i in CsHelper().execute(cmd): vals = i.lstrip().split() - if(vals[0] == 'inet'): - self.iplist[vals[1]] = self.dev + if (vals[0] == 'inet'): + self.iplist[vals[1]] = self.dev def configured(self): dev = self.address['device'] - if(self.address['cidr'] in self.iplist.keys()): - return True + if (self.address['cidr'] in self.iplist.keys()): + return True return False - + def ip(self): return str(self.address['cidr']) - + def hasIP(self, ip): return ip in self.address.values() def arpPing(self): cmd = "arping -c 1 -I %s -A -U -s %s %s" % (self.dev, self.address['public_ip'], self.address['public_ip']) - csHelper().execute(cmd) + CsHelper().execute(cmd) # Delete any ips that are configured but not in the bag def compare(self, bag): - if(len(self.iplist) > 0 and not self.dev in bag.keys()): - # Remove all IPs on this device - logging.info("Will remove all configured addresses on device %s", self.dev) - self.delete("all") - return False + if (len(self.iplist) > 0 and not self.dev in bag.keys()): + # Remove all IPs on this device + logging.info("Will remove all configured addresses on device %s", self.dev) + self.delete("all") + return False for ip in self.iplist: found = False for address in bag[self.dev]: self.setAddress(address) - if(self.hasIP(ip)): + if (self.hasIP(ip)): found = True - if(not found): + if (not found): self.delete(ip) def delete(self, ip): remove = [] - if(ip == "all"): + if (ip == "all"): logging.info("Removing addresses from device %s", self.dev) remove = self.iplist.keys() else: @@ -263,30 +312,37 @@ class csIp: cmd = "ip addr del dev %s %s" % (self.dev, ip) subprocess.call(cmd, shell=True) logging.info("Removed address %s from device %s", ip, self.dev) - - -# Main -logging.basicConfig(filename='/var/log/cloud.log',level=logging.DEBUG, format='%(asctime)s %(message)s') -db = dataBag() -db.setKey("ips") -db.load() -dbag = db.getDataBag() -for dev in csDevice('').list(): - ip = csIp(dev) - ip.compare(dbag) +def main(argv): + logging.basicConfig(filename='/var/log/cloud.log', + level=logging.DEBUG, format='%(asctime)s %(message)s') -for dev in dbag: - if dev == "id": - continue - ip = csIp(dev) - for address in dbag[dev]: - csRoute(dev).add(address) - ip.setAddress(address) - if ip.configured(): - logging.info("Address %s on device %s already configured", ip.ip(), dev) - else: - logging.info("Address %s on device %s not configured", ip.ip(), dev) - if csDevice(dev).waitForDevice(): - ip.configure() + db = dataBag() + db.setKey("ips") + db.load() + dbag = db.getDataBag() + for dev in CsDevice('').list(): + ip = CsIP(dev) + ip.compare(dbag) + + for dev in dbag: + if dev == "id": + continue + if dev == "eth0": + continue + ip = CsIP(dev) + for address in dbag[dev]: + csRoute(dev).add(address) + ip.setAddress(address) + if ip.configured(): + logging.info("Address %s on device %s already configured", ip.ip(), dev) + ip.post_configure() + else: + logging.info("Address %s on device %s not configured", ip.ip(), dev) + if CsDevice(dev).waitForDevice(): + ip.configure() + + +if __name__ == "__main__": + main(sys.argv) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py new file mode 100644 index 00000000000..6868a454ce7 --- /dev/null +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py @@ -0,0 +1,5 @@ +from pprint import pprint + +def merge(dbag, cmdline): + dbag.setdefault('config', []).append( cmdline ) + return dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py index 36b640f4a97..cd474c0fb91 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py @@ -14,5 +14,7 @@ def merge(dbag, ip): ip['device'] = 'eth' + str(ip['nic_dev_id']) ip['cidr'] = str(ipo.ip) + '/' + str(ipo.prefixlen) ip['network'] = str(ipo.network) + '/' + str(ipo.prefixlen) + if 'nw_type' not in ip.keys(): + ip['nw_type'] = 'public' dbag.setdefault('eth' + str(ip['nic_dev_id']), []).append( ip ) return dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/merge.py b/systemvm/patches/debian/config/opt/cloud/bin/merge.py index 4aa31a031e3..d61dc363965 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/merge.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/merge.py @@ -5,13 +5,14 @@ import os import logging import cs_ip import cs_guestnetwork +import cs_cmdline from pprint import pprint class dataBag: bdata = { } - DPATH = "/var/chef/data_bags/vr" + DPATH = "/etc/cloudstack" def load(self): data = self.bdata @@ -21,7 +22,6 @@ class dataBag: try: handle = open(self.fpath) except IOError: - print("FILE DOES NOT EXIST") logging.debug("Creating data bag type %s", self.key) data.update( { "id": self.key } ) else: @@ -51,16 +51,13 @@ class updateDataBag: qFile = {} fpath = '' bdata = { } - DPATH = "/var/chef/data_bags/vr" + DPATH = "/etc/cloudstack" def __init__(self,qFile): self.qFile = qFile self.process() def process(self): - if self.qFile.type == 'cl': - self.transformCL() - self.qFile.data = self.newData self.db = dataBag() self.db.setKey( self.qFile.type ) dbag = self.db.load( ) @@ -70,6 +67,8 @@ class updateDataBag: dbag = self.processIP(self.db.getDataBag()) if self.qFile.type == 'guestnetwork': dbag = self.processGuestNetwork(self.db.getDataBag()) + if self.qFile.type == 'cmdline': + dbag = self.processCL(self.db.getDataBag()) self.db.save(dbag) def processGuestNetwork(self, dbag): @@ -82,6 +81,7 @@ class updateDataBag: dp['one_to_one_nat'] = False dp['gateway'] = d['router_guest_gateway'] dp['nic_dev_id'] = d['device'][3] + dp['nw_type'] = 'guest' qf = loadQueueFile() qf.load({ 'ip_address' : [ dp ], 'type' : 'ips'}) return cs_guestnetwork.merge(dbag, self.qFile.data) @@ -91,31 +91,33 @@ class updateDataBag: dbag = cs_ip.merge(dbag, ip) return dbag - def transformCL(self): + def processCL(self, dbag): # Convert the ip stuff to an ip object and pass that into cs_ip_merge # "eth0ip": "192.168.56.32", # "eth0mask": "255.255.255.0", self.newData = [] - self.qFile.setType("ips") self.processCLItem('0') self.processCLItem('1') self.processCLItem('2') + return cs_cmdline.merge(dbag, self.qFile.data) def processCLItem(self, num): key = 'eth' + num + 'ip' dp = {} - if(key in self.qFile.data['cmdline']): - dp['public_ip'] = self.qFile.data['cmdline'][key] - dp['netmask'] = self.qFile.data['cmdline']['eth' + num + 'mask'] + if(key in self.qFile.data['cmd_line']): + dp['public_ip'] = self.qFile.data['cmd_line'][key] + dp['netmask'] = self.qFile.data['cmd_line']['eth' + num + 'mask'] dp['source_nat'] = False dp['add'] = True dp['one_to_one_nat'] = False - if('localgw' in self.qFile.data['cmdline']): - dp['gateway'] = self.qFile.data['cmdline']['localgw'] + if('localgw' in self.qFile.data['cmd_line']): + dp['gateway'] = self.qFile.data['cmd_line']['localgw'] else: dp['gateway'] = 'None' dp['nic_dev_id'] = num - self.newData = { 'ip_address' : [ dp ], 'type' : 'ips'} + dp['nw_type'] = 'control' + qf = loadQueueFile() + qf.load({ 'ip_address' : [ dp ], 'type' : 'ips'}) class loadQueueFile: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/test.sh b/systemvm/patches/debian/config/opt/cloud/bin/test.sh index e50edb15ea8..60fe35e9a45 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/test.sh +++ b/systemvm/patches/debian/config/opt/cloud/bin/test.sh @@ -1,5 +1,6 @@ #!/bin/sh +/opt/cloud/bin/update_config.py cmd_line.json /opt/cloud/bin/update_config.py gn0001.json /opt/cloud/bin/update_config.py ips0001.json /opt/cloud/bin/update_config.py ips0002.json