From 9385f071d7544ae3b104ac78c8fcede8c735ff6a Mon Sep 17 00:00:00 2001 From: Hugo Trippaers Date: Tue, 11 Nov 2014 11:04:16 +0100 Subject: [PATCH] Fix PEP8 compliance --- .../debian/config/opt/cloud/bin/configure.py | 88 +++++++------ .../config/opt/cloud/bin/cs/CsAddress.py | 71 ++++++----- .../debian/config/opt/cloud/bin/cs/CsApp.py | 36 +++--- .../config/opt/cloud/bin/cs/CsConfig.py | 21 +-- .../config/opt/cloud/bin/cs/CsDatabag.py | 5 +- .../debian/config/opt/cloud/bin/cs/CsDhcp.py | 37 +++--- .../debian/config/opt/cloud/bin/cs/CsFile.py | 11 +- .../config/opt/cloud/bin/cs/CsGuestNetwork.py | 5 +- .../config/opt/cloud/bin/cs/CsHelper.py | 38 ++++-- .../config/opt/cloud/bin/cs/CsNetfilter.py | 68 +++++----- .../config/opt/cloud/bin/cs/CsRedundant.py | 22 ++-- .../debian/config/opt/cloud/bin/cs/CsRoute.py | 5 +- .../debian/config/opt/cloud/bin/cs/CsRule.py | 1 + .../debian/config/opt/cloud/bin/cs_cmdline.py | 1 + .../debian/config/opt/cloud/bin/cs_dhcp.py | 11 +- .../opt/cloud/bin/cs_forwardingrules.py | 12 +- .../config/opt/cloud/bin/cs_guestnetwork.py | 7 +- .../debian/config/opt/cloud/bin/cs_ip.py | 27 ++-- .../config/opt/cloud/bin/cs_network_acl.py | 1 + .../config/opt/cloud/bin/cs_site2sitevpn.py | 3 +- .../debian/config/opt/cloud/bin/cs_vmdata.py | 3 +- .../debian/config/opt/cloud/bin/cs_vmp.py | 1 + .../debian/config/opt/cloud/bin/master.py | 4 +- .../debian/config/opt/cloud/bin/merge.py | 120 +++++++++--------- .../config/opt/cloud/bin/set_redundant.py | 6 +- .../config/opt/cloud/bin/update_config.py | 4 +- .../debian/config/opt/cloud/bin/vmdata.py | 31 +++-- 27 files changed, 345 insertions(+), 294 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index 2c74adf3936..4fac7e07694 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -38,6 +38,7 @@ from cs.CsFile import CsFile from cs.CsAddress import CsAddress from cs.CsApp import CsApache, CsPasswdSvc, CsDnsmasq + class CsPassword(CsDataBag): """ Update the password cache @@ -58,6 +59,7 @@ class CsPassword(CsDataBag): def __update(self, file, ip, password): file.search("%s=" % ip, "%s=%s" % (ip, password)) + class CsAcl(CsDataBag): """ Deal with Network acls @@ -70,9 +72,9 @@ class CsAcl(CsDataBag): self.ingess = [] self.egress = [] self.device = obj['device'] - self.ip = obj['nic_ip'] - self.netmask= obj['nic_netmask'] - self.cidr = "%s/%s" % (self.ip, self.netmask) + self.ip = obj['nic_ip'] + self.netmask = obj['nic_netmask'] + self.cidr = "%s/%s" % (self.ip, self.netmask) if "ingress_rules" in obj.keys(): self.ingress = obj['ingress_rules'] if "egress_rules" in obj.keys(): @@ -95,11 +97,11 @@ class CsAcl(CsDataBag): self.device = acl.device self.fw = acl.fw self.chain = config.get_ingress_chain(self.device, acl.ip) - self.dest = "-s %s" % rule['cidr'] + self.dest = "-s %s" % rule['cidr'] if direction == "egress": self.table = config.get_efress_table() self.chain = config.get_egress_chain(self.device, ip) - self.dest = "-d %s" % rule['cidr'] + self.dest = "-d %s" % rule['cidr'] self.type = "" self.type = rule['type'] self.icmp_type = "any" @@ -122,7 +124,6 @@ class CsAcl(CsDataBag): rule['last_port'] != rule['first_port']: self.dport = "%s:%s" % (self.dport, rule['last_port']) - def create(self): rstr = "" rstr = "%s -A %s -p %s %s" % (rstr, self.chain, self.protocol, self.dest) @@ -132,7 +133,6 @@ class CsAcl(CsDataBag): rstr = rstr.replace(" ", " ").lstrip() self.fw.append([self.table, "front", rstr]) - def process(self): for item in self.dbag: if item == "id": @@ -149,8 +149,8 @@ class CsVmMetadata(CsDataBag): logging.info("Processing metadata for %s" % ip) for item in self.dbag[ip]: folder = item[0] - file = item[1] - data = item[2] + file = item[1] + data = item[2] # process only valid data if folder != "userdata" and folder != "metadata": @@ -175,7 +175,7 @@ class CsVmMetadata(CsDataBag): def __createfile(self, ip, folder, file, data): dest = "/var/www/html/" + folder + "/" + ip + "/" + file metamanifestdir = "/var/www/html/" + folder + "/" + ip - metamanifest = metamanifestdir + "/meta-data" + metamanifest = metamanifestdir + "/meta-data" # base64 decode userdata if folder == "userdata" or folder == "user-data": @@ -198,12 +198,12 @@ class CsVmMetadata(CsDataBag): except OSError as e: # error 17 is already exists, we do it this way for concurrency if e.errno != 17: - print "failed to make directories " + metamanifestdir + " due to :" +e.strerror + print "failed to make directories " + metamanifestdir + " due to :" + e.strerror sys.exit(1) if os.path.exists(metamanifest): fh = open(metamanifest, "r+a") self.__exflock(fh) - if not file in fh.read(): + if file not in fh.read(): fh.write(file + '\n') self.__unflock(fh) fh.close() @@ -227,7 +227,7 @@ class CsVmMetadata(CsDataBag): if os.path.exists(htaccessFile): fh = open(htaccessFile, "r+a") self.__exflock(fh) - if not entry in fh.read(): + if entry not in fh.read(): fh.write(entry + '\n') self.__unflock(fh) fh.close() @@ -239,16 +239,16 @@ class CsVmMetadata(CsDataBag): self.__unflock(fh) fh.close() - entry="Options -Indexes\nOrder Deny,Allow\nDeny from all\nAllow from " + ip + entry = "Options -Indexes\nOrder Deny,Allow\nDeny from all\nAllow from " + ip htaccessFolder = "/var/www/html/" + folder + "/" + ip htaccessFile = htaccessFolder+"/.htaccess" try: - os.makedirs(htaccessFolder,0755) + os.makedirs(htaccessFolder, 0755) except OSError as e: # error 17 is already exists, we do it this way for sake of concurrency if e.errno != 17: - print "failed to make directories " + htaccessFolder + " due to :" +e.strerror + print "failed to make directories " + htaccessFolder + " due to :" + e.strerror sys.exit(1) fh = open(htaccessFile, "w") @@ -264,13 +264,13 @@ class CsVmMetadata(CsDataBag): fh = open(htaccessFile, "r+a") self.__exflock(fh) - if not entry in fh.read(): + if entry not in fh.read(): fh.write(entry + '\n') entry = "RewriteRule ^meta-data/$ ../" + folder + "/%{REMOTE_ADDR}/meta-data [L,NC,QSA]" fh.seek(0) - if not entry in fh.read(): + if entry not in fh.read(): fh.write(entry + '\n') self.__unflock(fh) fh.close() @@ -280,7 +280,7 @@ class CsVmMetadata(CsDataBag): flock(file, LOCK_EX) except IOError as e: print "failed to lock file" + file.name + " due to : " + e.strerror - sys.exit(1) #FIXME + sys.exit(1) # FIXME return True def __unflock(self, file): @@ -288,7 +288,7 @@ class CsVmMetadata(CsDataBag): flock(file, LOCK_UN) except IOError: print "failed to unlock file" + file.name + " due to : " + e.strerror - sys.exit(1) #FIXME + sys.exit(1) # FIXME return True @@ -328,28 +328,32 @@ class CsSite2SiteVpn(CsDataBag): logging.info("Removinf VPN configuration for %s", ip) CsHelper.execute("ipsec auto --down vpn-%s" % ip) CsHelper.execute("ipsec auto --delete vpn-%s" % ip) - vpnconffile = "%s/ipsec.vpn-%s.conf" % (self.VPNCONFDIR, ip) + vpnconffile = "%s/ipsec.vpn-%s.conf" % (self.VPNCONFDIR, ip) vpnsecretsfile = "%s/ipsec.vpn-%s.secrets" % (self.VPNCONFDIR, ip) os.remove(vpnconffile) os.remove(vpnsecretsfile) CsHelper.execute("ipsec auto --rereadall") def configure_iptables(self, dev, obj): - self.fw.append([ "", "front", "-A INPUT -i %s -p udp -m udp --dport 500 -j ACCEPT" % dev ]) - self.fw.append([ "", "front", "-A INPUT -i %s -p udp -m udp --dport 4500 -j ACCEPT" % dev ]) - self.fw.append([ "", "front", "-A INPUT -i %s -p esp -j ACCEPT" % dev ]) - self.fw.append([ "nat", "front", "-A POSTROUTING -t nat -o %s-m mark --set-xmark 0x525/0xffffffff -j ACCEPT" % dev ]) + self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 500 -j ACCEPT" % dev]) + self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 4500 -j ACCEPT" % dev]) + self.fw.append(["", "front", "-A INPUT -i %s -p esp -j ACCEPT" % dev]) + self.fw.append(["nat", "front", "-A POSTROUTING -t nat -o %s-m mark --set-xmark 0x525/0xffffffff -j ACCEPT" % dev]) for net in obj['peer_guest_cidr_list'].lstrip().rstrip().split(','): - self.fw.append([ "mangle", "front", "-A FORWARD -s %s -d %s -j MARK --set-xmark 0x525/0xffffffff" % (obj['local_guest_cidr'], net)]) - self.fw.append([ "mangle", "", "-A OUTPUT -s %s -d %s -j MARK --set-xmark 0x525/0xffffffff" % (obj['local_guest_cidr'], net)]) - self.fw.append([ "mangle", "front", "-A FORWARD -s %s -d %s -j MARK --set-xmark 0x524/0xffffffff" % (net, obj['local_guest_cidr'])]) - self.fw.append([ "mangle", "", "-A INPUT -s %s -d %s -j MARK --set-xmark 0x524/0xffffffff" % (net, obj['local_guest_cidr']) ]) + self.fw.append(["mangle", "front", + "-A FORWARD -s %s -d %s -j MARK --set-xmark 0x525/0xffffffff" % (obj['local_guest_cidr'], net)]) + self.fw.append(["mangle", "", + "-A OUTPUT -s %s -d %s -j MARK --set-xmark 0x525/0xffffffff" % (obj['local_guest_cidr'], net)]) + self.fw.append(["mangle", "front", + "-A FORWARD -s %s -d %s -j MARK --set-xmark 0x524/0xffffffff" % (net, obj['local_guest_cidr'])]) + self.fw.append(["mangle", "", + "-A INPUT -s %s -d %s -j MARK --set-xmark 0x524/0xffffffff" % (net, obj['local_guest_cidr'])]) def configure_ipsec(self, obj): - leftpeer = obj['local_public_ip'] + leftpeer = obj['local_public_ip'] rightpeer = obj['peer_gateway_ip'] - peerlist = obj['peer_guest_cidr_list'].lstrip().rstrip().replace(',', ' ') - vpnconffile = "%s/ipsec.vpn-%s.conf" % (self.VPNCONFDIR, rightpeer) + peerlist = obj['peer_guest_cidr_list'].lstrip().rstrip().replace(',', ' ') + vpnconffile = "%s/ipsec.vpn-%s.conf" % (self.VPNCONFDIR, rightpeer) vpnsecretsfile = "%s/ipsec.vpn-%s.secrets" % (self.VPNCONFDIR, rightpeer) if rightpeer in self.confips: self.confips.remove(rightpeer) @@ -390,7 +394,7 @@ class CsSite2SiteVpn(CsDataBag): hrs = int(val) / 3600 return "%sh" % hrs - + class CsForwardingRules(CsDataBag): def process(self): @@ -413,7 +417,7 @@ class CsForwardingRules(CsDataBag): if addy["public_ip"] == ip: return device return None - + def portsToString(self, ports, delimiter): ports_parts = ports.split(":", 2) if ports_parts[0] == ports_parts[1]: @@ -421,7 +425,6 @@ class CsForwardingRules(CsDataBag): else: return "%s%s%s" % (port_parts, delimiter, port_parts[1]) - def processForwardRule(self, rule): # FIXME this seems to be different for regular VRs? fwrule = "-A PREROUTING -d %s/32" % rule["public_ip"] @@ -432,21 +435,22 @@ class CsForwardingRules(CsDataBag): fwrule += " -j DNAT --to-destination %s" % rule["internal_ip"] if not rule["internal_ports"] == "any": fwrule += ":" + self.portsToString(rule["internal_ports"], "-") - self.fw.append(["nat","",fwrule]) - + self.fw.append(["nat", "", fwrule]) def processStaticNatRule(self, rule): # FIXME this needs ordering with the VPN no nat rule device = self.getDeviceByIp(rule["public_ip"]) - if device == None: + if device is None: raise Exception("Ip address %s has no device in the ips databag" % rule["public_ip"]) - self.fw.append(["nat","front","-A PREROUTING -d %s/32 -j DNAT --to-destination %s" % ( rule["public_ip"], rule["internal_ip"]) ]) - self.fw.append(["nat","front","-A POSTROUTING -o %s -s %s/32 -j SNAT --to-source %s" % ( device, rule["internal_ip"], rule["public_ip"]) ]) + self.fw.append(["nat", "front", + "-A PREROUTING -d %s/32 -j DNAT --to-destination %s" % (rule["public_ip"], rule["internal_ip"])]) + self.fw.append(["nat", "front", + "-A POSTROUTING -o %s -s %s/32 -j SNAT --to-source %s" % (device, rule["internal_ip"], rule["public_ip"])]) def main(argv): - config = CsConfig(False) - logging.basicConfig(filename= config.get_logger(), + config = CsConfig(False) + logging.basicConfig(filename=config.get_logger(), level=config.get_level(), format=config.get_format()) config.set_cl() diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py index 00097daec30..6de1c337aa8 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py @@ -24,7 +24,8 @@ import subprocess from CsRoute import CsRoute from CsRule import CsRule -VRRP_TYPES = [ 'guest', 'public' ] +VRRP_TYPES = ['guest', 'public'] + class CsAddress(CsDataBag): @@ -42,7 +43,7 @@ class CsAddress(CsDataBag): ret.append(CsInterface(ip)) return ret - def needs_vrrp(self,o): + def needs_vrrp(self, o): """ Returns if the ip needs to be managed by keepalived or not """ @@ -76,7 +77,7 @@ class CsAddress(CsDataBag): logging.info("Address %s on device %s not configured", ip.ip(), dev) if CsDevice(dev, self.config).waitfordevice(): ip.configure() - # This could go one level up but the ip type is stored in the + # This could go one level up but the ip type is stored in the # ip address object and not in the device object # Call only once if addcnt == 0: @@ -107,6 +108,7 @@ class CsAddress(CsDataBag): self.fw.append(["mangle", "", "-A VPN_STATS_%s -o %s -m mark --set-xmark 0x525/0xffffffff" % (dev, dev)]) self.fw.append(["mangle", "", "-A VPN_STATS_%s -i %s -m mark --set-xmark 0x524/0xffffffff" % (dev, dev)]) + class CsInterface: """ Hold one single ip """ def __init__(self, o): @@ -139,13 +141,14 @@ class CsInterface: return False def is_control(self): - if "nw_type" in self.address and self.address['nw_type'] in [ 'control' ]: + if "nw_type" in self.address and self.address['nw_type'] in ['control']: return True return False def to_str(self): pprint(self.address) + class CsDevice: """ Configure Network Devices """ def __init__(self, dev, config): @@ -175,10 +178,9 @@ class CsDevice: for line in open('/proc/net/dev'): vals = line.lstrip().split(':') if (not vals[0].startswith("eth")): - continue + continue self.devlist.append(vals[0]) - def waitfordevice(self): """ Wait up to 15 seconds for a device to become available """ count = 0 @@ -187,7 +189,7 @@ class CsDevice: return True time.sleep(1) count += 1 - self.buildlist(); + self.buildlist() logging.error("Device %s cannot be configured - device was not found", self.dev) return False @@ -195,7 +197,6 @@ class CsDevice: return self.devlist - class CsIP: def __init__(self, dev, config): @@ -221,7 +222,7 @@ class CsIP: def post_configure(self): """ The steps that must be done after a device is configured """ - if not self.get_type() in [ "control" ]: + if not self.get_type() in ["control"]: route = CsRoute(self.dev) route.routeTable() CsRule(self.dev).addMark() @@ -245,7 +246,7 @@ class CsIP: def set_mark(self): cmd = "-A PREROUTING -i %s -m state --state NEW -j CONNMARK --set-xmark 0x%s/0xffffffff" % \ - (self.getDevice(), self.getDevice()[3]) + (self.getDevice(), self.getDevice()[3]) self.fw.append(["mangle", "", cmd]) def get_type(self): @@ -259,7 +260,7 @@ class CsIP: return "unknown" def get_ip_address(self): - """ + """ Return ip address if known """ if "public_ip" in self.address: @@ -271,40 +272,41 @@ class CsIP: route.routeTable() route.add(self.address, method) # On deletion nw_type will no longer be known - if self.get_type() in [ "guest" ]: + if self.get_type() in ["guest"]: devChain = "ACL_INBOUND_%s" % (self.dev) CsDevice(self.dev, self.config).configure_rp() - self.fw.append(["nat", "front", - "-A POSTROUTING -s %s -o %s -j SNAT --to-source %s" % \ - (self.address['network'], self.dev, self.address['public_ip']) - ]) + self.fw.append(["nat", "front", + "-A POSTROUTING -s %s -o %s -j SNAT --to-source %s" % + (self.address['network'], self.dev, + self.address['public_ip']) + ]) self.fw.append(["mangle", "front", "-A %s -j ACCEPT" % devChain]) - self.fw.append(["", "front", - "-A FORWARD -o %s -d %s -j %s" % (self.dev, self.address['network'], devChain) - ]) + self.fw.append(["", "front", + "-A FORWARD -o %s -d %s -j %s" % (self.dev, self.address['network'], devChain) + ]) self.fw.append(["", "", "-A %s -j DROP" % devChain]) - self.fw.append(["mangle", "", - "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j %s" % \ - (self.dev, self.address['network'], self.address['public_ip'], devChain) - ]) + self.fw.append(["mangle", "", + "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j %s" % + (self.dev, self.address['network'], self.address['public_ip'], devChain) + ]) dns = CsDnsmasq(self) dns.add_firewall_rules() app = CsApache(self) app.setup() pwdsvc = CsPasswdSvc(self).setup() elif self.get_type() == "public": - if self.address["source_nat"] == True: + if self.address["source_nat"]: if self.cl.get_type() == "vpcrouter": vpccidr = self.cl.get_vpccidr() self.fw.append(["filter", "", "-A FORWARD -s %s ! -d %s -j ACCEPT" % (vpccidr, vpccidr)]) - self.fw.append(["nat","","-A POSTROUTING -j SNAT -o %s --to-source %s" % (self.dev, self.address['public_ip'])]) + self.fw.append(["nat", "", "-A POSTROUTING -j SNAT -o %s --to-source %s" % (self.dev, self.address['public_ip'])]) elif self.cl.get_type() == "router": logging.error("Not able to setup sourcenat for a regular router yet") else: logging.error("Unable to process source nat configuration for router of type %s" % type) - #route.flush() + # route.flush() def list(self): self.iplist = {} @@ -342,7 +344,7 @@ class CsIP: # Delete any ips that are configured but not in the bag def compare(self, bag): - if len(self.iplist) > 0 and (not self.dev in bag.keys() or len(bag[self.dev]) == 0): + if len(self.iplist) > 0 and (self.dev not in bag.keys() or len(bag[self.dev]) == 0): # Remove all IPs on this device logging.info("Will remove all configured addresses on device %s", self.dev) self.delete("all") @@ -351,7 +353,7 @@ class CsIP: # This condition should not really happen but did :) # It means an apache file got orphaned after a guest network address was deleted - if len(self.iplist) == 0 and (not self.dev in bag.keys() or len(bag[self.dev]) == 0): + if len(self.iplist) == 0 and (self.dev not in bag.keys() or len(bag[self.dev]) == 0): app = CsApache(self) app.remove() @@ -378,6 +380,7 @@ class CsIP: logging.info("Removed address %s from device %s", ip, self.dev) self.post_config_change("delete") + class CsRpsrfs: """ Configure rpsrfs if there is more than one cpu """ @@ -385,9 +388,11 @@ class CsRpsrfs: self.dev = dev def enable(self): - if not self.inKernel(): return + if not self.inKernel(): + return cpus = self.cpus() - if cpus < 2: return + if cpus < 2: + return val = format((1 << cpus) - 1, "x") filename = "/sys/class/net/%s/queues/rx-0/rps_cpus" % (self.dev) CsHelper.updatefile(filename, val, "w+") @@ -409,7 +414,9 @@ class CsRpsrfs: def cpus(self): count = 0 for line in open('/proc/cpuinfo'): - if "processor" not in line: continue + if "processor" not in line: + continue count += 1 - if count < 2: logging.debug("Single CPU machine") + if count < 2: + logging.debug("Single CPU machine") return count diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py index e6c7e178b7f..4aa988a0825 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsApp.py @@ -21,13 +21,15 @@ from CsFile import CsFile from CsProcess import CsProcess import CsHelper + class CsApp: def __init__(self, ip): - self.dev = ip.getDevice() - self.ip = ip.get_ip_address() - self.type = ip.get_type() + self.dev = ip.getDevice() + self.ip = ip.get_ip_address() + self.type = ip.get_type() self.fw = ip.fw + class CsApache(CsApp): """ Set up Apache """ @@ -37,10 +39,9 @@ class CsApache(CsApp): os.remove(file) CsHelper.service("apache2", "restart") - def setup(self): CsHelper.copy_if_needed("/etc/apache2/vhostexample.conf", - "/etc/apache2/conf.d/vhost%s.conf" % self.dev) + "/etc/apache2/conf.d/vhost%s.conf" % self.dev) file = CsFile("/etc/apache2/conf.d/vhost%s.conf" % (self.dev)) file.search("", "\t" % (self.ip)) @@ -54,8 +55,9 @@ class CsApache(CsApp): CsHelper.service("apache2", "restart") self.fw.append(["", "front", - "-A INPUT -i %s -d %s/32 -p tcp -m tcp -m state --state NEW --dport 80 -j ACCEPT" % (self.dev, self.ip) - ]) + "-A INPUT -i %s -d %s/32 -p tcp -m tcp -m state --state NEW --dport 80 -j ACCEPT" % (self.dev, self.ip) + ]) + class CsPasswdSvc(CsApp): """ @@ -64,8 +66,8 @@ class CsPasswdSvc(CsApp): def setup(self): self.fw.append(["", "front", - "-A INPUT -i %s -d %s/32 -p tcp -m tcp -m state --state NEW --dport 8080 -j ACCEPT" % (self.dev, self.ip) - ]) + "-A INPUT -i %s -d %s/32 -p tcp -m tcp -m state --state NEW --dport 8080 -j ACCEPT" % (self.dev, self.ip) + ]) proc = CsProcess(['/opt/cloud/bin/vpc_passwd_server', self.ip]) if not proc.find(): @@ -76,18 +78,16 @@ class CsDnsmasq(CsApp): """ Set up dnsmasq """ def add_firewall_rules(self): - """ Add the necessary firewall rules + """ Add the necessary firewall rules """ self.fw.append(["", "front", - "-A INPUT -i %s -p udp -m udp --dport 67 -j ACCEPT" % self.dev - ]) + "-A INPUT -i %s -p udp -m udp --dport 67 -j ACCEPT" % self.dev + ]) self.fw.append(["", "front", - "-A INPUT -i %s -d %s/32 -p udp -m udp --dport 53 -j ACCEPT" % (self.dev, self.ip) - ]) + "-A INPUT -i %s -d %s/32 -p udp -m udp --dport 53 -j ACCEPT" % (self.dev, self.ip) + ]) self.fw.append(["", "front", - "-A INPUT -i %s -d %s/32 -p tcp -m tcp --dport 53 -j ACCEPT" % ( self.dev, self.ip ) - ]) - - + "-A INPUT -i %s -d %s/32 -p tcp -m tcp --dport 53 -j ACCEPT" % (self.dev, self.ip) + ]) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py index 88e3d033243..e2e5f5d434f 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py @@ -19,15 +19,16 @@ from CsDatabag import CsCmdLine import logging + class CsConfig(object): """ - A class to cache all the stuff that the other classes need + A class to cache all the stuff that the other classes need """ - __LOG_FILE = "/var/log/cloud.log" - __LOG_LEVEL = "DEBUG" - __LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s" + __LOG_FILE = "/var/log/cloud.log" + __LOG_LEVEL = "DEBUG" + __LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s" - def __init__(self, load = False): + def __init__(self, load=False): if load: self.cl = self_set_cl() self.fw = [] @@ -55,18 +56,18 @@ class CsConfig(object): def get_ingress_chain(self, device, ip): if self.is_vpc: - return "ACL_INBOUND_%s" % device + return "ACL_INBOUND_%s" % device else: - return "FIREWALL_" % ip + return "FIREWALL_" % ip def get_egress_chain(self, device, ip): if self.is_vpc: - return "ACL_OUTBOUND_%s" % device + return "ACL_OUTBOUND_%s" % device else: - return "FW_EGRESS_RULES" + return "FW_EGRESS_RULES" def get_egress_table(self): if self.is_vpc: return 'mangle' else: - return ""; + return "" diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py index 230b623ea09..522bd956404 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py @@ -17,9 +17,10 @@ # under the License. from merge import dataBag + class CsDataBag(object): - def __init__(self, key, config = None): + def __init__(self, key, config=None): self.data = {} self.db = dataBag() self.db.setKey(key) @@ -46,6 +47,7 @@ class CsDataBag(object): """ self.db.save(self.dbag) + class CsCmdLine(CsDataBag): """ Get cmdline config parameters """ @@ -78,4 +80,3 @@ class CsCmdLine(CsDataBag): if "redundant_master" in self.dbag['config']: return self.dbag['config']['redundant_master'] == "true" return False - diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py index 15d6eecda4d..c8c93869508 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py @@ -20,12 +20,13 @@ from netaddr import * from CsGuestNetwork import CsGuestNetwork NO_PRELOAD = False -LEASES = "/var/lib/misc/dnsmasq.leases" +LEASES = "/var/lib/misc/dnsmasq.leases" DHCP_HOSTS = "/etc/dhcphosts.txt" -DHCP_OPTS = "/etc/dhcpopts.txt" +DHCP_OPTS = "/etc/dhcpopts.txt" DNSMASQ_CONF = "/etc/dnsmasq.conf" CLOUD_CONF = "/etc/dnsmasq.d/cloud.conf" + class CsDhcp(object): """ Manage dhcp entries """ @@ -51,10 +52,11 @@ class CsDhcp(object): dnsmasq.first_host = dnsmasqb4.first_host dnsmasq.configure_server() + class CsDnsMasq(object): - def __init__(self, preload = True): - self.list = [] + def __init__(self, preload=True): + self.list = [] self.hosts = [] self.leases = [] self.updated = False @@ -72,17 +74,17 @@ class CsDnsMasq(object): try: for line in open(LEASES): bits = line.strip().split(' ') - to = { "device" : bits[0], - "mac" : bits[1], - "ip" : bits[2], - "host" : bits[3], - "del" : False - } + to = {"device": bits[0], + "mac": bits[1], + "ip": bits[2], + "host": bits[3], + "del": False + } for l in clist: lbits = l.split(',') if lbits[0] == to['mac'] or \ lbits[1] == to['ip']: - to['del'] == True + to['del'] is True break self.leases.append(to) for o in self.leases: @@ -96,26 +98,25 @@ class CsDnsMasq(object): def configure_server(self): self.updated = self.updated | CsHelper.addifmissing(DNSMASQ_CONF, "dhcp-hostsfile=/etc/dhcphosts.txt") - #self.updated = self.updated | CsHelper.addifmissing(DNSMASQ_CONF, "dhcp-optsfile=%s:" % DHCP_OPTS) + # self.updated = self.updated | CsHelper.addifmissing(DNSMASQ_CONF, "dhcp-optsfile=%s:" % DHCP_OPTS) for i in self.devinfo: if not i['dnsmasq']: continue device = i['dev'] ip = i['ip'].split('/')[0] - line = "dhcp-range=interface:%s,set:interface-%s,%s,static" \ - % (device, device, ip) + line = "dhcp-range=interface:%s,set:interface-%s,%s,static" % (device, device, ip) self.updated = self.updated | CsHelper.addifmissing(CLOUD_CONF, line) # Next add the domain # if this is a guest network get it there otherwise use the value in resolv.conf gn = CsGuestNetwork(device) - line = "dhcp-option=tag:interface-%s,15,%s" % (device,gn.get_domain()) + line = "dhcp-option=tag:interface-%s,15,%s" % (device, gn.get_domain()) self.updated = self.updated | CsHelper.addifmissing(CLOUD_CONF, line) if self.updated: if self.first_host: CsHelper.service("dnsmasq", "restart") else: CsHelper.hup_dnsmasq("dnsmasq", "dnsmasq") - + def parse_dnsmasq(self): self.first_host = False try: @@ -157,7 +158,7 @@ class CsDnsMasq(object): b = line.split(',') handle.close() - def add(self,entry): + def add(self, entry): self.add_host(entry['ipv4_adress'], entry['host_name']) self.add_dnsmasq(entry['ipv4_adress'], entry['host_name'], entry['mac_address']) i = IPAddress(entry['ipv4_adress']) @@ -165,7 +166,7 @@ class CsDnsMasq(object): for v in self.devinfo: if i > v['network'].network and i < v['network'].broadcast: v['dnsmasq'] = True - + def add_dnsmasq(self, ip, host, mac): self.list.append("%s,%s,%s,infinite" % (mac, ip, host)) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py index 1417e2f340d..2943ac403c4 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py @@ -19,6 +19,7 @@ import logging import re import copy + class CsFile: """ File editors """ @@ -28,7 +29,7 @@ class CsFile: def load(self): self.new_config = [] - self.config = [] + self.config = [] try: for line in open(self.filename): self.new_config.append(line) @@ -66,7 +67,7 @@ class CsFile: token = string.split('=')[0] + '=' self.search(token, string) - def add(self, string, where = -1): + def add(self, string, where=-1): for index, line in enumerate(self.new_config): if line.strip() == string: return @@ -92,14 +93,14 @@ class CsFile: self.new_config = [w.replace(search, replace) for w in self.new_config] def search(self, search, replace): - found = False + found = False logging.debug("Searching for %s and replacing with %s" % (search, replace)) for index, line in enumerate(self.new_config): if line.lstrip().startswith("#"): continue if re.search(search, line): found = True - if not replace in line: + if replace not in line: self.new_config[index] = replace + "\n" if not found: - self.new_config.append(replace + "\n") + self.new_config.append(replace + "\n") diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py index 8f19969ea82..60730a072d8 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsGuestNetwork.py @@ -17,6 +17,7 @@ from merge import dataBag import CsHelper + class CsGuestNetwork: def __init__(self, device): self.data = {} @@ -26,9 +27,9 @@ class CsGuestNetwork: db.load() dbag = db.getDataBag() if device in dbag.keys() and len(dbag[device]) != 0: - self.data = dbag[device][0] + self.data = dbag[device][0] else: - self.guest = False + self.guest = False def is_guestnetwork(self): return self.guest diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py index 46f4c31b358..a353b3d8e2e 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsHelper.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" General helper functions +""" General helper functions for use in the configuation process """ @@ -27,6 +27,7 @@ import shutil from netaddr import * from pprint import pprint + def is_mounted(name): for i in execute("mount"): vals = i.lstrip().split() @@ -34,31 +35,37 @@ def is_mounted(name): return True return False + def mount_tmpfs(name): if not is_mounted(name): print "Mounting it" execute("mount tmpfs %s -t tmpfs" % name) + def umount_tmpfs(name): if is_mounted(name): execute("umount %s" % name) + def rm(name): os.remove(name) if os.path.isfile(name) else None + def rmdir(name): if name: shutil.rmtree(name, True) + def mkdir(name, mode, fatal): try: os.makedirs(name, mode) except OSError as e: if e.errno != 17: - print "failed to make directories " + name + " due to :" +e.strerror + print "failed to make directories " + name + " due to :" + e.strerror if(fatal): sys.exit(1) + def updatefile(filename, val, mode): """ add val to file """ handle = open(filename, 'r') @@ -71,11 +78,13 @@ def updatefile(filename, val, mode): handle.write(val) handle.close() + def bool_to_yn(val): if val: return "yes" return "no" + def get_device_info(): """ Returns all devices on system with their ipv4 ip netmask """ list = [] @@ -90,6 +99,7 @@ def get_device_info(): list.append(to) return list + def get_domain(): for line in open("/etc/resolv.conf"): vals = line.lstrip().split() @@ -97,8 +107,9 @@ def get_domain(): return vals[1] return "cloudnine.internal" + def get_device(ip): - """ Returns the device which has a specific ip + """ Returns the device which has a specific ip If the ip is not found returns an empty string """ for i in execute("ip addr show"): @@ -108,6 +119,7 @@ def get_device(ip): return vals[-1] return "" + def get_ip(device): """ Return first ip on an interface """ cmd = "ip addr show dev %s" % device @@ -117,6 +129,7 @@ def get_ip(device): return vals[1] return "" + def definedinfile(filename, val): """ Check if val is defined in the file """ for line in open(filename): @@ -124,22 +137,25 @@ def definedinfile(filename, val): return True return False + def addifmissing(filename, val): """ Add something to a file if it is not already there """ if not os.path.isfile(filename): - logging.debug("File %s doesn't exist, so create" % filename) - open(filename,"w").close() + logging.debug("File %s doesn't exist, so create" % filename) + open(filename, "w").close() if not definedinfile(filename, val): - updatefile(filename, val + "\n", "a") - logging.debug("Added %s to file %s" % (val, filename)) - return True + updatefile(filename, val + "\n", "a") + logging.debug("Added %s to file %s" % (val, filename)) + return True return False + def get_hostname(): for line in open("/etc/hostname"): return line.strip() + def execute(command): """ Execute command """ logging.debug("Executing %s" % command) @@ -147,6 +163,7 @@ def execute(command): result = p.communicate()[0] return result.splitlines() + def execute2(command): """ Execute command """ logging.debug("Executing %s" % command) @@ -154,15 +171,18 @@ def execute2(command): p.wait() return p + def service(name, op): execute("service %s %s" % (name, op)) logging.info("Service %s %s" % (name, op)) + def start_if_stopped(name): ret = execute2("service %s status" % name) if ret.returncode: execute2("service %s start" % name) + def hup_dnsmasq(name, user): pid = "" for i in execute("ps -ef | grep %s" % name): @@ -175,6 +195,7 @@ def hup_dnsmasq(name, user): else: service("dnsmasq", "start") + def copy_if_needed(src, dest): """ Copy a file if the destination does not already exist """ @@ -186,4 +207,3 @@ def copy_if_needed(src, dest): logging.Error("Could not copy %s to %s" % (src, dest)) else: logging.info("Copied %s to %s" % (src, dest)) - diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py index e9a1da8d48d..497c44402ea 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py @@ -20,6 +20,7 @@ from pprint import pprint from CsDatabag import CsDataBag, CsCmdLine import logging + class CsChain(object): def __init__(self): @@ -27,8 +28,8 @@ class CsChain(object): self.last_added = '' def add(self, table, chain): - if not table in self.chain.keys(): - self.chain.setdefault(table, []).append( chain ) + if table not in self.chain.keys(): + self.chain.setdefault(table, []).append(chain) else: self.chain[table].append(chain) self.last_added = chain @@ -40,12 +41,13 @@ class CsChain(object): return self.last_added def has_chain(self, table, chain): - if not table in self.chain.keys(): + if table not in self.chain.keys(): return False - if not chain in self.chain[table]: + if chain not in self.chain[table]: return False return True + class CsTable(object): def __init__(self): @@ -53,7 +55,7 @@ class CsTable(object): self.last_added = '' def add(self, name): - if not name in self.table: + if name not in self.table: self.table.append(name) self.last_added = name @@ -63,9 +65,10 @@ class CsTable(object): def last(self): return self.last_added + class CsNetfilters(object): - def __init__(self, load = True): + def __init__(self, load=True): self.rules = [] self.table = CsTable() self.chain = CsChain() @@ -74,17 +77,17 @@ class CsNetfilters(object): def get_all_rules(self): for i in CsHelper.execute("iptables-save"): - if i.startswith('*'): # Table + if i.startswith('*'): # Table self.table.add(i[1:]) - if i.startswith(':'): # Chain + if i.startswith(':'): # Chain self.chain.add(self.table.last(), i[1:].split(' ')[0]) - if i.startswith('-A'): # Rule + if i.startswith('-A'): # Rule rule = CsNetfilter() rule.parse(i) rule.set_table(self.table.last()) self.save(rule) - def save(self,rule): + def save(self, rule): self.rules.append(rule) def get(self): @@ -108,7 +111,7 @@ class CsNetfilters(object): for r in del_list: cmd = "iptables -t %s %s" % (r.get_table(), r.to_str(True)) CsHelper.execute(cmd) - #print "Delete rule %s from table %s" % (r.to_str(True), r.get_table()) + # print "Delete rule %s from table %s" % (r.to_str(True), r.get_table()) logging.info("Delete rule %s from table %s", r.to_str(True), r.get_table()) def compare(self, list): @@ -125,7 +128,7 @@ class CsNetfilters(object): if self.has_rule(new_rule): logging.debug("rule %s exists in table %s", fw[2], new_rule.get_table()) else: - #print "Add rule %s in table %s" % ( fw[2], new_rule.get_table()) + # print "Add rule %s in table %s" % ( fw[2], new_rule.get_table()) logging.info("Add rule %s in table %s", fw[2], new_rule.get_table()) # front means insert instead of append cpy = fw[2] @@ -136,15 +139,14 @@ class CsNetfilters(object): self.del_standard() self.get_unseen() - def add_chain(self, rule): """ Add the given chain if it is not already present """ if not self.has_chain(rule.get_table(), rule.get_chain()): - CsHelper.execute("iptables -t %s -N %s" % (rule.get_table(), rule.get_chain())) - self.chain.add(rule.get_table(), rule.get_chain()) + CsHelper.execute("iptables -t %s -N %s" % (rule.get_table(), rule.get_chain())) + self.chain.add(rule.get_table(), rule.get_chain()) def del_standard(self): - """ Del rules that are there but should not be deleted + """ Del rules that are there but should not be deleted These standard firewall rules vary according to the device type """ type = CsCmdLine("cmdline").get_type() @@ -152,9 +154,9 @@ class CsNetfilters(object): try: table = '' for i in open("/etc/iptables/iptables-%s" % type): - if i.startswith('*'): # Table + if i.startswith('*'): # Table table = i[1:].strip() - if i.startswith('-A'): # Rule + if i.startswith('-A'): # Rule self.del_rule(table, i.strip()) except IOError: # Nothing can be done @@ -171,19 +173,20 @@ class CsNetfilters(object): The rule will not actually be removed on the host """ self.rules[:] = [x for x in self.rules if not x == rule] + class CsNetfilter(object): - + def __init__(self): self.rule = {} self.table = '' self.chain = '' - self.seen = False + self.seen = False def parse(self, rule): self.rule = self.__convert_to_dict(rule) def unseen(self): - return self.seen == False + return self.seen is False def mark_seen(self): self.seen = True @@ -200,7 +203,7 @@ class CsNetfilter(object): rule = rule.replace('-m state', '-m2 state') rule = rule.replace('ESTABLISHED,RELATED', 'RELATED,ESTABLISHED') bits = rule.split(' ') - rule = dict(zip(bits[0::2],bits[1::2])) + rule = dict(zip(bits[0::2], bits[1::2])) if "-A" in rule.keys(): self.chain = rule["-A"] return rule @@ -222,12 +225,12 @@ class CsNetfilter(object): def get_rule(self): return self.rule - def to_str(self, delete = False): + def to_str(self, delete=False): """ Convert the rule back into aynactically correct iptables command """ - # Order is important - order = ['-A', '-s', '-d', '!_-d', '-i', '!_-i', '-p', '-m', '-m2', '--icmp-type', '--state', - '--dport', '--destination-port', '-o', '!_-o', '-j', '--set-xmark', '--checksum', - '--to-source', '--to-destination', '--mark' ] + # Order is important + order = ['-A', '-s', '-d', '!_-d', '-i', '!_-i', '-p', '-m', '-m2', '--icmp-type', '--state', + '--dport', '--destination-port', '-o', '!_-o', '-j', '--set-xmark', '--checksum', + '--to-source', '--to-destination', '--mark'] str = '' for k in order: if k in self.rule.keys(): @@ -245,22 +248,11 @@ class CsNetfilter(object): def __eq__(self, rule): if rule.get_table() != self.get_table(): return False - #if '-j' in self.get_rule().keys() and self.get_rule()['-j'] == "MARK" and self.get_rule()['--set-xmark'] == '0x524/0xffffffff' and \ - #'-j' in rule.get_rule().keys() and rule.get_rule()['-j'] == "MARK" and rule.get_rule()['--set-xmark'] == '0x524/0xffffffff': - #pprint(self.get_rule()) - #pprint(rule.get_rule()) - #pprint(self.get_chain()) - #pprint(rule.get_chain()) if rule.get_chain() != self.get_chain(): return False if len(rule.get_rule().items()) != len(self.get_rule().items()): return False common = set(rule.get_rule().items()) & set(self.get_rule().items()) - #if '-j' in self.get_rule().keys() and self.get_rule()['-j'] == "MARK" and self.get_rule()['--set-xmark'] == '0x524/0xffffffff': - #pprint(self.get_rule()) - #pprint(rule.get_rule()) - #pprint(common) if len(common) != len(rule.get_rule()): return False return True - diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py index 724f71795c4..c01a9d956c6 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py @@ -39,20 +39,20 @@ import CsHelper from CsFile import CsFile from CsConfig import CsConfig + class CsRedundant(object): CS_RAMDISK_DIR = "/ramdisk" - CS_ROUTER_DIR = "%s/rrouter" % CS_RAMDISK_DIR + CS_ROUTER_DIR = "%s/rrouter" % CS_RAMDISK_DIR CS_TEMPLATES = [ - "heartbeat.sh.templ", "check_heartbeat.sh.templ", - "arping_gateways.sh.templ" - ] - CS_TEMPLATES_DIR = "/opt/cloud/templates" - CONNTRACKD_BIN = "/usr/sbin/conntrackd" - CONNTRACKD_LOCK = "/var/lock/conntrack.lock" + "heartbeat.sh.templ", "check_heartbeat.sh.templ", + "arping_gateways.sh.templ" + ] + CS_TEMPLATES_DIR = "/opt/cloud/templates" + CONNTRACKD_BIN = "/usr/sbin/conntrackd" + CONNTRACKD_LOCK = "/var/lock/conntrack.lock" CONNTRACKD_CONFIG = "/etc/conntrackd/conntrackd.conf" - def __init__(self, config, address): self.cl = config.get_cmdline() self.address = address @@ -114,7 +114,7 @@ class CsRedundant(object): cron = CsFile("/etc/cron.d/heartbeat") cron.add("SHELL=/bin/bash", 0) cron.add("PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin", 1) - cron.add("*/1 * * * * root $SHELL %s/check_heartbeat.sh 2>&1 > /dev/null" % self.CS_ROUTER_DIR, -1) + cron.add("*/1 * * * * root $SHELL %s/check_heartbeat.sh 2>&1 > /dev/null" % self.CS_ROUTER_DIR, -1) cron.commit() def set_fault(self): @@ -166,7 +166,7 @@ class CsRedundant(object): return ads = [o for o in self.address.get_ips() if o.needs_vrrp()] for o in ads: - ## cmd2 = "ip link set %s up" % self.getDevice() + # cmd2 = "ip link set %s up" % self.getDevice() CsHelper.execute("ifconfig %s down" % o.get_device()) CsHelper.execute("ifconfig %s up" % o.get_device()) CsHelper.execute("arping -I %s -A %s -c 1" % (o.get_device(), o.get_ip())) @@ -192,7 +192,7 @@ class CsRedundant(object): """ lines = [] lines.append("\t\t\tIPv4_address %s\n" % "127.0.0.1") - lines.append("\t\t\tIPv4_address %s\n" % self.address.get_control_if().get_ip()) + lines.append("\t\t\tIPv4_address %s\n" % self.address.get_control_if().get_ip()) # FIXME - Do we need to also add any internal network gateways? return lines diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRoute.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRoute.py index b6825d76906..f44ce5897e9 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRoute.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRoute.py @@ -18,6 +18,7 @@ import CsHelper import logging + class CsRoute: """ Manage routes """ @@ -35,13 +36,13 @@ class CsRoute: CsHelper.execute("ip route flush table %s" % (self.table)) CsHelper.execute("ip route flush cache") - def add(self, address, method = "add"): + def add(self, address, method="add"): # ip route show dev eth1 table Table_eth1 10.0.2.0/24 if(method == "add"): cmd = "dev %s table %s %s" % (self.dev, self.table, address['network']) self.set_route(cmd, method) - def set_route(self, cmd, method = "add"): + def set_route(self, cmd, method="add"): """ Add a route is it is not already defined """ found = False for i in CsHelper.execute("ip route show " + cmd): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRule.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRule.py index fe9d708adad..e2ca806f029 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRule.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRule.py @@ -18,6 +18,7 @@ import CsHelper import logging + class CsRule: """ Manage iprules Supported Types: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py index e4445af134b..abdba3b1f39 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_cmdline.py @@ -1,5 +1,6 @@ from pprint import pprint + def merge(dbag, cmdline): if 'redundant_router' in cmdline['cmd_line']: cmdline['cmd_line']['redundant_router'] = "true" diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py index aa296270850..0bc2de16bed 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_dhcp.py @@ -1,15 +1,16 @@ from pprint import pprint from netaddr import * + def merge(dbag, data): # A duplicate ip address wil clobber the old value # This seems desirable .... - if "add" in data and data['add'] == False and \ - "ipv4_adress" in data : - if data['ipv4_adress'] in dbag: - del(dbag[data['ipv4_adress']]) - return dbag + if "add" in data and data['add'] is False and \ + "ipv4_adress" in data: + if data['ipv4_adress'] in dbag: + del(dbag[data['ipv4_adress']]) + return dbag else: dbag[data['ipv4_adress']] = data return dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py index 0a4eef671e4..343a6a84f4b 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py @@ -1,5 +1,6 @@ from pprint import pprint + def merge(dbag, rules): for rule in rules["rules"]: source_ip = rule["source_ip_address"] @@ -17,10 +18,10 @@ def merge(dbag, rules): newrule["public_ports"] = rule["source_port_range"] newrule["internal_ports"] = rule["destination_port_range"] newrule["protocol"] = rule["protocol"] - + if not revoke: if rules["type"] == "staticnatrules": - dbag[source_ip] = [ newrule ] + dbag[source_ip] = [newrule] elif rules["type"] == "forwardrules": index = -1 if source_ip in dbag.keys(): @@ -32,7 +33,7 @@ def merge(dbag, rules): else: dbag[source_ip].append(newrule) else: - dbag[source_ip] = [ newrule ] + dbag[source_ip] = [newrule] else: if rules["type"] == "staticnatrules": if source_ip in dbag.keys(): @@ -47,9 +48,9 @@ def merge(dbag, rules): if not index == -1: del dbag[source_ip][index] - return dbag + # Compare function checks only the public side, those must be equal the internal details could change def ruleCompare(ruleA, ruleB): if not ruleA["type"] == ruleB["type"]: @@ -57,4 +58,5 @@ def ruleCompare(ruleA, ruleB): if ruleA["type"] == "staticnat": return ruleA["public_ip"] == ruleB["public_ip"] elif ruleA["type"] == "forward": - return ruleA["public_ip"] == ruleB["public_ip"] and ruleA["public_ports"] == ruleB["public_ports"] and ruleA["protocol"] == ruleB["protocol"] \ No newline at end of file + return ruleA["public_ip"] == ruleB["public_ip"] and ruleA["public_ports"] == ruleB["public_ports"] \ + and ruleA["protocol"] == ruleB["protocol"] diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py index ced38e623bc..8cd061d3594 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_guestnetwork.py @@ -1,14 +1,15 @@ from pprint import pprint + def merge(dbag, gn): added = False for dev in dbag: if dev == "id": - continue + continue if len(dbag[dev]) == 0: continue if dbag[dev][0]['device'] == gn['device']: - dbag[dev].remove(dbag[dev][0]) + dbag[dev].remove(dbag[dev][0]) if gn['add']: - dbag.setdefault(gn['device'], []).append( gn ) + dbag.setdefault(gn['device'], []).append() return dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py index 575f4553204..5319d0563b2 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py @@ -18,24 +18,25 @@ from pprint import pprint from netaddr import * + def merge(dbag, ip): added = False for dev in dbag: if dev == "id": - continue + continue for address in dbag[dev]: if address['public_ip'] == ip['public_ip']: - dbag[dev].remove(address) + dbag[dev].remove(address) if ip['add']: - ipo = IPNetwork(ip['public_ip'] + '/' + ip['netmask']) - ip['device'] = 'eth' + str(ip['nic_dev_id']) - ip['broadcast'] = str(ipo.broadcast) - ip['cidr'] = str(ipo.ip) + '/' + str(ipo.prefixlen) - ip['network'] = str(ipo.network) + '/' + str(ipo.prefixlen) - if 'nw_type' not in ip.keys(): - ip['nw_type'] = 'public' - if ip['nw_type'] == 'control': - dbag['eth' + str(ip['nic_dev_id'])] = [ ip ] - else: - dbag.setdefault('eth' + str(ip['nic_dev_id']), []).append( ip ) + ipo = IPNetwork(ip['public_ip'] + '/' + ip['netmask']) + ip['device'] = 'eth' + str(ip['nic_dev_id']) + ip['broadcast'] = str(ipo.broadcast) + ip['cidr'] = str(ipo.ip) + '/' + str(ipo.prefixlen) + ip['network'] = str(ipo.network) + '/' + str(ipo.prefixlen) + if 'nw_type' not in ip.keys(): + ip['nw_type'] = 'public' + if ip['nw_type'] == 'control': + dbag['eth' + str(ip['nic_dev_id'])] = [ip] + else: + dbag.setdefault('eth' + str(ip['nic_dev_id']), []).append(ip) return dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_network_acl.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_network_acl.py index b469772e919..f77e404f0f6 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_network_acl.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_network_acl.py @@ -1,6 +1,7 @@ from pprint import pprint from netaddr import * + def merge(dbag, data): dbag[data['device']] = data return dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py index bfaa73ffdfc..02157b4194f 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py @@ -17,9 +17,10 @@ # under the License. from pprint import pprint + def merge(dbag, vpn): key = vpn['local_public_ip'] - op = vpn['create'] + op = vpn['create'] if key in dbag.keys() and not op: del(dbag[key]) else: diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_vmdata.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_vmdata.py index 32679caf123..24d2d5118ff 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_vmdata.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_vmdata.py @@ -1,5 +1,6 @@ from pprint import pprint + def merge(dbag, metadata): dbag[metadata["vm_ip_address"]] = metadata["vm_metadata"] - return dbag \ No newline at end of file + return dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_vmp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_vmp.py index 81143004495..4e637d4d641 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_vmp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_vmp.py @@ -1,6 +1,7 @@ from pprint import pprint from netaddr import * + def merge(dbag, data): """ Track vm passwords diff --git a/systemvm/patches/debian/config/opt/cloud/bin/master.py b/systemvm/patches/debian/config/opt/cloud/bin/master.py index 01cebadce68..ff8e92b18a7 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/master.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/master.py @@ -35,8 +35,8 @@ parser.add_option("-f", "--fault", help="Notify Fault") (options, args) = parser.parse_args() -config = CsConfig(False) -logging.basicConfig(filename= config.get_logger(), +config = CsConfig(False) +logging.basicConfig(filename=config.get_logger(), level=config.get_level(), format=config.get_format()) config.set_cl() diff --git a/systemvm/patches/debian/config/opt/cloud/bin/merge.py b/systemvm/patches/debian/config/opt/cloud/bin/merge.py index ee579eb46a5..1fd67a37208 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/merge.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/merge.py @@ -16,23 +16,24 @@ import cs_site2sitevpn from pprint import pprint + class dataBag: DPATH = "/etc/cloudstack" def __init__(self): - self.bdata = { } + self.bdata = {} def load(self): data = self.bdata if not os.path.exists(self.DPATH): - os.makedirs(self.DPATH) + os.makedirs(self.DPATH) self.fpath = self.DPATH + '/' + self.key + '.json' try: handle = open(self.fpath) except IOError: logging.debug("Creating data bag type %s", self.key) - data.update( { "id": self.key } ) + data.update({"id": self.key}) else: logging.debug("Loading data bag type %s", self.key) data = json.load(handle) @@ -55,61 +56,62 @@ class dataBag: def setKey(self, key): self.key = key + class updateDataBag: DPATH = "/etc/cloudstack" - def __init__(self,qFile): + def __init__(self, qFile): self.qFile = qFile self.fpath = '' self.bdata = {} self.process() def process(self): - self.db = dataBag() - if ( self.qFile.type == "staticnatrules" or self.qFile.type == "forwardrules"): + self.db = dataBag() + if (self.qFile.type == "staticnatrules" or self.qFile.type == "forwardrules"): self.db.setKey("forwardingrules") - else: - self.db.setKey( self.qFile.type ) - dbag = self.db.load( ) - logging.info("Command of type %s received", self.qFile.type) + else: + self.db.setKey(self.qFile.type) + dbag = self.db.load() + logging.info("Command of type %s received", self.qFile.type) + + if self.qFile.type == 'ips': + dbag = self.processIP(self.db.getDataBag()) + elif self.qFile.type == 'guestnetwork': + dbag = self.processGuestNetwork(self.db.getDataBag()) + elif self.qFile.type == 'cmdline': + dbag = self.processCL(self.db.getDataBag()) + elif self.qFile.type == 'vmpassword': + dbag = self.processVMpassword(self.db.getDataBag()) + elif self.qFile.type == 'networkacl': + dbag = self.process_network_acl(self.db.getDataBag()) + elif self.qFile.type == 'vmdata': + dbag = self.processVmData(self.db.getDataBag()) + elif self.qFile.type == 'dhcpentry': + dbag = self.process_dhcp_entry(self.db.getDataBag()) + elif self.qFile.type == 'staticnatrules' or self.qFile.type == 'forwardrules': + dbag = self.processForwardingRules(self.db.getDataBag()) + elif self.qFile.type == 'site2sitevpn': + dbag = self.process_site2sitevpn(self.db.getDataBag()) + else: + logging.error("Error I do not know what to do with file of type %s", self.qFile.type) + return + self.db.save(dbag) - if self.qFile.type == 'ips': - dbag = self.processIP(self.db.getDataBag()) - elif self.qFile.type == 'guestnetwork': - dbag = self.processGuestNetwork(self.db.getDataBag()) - elif self.qFile.type == 'cmdline': - dbag = self.processCL(self.db.getDataBag()) - elif self.qFile.type == 'vmpassword': - dbag = self.processVMpassword(self.db.getDataBag()) - elif self.qFile.type == 'networkacl': - dbag = self.process_network_acl(self.db.getDataBag()) - elif self.qFile.type == 'vmdata': - dbag = self.processVmData(self.db.getDataBag()) - elif self.qFile.type == 'dhcpentry': - dbag = self.process_dhcp_entry(self.db.getDataBag()) - elif self.qFile.type == 'staticnatrules' or self.qFile.type == 'forwardrules': - dbag = self.processForwardingRules(self.db.getDataBag()) - elif self.qFile.type == 'site2sitevpn': - dbag = self.process_site2sitevpn(self.db.getDataBag()) - else: - logging.error("Error I do not know what to do with file of type %s", self.qFile.type) - return - self.db.save(dbag) - def processGuestNetwork(self, dbag): d = self.qFile.data dp = {} - dp['public_ip'] = d['router_guest_ip'] - dp['netmask'] = d['router_guest_netmask'] - dp['source_nat'] = False - dp['add'] = d['add'] + dp['public_ip'] = d['router_guest_ip'] + dp['netmask'] = d['router_guest_netmask'] + dp['source_nat'] = False + dp['add'] = d['add'] dp['one_to_one_nat'] = False - dp['gateway'] = d['router_guest_gateway'] - dp['nic_dev_id'] = d['device'][3] - dp['nw_type'] = 'guest' + dp['gateway'] = d['router_guest_gateway'] + dp['nic_dev_id'] = d['device'][3] + dp['nw_type'] = 'guest' qf = loadQueueFile() - qf.load({ 'ip_address' : [ dp ], 'type' : 'ips'}) + qf.load({'ip_address': [dp], 'type': 'ips'}) if 'domain_name' not in d.keys() or d['domain_name'] == '': d['domain_name'] = "cloudnine.internal" return cs_guestnetwork.merge(dbag, self.qFile.data) @@ -150,26 +152,27 @@ class updateDataBag: def processCLItem(self, num, nw_type): key = 'eth' + num + 'ip' - dp = {} + dp = {} if(key in self.qFile.data['cmd_line']): - dp['public_ip'] = self.qFile.data['cmd_line'][key] - dp['netmask'] = self.qFile.data['cmd_line']['eth' + num + 'mask'] - dp['source_nat'] = False - dp['add'] = True - dp['one_to_one_nat'] = False - if('localgw' in self.qFile.data['cmd_line']): - dp['gateway'] = self.qFile.data['cmd_line']['localgw'] - else: - dp['gateway'] = 'None' - dp['nic_dev_id'] = num - dp['nw_type'] = nw_type - qf = loadQueueFile() - qf.load({ 'ip_address' : [ dp ], 'type' : 'ips'}) + dp['public_ip'] = self.qFile.data['cmd_line'][key] + dp['netmask'] = self.qFile.data['cmd_line']['eth' + num + 'mask'] + dp['source_nat'] = False + dp['add'] = True + dp['one_to_one_nat'] = False + if('localgw' in self.qFile.data['cmd_line']): + dp['gateway'] = self.qFile.data['cmd_line']['localgw'] + else: + dp['gateway'] = 'None' + dp['nic_dev_id'] = num + dp['nw_type'] = nw_type + qf = loadQueueFile() + qf.load({'ip_address': [dp], 'type': 'ips'}) def processVmData(self, dbag): cs_vmdata.merge(dbag, self.qFile.data) return dbag - + + class loadQueueFile: fileName = '' @@ -203,9 +206,9 @@ class loadQueueFile: def getType(self): return self.type - + def getData(self): - return self.data + return self.data def setPath(self, path): self.configCache = path @@ -215,4 +218,3 @@ class loadQueueFile: os.makedirs(path) timestamp = str(int(round(time.time()))) os.rename(origPath, path + "/" + self.fileName + "." + timestamp) - diff --git a/systemvm/patches/debian/config/opt/cloud/bin/set_redundant.py b/systemvm/patches/debian/config/opt/cloud/bin/set_redundant.py index b2d6db88010..8aae6d2d36d 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/set_redundant.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/set_redundant.py @@ -19,7 +19,7 @@ # This file is used by the tests to switch the redundancy status -from cs.CsConfig import CsConfig +from cs.CsConfig import CsConfig from optparse import OptionParser import logging @@ -33,8 +33,8 @@ parser.add_option("-d", "--disable", (options, args) = parser.parse_args() -config = CsConfig(False) -logging.basicConfig(filename= config.get_logger(), +config = CsConfig(False) +logging.basicConfig(filename=config.get_logger(), level=config.get_level(), format=config.get_format()) config.set_cl() diff --git a/systemvm/patches/debian/config/opt/cloud/bin/update_config.py b/systemvm/patches/debian/config/opt/cloud/bin/update_config.py index 606e6c2ad55..fb9e0597468 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/update_config.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/update_config.py @@ -9,10 +9,10 @@ import os import os.path import configure -logging.basicConfig(filename='/var/log/cloud.log',level=logging.DEBUG, format='%(asctime)s %(message)s') +logging.basicConfig(filename='/var/log/cloud.log', level=logging.DEBUG, format='%(asctime)s %(message)s') # first commandline argument should be the file to process -if ( len(sys.argv) != 2 ): +if (len(sys.argv) != 2): print "Invalid usage" sys.exit(1) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py b/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py index 30f2705c389..b9127a1b998 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/vmdata.py @@ -16,16 +16,20 @@ # specific language governing permissions and limitations # under the License. -import sys, getopt, json, os, base64 +import sys +import getopt +import json +import os +import base64 from fcntl import flock, LOCK_EX, LOCK_UN def main(argv): - fpath = '' + fpath = '' b64data = '' try: - opts, args = getopt.getopt(argv,"f:d:") + opts, args = getopt.getopt(argv, "f:d:") except getopt.GetoptError: print 'params: -f -d ' sys.exit(2) @@ -48,8 +52,8 @@ def main(argv): for ip in json_data: for item in json_data[ip]: folder = item[0] - file = item[1] - data = item[2] + file = item[1] + data = item[2] # process only valid data if folder != "userdata" and folder != "metadata": @@ -69,16 +73,18 @@ def main(argv): fh.close() os.remove(fpath) + def deletefile(ip, folder, file): datafile = "/var/www/html/" + folder + "/" + ip + "/" + file if os.path.exists(datafile): os.remove(datafile) + def createfile(ip, folder, file, data): dest = "/var/www/html/" + folder + "/" + ip + "/" + file metamanifestdir = "/var/www/html/" + folder + "/" + ip - metamanifest = metamanifestdir + "/meta-data" + metamanifest = metamanifestdir + "/meta-data" # base64 decode userdata if folder == "userdata" or folder == "user-data": @@ -101,12 +107,12 @@ def createfile(ip, folder, file, data): except OSError as e: # error 17 is already exists, we do it this way for concurrency if e.errno != 17: - print "failed to make directories " + metamanifestdir + " due to :" +e.strerror + print "failed to make directories " + metamanifestdir + " due to :" + e.strerror sys.exit(1) if os.path.exists(metamanifest): fh = open(metamanifest, "r+a") exflock(fh) - if not file in fh.read(): + if file not in fh.read(): fh.write(file + '\n') unflock(fh) fh.close() @@ -120,17 +126,18 @@ def createfile(ip, folder, file, data): if os.path.exists(metamanifest): os.chmod(metamanifest, 0644) + def htaccess(ip, folder, file): - entry="Options -Indexes\nOrder Deny,Allow\nDeny from all\nAllow from " + ip + entry = "Options -Indexes\nOrder Deny,Allow\nDeny from all\nAllow from " + ip htaccessFolder = "/var/www/html/" + folder + "/" + ip htaccessFile = htaccessFolder+"/.htaccess" try: - os.makedirs(htaccessFolder,0755) + os.makedirs(htaccessFolder, 0755) except OSError as e: # error 17 is already exists, we do it this way for sake of concurrency if e.errno != 17: - print "failed to make directories " + htaccessFolder + " due to :" +e.strerror + print "failed to make directories " + htaccessFolder + " due to :" + e.strerror sys.exit(1) fh = open(htaccessFile, "w") @@ -139,6 +146,7 @@ def htaccess(ip, folder, file): unflock(fh) fh.close() + def exflock(file): try: flock(file, LOCK_EX) @@ -147,6 +155,7 @@ def exflock(file): sys.exit(1) return True + def unflock(file): try: flock(file, LOCK_UN)