#!/usr/bin/python # Version @VERSION@ # # A plugin for executing script needed by vmops cloud import os, sys, time import XenAPIPlugin sys.path.append("/opt/xensource/sm/") import util from util import CommandException import hostvmstats import socket import stat import base64 import tempfile from os.path import exists as _exists from time import localtime as _localtime, asctime as _asctime vSwitchDBPidFile = "/var/run/openvswitch/ovsdb-server.pid" vSwitchDBDaemonName = "ovsdb-server" vSwitchPidFile = "/var/run/openvswitch/ovs-vswitchd.pid" vsctlPath = "/usr/bin/ovs-vsctl" vSwitchDaemonName = "ovs-vswitchd" logFile = "/var/log/ovsgre.log" fLog = None global result errors = \ {"NO_DB_PID_FILE" : "NO_DB_PID_FILE", \ "DB_NOT_RUN" : "DB_NOT_RUN", \ "NO_SWITCH_PID_FILE" : "NO_SWITCH_PID_FILE", \ "SWITCH_NOT_RUN" : "SWITCH_NOT_RUN", \ "NO_VSCTL" : "NO_VSCTL", \ "COMMAND_FAILED" : "COMMAND_FAILED", \ "TUNNEL_EXISTED" : "TUNNEL_EXISTED", \ "NO_INPORT" : "NO_INPORT", \ "NO_OFPORT" : "NO_OFPORT", \ "ERR_ARGS_NUM" : "ERR_ARGS_NUM", \ "ERROR_OP" : "ERROR_OP", \ "SUCCESS" : "SUCCESS", \ } class ovs_log(object): def __init__(self, name): n = "ovs-%s" % name; logfilename = "/var/run/cloud/" + n +".log" self.name = logfilename self.vmName = name self.bridge = "" self.domId = "" self.seqno = "" self.tag = "" self.vifs = "" self.macs = "" self.vlans = "" self.ofports = "" def write(self): log = open(self.name, "w") log.write("vmName=%s" % self.vmName) log.write("\n") log.write("bridge=%s" % self.bridge) log.write("\n") log.write("domId=%s" % self.domId) log.write("\n") log.write("seqno=%s" % self.seqno) log.write("\n") log.write("tag=%s" % self.tag) log.write("\n") log.write("vifs=%s" % self.vifs) log.write("\n") log.write("macs=%s" % self.macs) log.write("\n") log.write("vlans=%s" % self.vlans) log.write("\n") log.write("ofports=%s" % self.ofports) log.close() def read(self): try: lines = [line.rstrip() for line in open(self.name)] for i in lines: if "=" not in i: util.SMlog("invalid line(%s) in %s" % (i, self.name)) continue (key,value) = i.split("=") if key == "vmName": self.vmName = value elif key == "bridge": self.bridge = value elif key == "domId": self.domId = value elif key == "seqno": self.seqno = value elif key == "tag": self.tag = value elif key == "vifs": self.vifs = value elif key == "macs": self.macs = value elif key == "vlans": self.vlans = value elif key == "ofports": self.ofports = value except Exception, e: util.SMlog(e.__str__()) util.SMlog("Failed to open ovs log %s" % self.name); def get_common_info(self): self.read() return "%s,%s,%s,%s,%s" % (self.vmName, self.bridge, self.domId, self.seqno, self.tag) def remove(self): try: os.remove(self.name) except: util.SMlog("Failed to delete ovs log file " + self.name) def open_log (): global fLog try: if fLog == None: fLog = open (logFile, "a") except IOError, e: #print e pass def pr (str): global fLog if fLog != None: str = "[%s]:" % _asctime (_localtime()) + str + "\n" fLog.write (str) def close_log (): global fLog if fLog != None: fLog.close () def is_process_run (pidFile, name): try: fpid = open (pidFile, "r") pid = fpid.readline () fpid.close () except IOError, e: return -1 pid = pid[:-1] ps = os.popen ("ps -ae") for l in ps: if pid in l and name in l: ps.close () return 0 ps.close () return -2 def is_tool_exist (name): if _exists (name): return 0 return -1 def check_switch (): global result ret = is_process_run (vSwitchDBPidFile, vSwitchDBDaemonName); if ret < 0: if ret == -1: result = errors["NO_DB_PID_FILE"] if ret == -2: result = errors["DB_NOT_RUN"] return -1 ret = is_process_run (vSwitchPidFile, vSwitchDaemonName) if ret < 0: if ret == -1: result = errors["NO_SWITCH_PID_FILE"] if ret == -2: result = errors["SWITCH_NOT_RUN"] return -1 if is_tool_exist (vsctlPath) < 0: result = errors["NO_VSCTL"] return -1 return 0 def do_cmd (cmds, lines=False): cmd = "" for i in cmds: cmd += " " cmd += i pr("do command '%s'" % cmd) f = os.popen (cmd) if lines == True: res = f.readlines () else: res = f.readline () res = res[:-1] f.close () if lines == False: pr("command output '%s'" % res) return res ######################## GRE creation utils ########################## # UUID's format is 8-4-4-4-12 def is_uuid (uuid): list = uuid.split ("-") if len (list) != 5: return -1 if len (list[0]) != 8 or len (list[1]) != 4 \ or len (list[2]) != 4 or len (list[3]) != 4 \ or len (list[4]) != 12: return -1 return 0 def check_gre (bridge, remoteIP, greKey): ports = get_ports_on_bridge(bridge) if ports == None: return "[]" for i in ports: ifaces = get_interface_on_port(i) if ifaces == None: continue for j in ifaces: if j == '[]': continue options = get_field_of_interface(j, "options") if remoteIP in options and greKey in options: pr("WARNING: GRE tunnel for remote_ip=%s key=%s already here, \ interface(%s)" % (remoteIP, greKey, j)) return get_field_of_interface(j, "ofport") return "[]" def ovs_create_gre (session, args): global result bridge = args.pop("bridge") remoteIP = args.pop("remoteIP") greKey = args.pop("greKey") srcHost = args.pop("from") dstHost = args.pop("to") name = "%s-%s" % (srcHost, dstHost) res = check_gre(bridge, remoteIP, greKey) if res != "[]": result = "SUCCESS:%s" % res return result wait = [vsctlPath, "--timeout=30 wait-until bridge %s -- get bridge %s name" % \ (bridge, bridge)] res = do_cmd(wait) if bridge not in res: pr("WARNIING:Can't find bridge %s for creating tunnel!" % bridge) result = "COMMAND_FAILED_NO_BRIDGE" return result createInterface = [vsctlPath, "create interface", "name=%s" % name, \ 'type=gre options:"remote_ip=%s key=%s"' % (remoteIP, greKey)] ifaceUUID = do_cmd (createInterface) if is_uuid (ifaceUUID) < 0: pr("create interface failed, %s is not UUID" % ifaceUUID) result = "COMMAND_FAILED_CREATE_INTERFACE_FAILED" return result createPort = [vsctlPath, "create port", "name=%s" % name, \ "interfaces=[%s]" % ifaceUUID] portUUID = do_cmd (createPort) if is_uuid (portUUID) < 0: pr("create port failed, %s is not UUID" % portUUID) result = "COMMAND_FAILED_CREATE_PORT_FAILED" return result addBridge = [vsctlPath, "add bridge %s" % bridge, "ports %s" % portUUID] do_cmd (addBridge) wait = [vsctlPath, "--timeout=30 wait-until port %s -- get port %s name" % \ (name, name)] res = do_cmd(wait) if name in res: port = get_field_of_interface(name, "ofport"); noFlood = [vsctlPath, "ovs-ofctl mod-port %s %s noflood" % (bridge, \ name)] do_cmd(noFlood) result = "SUCCESS:%s" % port else: pr("create gre tunnel failed") result = "COMMAND_FAILED_CREATE_TUNNEL_FAILED" return result ######################## End GRE creation utils ########################## ######################## Flow creation utils ########################## def get_ports_on_bridge(bridge): listBr = [vsctlPath, "list br", bridge] res = do_cmd(listBr, True) for i in res: if "ports" in i: (x, str) = i.split(":") str = str.lstrip().rstrip() str = str[1:] str = str[:-1] return str.split(",") return None def get_filed_of_port(nameOruuid, field): listport = [vsctlPath, "list port", nameOruuid] res = do_cmd(listport, True) for i in res: if field in i: (x, r) = i.split(":") return r.lstrip().rstrip() return None def get_field_of_interface(nameOruuid, field): listIface = [vsctlPath, "list interface", nameOruuid] res = do_cmd(listIface, True) for i in res: if field in i: (x, r) = i.split(":") return r.lstrip().rstrip() return None def strip(str, direction="default"): str = str.lstrip().rstrip() if direction == "left": return str[1:] if direction == "right": return str[:-1] if direction == "both": str = str[1:] str = str[:-1] return str return str def get_vif_port(bridge, vifName): portUuids = get_ports_on_bridge(bridge) if portUuids == None: pr("No ports on bridge %s" % bridge) return None for i in portUuids: name = get_filed_of_port(i, "name") if name == None: pr("WARNING: no name found for %s" % name) continue name = strip(name, "both") if name == vifName: return get_field_of_interface(vifName, "ofport") return None def get_interface_on_port(nameOruuid): listPort = [vsctlPath, "list port", nameOruuid] res = do_cmd(listPort, True) for i in res: if "interfaces" in i: (x, str) = i.split(":") str = strip(str, "both") return str.split(",") return None def get_gre_ports(bridge): portUuids = get_ports_on_bridge(bridge) if portUuids == None: pr("WARNING:No ports on bridge %s" % bridge) return [] OfPorts = [] for i in portUuids: iface = get_filed_of_port(i, "interfaces") iface = strip(iface, "both") type = get_field_of_interface(iface, "type") if type == 'gre': port = get_field_of_interface(iface, "ofport") if port != '[]': OfPorts.append(port) return OfPorts def get_ofports_by_tag(bridge, tag): portUuids = get_ports_on_bridge(bridge) if portUuids == None: pr("WARNING:No ports on bridge %s" % bridge) return [] OfPorts = [] for i in portUuids: t = get_filed_of_port(i, "tag") if t != tag: pr("Skip port %s with tag=%s" % (i, t)) continue iface = get_filed_of_port(i, "interfaces") iface = strip(iface, "both") port = get_field_of_interface(iface, "ofport") if port != '[]': OfPorts.append(port) return OfPorts def format_flow(inPort, vlan, mac, outPut): flow = "in_port=%s dl_vlan=%s dl_dst=%s idle_timeout=0 hard_timeout=0 \ priority=10000 actions=strip_vlan,output:%s" % (inPort, vlan, mac, outPut) return flow def format_drop_flow(inPort): flow = "in_port=%s priority=1000 idle_timeout=0 hard_timeout=0 \ actions=drop" % inPort return flow def add_drop_flow(bridge, port): flow = format_drop_flow(port) add_flow(bridge, flow) def del_flow(bridge, mac): param = "dl_dst=%s" % mac flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] do_cmd(flow) def del_arp_and_dhcp_flow(bridge, vlan, inPort): param = "dl_type=0x0806 dl_vlan=%s in_port=%s" % (vlan, inPort) flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] do_cmd(flow) param = "dl_type=0x0800 nw_proto=17 tp_dst=68 dl_vlan=%s, in_port=%s" % (vlan, inPort) flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] do_cmd(flow) param = "dl_type=0x0800 nw_proto=17 tp_dst=67 dl_vlan=%s, in_port=%s" % (vlan, inPort) flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param] do_cmd(flow) def format_normal_flow(): flow = "priority=0 idle_timeout=0 hard_timeout=0 actions=normal" return flow def format_dhcp_flow(bridge, inPort, vlan, ports): outputs = '' for i in ports: str = "output:%s," % i outputs += str outputs = outputs[:-1] flow = "in_port=%s dl_vlan=%s dl_type=0x0800 nw_proto=17 tp_dst=67 idle_timeout=0 hard_timeout=0 \ priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs) return flow def format_dhcp_client_flow(bridge, inPort, vlan, ports): outputs = '' for i in ports: str = "output:%s," % i outputs += str outputs = outputs[:-1] flow = "in_port=%s dl_vlan=%s dl_type=0x0800 nw_proto=17 tp_dst=68 idle_timeout=0 hard_timeout=0 \ priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs) return flow def format_arp_flow(bridge, inPort, vlan, ports): outputs = '' for i in ports: str = "output:%s," % i outputs += str outputs = outputs[:-1] flow = "in_port=%s dl_vlan=%s dl_type=0x0806 idle_timeout=0 hard_timeout=0 \ priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs) return flow def create_flow (bridge, vifName, mac, vlans): global result output = get_vif_port(bridge, vifName) if output == None: pr("WARNING: cannot find ofport for %s" % vifName) return errors["NO_OFPORT"] return -1 if output == '[]': pr("WARNING: ofport is [] for %s" % vifName) return errors["NO_OFPORT"] return -1 #set remap here, remap has format e.g. 1/22/200/13/16 pr("") pr("Create flow for vlans=%s" % vlans) for v in vlans.split(","): try: (vlan, inPort) = v.split(":") flow = format_flow(inPort, vlan, mac, output) add_flow(bridge, flow) except Exception, e: pr(e.__str__()) pr("invalid map") # add normal flow make switch work as L2/L3 switch flow = format_normal_flow() add_flow(bridge, flow) inports = get_gre_ports(bridge) for i in inports: add_drop_flow(bridge, i) return errors["SUCCESS"] ######################## End Flow creation utils ########################## def set_tag(bridge, vifName, vlan): # The startVM command is slow, we may wait for a while for it creates vif on # open vswitch pr("Waiting for %s ..." % vifName) waitPortCmd = [vsctlPath, "--timeout=10 wait-until port %s -- get port %s name" % \ (vifName, vifName)] do_cmd (waitPortCmd) pr("%s is here" % vifName) if get_vif_port(bridge, vifName) == None: pr("WARNING: %s is not on bridge %s" % (vifName, bridge)) return 0 pr("Set tag") set_tagCmd = [vsctlPath, "set port", vifName, "tag=%s"%vlan] do_cmd (set_tagCmd) return 0 def ask_ports(bridge, vifNames): vifs = vifNames.split(",") if len(vifs) == 0: return ' ' ofports = [] for vif in vifs: op = get_vif_port(bridge, vif) if op == None: pr("ask_ports: no port(bridge:%s, vif:%s)" % (bridge, vif)) continue ofports.append(op) return ",".join(ofports) def delete_vm_flows(bridge, vmName, reCreate=True): log = ovs_log(vmName) log.read() macs = log.macs; for i in macs.split(","): del_flow(bridge, i) pr("Delete flows for %s" % i) vlans = log.vlans for v in vlans.split(","): try: (vlan, inPort) = v.split(":") del_arp_and_dhcp_flow(bridge, vlan, inPort) except Exception, e: pr(e.__str__()) pr("invalid map") if reCreate == False: return bridge = log.bridge tag = log.tag noneGreOfPorts = get_ofports_by_tag(bridge, tag) try: noneGreOfPorts.remove(log.ofports) except Exception, e: pr(e.__str__()) pr("ofport %s of %s is not on bridge %s" % (log.ofports, log.vmName, bridge)) if len(noneGreOfPorts) != 0: set_arp_and_dhcp_flow(bridge, vlans, tag, noneGreOfPorts) # add normal flow make switch work as L2/L3 switch flow = format_normal_flow() add_flow(bridge, flow) log.remove() def echo(fn): def wrapped(*v, **k): name = fn.__name__ util.SMlog("#### VMOPS enter %s ####" % name ) res = fn(*v, **k) util.SMlog("#### VMOPS exit %s ####" % name ) return res return wrapped def ovs_handle_rebooted_vm(session, vmName): curr_domid = '-1' log = ovs_log(vmName) log.read() (curr_domid, vifrs, hostuuid) = ovs_get_domid_vifrs_hostuuid(session, vmName) old_id = log.domId; if curr_domid == old_id: util.SMlog("OvsInfo:%s is normal" % vmName) return True util.SMlog("%s rebooted, reset flow for it" % vmName) vlans = log.vlans; bridge = log.bridge tag = log.tag for vifr in vifrs: vifName = "vif" + curr_domid + "." + vifr[0] set_tag(bridge, vifName, tag) create_flow(bridge, vifName, vifr[1], vlans) log.domId = curr_domid log.write() return True @echo def ovs_get_vm_log(session, args): host_uuid = args.pop('host_uuid') try: thishost = session.xenapi.host.get_by_uuid(host_uuid) hostrec = session.xenapi.host.get_record(thishost) vms = hostrec.get('resident_VMs') except Exception, e: util.SMlog("Failed to get host from uuid %s, exception: %s" % (host_uuid, e.__str__())) return ' ' result = [] try: for name in [session.xenapi.VM.get_name_label(x) for x in vms]: if 1 not in [ name.startswith(c) for c in ['r-', 'i-'] ]: continue ovs_handle_rebooted_vm(session, name) if name.startswith('i-'): log = ovs_log(name) info = log.get_common_info() result.append(info) except Exception, e: util.SMlog(e.__str__()) util.SMlog("OVs failed to get logs, better luck next time!") return ";".join(result) def ovs_write_vm_log(bridge, vmName, vmId, seqno, vifNames, macs, tag, vlans, ofports): log = ovs_log(vmName) log.read() log.bridge = bridge log.vmName = vmName log.domId = vmId log.seqno = seqno log.vifs = vifNames log.macs = macs log.tag = tag log.vlans = vlans log.ofports = ofports log.write() util.SMlog("Writing ovs log to " + log.name) @echo def ovs_delete_flow(session, args): bridge = args.pop('bridge') vmName = args.pop('vmName') delete_vm_flows(bridge, vmName) return 'SUCCESS' def ovs_get_domid_vifrs_hostuuid(session, vmName): def get_vif_field(name, field): return session.xenapi.VIF.get_record(name).get(field) try: vm = session.xenapi.VM.get_by_name_label(vmName) if len(vm) != 1: return (-1, [], "-1") vm_rec = session.xenapi.VM.get_record(vm[0]) vm_vifs = vm_rec.get('VIFs') vifrs = [] for vif in vm_vifs: rec = (get_vif_field(vif, 'device'), get_vif_field(vif, 'MAC')) vifrs.append(rec) domid = vm_rec.get('domid') host = vm_rec.get('resident_on') host_rec = session.xenapi.host.get_record(host) uuid = host_rec.get('uuid') util.SMlog("OVSINFO: (domid:%s, uuid:%s)" % (domid, uuid)) return (domid, vifrs, uuid) except: util.SMlog("### Failed to get domid or vif list for vm ##" + vmName) return (-1, [], "-1") def add_flow(bridge, flow): param = bridge + ' "%s"' % flow addflow = ["ovs-ofctl add-flow", param] do_cmd (addflow) def set_arp_and_dhcp_flow(bridge, vlans, tag, ofports): for v in vlans.split(","): try: (vlan, inPort) = v.split(":") arpFlow = format_arp_flow(bridge, inPort, vlan, ofports) add_flow(bridge, arpFlow) dhcpFlow = format_dhcp_flow(bridge, inPort, vlan, ofports) add_flow(bridge, dhcpFlow) dhcpClientFlow = format_dhcp_client_flow(bridge, inPort, vlan, ofports) add_flow(bridge, dhcpClientFlow) except Exception, e: pr(e.__str__()) pr("invalid map") @echo def ovs_set_arp_and_dhcp_flow(session, args): vlans = args.pop('vlans') bridge = args.pop('bridge') tag = args.pop('tag') pr("ovs_set_arp_and_dhcp_flow: bridge=%s, vlans=%s, tag=%s" % (bridge, vlans, tag)) if vlans == '[]': pr("No need to create arp and dhcp flow") return 'SUCCESS' ofports = get_ofports_by_tag(bridge, tag) if len(ofports) == 0: pr("No VMs, skip set arp and dhcp flow for tag=%s" % tag) return 'SUCCESS' set_arp_and_dhcp_flow(bridge, vlans, tag, ofports) return 'SUCCESS' @echo def ovs_set_tag_and_flow(session, args): bridge = args.pop('bridge') vmName = args.pop('vmName') vlans = args.pop('vlans') tag = args.pop('tag') seqno = args.pop('seqno') (domid, vifrs, hostuuid) = ovs_get_domid_vifrs_hostuuid(session, vmName) if domid == '-1': util.SMlog("### Failed to get domid for vm (-1): " + vmName) return 'NO_DOMID' if len(vifrs) == 0: return 'SUCCESS' pr("ovs_set_tag_and_flow: bridge=%s, vmName=%s, vlans=%s, tag=%s, seqno=%s" % (bridge, vmName, vlans, tag, seqno)) #delete old flows first delete_vm_flows(bridge, vmName, False) vifNames = [] vlans = strip(vlans, "both") macs = [] for vifr in vifrs: vifName = "vif" + domid + "." + vifr[0] vifNames.append(vifName) mac = vifr[1] macs.append(mac) set_tag(bridge, vifName, tag) create_flow(bridge, vifName, mac, vlans) vifs = ",".join(vifNames) ofports = ask_ports(bridge, vifs) ovs_write_vm_log(bridge, vmName, domid, seqno, vifs, ",".join(macs), tag, vlans, ofports) #see if there is rebooted vm to handle ovs_get_vm_log(session, {"host_uuid":hostuuid}) ovs_set_arp_and_dhcp_flow(session, {"bridge":bridge, "tag":tag, "vlans":vlans}) return 'SUCCESS' if __name__ == "__main__": open_log() XenAPIPlugin.dispatch({"ovs_create_gre":ovs_create_gre, "ovs_set_tag_and_flow":ovs_set_tag_and_flow, "ovs_get_vm_log":ovs_get_vm_log,"ovs_delete_flow":ovs_delete_flow}) close_log()