Frank c6683c2eab bug 7722: open vswitch -
refine ovs plugin, create a separate plugin instead of messing with vmops
refine gre tunnel, maintains tunnel in database instead of plugin
fix an arp issue cause by overlap vlan range
2011-01-11 02:55:36 -08:00

778 lines
19 KiB
Python
Executable File

#!/usr/bin/python
#
# A plugin for executing script needed by vmops cloud
import os, sys, time
import XenAPIPlugin
sys.path.append("/opt/xensource/sm/")
import util
from util import CommandException
import hostvmstats
import socket
import stat
import base64
import tempfile
from os.path import exists as _exists
from time import localtime as _localtime, asctime as _asctime
vSwitchDBPidFile = "/var/run/openvswitch/ovsdb-server.pid"
vSwitchDBDaemonName = "ovsdb-server"
vSwitchPidFile = "/var/run/openvswitch/ovs-vswitchd.pid"
vsctlPath = "/usr/bin/ovs-vsctl"
vSwitchDaemonName = "ovs-vswitchd"
logFile = "/var/log/ovsgre.log"
fLog = None
global result
errors = \
{"NO_DB_PID_FILE" : "NO_DB_PID_FILE", \
"DB_NOT_RUN" : "DB_NOT_RUN", \
"NO_SWITCH_PID_FILE" : "NO_SWITCH_PID_FILE", \
"SWITCH_NOT_RUN" : "SWITCH_NOT_RUN", \
"NO_VSCTL" : "NO_VSCTL", \
"COMMAND_FAILED" : "COMMAND_FAILED", \
"TUNNEL_EXISTED" : "TUNNEL_EXISTED", \
"NO_INPORT" : "NO_INPORT", \
"NO_OFPORT" : "NO_OFPORT", \
"ERR_ARGS_NUM" : "ERR_ARGS_NUM", \
"ERROR_OP" : "ERROR_OP", \
"SUCCESS" : "SUCCESS", \
}
class ovs_log(object):
def __init__(self, name):
n = "ovs-%s" % name;
logfilename = "/var/run/cloud/" + n +".log"
self.name = logfilename
self.vmName = name
self.bridge = ""
self.domId = ""
self.seqno = ""
self.tag = ""
self.vifs = ""
self.macs = ""
self.vlans = ""
self.ofports = ""
def write(self):
log = open(self.name, "w")
log.write("vmName=%s" % self.vmName)
log.write("\n")
log.write("bridge=%s" % self.bridge)
log.write("\n")
log.write("domId=%s" % self.domId)
log.write("\n")
log.write("seqno=%s" % self.seqno)
log.write("\n")
log.write("tag=%s" % self.tag)
log.write("\n")
log.write("vifs=%s" % self.vifs)
log.write("\n")
log.write("macs=%s" % self.macs)
log.write("\n")
log.write("vlans=%s" % self.vlans)
log.write("\n")
log.write("ofports=%s" % self.ofports)
log.close()
def read(self):
try:
lines = [line.rstrip() for line in open(self.name)]
for i in lines:
if "=" not in i:
util.SMlog("invalid line(%s) in %s" % (i, self.name))
continue
(key,value) = i.split("=")
if key == "vmName":
self.vmName = value
elif key == "bridge":
self.bridge = value
elif key == "domId":
self.domId = value
elif key == "seqno":
self.seqno = value
elif key == "tag":
self.tag = value
elif key == "vifs":
self.vifs = value
elif key == "macs":
self.macs = value
elif key == "vlans":
self.vlans = value
elif key == "ofports":
self.ofports = value
except Exception, e:
util.SMlog(e.__str__())
util.SMlog("Failed to open ovs log %s" % self.name);
def get_common_info(self):
self.read()
return "%s,%s,%s,%s,%s" % (self.vmName, self.bridge, self.domId,
self.seqno, self.tag)
def remove(self):
try:
os.remove(self.name)
except:
util.SMlog("Failed to delete ovs log file " + self.name)
def open_log ():
global fLog
try:
if fLog == None:
fLog = open (logFile, "a")
except IOError, e:
#print e
pass
def pr (str):
global fLog
if fLog != None:
str = "[%s]:" % _asctime (_localtime()) + str + "\n"
fLog.write (str)
def close_log ():
global fLog
if fLog != None:
fLog.close ()
def is_process_run (pidFile, name):
try:
fpid = open (pidFile, "r")
pid = fpid.readline ()
fpid.close ()
except IOError, e:
return -1
pid = pid[:-1]
ps = os.popen ("ps -ae")
for l in ps:
if pid in l and name in l:
ps.close ()
return 0
ps.close ()
return -2
def is_tool_exist (name):
if _exists (name):
return 0
return -1
def check_switch ():
global result
ret = is_process_run (vSwitchDBPidFile, vSwitchDBDaemonName);
if ret < 0:
if ret == -1: result = errors["NO_DB_PID_FILE"]
if ret == -2: result = errors["DB_NOT_RUN"]
return -1
ret = is_process_run (vSwitchPidFile, vSwitchDaemonName)
if ret < 0:
if ret == -1: result = errors["NO_SWITCH_PID_FILE"]
if ret == -2: result = errors["SWITCH_NOT_RUN"]
return -1
if is_tool_exist (vsctlPath) < 0:
result = errors["NO_VSCTL"]
return -1
return 0
def do_cmd (cmds, lines=False):
cmd = ""
for i in cmds:
cmd += " "
cmd += i
pr("do command '%s'" % cmd)
f = os.popen (cmd)
if lines == True:
res = f.readlines ()
else:
res = f.readline ()
res = res[:-1]
f.close ()
if lines == False:
pr("command output '%s'" % res)
return res
######################## GRE creation utils ##########################
# UUID's format is 8-4-4-4-12
def is_uuid (uuid):
list = uuid.split ("-")
if len (list) != 5:
return -1
if len (list[0]) != 8 or len (list[1]) != 4 \
or len (list[2]) != 4 or len (list[3]) != 4 \
or len (list[4]) != 12:
return -1
return 0
def check_gre (bridge, remoteIP, greKey):
ports = get_ports_on_bridge(bridge)
if ports == None:
return "[]"
for i in ports:
ifaces = get_interface_on_port(i)
if ifaces == None:
continue
for j in ifaces:
if j == '[]':
continue
options = get_field_of_interface(j, "options")
if remoteIP in options and greKey in options:
pr("WARNING: GRE tunnel for remote_ip=%s key=%s already here, \
interface(%s)" % (remoteIP, greKey, j))
return get_field_of_interface(j, "ofport")
return "[]"
def ovs_create_gre (session, args):
global result
bridge = args.pop("bridge")
remoteIP = args.pop("remoteIP")
greKey = args.pop("greKey")
srcHost = args.pop("from")
dstHost = args.pop("to")
name = "%s-%s" % (srcHost, dstHost)
res = check_gre(bridge, remoteIP, greKey)
if res != "[]":
result = "SUCCESS:%s" % res
return result
wait = [vsctlPath, "--timeout=30 wait-until bridge %s -- get bridge %s name" % \
(bridge, bridge)]
res = do_cmd(wait)
if bridge not in res:
pr("WARNIING:Can't find bridge %s for creating tunnel!" % bridge)
result = errors["COMMAND_FAILED"]
return result
createInterface = [vsctlPath, "create interface", "name=%s" % name, \
'type=gre options:"remote_ip=%s key=%s"' % (remoteIP, greKey)]
ifaceUUID = do_cmd (createInterface)
if is_uuid (ifaceUUID) < 0:
result = errors["COMMAND_FAILED"];
return result
createPort = [vsctlPath, "create port", "name=%s" % name, \
"interfaces=[%s]" % ifaceUUID]
portUUID = do_cmd (createPort)
if is_uuid (portUUID) < 0:
result = errors["COMMAND_FAILED"];
return result
addBridge = [vsctlPath, "add bridge %s" % bridge, "ports %s" % portUUID]
do_cmd (addBridge)
wait = [vsctlPath, "--timeout=5 wait-until port %s -- get port %s name" % \
(name, name)]
res = do_cmd(wait)
if name in res:
port = get_field_of_interface(name, "ofport");
result = "SUCCESS:%s" % port
else:
result = errors["COMMAND_FAILED"]
return result
######################## End GRE creation utils ##########################
######################## Flow creation utils ##########################
def get_ports_on_bridge(bridge):
listBr = [vsctlPath, "list br", bridge]
res = do_cmd(listBr, True)
for i in res:
if "ports" in i:
(x, str) = i.split(":")
str = str.lstrip().rstrip()
str = str[1:]
str = str[:-1]
return str.split(",")
return None
def get_filed_of_port(nameOruuid, field):
listport = [vsctlPath, "list port", nameOruuid]
res = do_cmd(listport, True)
for i in res:
if field in i:
(x, r) = i.split(":")
return r.lstrip().rstrip()
return None
def get_field_of_interface(nameOruuid, field):
listIface = [vsctlPath, "list interface", nameOruuid]
res = do_cmd(listIface, True)
for i in res:
if field in i:
(x, r) = i.split(":")
return r.lstrip().rstrip()
return None
def strip(str, direction="default"):
str = str.lstrip().rstrip()
if direction == "left":
return str[1:]
if direction == "right":
return str[:-1]
if direction == "both":
str = str[1:]
str = str[:-1]
return str
return str
def get_vif_port(bridge, vifName):
portUuids = get_ports_on_bridge(bridge)
if portUuids == None:
pr("No ports on bridge %s" % bridge)
return None
for i in portUuids:
name = get_filed_of_port(i, "name")
if name == None:
pr("WARNING: no name found for %s" % name)
continue
name = strip(name, "both")
if name == vifName:
return get_field_of_interface(vifName, "ofport")
return None
def get_interface_on_port(nameOruuid):
listPort = [vsctlPath, "list port", nameOruuid]
res = do_cmd(listPort, True)
for i in res:
if "interfaces" in i:
(x, str) = i.split(":")
str = strip(str, "both")
return str.split(",")
return None
def get_ofports_by_tag(bridge, tag):
portUuids = get_ports_on_bridge(bridge)
if portUuids == None:
pr("WARNING:No ports on bridge %s" % bridge)
return []
OfPorts = []
for i in portUuids:
t = get_filed_of_port(i, "tag")
if t != tag:
pr("Skip port %s with tag=%s" % (i, t))
continue
iface = get_filed_of_port(i, "interfaces")
iface = strip(iface, "both")
port = get_field_of_interface(iface, "ofport")
if port != '[]':
OfPorts.append(port)
return OfPorts
def format_flow(inPort, vlan, mac, outPut):
flow = "in_port=%s dl_vlan=%s dl_dst=%s idle_timeout=0 hard_timeout=0 \
priority=10000 actions=strip_vlan,output:%s" % (inPort, vlan, mac, outPut)
return flow
def format_drop_flow(inPort, vlan):
flow = "in_port=%s dl_vlan=%s priority=0 idle_timeout=0 hard_timeout=0 \
actions=drop" % (inPort, vlan)
return flow
def del_flow(bridge, mac):
param = "dl_dst=%s" % mac
flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param]
do_cmd(flow)
def del_arp_and_dhcp_flow(bridge, vlan, inPort):
param = "dl_type=0x0806 dl_vlan=%s in_port=%s" % (vlan, inPort)
flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param]
do_cmd(flow)
param = "dl_type=0x0800 nw_proto=17 tp_dst=68 dl_vlan=%s, in_port=%s" % (vlan, inPort)
flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param]
do_cmd(flow)
param = "dl_type=0x0800 nw_proto=17 tp_dst=67 dl_vlan=%s, in_port=%s" % (vlan, inPort)
flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param]
do_cmd(flow)
def del_drop_flow(bridge, vlan):
param = "priority=0 dl_vlan=%s" % vlan
flow = ["ovs-ofctl del-flows %s" % bridge, '"%s"' % param]
do_cmd(flow)
def format_normal_flow():
flow = "priority=0 idle_timeout=0 hard_timeout=0 actions=normal"
return flow
def format_dhcp_flow(bridge, inPort, vlan, ports):
outputs = ''
for i in ports:
str = "output:%s," % i
outputs += str
outputs = outputs[:-1]
flow = "in_port=%s dl_vlan=%s dl_type=0x0800 nw_proto=17 tp_dst=67 idle_timeout=0 hard_timeout=0 \
priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs)
return flow
def format_dhcp_client_flow(bridge, inPort, vlan, ports):
outputs = ''
for i in ports:
str = "output:%s," % i
outputs += str
outputs = outputs[:-1]
flow = "in_port=%s dl_vlan=%s dl_type=0x0800 nw_proto=17 tp_dst=68 idle_timeout=0 hard_timeout=0 \
priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs)
return flow
def format_arp_flow(bridge, inPort, vlan, ports):
outputs = ''
for i in ports:
str = "output:%s," % i
outputs += str
outputs = outputs[:-1]
flow = "in_port=%s dl_vlan=%s dl_type=0x0806 idle_timeout=0 hard_timeout=0 \
priority=10000 actions=strip_vlan,%s" % (inPort, vlan, outputs)
return flow
def create_flow (bridge, vifName, mac, vlans):
global result
output = get_vif_port(bridge, vifName)
if output == None:
pr("WARNING: cannot find ofport for %s" % vifName)
return errors["NO_OFPORT"]
return -1
if output == '[]':
pr("WARNING: ofport is [] for %s" % vifName)
return errors["NO_OFPORT"]
return -1
#set remap here, remap has format e.g. 1/22/200/13/16
pr("")
pr("Create flow for vlans=%s" % vlans)
for v in vlans.split(","):
try:
(vlan, inPort) = v.split(":")
flow = format_drop_flow(inPort, vlan)
add_flow(bridge, flow)
flow = format_flow(inPort, vlan, mac, output)
add_flow(bridge, flow)
except Exception, e:
pr(e.__str__())
pr("invalid map")
# add normal flow make switch work as L2/L3 switch
flow = format_normal_flow()
add_flow(bridge, flow)
return errors["SUCCESS"]
######################## End Flow creation utils ##########################
def set_tag(bridge, vifName, vlan):
# The startVM command is slow, we may wait for a while for it creates vif on
# open vswitch
pr("Waiting for %s ..." % vifName)
waitPortCmd = [vsctlPath, "--timeout=10 wait-until port %s -- get port %s name" % \
(vifName, vifName)]
do_cmd (waitPortCmd)
pr("%s is here" % vifName)
if get_vif_port(bridge, vifName) == None:
pr("WARNING: %s is not on bridge %s" % (vifName, bridge))
return 0
pr("Set tag")
set_tagCmd = [vsctlPath, "set port", vifName, "tag=%s"%vlan]
do_cmd (set_tagCmd)
return 0
def ask_ports(bridge, vifNames):
vifs = vifNames.split(",")
if len(vifs) == 0:
return ' '
ofports = []
for vif in vifs:
op = get_vif_port(bridge, vif)
if op == None:
pr("ask_ports: no port(bridge:%s, vif:%s)" % (bridge, vif))
continue
ofports.append(op)
return ",".join(ofports)
def delete_vm_flows(bridge, vmName, reCreate=True):
log = ovs_log(vmName)
log.read()
macs = log.macs;
for i in macs.split(","):
del_flow(bridge, i)
pr("Delete flows for %s" % i)
vlans = log.vlans
for v in vlans.split(","):
try:
(vlan, inPort) = v.split(":")
del_arp_and_dhcp_flow(bridge, vlan, inPort)
except Exception, e:
pr(e.__str__())
pr("invalid map")
for v in vlans:
try:
(vlan, inport) = v.split(":")
del_drop_flow(bridge, vlan)
except Exception, e:
pr(e.__str__())
pr("invalid map")
if reCreate == False:
return
bridge = log.bridge
tag = log.tag
noneGreOfPorts = get_ofports_by_tag(bridge, tag)
try:
noneGreOfPorts.remove(log.ofports)
except Exception, e:
pr(e.__str__())
pr("ofport %s of %s is not on bridge %s" % (log.ofports, log.vmName,
bridge))
if len(noneGreOfPorts) != 0:
set_arp_and_dhcp_flow(bridge, vlans, tag, noneGreOfPorts)
# add normal flow make switch work as L2/L3 switch
flow = format_normal_flow()
add_flow(bridge, flow)
log.remove()
def echo(fn):
def wrapped(*v, **k):
name = fn.__name__
util.SMlog("#### VMOPS enter %s ####" % name )
res = fn(*v, **k)
util.SMlog("#### VMOPS exit %s ####" % name )
return res
return wrapped
def ovs_handle_rebooted_vm(session, vmName):
curr_domid = '-1'
log = ovs_log(vmName)
log.read()
(curr_domid, vifrs, hostuuid) = ovs_get_domid_vifrs_hostuuid(session, vmName)
old_id = log.domId;
if curr_domid == old_id:
util.SMlog("OvsInfo:%s is normal" % vmName)
return True
util.SMlog("%s rebooted, reset flow for it" % vmName)
vlans = log.vlans;
bridge = log.bridge
tag = log.tag
for vifr in vifrs:
vifName = "vif" + curr_domid + "." + vifr[0]
set_tag(bridge, vifName, tag)
create_flow(bridge, vifName, vifr[1], vlans)
log.domId = curr_domid
log.write()
return True
@echo
def ovs_get_vm_log(session, args):
host_uuid = args.pop('host_uuid')
try:
thishost = session.xenapi.host.get_by_uuid(host_uuid)
hostrec = session.xenapi.host.get_record(thishost)
vms = hostrec.get('resident_VMs')
except Exception, e:
util.SMlog("Failed to get host from uuid %s, exception: %s" % (host_uuid, e.__str__()))
return ' '
result = []
try:
for name in [session.xenapi.VM.get_name_label(x) for x in vms]:
if 1 not in [ name.startswith(c) for c in ['r-', 'i-'] ]:
continue
ovs_handle_rebooted_vm(session, name)
if name.startswith('i-'):
log = ovs_log(name)
info = log.get_common_info()
result.append(info)
except Exception, e:
util.SMlog(e.__str__())
util.SMlog("OVs failed to get logs, better luck next time!")
return ";".join(result)
def ovs_write_vm_log(bridge, vmName, vmId, seqno, vifNames, macs, tag, vlans, ofports):
log = ovs_log(vmName)
log.read()
log.bridge = bridge
log.vmName = vmName
log.domId = vmId
log.seqno = seqno
log.vifs = vifNames
log.macs = macs
log.tag = tag
log.vlans = vlans
log.ofports = ofports
log.write()
util.SMlog("Writing ovs log to " + log.name)
@echo
def ovs_delete_flow(session, args):
bridge = args.pop('bridge')
vmName = args.pop('vmName')
delete_vm_flows(bridge, vmName)
return 'SUCCESS'
def ovs_get_domid_vifrs_hostuuid(session, vmName):
def get_vif_field(name, field):
return session.xenapi.VIF.get_record(name).get(field)
try:
vm = session.xenapi.VM.get_by_name_label(vmName)
if len(vm) != 1:
return 'NO_VM'
vm_rec = session.xenapi.VM.get_record(vm[0])
vm_vifs = vm_rec.get('VIFs')
vifrs = []
for vif in vm_vifs:
rec = (get_vif_field(vif, 'device'), get_vif_field(vif, 'MAC'))
vifrs.append(rec)
domid = vm_rec.get('domid')
host = vm_rec.get('resident_on')
host_rec = session.xenapi.host.get_record(host)
uuid = host_rec.get('uuid')
util.SMlog("OVSINFO: (domid:%s, uuid:%s)" % (domid, uuid))
return (domid, vifrs, uuid)
except:
util.SMlog("### Failed to get domid or vif list for vm ##" + vmName)
return (-1, [], "-1")
def add_flow(bridge, flow):
param = bridge + ' "%s"' % flow
addflow = ["ovs-ofctl add-flow", param]
do_cmd (addflow)
def set_arp_and_dhcp_flow(bridge, vlans, tag, ofports):
for v in vlans.split(","):
try:
(vlan, inPort) = v.split(":")
arpFlow = format_arp_flow(bridge, inPort, vlan, ofports)
add_flow(bridge, arpFlow)
dhcpFlow = format_dhcp_flow(bridge, inPort, vlan, ofports)
add_flow(bridge, dhcpFlow)
dhcpClientFlow = format_dhcp_client_flow(bridge, inPort, vlan, ofports)
add_flow(bridge, dhcpClientFlow)
except Exception, e:
pr(e.__str__())
pr("invalid map")
@echo
def ovs_set_arp_and_dhcp_flow(session, args):
vlans = args.pop('vlans')
bridge = args.pop('bridge')
tag = args.pop('tag')
pr("ovs_set_arp_and_dhcp_flow: bridge=%s, vlans=%s, tag=%s" % (bridge,
vlans, tag))
if vlans == '[]':
pr("No need to create arp and dhcp flow")
return 'SUCCESS'
ofports = get_ofports_by_tag(bridge, tag)
if len(ofports) == 0:
pr("No VMs, skip set arp and dhcp flow for tag=%s" % tag)
return 'SUCCESS'
set_arp_and_dhcp_flow(bridge, vlans, tag, ofports)
return 'SUCCESS'
@echo
def ovs_set_tag_and_flow(session, args):
bridge = args.pop('bridge')
vmName = args.pop('vmName')
vlans = args.pop('vlans')
tag = args.pop('tag')
seqno = args.pop('seqno')
(domid, vifrs, hostuuid) = ovs_get_domid_vifrs_hostuuid(session, vmName)
if domid == '-1':
util.SMlog("### Failed to get domid for vm (-1): " + vmName)
return 'NO_DOMID'
if len(vifrs) == 0:
return 'SUCCESS'
pr("ovs_set_tag_and_flow: bridge=%s, vmName=%s, vlans=%s, tag=%s, seqno=%s" %
(bridge, vmName, vlans, tag, seqno))
#delete old flows first
delete_vm_flows(bridge, vmName, False)
vifNames = []
vlans = strip(vlans, "both")
macs = []
for vifr in vifrs:
vifName = "vif" + domid + "." + vifr[0]
vifNames.append(vifName)
mac = vifr[1]
macs.append(mac)
set_tag(bridge, vifName, tag)
create_flow(bridge, vifName, mac, vlans)
vifs = ",".join(vifNames)
ofports = ask_ports(bridge, vifs)
ovs_write_vm_log(bridge, vmName, domid, seqno, vifs, ",".join(macs), tag, vlans, ofports)
#see if there is rebooted vm to handle
ovs_get_vm_log(session, {"host_uuid":hostuuid})
ovs_set_arp_and_dhcp_flow(session, {"bridge":bridge, "tag":tag, "vlans":vlans})
return 'SUCCESS'
if __name__ == "__main__":
open_log()
XenAPIPlugin.dispatch({"ovs_create_gre":ovs_create_gre, "ovs_set_tag_and_flow":ovs_set_tag_and_flow, "ovs_get_vm_log":ovs_get_vm_log,"ovs_delete_flow":ovs_delete_flow})
close_log()