adding distributed routing support for KVM OVS

some check style error fixes
This commit is contained in:
Murali Reddy 2014-03-11 13:05:03 +05:30
parent 423a748807
commit 36541a2f4c
6 changed files with 373 additions and 10 deletions

View File

@ -78,6 +78,8 @@ import com.cloud.agent.api.OvsDestroyTunnelCommand;
import com.cloud.agent.api.OvsFetchInterfaceAnswer;
import com.cloud.agent.api.OvsFetchInterfaceCommand;
import com.cloud.agent.api.OvsSetupBridgeCommand;
import com.cloud.agent.api.OvsVpcPhysicalTopologyConfigCommand;
import com.cloud.agent.api.OvsVpcRoutingPolicyConfigCommand;
import com.cloud.agent.api.PingCommand;
import com.cloud.agent.api.PingRoutingCommand;
import com.cloud.agent.api.PingRoutingWithNwGroupsCommand;
@ -1360,6 +1362,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
return execute((OvsCreateTunnelCommand)cmd);
} else if (cmd instanceof OvsDestroyTunnelCommand) {
return execute((OvsDestroyTunnelCommand)cmd);
} else if (cmd instanceof OvsVpcPhysicalTopologyConfigCommand) {
return execute((OvsVpcPhysicalTopologyConfigCommand) cmd);
} else if (cmd instanceof OvsVpcRoutingPolicyConfigCommand) {
return execute((OvsVpcRoutingPolicyConfigCommand) cmd);
} else {
s_logger.warn("Unsupported command ");
return Answer.createUnsupportedCommandAnswer(cmd);
@ -1401,6 +1407,47 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
return new Answer(cmd, true, null);
}
public Answer execute(OvsVpcPhysicalTopologyConfigCommand cmd) {
String bridge = cmd.getBridgeName();
try {
Script command = new Script(_ovsTunnelPath, _timeout, s_logger);
command.add("configure_ovs_bridge_for_network_topology");
command.add("--bridge", bridge);
command.add("--config", cmd.getVpcConfigInJson());
String result = command.execute();
if (result.equalsIgnoreCase("SUCCESS")) {
return new Answer(cmd, true, result);
} else {
return new Answer(cmd, false, result);
}
} catch (Exception e) {
s_logger.warn("caught exception while updating host with latest routing polcies", e);
return new Answer(cmd, false, e.getMessage());
}
}
public Answer execute(OvsVpcRoutingPolicyConfigCommand cmd) {
try {
Script command = new Script(_ovsTunnelPath, _timeout, s_logger);
command.add("configure_ovs_bridge_for_routing_policies");
command.add("--bridge", cmd.getBridgeName());
command.add("--config", cmd.getVpcConfigInJson());
String result = command.execute();
if (result.equalsIgnoreCase("SUCCESS")) {
return new Answer(cmd, true, result);
} else {
return new Answer(cmd, false, result);
}
} catch (Exception e) {
s_logger.warn("caught exception while updating host with latest VPC topology", e);
return new Answer(cmd, false, e.getMessage());
}
}
private synchronized void destroyTunnelNetwork(String bridge) {
try {
findOrCreateTunnelNetwork(bridge);

View File

@ -88,7 +88,6 @@ import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import com.cloud.agent.IAgentControl;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.AttachIsoCommand;
import com.cloud.agent.api.AttachVolumeAnswer;
@ -149,6 +148,7 @@ import com.cloud.agent.api.OvsSetTagAndFlowAnswer;
import com.cloud.agent.api.OvsSetTagAndFlowCommand;
import com.cloud.agent.api.OvsSetupBridgeCommand;
import com.cloud.agent.api.OvsVpcPhysicalTopologyConfigCommand;
import com.cloud.agent.api.OvsVpcRoutingPolicyConfigCommand;
import com.cloud.agent.api.PerformanceMonitorAnswer;
import com.cloud.agent.api.PerformanceMonitorCommand;
import com.cloud.agent.api.PingCommand;

View File

@ -18,7 +18,6 @@ package com.cloud.agent.api;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import java.util.UUID;
/**
* This command represents logical view of VM's connectivity in VPC.

View File

@ -16,11 +16,25 @@
// under the License.
package com.cloud.network.ovs;
import com.amazonaws.services.ec2.model.NetworkAcl;
import com.cloud.agent.api.*;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.OvsCreateTunnelAnswer;
import com.cloud.agent.api.OvsCreateTunnelCommand;
import com.cloud.agent.api.OvsDestroyBridgeCommand;
import com.cloud.agent.api.OvsDestroyTunnelCommand;
import com.cloud.agent.api.OvsFetchInterfaceAnswer;
import com.cloud.agent.api.OvsFetchInterfaceCommand;
import com.cloud.agent.api.OvsSetupBridgeCommand;
import com.cloud.agent.api.OvsVpcPhysicalTopologyConfigCommand;
import com.cloud.agent.api.OvsVpcRoutingPolicyConfigCommand;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkVO;
import com.cloud.network.vpc.*;
import com.cloud.network.vpc.NetworkACLVO;
import com.cloud.network.vpc.NetworkACLItemDao;
import com.cloud.network.vpc.NetworkACLItemVO;
import com.cloud.network.vpc.dao.VpcDao;
import com.cloud.network.vpc.VpcManager;
import com.cloud.network.vpc.VpcVO;
import com.cloud.network.vpc.dao.NetworkACLDao;
import com.cloud.vm.dao.VMInstanceDao;
import com.cloud.vm.Nic;
@ -63,7 +77,6 @@ import com.cloud.network.ovs.dao.OvsTunnelInterfaceVO;
import com.cloud.network.ovs.dao.OvsTunnelNetworkDao;
import com.cloud.network.ovs.dao.OvsTunnelNetworkVO;
import com.cloud.network.ovs.dao.OvsTunnel;
import com.cloud.network.vpc.dao.VpcDao;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;

View File

@ -174,6 +174,7 @@ def _build_flow_expr(**kwargs):
dl_dst = 'dl_dst' in kwargs and ",dl_dst=%s" % kwargs['dl_dst'] or ''
nw_src = 'nw_src' in kwargs and ",nw_src=%s" % kwargs['nw_src'] or ''
nw_dst = 'nw_dst' in kwargs and ",nw_dst=%s" % kwargs['nw_dst'] or ''
table = 'table' in kwargs and ",table=%s" % kwargs['table'] or ''
proto = 'proto' in kwargs and ",%s" % kwargs['proto'] or ''
ip = ('nw_src' in kwargs or 'nw_dst' in kwargs) and ',ip' or ''
flow = (flow + in_port + dl_type + dl_src + dl_dst +
@ -217,3 +218,228 @@ def del_all_flows(bridge):
def del_port(bridge, port):
delPort = [VSCTL_PATH, "del-port", bridge, port]
do_cmd(delPort)
def get_network_id_for_vif(vif_name):
domain_id, device_id = vif_name[3:len(vif_name)].split(".")
dom_uuid = do_cmd([XE_PATH, "vm-list", "dom-id=%s" % domain_id, "--minimal"])
vif_uuid = do_cmd([XE_PATH, "vif-list", "vm-uuid=%s" % dom_uuid, "device=%s" % device_id, "--minimal"])
vnet = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=other-config",
"param-key=cloudstack-network-id"])
return vnet
def get_network_id_for_tunnel_port(tunnelif_name):
vnet = do_cmd([VSCTL_PATH, "get", "interface", tunnelif_name, "options:cloudstack-network-id"])
return vnet
def clear_flooding_rules_for_port(bridge, ofport):
del_flows(bridge, in_port=ofport, table=2)
def add_flooding_rules_for_port(bridge, in_ofport, out_ofports):
action = "".join("output:%s," %ofport for ofport in out_ofports)[:-1]
add_flow(bridge, priority=1100, in_port=in_ofport, table=1, actions=action)
def get_ofport_for_vif(vif_name):
return do_cmd([VSCTL_PATH, "get", "interface", vif_name, "ofport"])
def get_macaddress_of_vif(vif_name):
domain_id, device_id = vif_name[3:len(vif_name)].split(".")
dom_uuid = do_cmd([XE_PATH, "vm-list", "dom-id=%s" % domain_id, "--minimal"])
vif_uuid = do_cmd([XE_PATH, "vif-list", "vm-uuid=%s" % dom_uuid, "device=%s" % device_id, "--minimal"])
mac = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=MAC"])
return mac
def get_vif_name_from_macaddress(macaddress):
vif_uuid = do_cmd([XE_PATH, "vif-list", "MAC=%s" % macaddress, "--minimal"])
vif_device_id = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=device"])
vm_uuid = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid, "param-name=vm-uuid"])
vm_domain_id = do_cmd([XE_PATH, "vm-param-get", "uuid=%s" % vm_uuid, "param-name=dom-id"])
return "vif"+vm_domain_id+"."+vif_device_id
def add_mac_lookup_table_entry(bridge, mac_address, out_of_port):
add_flow(bridge, priority=1100, dl_dst=mac_address, table=1, actions="output:%s" % out_of_port)
def delete_mac_lookup_table_entry(bridge, mac_address):
del_flows(bridge, dl_dst=mac_address, table=1)
def add_ip_lookup_table_entry(bridge, ip, dst_tier_gateway_mac, dst_vm_mac):
action_str = "mod_dl_sr:%s" % dst_tier_gateway_mac + ",mod_dl_dst:%s" % dst_vm_mac +",resubmit(,5)"
addflow = [OFCTL_PATH, "add-flow", bridge, "table=4", "nw_dst=%s" % ip, "actions=%s" %action_str]
do_cmd(addflow)
def get_vms_on_host(vpc, host_id):
all_vms = vpc.vms
vms_on_host = []
for vm in all_vms:
if vm.hostid == host_id:
vms_on_host.append(vm)
return vms_on_host
def get_network_details(vpc, network_uuid):
tiers = vpc.tiers
for tier in tiers:
if tier.networkuuid == network_uuid:
return tier
return None
class jsonLoader(object):
def __init__(self, obj):
for k in obj:
v = obj[k]
if isinstance(v, dict):
setattr(self, k, jsonLoader(v))
elif isinstance(v, (list, tuple)):
if len(v) > 0 and isinstance(v[0], dict):
setattr(self, k, [jsonLoader(elem) for elem in v])
else:
setattr(self, k, v)
else:
setattr(self, k, v)
def __getattr__(self, val):
if val in self.__dict__:
return self.__dict__[val]
else:
return None
def __repr__(self):
return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v)
in self.__dict__.iteritems()))
def __str__(self):
return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v)
in self.__dict__.iteritems()))
def configure_bridge_for_network_topology(bridge, this_host_id, json_config):
vpconfig = jsonLoader(json.loads(json_config)).vpc
if vpconfig is None:
logging.debug("WARNING:Can't find VPC info in json config file")
return "FAILURE:IMPROPER_JSON_CONFG_FILE"
# get the list of Vm's in the VPC from the JSON config
this_host_vms = get_vms_on_host(vpconfig, this_host_id)
for vm in this_host_vms:
for nic in vm.nics:
mac_addr = nic.macaddress
ip = nic.ipaddress
vif_name = get_vif_name_from_macaddress(mac_addr)
of_port = get_ofport_for_vif(vif_name)
network = get_network_details(vpconfig, nic.networkuuid)
# Add flow rule in L2 look up table, if the destination mac = MAC of the nic send packet on the found OFPORT
add_mac_lookup_table_entry(bridge, mac_addr, of_port)
# Add flow rule in L3 look up table: if the destination IP = VM's IP then modify the packet
# to set DST MAC = VM's MAC, SRC MAC=tier gateway MAC and send to egress table
add_ip_lookup_table_entry(bridge, ip, network.gatewaymac, mac_addr)
# Add flow entry to send with intra tier traffic from the NIC to L2 lookup path)
addflow = [OFCTL_PATH, "add-flow", bridge, "table=0", "in_port=%s" % of_port,
"nw_dst=%s" %network.cidr, "actions=resubmit(,1)"]
do_cmd(addflow)
#add flow entry to send inter-tier traffic from the NIC to egress ACL table(to L3 lookup path)
addflow = [OFCTL_PATH, "add-flow", bridge, "table=0", "in_port=%s" % of_port,
"dl_dst=%s" %network.gatewaymac, "nw_dst=%s" %vpconfig.cidr, "actions=resubmit(,3)"]
do_cmd(addflow)
# get the list of hosts on which VPC spans from the JSON config
vpc_spanning_hosts = vpconfig.hosts
for host in vpc_spanning_hosts:
if this_host_id == host.hostid:
continue
other_host_vms = get_vms_on_host(vpconfig, host.hostid)
for vm in other_host_vms:
for nic in vm.nics:
mac_addr = nic.macaddress
ip = nic.ipaddress
network = get_network_details(vpconfig, nic.networkuuid)
gre_key = network.grekey
# generate tunnel name from tunnel naming convention
tunnel_name = "t%s-%s-%s" % (gre_key, this_host_id, host.hostid)
of_port = get_ofport_for_vif(tunnel_name)
# Add flow rule in L2 look up table, if the destination mac = MAC of the nic send packet tunnel port
add_mac_lookup_table_entry(bridge, mac_addr, of_port)
# Add flow tule in L3 look up table: if the destination IP = VM's IP then modify the packet
# set DST MAC = VM's MAC, SRC MAC=tier gateway MAC and send to egress table
add_ip_lookup_table_entry(bridge, ip, network.gatewaymac, mac_addr)
return "SUCCESS: successfully configured bridge as per the VPC topology"
def get_acl(vpcconfig, required_acl_id):
acls = vpcconfig.acls
for acl in acls:
if acl.id == required_acl_id:
return acl
return None
def configure_ovs_bridge_for_routing_policies(bridge, json_config):
vpconfig = jsonLoader(json.loads(json_config)).vpc
if vpconfig is None:
logging.debug("WARNING:Can't find VPC info in json config file")
return "FAILURE:IMPROPER_JSON_CONFG_FILE"
# First flush current egress ACL's before re-applying the ACL's
del_flows(bridge, table=3)
egress_rules_added = False
ingress_rules_added = False
tiers = vpconfig.tiers
for tier in tiers:
tier_cidr = tier.cidr
acl = get_acl(vpconfig, tier.aclid)
acl_items = acl.aclitems
for acl_item in acl_items:
number = acl_item.number
action = acl_item.action
direction = acl_item.direction
source_port_start = acl_item.sourceportstart
source_port_end = acl_item.sourceportend
protocol = acl_item.protocol
source_cidrs = acl_item.sourcecidrs
acl_priority = 1000 + number
for source_cidr in source_cidrs:
if direction is "ingress":
ingress_rules_added = True
# add flow rule to do action (allow/deny) for flows where source IP of the packet is in
# source_cidr and destination ip is in tier_cidr
port = source_port_start
while (port < source_port_end):
if action is "deny":
add_flow(bridge, priority= acl_priority, table=5, nw_src=source_cidr, nw_dst=tier_cidr, tp_dst=port,
nw_proto=protocol, actions='drop')
if action is "allow":
add_flow(bridge, priority= acl_priority,table=5, nw_src=source_cidr, nw_dst=tier_cidr, tp_dst=port,
nw_proto=protocol, actions='resubmit(,1)')
port = port + 1
elif direction in "egress":
egress_rules_added = True
# add flow rule to do action (allow/deny) for flows where destination IP of the packet is in
# source_cidr and source ip is in tier_cidr
port = source_port_start
while (port < source_port_end):
if action is "deny":
add_flow(bridge, priority= acl_priority, table=5, nw_src=tier_cidr, nw_dst=source_cidr, tp_dst=port,
nw_proto=protocol, actions='drop')
if action is "allow":
add_flow(bridge, priority= acl_priority, table=5, nw_src=tier_cidr, nw_dst=source_cidr, tp_dst=port,
nw_proto=protocol, actions='resubmit(,1)')
port = port + 1
if egress_rules_added is False:
# add a default rule in egress table to forward packet to L3 lookup table
add_flow(bridge, priority=0, table=3, actions='resubmit(,4)')
if ingress_rules_added is False:
# add a default rule in egress table drop packets
add_flow(bridge, priority=0, table=5, actions='drop')

View File

@ -27,6 +27,7 @@ import os
import sys
import subprocess
import time
import simplejson as json
from optparse import OptionParser, OptionGroup, OptParseError, BadOptionError, OptionError, OptionConflictError, OptionValueError
from time import localtime as _localtime, asctime as _asctime
@ -72,6 +73,58 @@ def setup_ovs_bridge(bridge, key, cs_host_id):
logging.debug("Setup_ovs_bridge completed with result:%s" % result)
return result
@echo
def setup_ovs_bridge_for_distributed_routing(bridge, cs_host_id):
res = lib.check_switch()
if res != "SUCCESS":
return "FAILURE:%s" % res
logging.debug("About to manually create the bridge:%s" % bridge)
res = lib.do_cmd([lib.VSCTL_PATH, "--", "--may-exist", "add-br", bridge])
logging.debug("Bridge has been manually created:%s" % res)
# Non empty result means something went wrong
if res:
result = "FAILURE:%s" % res
else:
# Verify the bridge actually exists
res = lib.do_cmd([lib.VSCTL_PATH, "list", "bridge", bridge])
res = lib.do_cmd([lib.VSCTL_PATH, "set", "bridge", bridge, "other_config:is-ovs_vpc_distributed_vr_network=True"])
conf_hosts = lib.do_cmd([lib.VSCTL_PATH, "get","bridge", bridge,"other:ovs-host-setup"])
conf_hosts = cs_host_id + (conf_hosts and ',%s' % conf_hosts or '')
lib.do_cmd([lib.VSCTL_PATH, "set", "bridge", bridge,
"other_config:ovs-host-setup=%s" % conf_hosts])
# add a default flow rule to send broadcast and multi-cast packets to L2 flooding table
lib.add_flow(bridge, priority=1000, dl_dst='ff:ff:ff:ff:ff:ff', table=0, actions='resubmit(,2)')
lib.add_flow(bridge, priority=1000, nw_dst='224.0.0.0/24', table=0, actions='resubmit(,2)')
# add a default flow rule to send uni-cast traffic to L2 lookup table
lib.add_flow(bridge, priority=0, table=0, actions='resubmit(,1)')
# add a default rule to send unknown mac address to L2 flooding table
lib.add_flow(bridge, priority=0, table=1, actions='resubmit(,2)')
# add a default rule in L2 flood table to drop packet
lib.add_flow(bridge, priority=0, table=2, actions='drop')
# add a default rule in egress table to forward packet to L3 lookup table
lib.add_flow(bridge, priority=0, table=3, actions='resubmit(,4)')
# add a default rule in L3 lookup table to forward packet to L2 lookup table
lib.add_flow(bridge, priority=0, table=4, actions='resubmit(,1)')
# add a default rule in ingress table to drop in bound packets
lib.add_flow(bridge, priority=0, table=5, actions='drop')
result = "SUCCESS: successfully setup bridge with flow rules"
logging.debug("Setup_ovs_bridge completed with result:%s" % result)
return result
def destroy_ovs_bridge(bridge):
res = lib.check_switch()
@ -163,12 +216,30 @@ def create_tunnel(bridge, remote_ip, key, src_host, dst_host):
# Ensure no trailing LF
if tun_ofport.endswith('\n'):
tun_ofport = tun_ofport[:-1]
# add flow entryies for dropping broadcast coming in from gre tunnel
lib.add_flow(bridge, priority=1000, in_port=tun_ofport,
ovs_tunnel_network = lib.do_cmd([lib.VSCTL_PATH, "get", "bridge", bridge, "other_config:is-ovs-tun-network"])
ovs_vpc_distributed_vr_network = lib.do_cmd([lib.VSCTL_PATH, "get", "bridge", bridge,
"other_config:is-ovs_vpc_distributed_vr_network"])
if ovs_tunnel_network == 'True':
# add flow entryies for dropping broadcast coming in from gre tunnel
lib.add_flow(bridge, priority=1000, in_port=tun_ofport,
dl_dst='ff:ff:ff:ff:ff:ff', actions='drop')
lib.add_flow(bridge, priority=1000, in_port=tun_ofport,
lib.add_flow(bridge, priority=1000, in_port=tun_ofport,
nw_dst='224.0.0.0/24', actions='drop')
drop_flow_setup = True
drop_flow_setup = True
if ovs_vpc_distributed_vr_network == 'True':
# add flow rules for dropping broadcast coming in from tunnel ports
lib.add_flow(bridge, priority=1000, in_port=tun_ofport, table=0,
dl_dst='ff:ff:ff:ff:ff:ff', actions='drop')
lib.add_flow(bridge, priority=1000, in_port=tun_ofport, table=0,
nw_dst='224.0.0.0/24', actions='drop')
# add flow rule to send the traffic from tunnel ports to L2 switching table only
lib.add_flow(bridge, priority=1000, in_port=tun_ofport, table=0, actions='resubmit(,1)')
lib.do_cmd([lib.VSCTL_PATH, "set", "interface", name, "options:cloudstack-network-id=%s" % network_uuid])
logging.debug("Broadcast drop rules added")
# return "SUCCESS:%s" % name
return 'true'
@ -210,6 +281,7 @@ if __name__ == '__main__':
parser.add_option("--src_host", dest="src_host")
parser.add_option("--dst_host", dest="dst_host")
parser.add_option("--iface_name", dest="iface_name")
parser.ad_option("--config", dest="config")
(option, args) = parser.parse_args()
if len(args) == 0:
logging.debug("No command to execute")
@ -223,6 +295,12 @@ if __name__ == '__main__':
create_tunnel(option.bridge, option.remote_ip, option.key, option.src_host, option.dst_host)
elif cmd == "destroy_tunnel":
destroy_tunnel(option.bridge, option.iface_name)
elif cmd == "setup_ovs_bridge_for_distributed_routing":
setup_ovs_bridge_for_distributed_routing(bridge, cs_host_id)
elif cmd == "configure_ovs_bridge_for_network_topology":
configure_bridge_for_network_topology(brdige, cs_host_id, config)
elif cmd == "configure_ovs_bridge_for_routing_policies":
configure_ovs_bridge_for_routing_policies(bridge, config)
else:
logging.debug("Unknown command: " + cmd)
sys.exit(1)