From a0ade283b7a92af4a8ca402a04c80a7200092bd7 Mon Sep 17 00:00:00 2001 From: Hugo Trippaers Date: Thu, 10 Jan 2013 18:30:07 +0100 Subject: [PATCH] Summary: Add initial support for OpenVswitch to the KVM hypervisor Create OvsVifDriver to deal with openvswitch specifics for plugging intefaces Create a parameter to set the bridge type to use in LibvirtComputingResource. Create several functions to get bridge information from openvswitch Add a check to detect the libvirt version and throw an exception when the version is to low ( < 0.9.11 ) Fix classpath loading in Script.findScript to deal with missing path separators at the end. Add notification to the BridgeVifDriver that lswitch broadcast type is not supported. --- .../kvm/resource/BridgeVifDriver.java | 3 + .../resource/LibvirtComputingResource.java | 97 +++++++- .../hypervisor/kvm/resource/LibvirtVMDef.java | 25 ++ .../hypervisor/kvm/resource/OvsVifDriver.java | 213 ++++++++++++++++++ utils/src/com/cloud/utils/script/Script.java | 20 +- 5 files changed, 348 insertions(+), 10 deletions(-) create mode 100644 plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index e6f2f7f376a..031721e8a3d 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -85,6 +85,9 @@ public class BridgeVifDriver extends VifDriverBase { URI broadcastUri = nic.getBroadcastUri(); vlanId = broadcastUri.getHost(); } + else if (nic.getBroadcastType() == Networks.BroadcastDomainType.Lswitch) { + throw new InternalErrorException("Nicira NVP Logicalswitches are not supported by the BridgeVifDriver"); + } String trafficLabel = nic.getName(); if (nic.getType() == Networks.TrafficType.Guest) { if (nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index b52e2d8a0b0..fade750c5f8 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -362,10 +362,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements private String _pingTestPath; private int _dom0MinMem; + + protected enum BridgeType { + NATIVE, OPENVSWITCH + } protected enum defineOps { UNDEFINE_VM, DEFINE_VM } + + protected BridgeType _bridgeType; private String getEndIpFromStartIp(String startIp, int numIps) { String[] tokens = startIp.split("[.]"); @@ -474,6 +480,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements if (storageScriptsDir == null) { storageScriptsDir = getDefaultStorageScriptsDir(); } + + String bridgeType = (String) params.get("network.bridge.type"); + if (bridgeType == null) { + _bridgeType = BridgeType.NATIVE; + } + else { + _bridgeType = BridgeType.valueOf(bridgeType.toUpperCase()); + } params.put("domr.scripts.dir", domrScriptsDir); @@ -650,11 +664,19 @@ public class LibvirtComputingResource extends ServerResourceBase implements LibvirtConnection.initialize(_hypervisorURI); Connect conn = null; - try { - conn = LibvirtConnection.getConnection(); - } catch (LibvirtException e) { - throw new CloudRuntimeException(e.getMessage()); - } + try { + conn = LibvirtConnection.getConnection(); + + if (_bridgeType == BridgeType.OPENVSWITCH) { + if (conn.getLibVirVersion() < (9 * 1000 + 11)) { + throw new ConfigurationException( + "LibVirt version 0.9.11 required for openvswitch support, but version " + + conn.getLibVirVersion() + " detected"); + } + } + } catch (LibvirtException e) { + throw new CloudRuntimeException(e.getMessage()); + } /* Does node support HVM guest? If not, exit */ if (!IsHVMEnabled(conn)) { @@ -697,7 +719,15 @@ public class LibvirtComputingResource extends ServerResourceBase implements } } - getPifs(); + switch (_bridgeType) { + case NATIVE: + getPifs(); + break; + case OPENVSWITCH: + getOvsPifs(); + break; + } + if (_pifs.get("private") == null) { s_logger.debug("Failed to get private nic name"); throw new ConfigurationException("Failed to get private nic name"); @@ -796,6 +826,26 @@ public class LibvirtComputingResource extends ServerResourceBase implements } s_logger.debug("done looking for pifs, no more bridges"); } + + private void getOvsPifs() { + String cmdout = Script.runSimpleBashScript("ovs-vsctl list-br | sed '{:q;N;s/\\n/%/g;t q}'"); + s_logger.debug("cmdout was " + cmdout); + List bridges = Arrays.asList(cmdout.split("%")); + for (String bridge : bridges) { + s_logger.debug("looking for pif for bridge " + bridge); + //String pif = getOvsPif(bridge); + // Not really interested in the pif name at this point for ovs bridges + String pif = bridge; + if(_publicBridgeName != null && bridge.equals(_publicBridgeName)){ + _pifs.put("public", pif); + } + if (_guestBridgeName != null && bridge.equals(_guestBridgeName)) { + _pifs.put("private", pif); + } + _pifs.put(bridge, pif); + } + s_logger.debug("done looking for pifs, no more bridges"); + } private String getPif(String bridge) { String pif = Script.runSimpleBashScript("brctl show | grep " + bridge + " | awk '{print $4}'"); @@ -808,11 +858,29 @@ public class LibvirtComputingResource extends ServerResourceBase implements return pif; } + private String getOvsPif(String bridge) { + String pif = Script.runSimpleBashScript("ovs-vsctl list-ports " + bridge); + return pif; + } + private boolean checkNetwork(String networkName) { if (networkName == null) { return true; } + if (_bridgeType == BridgeType.OPENVSWITCH) { + return checkOvsNetwork(networkName); + } + else { + return checkBridgeNetwork(networkName); + } + } + + private boolean checkBridgeNetwork(String networkName) { + if (networkName == null) { + return true; + } + String name = Script.runSimpleBashScript("brctl show | grep " + networkName + " | awk '{print $4}'"); if (name == null) { @@ -821,6 +889,23 @@ public class LibvirtComputingResource extends ServerResourceBase implements return true; } } + + private boolean checkOvsNetwork(String networkName) { + s_logger.debug("Checking if network " + networkName + " exists as openvswitch bridge"); + if (networkName == null) { + return true; + } + + Script command = new Script("/bin/sh", _timeout); + command.add("-c"); + command.add("ovs-vsctl br-exists " + networkName); + String result = command.execute(null); + if ("Ok".equals(result)) { + return true; + } else { + return false; + } + } private String getVnetId(String vnetId) { return vnetId; diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index ba6c715e455..17f6eef0b8e 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -646,6 +646,8 @@ public class LibvirtVMDef { private String _ipAddr; private String _scriptPath; private nicModel _model; + private String _virtualPortType; + private String _virtualPortInterfaceId; public void defBridgeNet(String brName, String targetBrName, String macAddr, nicModel model) { @@ -695,6 +697,22 @@ public class LibvirtVMDef { public String getMacAddress() { return _macAddr; } + + public void setVirtualPortType(String virtualPortType) { + _virtualPortType = virtualPortType; + } + + public String getVirtualPortType() { + return _virtualPortType; + } + + public void setVirtualPortInterfaceId(String virtualPortInterfaceId) { + _virtualPortInterfaceId = virtualPortInterfaceId; + } + + public String getVirtualPortInterfaceId() { + return _virtualPortInterfaceId; + } @Override public String toString() { @@ -714,6 +732,13 @@ public class LibvirtVMDef { if (_model != null) { netBuilder.append("\n"); } + if (_virtualPortType != null) { + netBuilder.append("\n"); + if (_virtualPortInterfaceId != null) { + netBuilder.append("\n"); + } + netBuilder.append("\n"); + } netBuilder.append("\n"); return netBuilder.toString(); } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java new file mode 100644 index 00000000000..6c3e4963ec5 --- /dev/null +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.hypervisor.kvm.resource; + +import java.net.URI; +import java.util.Map; + +import javax.naming.ConfigurationException; + +import org.apache.log4j.Logger; +import org.libvirt.LibvirtException; + +import com.cloud.agent.api.to.NicTO; +import com.cloud.exception.InternalErrorException; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; +import com.cloud.network.Networks; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.net.NetUtils; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; + +public class OvsVifDriver extends VifDriverBase { + private static final Logger s_logger = Logger + .getLogger(BridgeVifDriver.class); + private int _timeout; + private String _modifyVlanPath; + + @Override + public void configure(Map params) throws ConfigurationException { + super.configure(params); + + String networkScriptsDir = (String) params.get("network.scripts.dir"); + if (networkScriptsDir == null) { + networkScriptsDir = "scripts/vm/network/vnet"; + } + + String value = (String) params.get("scripts.timeout"); + _timeout = NumbersUtil.parseInt(value, 30 * 60) * 1000; + + _modifyVlanPath = Script.findScript(networkScriptsDir, "modifyvlan.sh"); + if (_modifyVlanPath == null) { + throw new ConfigurationException("Unable to find modifyvlan.sh"); + } + + createControlNetwork(_bridges.get("linklocal")); + } + + @Override + public InterfaceDef plug(NicTO nic, String guestOsType) + throws InternalErrorException, LibvirtException { + s_logger.debug("plugging nic=" + nic); + + LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef(); + intf.setVirtualPortType("openvswitch"); + + String vlanId = null; + String logicalSwitchUuid = null; + if (nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan) { + URI broadcastUri = nic.getBroadcastUri(); + vlanId = broadcastUri.getHost(); + } + else if (nic.getBroadcastType() == Networks.BroadcastDomainType.Lswitch) { + logicalSwitchUuid = nic.getBroadcastUri().getSchemeSpecificPart(); + } + String trafficLabel = nic.getName(); + if (nic.getType() == Networks.TrafficType.Guest) { + if (nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan + && !vlanId.equalsIgnoreCase("untagged")) { + if(trafficLabel != null && !trafficLabel.isEmpty()) { + s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel); + String brName = createVlanBr(vlanId, _pifs.get(trafficLabel)); + intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType)); + } else { + String brName = createVlanBr(vlanId, _pifs.get("private")); + intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType)); + } + } else if (nic.getBroadcastType() == Networks.BroadcastDomainType.Lswitch) { + s_logger.debug("nic " + nic + " needs to be connected to LogicalSwitch " + logicalSwitchUuid); + intf.setVirtualPortInterfaceId(nic.getUuid()); + String brName = (trafficLabel != null && !trafficLabel.isEmpty()) ? _pifs.get(trafficLabel) : _pifs.get("private"); + intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType)); + } + else { + intf.defBridgeNet(_bridges.get("guest"), null, nic.getMac(), getGuestNicModel(guestOsType)); + } + } else if (nic.getType() == Networks.TrafficType.Control) { + /* Make sure the network is still there */ + createControlNetwork(_bridges.get("linklocal")); + intf.defBridgeNet(_bridges.get("linklocal"), null, nic.getMac(), getGuestNicModel(guestOsType)); + } else if (nic.getType() == Networks.TrafficType.Public) { + if (nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan + && !vlanId.equalsIgnoreCase("untagged")) { + if(trafficLabel != null && !trafficLabel.isEmpty()){ + s_logger.debug("creating a vlan dev and bridge for public traffic per traffic label " + trafficLabel); + String brName = createVlanBr(vlanId, _pifs.get(trafficLabel)); + intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType)); + } else { + String brName = createVlanBr(vlanId, _pifs.get("public")); + intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType)); + } + } else { + intf.defBridgeNet(_bridges.get("public"), null, nic.getMac(), getGuestNicModel(guestOsType)); + } + } else if (nic.getType() == Networks.TrafficType.Management) { + intf.defBridgeNet(_bridges.get("private"), null, nic.getMac(), getGuestNicModel(guestOsType)); + } else if (nic.getType() == Networks.TrafficType.Storage) { + String storageBrName = nic.getName() == null ? _bridges.get("private") + : nic.getName(); + intf.defBridgeNet(storageBrName, null, nic.getMac(), getGuestNicModel(guestOsType)); + } + return intf; + } + + @Override + public void unplug(InterfaceDef iface) { + // Libvirt apparently takes care of this, see BridgeVifDriver unplug + } + + private String setVnetBrName(String pifName, String vnetId) { + String brName = "br" + pifName + "-"+ vnetId; + String oldStyleBrName = "cloudVirBr" + vnetId; + + if (isBridgeExists(oldStyleBrName)) { + s_logger.info("Using old style bridge name for vlan " + vnetId + " because existing bridge " + oldStyleBrName + " was found"); + brName = oldStyleBrName; + } + + return brName; + } + + private String createVlanBr(String vlanId, String nic) + throws InternalErrorException { + String brName = setVnetBrName(nic, vlanId); + createVnet(vlanId, nic, brName); + return brName; + } + + private void createVnet(String vnetId, String pif, String brName) + throws InternalErrorException { + final Script command = new Script(_modifyVlanPath, _timeout, s_logger); + command.add("-v", vnetId); + command.add("-p", pif); + command.add("-b", brName); + command.add("-o", "add"); + + final String result = command.execute(); + if (result != null) { + throw new InternalErrorException("Failed to create vnet " + vnetId + + ": " + result); + } + } + + private void deleteExitingLinkLocalRoutTable(String linkLocalBr) { + Script command = new Script("/bin/bash", _timeout); + command.add("-c"); + command.add("ip route | grep " + NetUtils.getLinkLocalCIDR()); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = command.execute(parser); + boolean foundLinkLocalBr = false; + if (result == null && parser.getLines() != null) { + String[] lines = parser.getLines().split("\\n"); + for (String line : lines) { + String[] tokens = line.split(" "); + if (!tokens[2].equalsIgnoreCase(linkLocalBr)) { + Script.runSimpleBashScript("ip route del " + NetUtils.getLinkLocalCIDR()); + } else { + foundLinkLocalBr = true; + } + } + } + if (!foundLinkLocalBr) { + Script.runSimpleBashScript("ifconfig " + linkLocalBr + " 169.254.0.1;" + "ip route add " + + NetUtils.getLinkLocalCIDR() + " dev " + linkLocalBr + " src " + NetUtils.getLinkLocalGateway()); + } + } + + private void createControlNetwork(String privBrName) { + deleteExitingLinkLocalRoutTable(privBrName); + if (!isBridgeExists(privBrName)) { + Script.runSimpleBashScript("ovs-vsctl add-br " + privBrName + "; ifconfig " + privBrName + " up; ifconfig " + + privBrName + " 169.254.0.1", _timeout); + } + + } + + private boolean isBridgeExists(String bridgeName) { + Script command = new Script("/bin/sh", _timeout); + command.add("-c"); + command.add("ovs-vsctl br-exists " + bridgeName); + String result = command.execute(null); + if ("Ok".equals(result)) { + return true; + } else { + return false; + } + } +} diff --git a/utils/src/com/cloud/utils/script/Script.java b/utils/src/com/cloud/utils/script/Script.java index 1444f83f425..d82d1d00957 100755 --- a/utils/src/com/cloud/utils/script/Script.java +++ b/utils/src/com/cloud/utils/script/Script.java @@ -195,7 +195,7 @@ public class Script implements Callable { } Task task = null; - if (interpreter.drain()) { + if (interpreter != null && interpreter.drain()) { task = new Task(interpreter, ir); s_executors.execute(task); } @@ -204,8 +204,13 @@ public class Script implements Callable { try { if (_process.waitFor() == 0) { _logger.debug("Execution is successful."); - - return interpreter.drain() ? task.getResult() : interpreter.interpret(ir); + if (interpreter != null) { + return interpreter.drain() ? task.getResult() : interpreter.interpret(ir); + } + else { + // null return is ok apparently + return (_process.exitValue() == 0) ? "Ok" : "Failed, exit code " + _process.exitValue(); + } } else { break; } @@ -239,7 +244,14 @@ public class Script implements Callable { BufferedReader reader = new BufferedReader(new InputStreamReader(_process.getInputStream()), 128); - String error = interpreter.processError(reader); + String error; + if (interpreter != null) { + error = interpreter.processError(reader); + } + else { + error = "Non zero exit code : " + _process.exitValue(); + } + if (_logger.isDebugEnabled()) { _logger.debug(error); }