apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Enable DPDK support on KVM (#2839)

Browse Source 
* Enable DPDK support on KVM

* Allow DPDK deployments on user VMs only

* Fix port name ordering
This commit is contained in:
Nicolas Vazquez 2018-11-07 09:29:01 -03:00 committed by GitHub
parent 604d2d1bd2
commit 4de4eabd18
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 422 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
agent/conf/agent.properties
View File

@ -108,6 +108,10 @@ domr.scripts.dir=scripts/network/domr/kvm
# openvswitch = com.cloud.hypervisor.kvm.resource.OvsVifDriver
#libvirt.vif.driver=com.cloud.hypervisor.kvm.resource.BridgeVifDriver
# Set DPDK Support on OpenVswitch
#openvswitch.dpdk.enabled=true
#openvswitch.dpdk.ovs.path=/var/run/openvswitch
# set the hypervisor type, values are: kvm, lxc
hypervisor.type=kvm

9
api/src/main/java/com/cloud/agent/api/to/NicTO.java
View File

@ -30,6 +30,7 @@ public class NicTO extends NetworkTO {
String nicUuid;
List<String> nicSecIps;
Map<NetworkOffering.Detail, String> details;
boolean dpdkDisabled;
public NicTO() {
super();
@ -109,4 +110,12 @@ public class NicTO extends NetworkTO {
public void setDetails(final Map<NetworkOffering.Detail, String> details) {
this.details = details;
}
public boolean isDpdkDisabled() {
return dpdkDisabled;
}
public void setDpdkDisabled(boolean dpdkDisabled) {
this.dpdkDisabled = dpdkDisabled;
}
}

8
api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java
View File

@ -73,6 +73,7 @@ public class VirtualMachineTO {
Double cpuQuotaPercentage = null;
Map<String, String> guestOsDetails = new HashMap<String, String>();
Map<String, String> extraConfig = new HashMap<>();
public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader,
String os, boolean enableHA, boolean limitCpuUse, String vncPassword) {
@ -350,4 +351,11 @@ public class VirtualMachineTO {
public void setCpuQuotaPercentage(Double cpuQuotaPercentage) {
this.cpuQuotaPercentage = cpuQuotaPercentage;
}
public void addExtraConfig(String key, String value) {
extraConfig.put(key, value);
}
public Map<String, String> getExtraConfig() {
return extraConfig;
}
}

1
api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
View File

@ -117,6 +117,7 @@ public class ApiConstants {
public static final String END_PORT = "endport";
public static final String ENTRY_TIME = "entrytime";
public static final String EXPIRES = "expires";
public static final String EXTRA_CONFIG = "extraconfig";
public static final String EXTRA_DHCP_OPTION = "extradhcpoption";
public static final String EXTRA_DHCP_OPTION_NAME = "extradhcpoptionname";
public static final String EXTRA_DHCP_OPTION_CODE = "extradhcpoptioncode";

7
api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java
View File

@ -200,6 +200,9 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
" an optional parameter used to create additional data disks from datadisk templates; can't be specified with diskOfferingId parameter")
private Map dataDiskTemplateToDiskOfferingList;
@Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", length = 5120)
private String extraConfig;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -482,6 +485,10 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
return dataDiskTemplateToDiskOfferingMap;
}
public String getExtraConfig() {
return extraConfig;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////

6
api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java
View File

@ -128,6 +128,8 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction
+ " Example: dhcpoptionsnetworklist[0].dhcp:114=url&dhcpoptionsetworklist[0].networkid=networkid&dhcpoptionsetworklist[0].dhcp:66=www.test.com")
private Map dhcpOptionsNetworkList;
@Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", authorized = { RoleType.Admin }, length = 5120)
private String extraConfig;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
@ -221,6 +223,10 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction
return dhcpOptionsMap;
}
public String getExtraConfig() {
return extraConfig;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////

12
engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
View File

@ -39,6 +39,7 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.context.CallContext;
@ -1112,6 +1113,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx);
addExtraConfig(vmTO);
work = _workDao.findById(work.getId());
if (work == null || work.getStep() != Step.Prepare) {
throw new ConcurrentOperationException("Work steps have been changed: " + work);
@ -1276,6 +1279,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
}
private void addExtraConfig(VirtualMachineTO vmTO) {
Map<String, String> details = vmTO.getDetails();
for (String key : details.keySet()) {
if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
vmTO.addExtraConfig(key, details.get(key));
}
}
}
// for managed storage on KVM, need to make sure the path field of the volume in question is populated with the IQN
private void handlePath(final DiskTO[] disks, final HypervisorType hypervisorType) {
if (hypervisorType != HypervisorType.KVM) {

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java
View File

@ -207,7 +207,7 @@ public class BridgeVifDriver extends VifDriverBase {
}
@Override
public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException {
public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
if (s_logger.isDebugEnabled()) {
s_logger.debug("nic=" + nic);

5
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/DirectVifDriver.java
View File

@ -26,6 +26,8 @@ import com.cloud.agent.api.to.NicTO;
import com.cloud.exception.InternalErrorException;
import com.cloud.network.Networks;
import java.util.Map;
public class DirectVifDriver extends VifDriverBase {
private static final Logger s_logger = Logger.getLogger(DirectVifDriver.class);
@ -36,12 +38,13 @@ public class DirectVifDriver extends VifDriverBase {
*
* @param nic
* @param guestOsType
* @param extraConfig
* @return
* @throws InternalErrorException
* @throws LibvirtException
*/
@Override
public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException {
public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef();
if (nic.getType() == Networks.TrafficType.Guest) {

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/IvsVifDriver.java
View File

@ -77,7 +77,7 @@ public class IvsVifDriver extends VifDriverBase {
}
@Override
public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException {
public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef();
String vNetId = null;

81
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -48,6 +48,7 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import com.cloud.resource.RequestWrapper;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
@ -56,6 +57,7 @@ import org.apache.cloudstack.utils.linux.CPUStat;
import org.apache.cloudstack.utils.linux.MemStat;
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
import org.apache.cloudstack.utils.security.KeyStoreUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
@ -521,6 +523,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
protected StorageSubsystemCommandHandler storageHandler;
protected boolean dpdkSupport = false;
protected String dpdkOvsPath;
protected static final String DPDK_NUMA = ApiConstants.EXTRA_CONFIG + "-dpdk-numa";
protected static final String DPDK_HUGE_PAGES = ApiConstants.EXTRA_CONFIG + "-dpdk-hugepages";
protected static final String DPDK_INTERFACE_PREFIX = ApiConstants.EXTRA_CONFIG + "-dpdk-interface-";
private String getEndIpFromStartIp(final String startIp, final int numIps) {
final String[] tokens = startIp.split("[.]");
assert tokens.length == 4;
@ -637,6 +645,15 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
_bridgeType = BridgeType.valueOf(bridgeType.toUpperCase());
}
String dpdk = (String) params.get("openvswitch.dpdk.enabled");
if (_bridgeType == BridgeType.OPENVSWITCH && Boolean.parseBoolean(dpdk)) {
dpdkSupport = true;
dpdkOvsPath = (String) params.get("openvswitch.dpdk.ovs.path");
if (dpdkOvsPath != null && !dpdkOvsPath.endsWith("/")) {
dpdkOvsPath += "/";
}
}
params.put("domr.scripts.dir", domrScriptsDir);
_virtRouterResource = new VirtualRoutingResource(this);
@ -1634,7 +1651,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
}
final Domain vm = getDomain(conn, vmName);
vm.attachDevice(getVifDriver(nicTO.getType()).plug(nicTO, "Other PV", "").toString());
vm.attachDevice(getVifDriver(nicTO.getType()).plug(nicTO, "Other PV", "", null).toString());
}
@ -2039,6 +2056,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
vm.setDomDescription(vmTO.getOs());
vm.setPlatformEmulator(vmTO.getPlatformEmulator());
Map<String, String> extraConfig = vmTO.getExtraConfig();
if (dpdkSupport && (!extraConfig.containsKey(DPDK_NUMA) || !extraConfig.containsKey(DPDK_HUGE_PAGES))) {
s_logger.info("DPDK is enabled but it needs extra configurations for CPU NUMA and Huge Pages for VM deployment");
}
final GuestDef guest = new GuestDef();
if (HypervisorType.LXC == _hypervisorType && VirtualMachine.Type.User == vmTO.getType()) {
@ -2072,21 +2094,23 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
grd.setVcpuNum(vcpus);
vm.addComp(grd);
final CpuModeDef cmd = new CpuModeDef();
cmd.setMode(_guestCpuMode);
cmd.setModel(_guestCpuModel);
if (vmTO.getType() == VirtualMachine.Type.User) {
cmd.setFeatures(_cpuFeatures);
if (!extraConfig.containsKey(DPDK_NUMA)) {
final CpuModeDef cmd = new CpuModeDef();
cmd.setMode(_guestCpuMode);
cmd.setModel(_guestCpuModel);
if (vmTO.getType() == VirtualMachine.Type.User) {
cmd.setFeatures(_cpuFeatures);
}
// multi cores per socket, for larger core configs
if (vcpus % 6 == 0) {
final int sockets = vcpus / 6;
cmd.setTopology(6, sockets);
} else if (vcpus % 4 == 0) {
final int sockets = vcpus / 4;
cmd.setTopology(4, sockets);
}
vm.addComp(cmd);
}
// multi cores per socket, for larger core configs
if (vcpus % 6 == 0) {
final int sockets = vcpus / 6;
cmd.setTopology(6, sockets);
} else if (vcpus % 4 == 0) {
final int sockets = vcpus / 4;
cmd.setTopology(4, sockets);
}
vm.addComp(cmd);
if (_hypervisorLibvirtVersion >= 9000) {
final CpuTuneDef ctd = new CpuTuneDef();
@ -2192,9 +2216,29 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
vm.addComp(devices);
addExtraConfigComponent(extraConfig, vm);
return vm;
}
/**
* Add extra configurations (if any) as a String component to the domain XML
*/
protected void addExtraConfigComponent(Map<String, String> extraConfig, LibvirtVMDef vm) {
if (MapUtils.isNotEmpty(extraConfig)) {
StringBuilder extraConfigBuilder = new StringBuilder();
for (String key : extraConfig.keySet()) {
if (!key.startsWith(DPDK_INTERFACE_PREFIX)) {
extraConfigBuilder.append(extraConfig.get(key));
}
}
String comp = extraConfigBuilder.toString();
if (org.apache.commons.lang.StringUtils.isNotBlank(comp)) {
vm.addComp(comp);
}
}
}
public void createVifs(final VirtualMachineTO vmSpec, final LibvirtVMDef vm) throws InternalErrorException, LibvirtException {
final NicTO[] nics = vmSpec.getNics();
final Map <String, String> params = vmSpec.getDetails();
@ -2202,10 +2246,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
if (params != null && params.get("nicAdapter") != null && !params.get("nicAdapter").isEmpty()) {
nicAdapter = params.get("nicAdapter");
}
Map<String, String> extraConfig = vmSpec.getExtraConfig();
for (int i = 0; i < nics.length; i++) {
for (final NicTO nic : vmSpec.getNics()) {
if (nic.getDeviceId() == i) {
createVif(vm, nic, nicAdapter);
createVif(vm, nic, nicAdapter, extraConfig);
}
}
}
@ -2400,7 +2445,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
}
private void createVif(final LibvirtVMDef vm, final NicTO nic, final String nicAdapter) throws InternalErrorException, LibvirtException {
private void createVif(final LibvirtVMDef vm, final NicTO nic, final String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
if (nic.getType().equals(TrafficType.Guest) && nic.getBroadcastType().equals(BroadcastDomainType.Vsp)) {
String vrIp = nic.getBroadcastUri().getPath().substring(1);
@ -2415,7 +2460,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
s_logger.error("LibvirtVMDef object get devices with null result");
throw new InternalErrorException("LibvirtVMDef object get devices with null result");
}
vm.getDevices().addDevice(getVifDriver(nic.getType(), nic.getName()).plug(nic, vm.getPlatformEmulator(), nicAdapter));
vm.getDevices().addDevice(getVifDriver(nic.getType(), nic.getName()).plug(nic, vm.getPlatformEmulator(), nicAdapter, extraConfig));
}
public boolean cleanupDisk(Map<String, String> volumeToDisconnect) {

5
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java
View File

@ -182,6 +182,11 @@ public class LibvirtDomainXMLParser {
} else if (type.equalsIgnoreCase("ethernet")) {
String scriptPath = getAttrValue("script", "path", nic);
def.defEthernet(dev, mac, NicModel.valueOf(model.toUpperCase()), scriptPath, networkRateKBps);
} else if (type.equals("vhostuser")) {
String sourcePort = getAttrValue("source", "path", nic);
String[] sourcePathParts = sourcePort.split("/");
String port = sourcePathParts[sourcePathParts.length - 1];
def.setDpdkSourcePort(port);
}
if (StringUtils.isNotBlank(slot)) {

35
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java
View File

@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import com.google.common.collect.Maps;
@ -900,7 +901,7 @@ public class LibvirtVMDef {
public static class InterfaceDef {
enum GuestNetType {
BRIDGE("bridge"), DIRECT("direct"), NETWORK("network"), USER("user"), ETHERNET("ethernet"), INTERNAL("internal");
BRIDGE("bridge"), DIRECT("direct"), NETWORK("network"), USER("user"), ETHERNET("ethernet"), INTERNAL("internal"), VHOSTUSER("vhostuser");
String _type;
GuestNetType(String type) {
@ -933,7 +934,7 @@ public class LibvirtVMDef {
private GuestNetType _netType; /*
* bridge, ethernet, network, user,
* internal
* internal, vhostuser
*/
private HostNicType _hostNetType; /* Only used by agent java code */
private String _netSourceMode;
@ -950,6 +951,9 @@ public class LibvirtVMDef {
private boolean _pxeDisable = false;
private boolean _linkStateUp = true;
private Integer _slot;
private String _dpdkSourcePath;
private String _dpdkSourcePort;
private String _dpdkExtraLines;
public void defBridgeNet(String brName, String targetBrName, String macAddr, NicModel model) {
defBridgeNet(brName, targetBrName, macAddr, model, 0);
@ -964,6 +968,16 @@ public class LibvirtVMDef {
_networkRateKBps = networkRateKBps;
}
public void defDpdkNet(String dpdkSourcePath, String dpdkPort, String macAddress, NicModel model, Integer networkRateKBps, String extra) {
_netType = GuestNetType.VHOSTUSER;
_dpdkSourcePath = dpdkSourcePath;
_dpdkSourcePort = dpdkPort;
_macAddr = macAddress;
_model = model;
_networkRateKBps = networkRateKBps;
_dpdkExtraLines = extra;
}
public void defDirectNet(String sourceName, String targetName, String macAddr, NicModel model, String sourceMode) {
defDirectNet(sourceName, targetName, macAddr, model, sourceMode, 0);
}
@ -1089,6 +1103,13 @@ public class LibvirtVMDef {
return _linkStateUp;
}
public String getDpdkSourcePort() {
return _dpdkSourcePort;
}
public void setDpdkSourcePort(String port) {
_dpdkSourcePort = port;
}
@Override
public String toString() {
StringBuilder netBuilder = new StringBuilder();
@ -1099,6 +1120,8 @@ public class LibvirtVMDef {
netBuilder.append("<source network='" + _sourceName + "'/>\n");
} else if (_netType == GuestNetType.DIRECT) {
netBuilder.append("<source dev='" + _sourceName + "' mode='" + _netSourceMode + "'/>\n");
} else if (_netType == GuestNetType.VHOSTUSER) {
netBuilder.append("<source type='unix' path='"+ _dpdkSourcePath + _dpdkSourcePort + "' mode='client'/>\n");
}
if (_networkName != null) {
netBuilder.append("<target dev='" + _networkName + "'/>\n");
@ -1132,7 +1155,13 @@ public class LibvirtVMDef {
netBuilder.append("<vlan trunk='no'>\n<tag id='" + _vlanTag + "'/>\n</vlan>");
}
netBuilder.append("<link state='" + (_linkStateUp ? "up" : "down") +"'/>\n");
if (StringUtils.isNotBlank(_dpdkExtraLines)) {
netBuilder.append(_dpdkExtraLines);
}
if (_netType != GuestNetType.VHOSTUSER) {
netBuilder.append("<link state='" + (_linkStateUp ? "up" : "down") +"'/>\n");
}
if (_slot != null) {
netBuilder.append(String.format("<address type='pci' domain='0x0000' bus='0x00' slot='0x%02x' function='0x0'/>\n", _slot));

84
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java
View File

@ -24,6 +24,8 @@ import java.util.Map;
import javax.naming.ConfigurationException;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.libvirt.LibvirtException;
@ -40,6 +42,8 @@ public class OvsVifDriver extends VifDriverBase {
private static final Logger s_logger = Logger.getLogger(OvsVifDriver.class);
private int _timeout;
protected static final String DPDK_PORT_PREFIX = "csdpdk-";
@Override
public void configure(Map<String, Object> params) throws ConfigurationException {
super.configure(params);
@ -76,12 +80,65 @@ public class OvsVifDriver extends VifDriverBase {
s_logger.debug("done looking for pifs, no more bridges");
}
/**
* Get the latest DPDK port number created on a DPDK enabled host
*/
protected int getDpdkLatestPortNumberUsed() {
s_logger.debug("Checking the last DPDK port created");
String cmd = "ovs-vsctl show | grep Port | grep " + DPDK_PORT_PREFIX + " | " +
"awk '{ print $2 }' | sort -rV | head -1";
String port = Script.runSimpleBashScript(cmd);
int portNumber = 0;
if (StringUtils.isNotBlank(port)) {
String unquotedPort = port.replace("\"", "");
String dpdkPortNumber = unquotedPort.split(DPDK_PORT_PREFIX)[1];
portNumber = Integer.valueOf(dpdkPortNumber);
}
return portNumber;
}
/**
* Get the next DPDK port name to be created
*/
protected String getNextDpdkPort() {
int portNumber = getDpdkLatestPortNumberUsed();
return DPDK_PORT_PREFIX + String.valueOf(portNumber + 1);
}
/**
* Add OVS port (if it does not exist) to bridge with DPDK support
*/
protected void addDpdkPort(String bridgeName, String port, String vlan) {
String cmd = String.format("ovs-vsctl add-port %s %s " +
"vlan_mode=access tag=%s " +
"-- set Interface %s type=dpdkvhostuser", bridgeName, port, vlan, port);
s_logger.debug("DPDK property enabled, executing: " + cmd);
Script.runSimpleBashScript(cmd);
}
/**
* Check for additional extra 'dpdk-interface' configurations, return them appended
*/
private String getExtraDpdkProperties(Map<String, String> extraConfig) {
StringBuilder stringBuilder = new StringBuilder();
for (String key : extraConfig.keySet()) {
if (key.startsWith(LibvirtComputingResource.DPDK_INTERFACE_PREFIX)) {
stringBuilder.append(extraConfig.get(key));
}
}
return stringBuilder.toString();
}
@Override
public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException {
public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
s_logger.debug("plugging nic=" + nic);
LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef();
intf.setVirtualPortType("openvswitch");
if (!_libvirtComputingResource.dpdkSupport || nic.isDpdkDisabled()) {
// Let libvirt handle OVS ports creation when DPDK property is disabled or when it is enabled but disabled for the nic
// For DPDK support, libvirt does not handle ports creation, invoke 'addDpdkPort' method
intf.setVirtualPortType("openvswitch");
}
String vlanId = null;
String logicalSwitchUuid = null;
@ -99,9 +156,19 @@ public class OvsVifDriver extends VifDriverBase {
if ((nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan || nic.getBroadcastType() == Networks.BroadcastDomainType.Pvlan) &&
!vlanId.equalsIgnoreCase("untagged")) {
if (trafficLabel != null && !trafficLabel.isEmpty()) {
s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel);
intf.defBridgeNet(_pifs.get(trafficLabel), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);
intf.setVlanTag(Integer.parseInt(vlanId));
if (_libvirtComputingResource.dpdkSupport && !nic.isDpdkDisabled()) {
s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
if (StringUtils.isBlank(_libvirtComputingResource.dpdkOvsPath)) {
throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
}
String port = getNextDpdkPort();
addDpdkPort(_pifs.get(trafficLabel), port, vlanId);
intf.defDpdkNet(_libvirtComputingResource.dpdkOvsPath, port, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), 0, getExtraDpdkProperties(extraConfig));
} else {
s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel);
intf.defBridgeNet(_pifs.get(trafficLabel), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);
intf.setVlanTag(Integer.parseInt(vlanId));
}
} else {
intf.defBridgeNet(_pifs.get("private"), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);
intf.setVlanTag(Integer.parseInt(vlanId));
@ -153,6 +220,13 @@ public class OvsVifDriver extends VifDriverBase {
@Override
public void unplug(InterfaceDef iface) {
// Libvirt apparently takes care of this, see BridgeVifDriver unplug
if (_libvirtComputingResource.dpdkSupport) {
// If DPDK is enabled, we'll need to cleanup the port as libvirt won't
String dpdkPort = iface.getDpdkSourcePort();
String cmd = String.format("ovs-vsctl del-port %s", dpdkPort);
s_logger.debug("Removing DPDK port: " + dpdkPort);
Script.runSimpleBashScript(cmd);
}
}

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java
View File

@ -32,7 +32,7 @@ public interface VifDriver {
public void configure(Map<String, Object> params) throws ConfigurationException;
public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException;
public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException;
public void unplug(LibvirtVMDef.InterfaceDef iface);

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriverBase.java
View File

@ -42,7 +42,7 @@ public abstract class VifDriverBase implements VifDriver {
}
@Override
public abstract LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException;
public abstract LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException;
@Override
public abstract void unplug(LibvirtVMDef.InterfaceDef iface);

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java
View File

@ -61,7 +61,7 @@ public final class LibvirtPlugNicCommandWrapper extends CommandWrapper<PlugNicCo
nicnum++;
}
final VifDriver vifDriver = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName());
final InterfaceDef interfaceDef = vifDriver.plug(nic, "Other PV", "");
final InterfaceDef interfaceDef = vifDriver.plug(nic, "Other PV", "", null);
vm.attachDevice(interfaceDef.toString());
return new PlugNicAnswer(command, true, "success");

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java
View File

@ -64,7 +64,7 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp
final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vm.getName());
for (final NicTO nic : nics) {
libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).plug(nic, null, "");
libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).plug(nic, null, "", null);
}
/* setup disks, e.g for iso */

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReplugNicCommandWrapper.java
View File

@ -65,7 +65,7 @@ public final class LibvirtReplugNicCommandWrapper extends CommandWrapper<ReplugN
InterfaceDef oldPluggedNic = findPluggedNic(libvirtComputingResource, nic, vmName, conn);
final VifDriver newVifDriver = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName());
final InterfaceDef interfaceDef = newVifDriver.plug(nic, "Other PV", oldPluggedNic.getModel().toString());
final InterfaceDef interfaceDef = newVifDriver.plug(nic, "Other PV", oldPluggedNic.getModel().toString(), null);
interfaceDef.setSlot(oldPluggedNic.getSlot());
interfaceDef.setDevName(oldPluggedNic.getDevName());

1
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java
View File

@ -69,6 +69,7 @@ public final class LibvirtStartCommandWrapper extends CommandWrapper<StartComman
for (final NicTO nic : nics) {
if (vmSpec.getType() != VirtualMachine.Type.User) {
nic.setPxeDisable(true);
nic.setDpdkDisabled(true);
}
}

32
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java
View File

@ -26,7 +26,9 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.UUID;
import java.util.Vector;
@ -176,8 +178,10 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ -189,6 +193,8 @@ public class LibvirtComputingResourceTest {
private LibvirtComputingResource libvirtComputingResource;
@Mock
VirtualMachineTO vmTO;
@Mock
LibvirtVMDef vmDef;
String hyperVisorType = "kvm";
Random random = new Random();
@ -3165,7 +3171,7 @@ public class LibvirtComputingResourceTest {
when(libvirtComputingResource.getVifDriver(nic.getType(), nic.getName())).thenReturn(vifDriver);
when(vifDriver.plug(nic, "Other PV", "")).thenReturn(interfaceDef);
when(vifDriver.plug(nic, "Other PV", "", null)).thenReturn(interfaceDef);
when(interfaceDef.toString()).thenReturn("Interface");
final String interfaceDefStr = interfaceDef.toString();
@ -3188,7 +3194,7 @@ public class LibvirtComputingResourceTest {
verify(libvirtUtilitiesHelper, times(1)).getConnectionByVmName(command.getVmName());
verify(libvirtComputingResource, times(1)).getDomain(conn, instanceName);
verify(libvirtComputingResource, times(1)).getVifDriver(nic.getType(), nic.getName());
verify(vifDriver, times(1)).plug(nic, "Other PV", "");
verify(vifDriver, times(1)).plug(nic, "Other PV", "", null);
} catch (final LibvirtException e) {
fail(e.getMessage());
} catch (final InternalErrorException e) {
@ -3262,7 +3268,7 @@ public class LibvirtComputingResourceTest {
when(libvirtComputingResource.getVifDriver(nic.getType(), nic.getName())).thenReturn(vifDriver);
when(vifDriver.plug(nic, "Other PV", "")).thenThrow(InternalErrorException.class);
when(vifDriver.plug(nic, "Other PV", "", null)).thenThrow(InternalErrorException.class);
} catch (final LibvirtException e) {
fail(e.getMessage());
@ -3281,7 +3287,7 @@ public class LibvirtComputingResourceTest {
verify(libvirtUtilitiesHelper, times(1)).getConnectionByVmName(command.getVmName());
verify(libvirtComputingResource, times(1)).getDomain(conn, instanceName);
verify(libvirtComputingResource, times(1)).getVifDriver(nic.getType(), nic.getName());
verify(vifDriver, times(1)).plug(nic, "Other PV", "");
verify(vifDriver, times(1)).plug(nic, "Other PV", "", null);
} catch (final LibvirtException e) {
fail(e.getMessage());
} catch (final InternalErrorException e) {
@ -5217,4 +5223,22 @@ public class LibvirtComputingResourceTest {
assertFalse(ans instanceof UnsupportedAnswer);
assertTrue(ans instanceof Answer);
}
@Test
public void testAddExtraConfigComponentEmptyExtraConfig() {
libvirtComputingResource = new LibvirtComputingResource();
libvirtComputingResource.addExtraConfigComponent(new HashMap<>(), vmDef);
Mockito.verify(vmDef, never()).addComp(any());
}
@Test
public void testAddExtraConfigComponentNotEmptyExtraConfig() {
libvirtComputingResource = new LibvirtComputingResource();
Map<String, String> extraConfig = new HashMap<>();
extraConfig.put("extraconfig-1", "value1");
extraConfig.put("extraconfig-2", "value2");
extraConfig.put("extraconfig-3", "value3");
libvirtComputingResource.addExtraConfigComponent(extraConfig, vmDef);
Mockito.verify(vmDef, times(1)).addComp(any());
}
}

75
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/OvsVifDriverTest.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.hypervisor.kvm.resource;
import com.cloud.utils.script.Script;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Matchers;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@PrepareForTest({ Script.class })
@RunWith(PowerMockRunner.class)
public class OvsVifDriverTest {
private static final int dpdkPortNumber = 7;
private OvsVifDriver driver = new OvsVifDriver();
@Before
public void initMocks() {
MockitoAnnotations.initMocks(this);
PowerMockito.mockStatic(Script.class);
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).thenReturn(null);
}
@Test
public void testGetDpdkLatestPortNumberUsedNoDpdkPorts() {
Assert.assertEquals(0, driver.getDpdkLatestPortNumberUsed());
}
@Test
public void testGetDpdkLatestPortNumberUsedExistingDpdkPorts() {
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
thenReturn(OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
Assert.assertEquals(dpdkPortNumber, driver.getDpdkLatestPortNumberUsed());
}
@Test
public void testGetNextDpdkPortNoDpdkPorts() {
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
thenReturn(null);
String expectedPortName = OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(1);
Assert.assertEquals(expectedPortName, driver.getNextDpdkPort());
}
@Test
public void testGetNextDpdkPortExistingDpdkPorts() {
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
thenReturn(OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
String expectedPortName = OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber + 1);
Assert.assertEquals(expectedPortName, driver.getNextDpdkPort());
}
}

85
server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
View File

@ -44,6 +44,7 @@ import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import com.cloud.user.AccountVO;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
import org.apache.cloudstack.affinity.AffinityGroupService;
@ -514,6 +515,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
private static final ConfigKey<Boolean> AllowDeployVmIfGivenHostFails = new ConfigKey<Boolean>("Advanced", Boolean.class, "allow.deploy.vm.if.deploy.on.given.host.fails", "false",
"allow vm to deploy on different host if vm fails to deploy on the given host ", true);
private static final ConfigKey<Boolean> EnableAdditionalVmConfig = new ConfigKey<>("Advanced", Boolean.class, "enable.additional.vm.configuration",
"false", "allow additional arbitrary configuration to vm", true, ConfigKey.Scope.Account);
@Override
public UserVmVO getVirtualMachine(long vmId) {
@ -2388,8 +2391,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
Map<String,String> details = cmd.getDetails();
List<Long> securityGroupIdList = getSecurityGroupIdList(cmd);
boolean cleanupDetails = cmd.isCleanupDetails();
String extraConfig = cmd.getExtraConfig();
UserVmVO vmInstance = _vmDao.findById(cmd.getId());
long accountId = vmInstance.getAccountId();
if (isDisplayVm != null && isDisplayVm != vmInstance.isDisplay()) {
updateDisplayVmFlag(isDisplayVm, id, vmInstance);
@ -2397,9 +2402,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
if (cleanupDetails){
userVmDetailsDao.removeDetails(id);
}
else if (MapUtils.isNotEmpty(details)) {
vmInstance.setDetails(details);
_vmDao.saveDetails(vmInstance);
else {
if (MapUtils.isNotEmpty(details)) {
vmInstance.setDetails(details);
_vmDao.saveDetails(vmInstance);
}
if (StringUtils.isNotBlank(extraConfig) && EnableAdditionalVmConfig.valueIn(accountId)) {
AccountVO account = _accountDao.findById(accountId);
addExtraConfig(vmInstance, account, extraConfig);
}
}
return updateVirtualMachine(id, displayName, group, ha, isDisplayVm, osTypeId, userData, isDynamicallyScalable,
cmd.getHttpMethod(), cmd.getCustomId(), hostName, cmd.getInstanceName(), securityGroupIdList, cmd.getDhcpOptionsMap());
@ -4847,9 +4858,75 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
_tmplService.attachIso(tmpl.getId(), vm.getId());
}
}
// Add extraConfig to user_vm_details table
Account caller = CallContext.current().getCallingAccount();
Long callerId = caller.getId();
String extraConfig = cmd.getExtraConfig();
if (StringUtils.isNotBlank(extraConfig) && EnableAdditionalVmConfig.valueIn(callerId) ) {
addExtraConfig(vm, caller, extraConfig);
}
return vm;
}
/**
* Persist extra configurations as details for VMware VMs
*/
protected void persistExtraConfigVmware(String decodedUrl, UserVm vm) {
String[] configDataArr = decodedUrl.split("\\r?\\n");
for (String config: configDataArr) {
String[] keyValue = config.split("=");
try {
userVmDetailsDao.addDetail(vm.getId(), keyValue[0], keyValue[1], true);
} catch (ArrayIndexOutOfBoundsException e) {
throw new CloudRuntimeException("Issue occurred during parsing of:" + config);
}
}
}
/**
* Persist extra configurations as details for hypervisors except Vmware
*/
protected void persistExtraConfigNonVmware(String decodedUrl, UserVm vm) {
String[] extraConfigs = decodedUrl.split("\n\n");
for (String cfg : extraConfigs) {
int i = 1;
String[] cfgParts = cfg.split("\n");
String extraConfigKey = ApiConstants.EXTRA_CONFIG;
String extraConfigValue;
if (cfgParts[0].matches("\\S+:$")) {
extraConfigKey += "-" + cfgParts[0].substring(0,cfgParts[0].length() - 1);
extraConfigValue = cfg.replace(cfgParts[0] + "\n", "");
} else {
extraConfigKey += "-" + String.valueOf(i);
extraConfigValue = cfg;
}
userVmDetailsDao.addDetail(vm.getId(), extraConfigKey, extraConfigValue, true);
i++;
}
}
protected void addExtraConfig(UserVm vm, Account caller, String extraConfig) {
String decodedUrl = decodeExtraConfig(extraConfig);
HypervisorType hypervisorType = vm.getHypervisorType();
if (hypervisorType == HypervisorType.VMware) {
persistExtraConfigVmware(decodedUrl, vm);
} else {
persistExtraConfigNonVmware(decodedUrl, vm);
}
}
protected String decodeExtraConfig(String encodeString) {
String decodedUrl;
try {
decodedUrl = URLDecoder.decode(encodeString, "UTF-8");
} catch (UnsupportedEncodingException e) {
throw new CloudRuntimeException("Failed to provided decode URL string: " + e.getMessage());
}
return decodedUrl;
}
protected List<Long> getSecurityGroupIdList(SecurityGroupAction cmd) {
if (cmd.getSecurityGroupNameList() != null && cmd.getSecurityGroupIdList() != null) {
throw new InvalidParameterValueException("securitygroupids parameter is mutually exclusive with securitygroupnames parameter");
@ -6394,7 +6471,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {EnableDynamicallyScaleVm, AllowUserExpungeRecoverVm, VmIpFetchWaitInterval, VmIpFetchTrialMax, VmIpFetchThreadPoolMax,
VmIpFetchTaskWorkers, AllowDeployVmIfGivenHostFails};
VmIpFetchTaskWorkers, AllowDeployVmIfGivenHostFails, EnableAdditionalVmConfig};
}
@Override