apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-11-02 20:02:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

KVM: DPDK live migrations (#3365)

Browse Source 
* DPDK live migrations

* Remove DPDK created ports if VM migration fails or prepare migration fails

* Rename DPDK classes lowercase
This commit is contained in:
Nicolas Vazquez 2019-06-25 12:23:09 -03:00 committed by GitHub
parent 1ef05984c6
commit a75444a585
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 980 additions and 350 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
api/src/main/java/com/cloud/agent/api/to/DpdkTO.java Normal file
View File

@ -0,0 +1,45 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.agent.api.to;
public class DpdkTO {
private String path;
private String port;
private String mode;
public DpdkTO() {
}
public DpdkTO(String path, String port, String mode) {
this.path = path;
this.port = port;
this.mode = mode;
}
public String getPath() {
return path;
}
public String getPort() {
return port;
}
public String getMode() {
return mode;
}
}

10
api/src/main/java/com/cloud/agent/api/to/NicTO.java
View File

@ -30,7 +30,7 @@ public class NicTO extends NetworkTO {
String nicUuid;
List<String> nicSecIps;
Map<NetworkOffering.Detail, String> details;
boolean dpdkDisabled;
boolean dpdkEnabled;
public NicTO() {
super();
@ -111,11 +111,11 @@ public class NicTO extends NetworkTO {
this.details = details;
}
public boolean isDpdkDisabled() {
return dpdkDisabled;
public boolean isDpdkEnabled() {
return dpdkEnabled;
}
public void setDpdkDisabled(boolean dpdkDisabled) {
this.dpdkDisabled = dpdkDisabled;
public void setDpdkEnabled(boolean dpdkEnabled) {
this.dpdkEnabled = dpdkEnabled;
}
}

10
core/src/main/java/com/cloud/agent/api/MigrateCommand.java
View File

@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.cloud.agent.api.to.DpdkTO;
import com.cloud.agent.api.to.VirtualMachineTO;
public class MigrateCommand extends Command {
@ -37,6 +38,15 @@ public class MigrateCommand extends Command {
private VirtualMachineTO vmTO;
private boolean executeInSequence = false;
private List<MigrateDiskInfo> migrateDiskInfoList = new ArrayList<>();
private Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
public Map<String, DpdkTO> getDpdkInterfaceMapping() {
return dpdkInterfaceMapping;
}
public void setDpdkInterfaceMapping(Map<String, DpdkTO> dpdkInterfaceMapping) {
this.dpdkInterfaceMapping = dpdkInterfaceMapping;
}
protected MigrateCommand() {
}

16
core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java
View File

@ -19,7 +19,15 @@
package com.cloud.agent.api;
import com.cloud.agent.api.to.DpdkTO;
import java.util.HashMap;
import java.util.Map;
public class PrepareForMigrationAnswer extends Answer {
private Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
protected PrepareForMigrationAnswer() {
}
@ -34,4 +42,12 @@ public class PrepareForMigrationAnswer extends Answer {
public PrepareForMigrationAnswer(PrepareForMigrationCommand cmd) {
super(cmd, true, null);
}
public void setDpdkInterfaceMapping(Map<String, DpdkTO> mapping) {
this.dpdkInterfaceMapping = mapping;
}
public Map<String, DpdkTO> getDpdkInterfaceMapping() {
return this.dpdkInterfaceMapping;
}
}

10
core/src/main/java/com/cloud/agent/api/StopCommand.java
View File

@ -19,6 +19,7 @@
package com.cloud.agent.api;
import com.cloud.agent.api.to.DpdkTO;
import com.cloud.agent.api.to.GPUDeviceTO;
import com.cloud.vm.VirtualMachine;
@ -34,6 +35,15 @@ public class StopCommand extends RebootCommand {
boolean checkBeforeCleanup = false;
String controlIp = null;
boolean forceStop = false;
private Map<String, DpdkTO> dpdkInterfaceMapping;
public Map<String, DpdkTO> getDpdkInterfaceMapping() {
return dpdkInterfaceMapping;
}
public void setDpdkInterfaceMapping(Map<String, DpdkTO> dpdkInterfaceMapping) {
this.dpdkInterfaceMapping = dpdkInterfaceMapping;
}
/**
* On KVM when using iSCSI-based managed storage, if the user shuts a VM down from the guest OS (as opposed to doing so from CloudStack),
* we need to pass to the KVM agent a list of applicable iSCSI volumes that need to be disconnected.

29
engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
View File

@ -39,7 +39,8 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.api.ApiConstants;
import com.cloud.agent.api.PrepareForMigrationAnswer;
import com.cloud.agent.api.to.DpdkTO;
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd;
import org.apache.cloudstack.api.command.admin.volume.MigrateVolumeCmdByAdmin;
@ -1118,8 +1119,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx);
addExtraConfig(vmTO);
work = _workDao.findById(work.getId());
if (work == null || work.getStep() != Step.Prepare) {
throw new ConcurrentOperationException("Work steps have been changed: " + work);
@ -1284,15 +1283,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
}
private void addExtraConfig(VirtualMachineTO vmTO) {
Map<String, String> details = vmTO.getDetails();
for (String key : details.keySet()) {
if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
vmTO.addExtraConfig(key, details.get(key));
}
}
}
// for managed storage on KVM, need to make sure the path field of the volume in question is populated with the IQN
private void handlePath(final DiskTO[] disks, final HypervisorType hypervisorType) {
if (hypervisorType != HypervisorType.KVM) {
@ -2362,6 +2352,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
boolean migrated = false;
Map<String, DpdkTO> dpdkInterfaceMapping = null;
try {
final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
@ -2370,6 +2361,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
mc.setAutoConvergence(kvmAutoConvergence);
mc.setHostGuid(dest.getHost().getGuid());
dpdkInterfaceMapping = ((PrepareForMigrationAnswer) pfma).getDpdkInterfaceMapping();
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
mc.setDpdkInterfaceMapping(dpdkInterfaceMapping);
}
try {
final Answer ma = _agentMgr.send(vm.getLastHostId(), mc);
if (ma == null || !ma.getResult()) {
@ -2396,7 +2392,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
if (!checkVmOnHost(vm, dstHostId)) {
s_logger.error("Unable to complete migration for " + vm);
try {
_agentMgr.send(srcHostId, new Commands(cleanup(vm)), null);
_agentMgr.send(srcHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
} catch (final AgentUnavailableException e) {
s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId);
}
@ -2417,7 +2413,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
"Unable to migrate vm " + vm.getInstanceName() + " from host " + fromHost.getName() + " in zone " + dest.getDataCenter().getName() + " and pod " +
dest.getPod().getName(), "Migrate Command failed. Please check logs.");
try {
_agentMgr.send(dstHostId, new Commands(cleanup(vm)), null);
_agentMgr.send(dstHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
} catch (final AgentUnavailableException ae) {
s_logger.info("Looks like the destination Host is unavailable for cleanup");
}
@ -3106,9 +3102,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
}
public Command cleanup(final VirtualMachine vm) {
public Command cleanup(final VirtualMachine vm, Map<String, DpdkTO> dpdkInterfaceMapping) {
StopCommand cmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
cmd.setControlIp(getControlNicIpForVM(vm));
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
cmd.setDpdkInterfaceMapping(dpdkInterfaceMapping);
}
return cmd;
}

8
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKDriver.java → plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DpdkDriver.java
View File

@ -22,7 +22,7 @@ import com.cloud.utils.component.Adapter;
import java.util.Map;
public interface DPDKDriver extends Adapter {
public interface DpdkDriver extends Adapter {
/**
* Get the next DPDK port name to be created
@ -37,17 +37,17 @@ public interface DPDKDriver extends Adapter {
/**
* Add OVS port (if it does not exist) to bridge with DPDK support
*/
void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.VHostUserMode vHostUserMode, String dpdkOvsPath);
void addDpdkPort(String bridgeName, String port, String vlan, DpdkHelper.VHostUserMode vHostUserMode, String dpdkOvsPath);
/**
* Since DPDK user client/server mode, retrieve the guest interfaces mode from the DPDK vHost User mode
*/
String getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode dpdKvHostUserMode);
String getGuestInterfacesModeFromDpdkVhostUserMode(DpdkHelper.VHostUserMode dpdKvHostUserMode);
/**
* Get DPDK vHost User mode from extra config. If it is not present, server is returned as default
*/
DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraConfig);
DpdkHelper.VHostUserMode getDpdkvHostUserMode(Map<String, String> extraConfig);
/**
* Check for additional extra 'dpdk-interface' configurations, return them appended

26
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKDriverImpl.java → plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DpdkDriverImpl.java
View File

@ -25,15 +25,15 @@ import org.apache.log4j.Logger;
import java.util.Map;
public class DPDKDriverImpl extends AdapterBase implements DPDKDriver {
public class DpdkDriverImpl extends AdapterBase implements DpdkDriver {
static final String DPDK_PORT_PREFIX = "csdpdk-";
private final String dpdkPortVhostUserType = "dpdkvhostuser";
private final String dpdkPortVhostUserClientType = "dpdkvhostuserclient";
private static final Logger s_logger = Logger.getLogger(DPDKDriver.class);
private static final Logger s_logger = Logger.getLogger(DpdkDriver.class);
public DPDKDriverImpl() {
public DpdkDriverImpl() {
}
/**
@ -64,8 +64,8 @@ public class DPDKDriverImpl extends AdapterBase implements DPDKDriver {
/**
* Add OVS port (if it does not exist) to bridge with DPDK support
*/
public void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.VHostUserMode vHostUserMode, String dpdkOvsPath) {
String type = vHostUserMode == DPDKHelper.VHostUserMode.SERVER ?
public void addDpdkPort(String bridgeName, String port, String vlan, DpdkHelper.VHostUserMode vHostUserMode, String dpdkOvsPath) {
String type = vHostUserMode == DpdkHelper.VHostUserMode.SERVER ?
dpdkPortVhostUserType :
dpdkPortVhostUserClientType;
@ -74,7 +74,7 @@ public class DPDKDriverImpl extends AdapterBase implements DPDKDriver {
"vlan_mode=access tag=%s " +
"-- set Interface %s type=%s", bridgeName, port, vlan, port, type));
if (vHostUserMode == DPDKHelper.VHostUserMode.CLIENT) {
if (vHostUserMode == DpdkHelper.VHostUserMode.CLIENT) {
stringBuilder.append(String.format(" options:vhost-server-path=%s/%s",
dpdkOvsPath, port));
}
@ -87,17 +87,17 @@ public class DPDKDriverImpl extends AdapterBase implements DPDKDriver {
/**
* Since DPDK user client/server mode, retrieve the guest interfaces mode from the DPDK vHost User mode
*/
public String getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode dpdKvHostUserMode) {
return dpdKvHostUserMode == DPDKHelper.VHostUserMode.CLIENT ? "server" : "client";
public String getGuestInterfacesModeFromDpdkVhostUserMode(DpdkHelper.VHostUserMode dpdKvHostUserMode) {
return dpdKvHostUserMode == DpdkHelper.VHostUserMode.CLIENT ? "server" : "client";
}
/**
* Get DPDK vHost User mode from extra config. If it is not present, server is returned as default
*/
public DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraConfig) {
return extraConfig.containsKey(DPDKHelper.DPDK_VHOST_USER_MODE) ?
DPDKHelper.VHostUserMode.fromValue(extraConfig.get(DPDKHelper.DPDK_VHOST_USER_MODE)) :
DPDKHelper.VHostUserMode.SERVER;
public DpdkHelper.VHostUserMode getDpdkvHostUserMode(Map<String, String> extraConfig) {
return extraConfig.containsKey(DpdkHelper.DPDK_VHOST_USER_MODE) ?
DpdkHelper.VHostUserMode.fromValue(extraConfig.get(DpdkHelper.DPDK_VHOST_USER_MODE)) :
DpdkHelper.VHostUserMode.SERVER;
}
/**
@ -106,7 +106,7 @@ public class DPDKDriverImpl extends AdapterBase implements DPDKDriver {
public String getExtraDpdkProperties(Map<String, String> extraConfig) {
StringBuilder stringBuilder = new StringBuilder();
for (String key : extraConfig.keySet()) {
if (key.startsWith(DPDKHelper.DPDK_INTERFACE_PREFIX)) {
if (key.startsWith(DpdkHelper.DPDK_INTERFACE_PREFIX)) {
stringBuilder.append(extraConfig.get(key));
}
}

13
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -46,7 +46,7 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
import com.cloud.resource.RequestWrapper;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
@ -2070,7 +2070,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
vm.setPlatformEmulator(vmTO.getPlatformEmulator());
Map<String, String> extraConfig = vmTO.getExtraConfig();
if (dpdkSupport && (!extraConfig.containsKey(DPDKHelper.DPDK_NUMA) || !extraConfig.containsKey(DPDKHelper.DPDK_HUGE_PAGES))) {
if (dpdkSupport && (!extraConfig.containsKey(DpdkHelper.DPDK_NUMA) || !extraConfig.containsKey(DpdkHelper.DPDK_HUGE_PAGES))) {
s_logger.info("DPDK is enabled but it needs extra configurations for CPU NUMA and Huge Pages for VM deployment");
}
@ -2107,7 +2107,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
grd.setVcpuNum(vcpus);
vm.addComp(grd);
if (!extraConfig.containsKey(DPDKHelper.DPDK_NUMA)) {
if (!extraConfig.containsKey(DpdkHelper.DPDK_NUMA)) {
final CpuModeDef cmd = new CpuModeDef();
cmd.setMode(_guestCpuMode);
cmd.setModel(_guestCpuModel);
@ -2235,7 +2235,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
if (MapUtils.isNotEmpty(extraConfig)) {
StringBuilder extraConfigBuilder = new StringBuilder();
for (String key : extraConfig.keySet()) {
if (!key.startsWith(DPDKHelper.DPDK_INTERFACE_PREFIX) && !key.equals(DPDKHelper.DPDK_VHOST_USER_MODE)) {
if (!key.startsWith(DpdkHelper.DPDK_INTERFACE_PREFIX) && !key.equals(DpdkHelper.DPDK_VHOST_USER_MODE)) {
extraConfigBuilder.append(extraConfig.get(key));
}
}
@ -2706,7 +2706,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem);
final String capabilities = String.join(",", info.getCapabilities());
String capabilities = String.join(",", info.getCapabilities());
if (dpdkSupport) {
capabilities += ",dpdk";
}
final StartupRoutingCommand cmd =
new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, _hypervisorType,

8
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java
View File

@ -224,9 +224,13 @@ public class LibvirtDomainXMLParser {
def.defEthernet(dev, mac, NicModel.valueOf(model.toUpperCase()), scriptPath, networkRateKBps);
} else if (type.equals("vhostuser")) {
String sourcePort = getAttrValue("source", "path", nic);
String[] sourcePathParts = sourcePort.split("/");
String port = sourcePathParts[sourcePathParts.length - 1];
String mode = getAttrValue("source", "mode", nic);
int lastSlashIndex = sourcePort.lastIndexOf("/");
String ovsPath = sourcePort.substring(0,lastSlashIndex);
String port = sourcePort.substring(lastSlashIndex + 1);
def.setDpdkSourcePort(port);
def.setDpdkOvsPath(ovsPath);
def.setInterfaceMode(mode);
}
if (StringUtils.isNotBlank(slot)) {

30
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java
View File

@ -963,7 +963,7 @@ public class LibvirtVMDef {
}
public static class InterfaceDef {
enum GuestNetType {
public enum GuestNetType {
BRIDGE("bridge"), DIRECT("direct"), NETWORK("network"), USER("user"), ETHERNET("ethernet"), INTERNAL("internal"), VHOSTUSER("vhostuser");
String _type;
@ -1176,10 +1176,24 @@ public class LibvirtVMDef {
_dpdkSourcePort = port;
}
@Override
public String toString() {
public String getDpdkOvsPath() {
return _dpdkSourcePath;
}
public void setDpdkOvsPath(String path) {
_dpdkSourcePath = path;
}
public String getInterfaceMode() {
return _interfaceMode;
}
public void setInterfaceMode(String mode) {
_interfaceMode = mode;
}
public String getContent() {
StringBuilder netBuilder = new StringBuilder();
netBuilder.append("<interface type='" + _netType + "'>\n");
if (_netType == GuestNetType.BRIDGE) {
netBuilder.append("<source bridge='" + _sourceName + "'/>\n");
} else if (_netType == GuestNetType.NETWORK) {
@ -1233,6 +1247,14 @@ public class LibvirtVMDef {
if (_slot != null) {
netBuilder.append(String.format("<address type='pci' domain='0x0000' bus='0x00' slot='0x%02x' function='0x0'/>\n", _slot));
}
return netBuilder.toString();
}
@Override
public String toString() {
StringBuilder netBuilder = new StringBuilder();
netBuilder.append("<interface type='" + _netType + "'>\n");
netBuilder.append(getContent());
netBuilder.append("</interface>\n");
return netBuilder.toString();
}

52
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java
View File

@ -24,9 +24,9 @@ import java.util.Map;
import javax.naming.ConfigurationException;
import com.cloud.hypervisor.kvm.dpdk.DPDKDriver;
import com.cloud.hypervisor.kvm.dpdk.DPDKDriverImpl;
import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
import com.cloud.hypervisor.kvm.dpdk.DpdkDriver;
import com.cloud.hypervisor.kvm.dpdk.DpdkDriverImpl;
import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@ -44,7 +44,7 @@ import com.cloud.utils.script.Script;
public class OvsVifDriver extends VifDriverBase {
private static final Logger s_logger = Logger.getLogger(OvsVifDriver.class);
private int _timeout;
private DPDKDriver dpdkDriver;
private DpdkDriver dpdkDriver;
@Override
public void configure(Map<String, Object> params) throws ConfigurationException {
@ -59,7 +59,7 @@ public class OvsVifDriver extends VifDriverBase {
String dpdk = (String) params.get("openvswitch.dpdk.enabled");
if (StringUtils.isNotBlank(dpdk) && Boolean.parseBoolean(dpdk)) {
dpdkDriver = new DPDKDriverImpl();
dpdkDriver = new DpdkDriverImpl();
}
String value = (String)params.get("scripts.timeout");
@ -87,12 +87,34 @@ public class OvsVifDriver extends VifDriverBase {
s_logger.debug("done looking for pifs, no more bridges");
}
/**
* Plug interface with DPDK support:
* - Create a new port with DPDK support for the interface
* - Set the 'intf' path to the new port
*/
protected void plugDPDKInterface(InterfaceDef intf, String trafficLabel, Map<String, String> extraConfig,
String vlanId, String guestOsType, NicTO nic, String nicAdapter) {
s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
String dpdkOvsPath = _libvirtComputingResource.dpdkOvsPath;
if (StringUtils.isBlank(dpdkOvsPath)) {
throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
}
String port = dpdkDriver.getNextDpdkPort();
DpdkHelper.VHostUserMode dpdKvHostUserMode = dpdkDriver.getDpdkvHostUserMode(extraConfig);
dpdkDriver.addDpdkPort(_pifs.get(trafficLabel), port, vlanId, dpdKvHostUserMode, dpdkOvsPath);
String interfaceMode = dpdkDriver.getGuestInterfacesModeFromDpdkVhostUserMode(dpdKvHostUserMode);
intf.defDpdkNet(dpdkOvsPath, port, nic.getMac(),
getGuestNicModel(guestOsType, nicAdapter), 0,
dpdkDriver.getExtraDpdkProperties(extraConfig),
interfaceMode);
}
@Override
public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
s_logger.debug("plugging nic=" + nic);
LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef();
if (!_libvirtComputingResource.dpdkSupport || nic.isDpdkDisabled()) {
if (!_libvirtComputingResource.dpdkSupport || !nic.isDpdkEnabled()) {
// Let libvirt handle OVS ports creation when DPDK property is disabled or when it is enabled but disabled for the nic
// For DPDK support, libvirt does not handle ports creation, invoke 'addDpdkPort' method
intf.setVirtualPortType("openvswitch");
@ -114,20 +136,8 @@ public class OvsVifDriver extends VifDriverBase {
if ((nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan || nic.getBroadcastType() == Networks.BroadcastDomainType.Pvlan) &&
!vlanId.equalsIgnoreCase("untagged")) {
if (trafficLabel != null && !trafficLabel.isEmpty()) {
if (_libvirtComputingResource.dpdkSupport && !nic.isDpdkDisabled()) {
s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
String dpdkOvsPath = _libvirtComputingResource.dpdkOvsPath;
if (StringUtils.isBlank(dpdkOvsPath)) {
throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
}
String port = dpdkDriver.getNextDpdkPort();
DPDKHelper.VHostUserMode dpdKvHostUserMode = dpdkDriver.getDPDKvHostUserMode(extraConfig);
dpdkDriver.addDpdkPort(_pifs.get(trafficLabel), port, vlanId, dpdKvHostUserMode, dpdkOvsPath);
String interfaceMode = dpdkDriver.getGuestInterfacesModeFromDPDKVhostUserMode(dpdKvHostUserMode);
intf.defDpdkNet(dpdkOvsPath, port, nic.getMac(),
getGuestNicModel(guestOsType, nicAdapter), 0,
dpdkDriver.getExtraDpdkProperties(extraConfig),
interfaceMode);
if (_libvirtComputingResource.dpdkSupport && nic.isDpdkEnabled()) {
plugDPDKInterface(intf, trafficLabel, extraConfig, vlanId, guestOsType, nic, nicAdapter);
} else {
s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel);
intf.defBridgeNet(_pifs.get(trafficLabel), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);
@ -180,7 +190,7 @@ public class OvsVifDriver extends VifDriverBase {
@Override
public void unplug(InterfaceDef iface) {
// Libvirt apparently takes care of this, see BridgeVifDriver unplug
if (_libvirtComputingResource.dpdkSupport) {
if (_libvirtComputingResource.dpdkSupport && StringUtils.isNotBlank(iface.getDpdkSourcePort())) {
// If DPDK is enabled, we'll need to cleanup the port as libvirt won't
String dpdkPort = iface.getDpdkSourcePort();
String cmd = String.format("ovs-vsctl del-port %s", dpdkPort);

76
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java
View File

@ -43,6 +43,7 @@ import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import com.cloud.agent.api.to.DpdkTO;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@ -153,6 +154,11 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
xmlDesc = replaceStorage(xmlDesc, mapMigrateStorage, migrateStorageManaged);
}
Map<String, DpdkTO> dpdkPortsMapping = command.getDpdkInterfaceMapping();
if (MapUtils.isNotEmpty(dpdkPortsMapping)) {
xmlDesc = replaceDpdkInterfaces(xmlDesc, dpdkPortsMapping);
}
dconn = libvirtUtilitiesHelper.retrieveQemuConnection(destinationUri);
//run migration in thread so we can monitor it
@ -283,6 +289,76 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
return new MigrateAnswer(command, result == null, result, null);
}
/**
* Replace DPDK source path and target before migrations
*/
protected String replaceDpdkInterfaces(String xmlDesc, Map<String, DpdkTO> dpdkPortsMapping) throws TransformerException, ParserConfigurationException, IOException, SAXException {
InputStream in = IOUtils.toInputStream(xmlDesc);
DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
Document doc = docBuilder.parse(in);
// Get the root element
Node domainNode = doc.getFirstChild();
NodeList domainChildNodes = domainNode.getChildNodes();
for (int i = 0; i < domainChildNodes.getLength(); i++) {
Node domainChildNode = domainChildNodes.item(i);
if ("devices".equals(domainChildNode.getNodeName())) {
NodeList devicesChildNodes = domainChildNode.getChildNodes();
for (int x = 0; x < devicesChildNodes.getLength(); x++) {
Node deviceChildNode = devicesChildNodes.item(x);
if ("interface".equals(deviceChildNode.getNodeName())) {
Node interfaceNode = deviceChildNode;
NamedNodeMap attributes = interfaceNode.getAttributes();
Node interfaceTypeAttr = attributes.getNamedItem("type");
if ("vhostuser".equals(interfaceTypeAttr.getNodeValue())) {
NodeList diskChildNodes = interfaceNode.getChildNodes();
String mac = null;
for (int y = 0; y < diskChildNodes.getLength(); y++) {
Node diskChildNode = diskChildNodes.item(y);
if (!"mac".equals(diskChildNode.getNodeName())) {
continue;
}
mac = diskChildNode.getAttributes().getNamedItem("address").getNodeValue();
}
if (StringUtils.isNotBlank(mac)) {
DpdkTO to = dpdkPortsMapping.get(mac);
for (int z = 0; z < diskChildNodes.getLength(); z++) {
Node diskChildNode = diskChildNodes.item(z);
if ("target".equals(diskChildNode.getNodeName())) {
Node targetNode = diskChildNode;
Node targetNodeAttr = targetNode.getAttributes().getNamedItem("dev");
targetNodeAttr.setNodeValue(to.getPort());
} else if ("source".equals(diskChildNode.getNodeName())) {
Node sourceNode = diskChildNode;
NamedNodeMap attrs = sourceNode.getAttributes();
Node path = attrs.getNamedItem("path");
path.setNodeValue(to.getPath() + "/" + to.getPort());
Node mode = attrs.getNamedItem("mode");
mode.setNodeValue(to.getMode());
}
}
}
}
}
}
}
}
return getXml(doc);
}
/**
* In case of a local file, it deletes the file on the source host/storage pool. Otherwise (for instance iScsi) it disconnects the disk on the source storage pool. </br>
* This method must be executed after a successful migration to a target storage pool, cleaning up the source storage.

36
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java
View File

@ -22,20 +22,28 @@ package com.cloud.hypervisor.kvm.resource.wrapper;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.PrepareForMigrationAnswer;
import com.cloud.agent.api.PrepareForMigrationCommand;
import com.cloud.agent.api.to.DpdkTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.NicTO;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.exception.InternalErrorException;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef.GuestNetType;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import com.cloud.storage.Volume;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;
import org.libvirt.Connect;
import org.libvirt.LibvirtException;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;
@ResourceWrapper(handles = PrepareForMigrationCommand.class)
public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapper<PrepareForMigrationCommand, Answer, LibvirtComputingResource> {
@ -56,6 +64,8 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp
final NicTO[] nics = vm.getNics();
Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
boolean skipDisconnect = false;
final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
@ -63,8 +73,13 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp
final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vm.getName());
for (final NicTO nic : nics) {
libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).plug(nic, null, "", null);
LibvirtVMDef.InterfaceDef interfaceDef = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).plug(nic, null, "", vm.getExtraConfig());
if (interfaceDef != null && interfaceDef.getNetType() == GuestNetType.VHOSTUSER) {
DpdkTO to = new DpdkTO(interfaceDef.getDpdkOvsPath(), interfaceDef.getDpdkSourcePort(), interfaceDef.getInterfaceMode());
dpdkInterfaceMapping.put(nic.getMac(), to);
}
}
/* setup disks, e.g for iso */
@ -81,12 +96,19 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp
return new PrepareForMigrationAnswer(command, "failed to connect physical disks to host");
}
return new PrepareForMigrationAnswer(command);
} catch (final LibvirtException e) {
return new PrepareForMigrationAnswer(command, e.toString());
} catch (final InternalErrorException e) {
return new PrepareForMigrationAnswer(command, e.toString());
} catch (final URISyntaxException e) {
PrepareForMigrationAnswer answer = new PrepareForMigrationAnswer(command);
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
answer.setDpdkInterfaceMapping(dpdkInterfaceMapping);
}
return answer;
} catch (final LibvirtException | CloudRuntimeException | InternalErrorException | URISyntaxException e) {
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
for (DpdkTO to : dpdkInterfaceMapping.values()) {
String cmd = String.format("ovs-vsctl del-port %s", to.getPort());
s_logger.debug("Removing DPDK port: " + to.getPort());
Script.runSimpleBashScript(cmd);
}
}
return new PrepareForMigrationAnswer(command, e.toString());
} finally {
if (!skipDisconnect) {

1
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java
View File

@ -69,7 +69,6 @@ public final class LibvirtStartCommandWrapper extends CommandWrapper<StartComman
for (final NicTO nic : nics) {
if (vmSpec.getType() != VirtualMachine.Type.User) {
nic.setPxeDisable(true);
nic.setDpdkDisabled(true);
}
}

26
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java
View File

@ -23,8 +23,12 @@ import java.io.File;
import java.util.List;
import java.util.Map;
import com.cloud.agent.api.to.DpdkTO;
import com.cloud.utils.Pair;
import com.cloud.utils.script.Script;
import com.cloud.utils.ssh.SshHelper;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;
import org.libvirt.Connect;
import org.libvirt.Domain;
@ -106,11 +110,23 @@ public final class LibvirtStopCommandWrapper extends CommandWrapper<StopCommand,
}
}
for (final InterfaceDef iface : ifaces) {
// We don't know which "traffic type" is associated with
// each interface at this point, so inform all vif drivers
for (final VifDriver vifDriver : libvirtComputingResource.getAllVifDrivers()) {
vifDriver.unplug(iface);
if (CollectionUtils.isEmpty(ifaces)) {
Map<String, DpdkTO> dpdkInterfaceMapping = command.getDpdkInterfaceMapping();
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
for (DpdkTO to : dpdkInterfaceMapping.values()) {
String portToRemove = to.getPort();
String cmd = String.format("ovs-vsctl del-port %s", portToRemove);
s_logger.debug("Removing DPDK port: " + portToRemove);
Script.runSimpleBashScript(cmd);
}
}
} else {
for (final InterfaceDef iface : ifaces) {
// We don't know which "traffic type" is associated with
// each interface at this point, so inform all vif drivers
for (final VifDriver vifDriver : libvirtComputingResource.getAllVifDrivers()) {
vifDriver.unplug(iface);
}
}
}
}

42
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/dpdk/DPDKDriverTest.java → plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/dpdk/DpdkDriverTest.java
View File

@ -35,11 +35,11 @@ import java.util.Map;
@PrepareForTest({ Script.class })
@RunWith(PowerMockRunner.class)
public class DPDKDriverTest {
public class DpdkDriverTest {
private static final int dpdkPortNumber = 7;
private DPDKDriver driver = new DPDKDriverImpl();
private DpdkDriver driver = new DpdkDriverImpl();
private Map<String, String> extraConfig;
@ -59,7 +59,7 @@ public class DPDKDriverTest {
@Test
public void testGetDpdkLatestPortNumberUsedExistingDpdkPorts() {
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
thenReturn(DPDKDriverImpl.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
thenReturn(DpdkDriverImpl.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
Assert.assertEquals(dpdkPortNumber, driver.getDpdkLatestPortNumberUsed());
}
@ -67,47 +67,47 @@ public class DPDKDriverTest {
public void testGetNextDpdkPortNoDpdkPorts() {
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
thenReturn(null);
String expectedPortName = DPDKDriverImpl.DPDK_PORT_PREFIX + String.valueOf(1);
String expectedPortName = DpdkDriverImpl.DPDK_PORT_PREFIX + String.valueOf(1);
Assert.assertEquals(expectedPortName, driver.getNextDpdkPort());
}
@Test
public void testGetNextDpdkPortExistingDpdkPorts() {
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
thenReturn(DPDKDriverImpl.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
String expectedPortName = DPDKDriverImpl.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber + 1);
thenReturn(DpdkDriverImpl.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
String expectedPortName = DpdkDriverImpl.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber + 1);
Assert.assertEquals(expectedPortName, driver.getNextDpdkPort());
}
@Test
public void testGetGuestInterfacesModeFromDPDKVhostUserModeClientDPDK() {
String guestMode = driver.getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode.CLIENT);
public void testGetGuestInterfacesModeFromDpdkVhostUserModeClientDpdk() {
String guestMode = driver.getGuestInterfacesModeFromDpdkVhostUserMode(DpdkHelper.VHostUserMode.CLIENT);
Assert.assertEquals("server", guestMode);
}
@Test
public void testGetGuestInterfacesModeFromDPDKVhostUserModeServerDPDK() {
String guestMode = driver.getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode.SERVER);
public void testGetGuestInterfacesModeFromDpdkVhostUserModeServerDpdk() {
String guestMode = driver.getGuestInterfacesModeFromDpdkVhostUserMode(DpdkHelper.VHostUserMode.SERVER);
Assert.assertEquals("client", guestMode);
}
@Test
public void testGetDPDKvHostUserModeServerExtraConfig() {
extraConfig.put(DPDKHelper.DPDK_VHOST_USER_MODE, DPDKHelper.VHostUserMode.SERVER.toString());
DPDKHelper.VHostUserMode dpdKvHostUserMode = driver.getDPDKvHostUserMode(extraConfig);
Assert.assertEquals(DPDKHelper.VHostUserMode.SERVER, dpdKvHostUserMode);
public void testGetDpdkvHostUserModeServerExtraConfig() {
extraConfig.put(DpdkHelper.DPDK_VHOST_USER_MODE, DpdkHelper.VHostUserMode.SERVER.toString());
DpdkHelper.VHostUserMode dpdKvHostUserMode = driver.getDpdkvHostUserMode(extraConfig);
Assert.assertEquals(DpdkHelper.VHostUserMode.SERVER, dpdKvHostUserMode);
}
@Test
public void testGetDPDKvHostUserModeServerClientExtraConfig() {
extraConfig.put(DPDKHelper.DPDK_VHOST_USER_MODE, DPDKHelper.VHostUserMode.CLIENT.toString());
DPDKHelper.VHostUserMode dpdKvHostUserMode = driver.getDPDKvHostUserMode(extraConfig);
Assert.assertEquals(DPDKHelper.VHostUserMode.CLIENT, dpdKvHostUserMode);
public void testGetDpdkvHostUserModeServerClientExtraConfig() {
extraConfig.put(DpdkHelper.DPDK_VHOST_USER_MODE, DpdkHelper.VHostUserMode.CLIENT.toString());
DpdkHelper.VHostUserMode dpdKvHostUserMode = driver.getDpdkvHostUserMode(extraConfig);
Assert.assertEquals(DpdkHelper.VHostUserMode.CLIENT, dpdKvHostUserMode);
}
@Test
public void testGetDPDKvHostUserModeServerEmptyExtraConfig() {
DPDKHelper.VHostUserMode dpdKvHostUserMode = driver.getDPDKvHostUserMode(extraConfig);
Assert.assertEquals(DPDKHelper.VHostUserMode.SERVER, dpdKvHostUserMode);
public void testGetDpdkvHostUserModeServerEmptyExtraConfig() {
DpdkHelper.VHostUserMode dpdKvHostUserMode = driver.getDpdkvHostUserMode(extraConfig);
Assert.assertEquals(DpdkHelper.VHostUserMode.SERVER, dpdKvHostUserMode);
}
}

153
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java
View File

@ -37,6 +37,7 @@ import javax.xml.transform.TransformerException;
import org.junit.Assert;
import org.junit.Before;
import com.cloud.agent.api.to.DpdkTO;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.libvirt.Connect;
@ -302,6 +303,146 @@ public class LibvirtMigrateCommandWrapperTest {
" </devices>\n" +
"</domain>";
private String sourceDPDKVMToMigrate =
"<domain type='kvm' id='17'>\n" +
" <name>i-2-33-VM</name>\n" +
" <uuid>14c5c052-46cb-4301-a00a-28f6cc1dc605</uuid>\n" +
" <description>Other PV (64-bit)</description>\n" +
" <memory unit='KiB'>9437184</memory>\n" +
" <currentMemory unit='KiB'>9437184</currentMemory>\n" +
" <memoryBacking>\n" +
" <hugepages/>\n" +
" </memoryBacking>\n" +
" <vcpu placement='static'>2</vcpu>\n" +
" <cputune>\n" +
" <shares>4000</shares>\n" +
" </cputune>\n" +
" <resource>\n" +
" <partition>/machine</partition>\n" +
" </resource>\n" +
" <sysinfo type='smbios'>\n" +
" <system>\n" +
" <entry name='manufacturer'>Apache Software Foundation</entry>\n" +
" <entry name='product'>CloudStack KVM Hypervisor</entry>\n" +
" <entry name='uuid'>14c5c052-46cb-4301-a00a-28f6cc1dc605</entry>\n" +
" </system>\n" +
" </sysinfo>\n" +
" <os>\n" +
" <type arch='x86_64' machine='pc-i440fx-rhel7.5.0'>hvm</type>\n" +
" <boot dev='cdrom'/>\n" +
" <boot dev='hd'/>\n" +
" <smbios mode='sysinfo'/>\n" +
" </os>\n" +
" <features>\n" +
" <acpi/>\n" +
" <apic/>\n" +
" <pae/>\n" +
" </features>\n" +
" <cpu mode='host-passthrough' check='none'>\n" +
" <numa>\n" +
" <cell id='0' cpus='0' memory='9437184' unit='KiB' memAccess='shared'/>\n" +
" </numa>\n" +
" </cpu>\n" +
" <clock offset='utc'>\n" +
" <timer name='kvmclock'/>\n" +
" </clock>\n" +
" <on_poweroff>destroy</on_poweroff>\n" +
" <on_reboot>restart</on_reboot>\n" +
" <on_crash>destroy</on_crash>\n" +
" <devices>\n" +
" <emulator>/usr/libexec/qemu-kvm</emulator>\n" +
" <disk type='network' device='disk'>\n" +
" <driver name='qemu' type='raw' cache='none'/>\n" +
" <auth username='cloudstack'>\n" +
" <secret type='ceph' uuid='66afbc07-6fdb-385a-ae25-a9acfbc3684d'/>\n" +
" </auth>\n" +
" <source protocol='rbd' name='cloudstack/afb1d2e4-01fe-4694-940b-fcf052afa279'>\n" +
" <host name='VLAB01-CEPH-MON.ceph.local' port='6789'/>\n" +
" </source>\n" +
" <target dev='vda' bus='virtio'/>\n" +
" <serial>afb1d2e401fe4694940b</serial>\n" +
" <alias name='virtio-disk0'/>\n" +
" <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>\n" +
" </disk>\n" +
" <disk type='file' device='cdrom'>\n" +
" <target dev='hdc' bus='ide'/>\n" +
" <readonly/>\n" +
" <alias name='ide0-1-0'/>\n" +
" <address type='drive' controller='0' bus='1' target='0' unit='0'/>\n" +
" </disk>\n" +
" <controller type='usb' index='0' model='piix3-uhci'>\n" +
" <alias name='usb'/>\n" +
" <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>\n" +
" </controller>\n" +
" <controller type='pci' index='0' model='pci-root'>\n" +
" <alias name='pci.0'/>\n" +
" </controller>\n" +
" <controller type='ide' index='0'>\n" +
" <alias name='ide'/>\n" +
" <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>\n" +
" </controller>\n" +
" <controller type='virtio-serial' index='0'>\n" +
" <alias name='virtio-serial0'/>\n" +
" <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>\n" +
" </controller>\n" +
" <interface type='vhostuser'>\n" +
" <mac address='02:00:18:91:00:10'/>\n" +
" <source type='unix' path='/var/run/libvirt-vhost-user/csdpdk-1' mode='server'/>\n" +
" <target dev='csdpdk-1'/>\n" +
" <model type='virtio'/>\n" +
" <alias name='net0'/>\n" +
" <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>\n" +
" </interface>\n" +
" <serial type='pty'>\n" +
" <source path='/dev/pts/0'/>\n" +
" <target type='isa-serial' port='0'>\n" +
" <model name='isa-serial'/>\n" +
" </target>\n" +
" <alias name='serial0'/>\n" +
" </serial>\n" +
" <console type='pty' tty='/dev/pts/0'>\n" +
" <source path='/dev/pts/0'/>\n" +
" <target type='serial' port='0'/>\n" +
" <alias name='serial0'/>\n" +
" </console>\n" +
" <channel type='unix'>\n" +
" <source mode='bind' path='/var/lib/libvirt/qemu/i-2-33-VM.org.qemu.guest_agent.0'/>\n" +
" <target type='virtio' name='org.qemu.guest_agent.0' state='disconnected'/>\n" +
" <alias name='channel0'/>\n" +
" <address type='virtio-serial' controller='0' bus='0' port='1'/>\n" +
" </channel>\n" +
" <input type='tablet' bus='usb'>\n" +
" <alias name='input0'/>\n" +
" <address type='usb' bus='0' port='1'/>\n" +
" </input>\n" +
" <input type='mouse' bus='ps2'>\n" +
" <alias name='input1'/>\n" +
" </input>\n" +
" <input type='keyboard' bus='ps2'>\n" +
" <alias name='input2'/>\n" +
" </input>\n" +
" <graphics type='vnc' port='5900' autoport='yes' listen='198.19.254.10'>\n" +
" <listen type='address' address='198.19.254.10'/>\n" +
" </graphics>\n" +
" <video>\n" +
" <model type='cirrus' vram='16384' heads='1' primary='yes'/>\n" +
" <alias name='video0'/>\n" +
" <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>\n" +
" </video>\n" +
" <watchdog model='i6300esb' action='none'>\n" +
" <alias name='watchdog0'/>\n" +
" <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>\n" +
" </watchdog>\n" +
" <memballoon model='none'>\n" +
" <alias name='balloon0'/>\n" +
" </memballoon>\n" +
" </devices>\n" +
" <seclabel type='dynamic' model='dac' relabel='yes'>\n" +
" <label>+0:+0</label>\n" +
" <imagelabel>+0:+0</imagelabel>\n" +
" </seclabel>\n" +
"</domain>";
LibvirtMigrateCommandWrapper libvirtMigrateCmdWrapper = new LibvirtMigrateCommandWrapper();
final String memInfo = "MemTotal: 5830236 kB\n" +
@ -636,4 +777,16 @@ public class LibvirtMigrateCommandWrapperTest {
assertFalse(newXml.contains("/mnt/" + sourcePoolUuid + "/" + disk1SourceFilename));
assertFalse(newXml.contains("/mnt/" + sourcePoolUuid + "/" + disk2SourceFilename));
}
@Test
public void testReplaceDPDKPorts() throws ParserConfigurationException, IOException, SAXException, TransformerException {
final LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper();
Map<String, DpdkTO> dpdkPortMapping = new HashMap<>();
DpdkTO to = new DpdkTO("/var/run/libvirt-vhost-user", "csdpdk-7", "client");
dpdkPortMapping.put("02:00:18:91:00:10", to);
String replaced = lw.replaceDpdkInterfaces(sourceDPDKVMToMigrate, dpdkPortMapping);
Assert.assertTrue(replaced.contains("csdpdk-7"));
Assert.assertFalse(replaced.contains("csdpdk-1"));
}
}

33
server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java
View File

@ -22,7 +22,9 @@ import java.util.UUID;
import javax.inject.Inject;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import com.cloud.agent.api.Command;
@ -126,6 +128,34 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
return to;
}
/**
* Add extra configuration from VM details. Extra configuration is stored as details starting with 'extraconfig'
*/
private void addExtraConfig(Map<String, String> details, VirtualMachineTO to) {
for (String key : details.keySet()) {
if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
to.addExtraConfig(key, details.get(key));
}
}
}
/**
* Add extra configurations from service offering to the VM TO.
* Extra configuration keys are expected in formats:
* - "extraconfig-N"
* - "extraconfig-CONFIG_NAME"
*/
protected void addServiceOfferingExtraConfiguration(ServiceOffering offering, VirtualMachineTO to) {
List<ServiceOfferingDetailsVO> details = _serviceOfferingDetailsDao.listDetails(offering.getId());
if (CollectionUtils.isNotEmpty(details)) {
for (ServiceOfferingDetailsVO detail : details) {
if (detail.getName().startsWith(ApiConstants.EXTRA_CONFIG)) {
to.addExtraConfig(detail.getName(), detail.getValue());
}
}
}
}
protected VirtualMachineTO toVirtualMachineTO(VirtualMachineProfile vmProfile) {
ServiceOffering offering = _serviceOfferingDao.findById(vmProfile.getId(), vmProfile.getServiceOfferingId());
VirtualMachine vm = vmProfile.getVirtualMachine();
@ -169,8 +199,11 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
Map<String, String> detailsInVm = _userVmDetailsDao.listDetailsKeyPairs(vm.getId());
if (detailsInVm != null) {
to.setDetails(detailsInVm);
addExtraConfig(detailsInVm, to);
}
addServiceOfferingExtraConfiguration(offering, to);
// Set GPU details
ServiceOfferingDetailsVO offeringDetail = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString());
if (offeringDetail != null) {

39
server/src/main/java/com/cloud/hypervisor/KVMGuru.java
View File

@ -18,13 +18,12 @@ package com.cloud.hypervisor;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.NicTO;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingDetailsVO;
import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.GuestOSHypervisorVO;
import com.cloud.storage.GuestOSVO;
@ -34,16 +33,14 @@ import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineProfile;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.List;
import java.util.Map;
public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
@ -54,7 +51,7 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
@Inject
HostDao _hostDao;
@Inject
DPDKHelper dpdkHelper;
DpdkHelper dpdkHelper;
public static final Logger s_logger = Logger.getLogger(KVMGuru.class);
@ -114,12 +111,18 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
public VirtualMachineTO implement(VirtualMachineProfile vm) {
VirtualMachineTO to = toVirtualMachineTO(vm);
setVmQuotaPercentage(to, vm);
addServiceOfferingExtraConfiguration(to, vm);
if (dpdkHelper.isDPDKvHostUserModeSettingOnServiceOffering(vm)) {
if (dpdkHelper.isDpdkvHostUserModeSettingOnServiceOffering(vm)) {
dpdkHelper.setDpdkVhostUserMode(to, vm);
}
if (to.getType() == VirtualMachine.Type.User && MapUtils.isNotEmpty(to.getExtraConfig()) &&
to.getExtraConfig().containsKey(DpdkHelper.DPDK_NUMA) && to.getExtraConfig().containsKey(DpdkHelper.DPDK_HUGE_PAGES)) {
for (final NicTO nic : to.getNics()) {
nic.setDpdkEnabled(true);
}
}
// Determine the VM's OS description
GuestOSVO guestOS = _guestOsDao.findByIdIncludingRemoved(vm.getVirtualMachine().getGuestOSId());
to.setOs(guestOS.getDisplayName());
@ -137,24 +140,6 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
return to;
}
/**
* Add extra configurations from service offering to the VM TO.
* Extra configuration keys are expected in formats:
* - "extraconfig-N"
* - "extraconfig-CONFIG_NAME"
*/
protected void addServiceOfferingExtraConfiguration(VirtualMachineTO to, VirtualMachineProfile vmProfile) {
ServiceOffering offering = vmProfile.getServiceOffering();
List<ServiceOfferingDetailsVO> details = _serviceOfferingDetailsDao.listDetails(offering.getId());
if (CollectionUtils.isNotEmpty(details)) {
for (ServiceOfferingDetailsVO detail : details) {
if (detail.getName().startsWith(ApiConstants.EXTRA_CONFIG)) {
to.addExtraConfig(detail.getName(), detail.getValue());
}
}
}
}
@Override
public Pair<Boolean, Long> getCommandHostDelegation(long hostId, Command cmd) {
if (cmd instanceof StorageSubSystemCommand) {

68
server/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKHelperImpl.java
View File

@ -1,68 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.hypervisor.kvm.dpdk;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingDetailsVO;
import com.cloud.service.dao.ServiceOfferingDetailsDao;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachineProfile;
import org.apache.log4j.Logger;
import javax.inject.Inject;
public class DPDKHelperImpl implements DPDKHelper {
@Inject
ServiceOfferingDetailsDao serviceOfferingDetailsDao;
public static final Logger s_logger = Logger.getLogger(DPDKHelperImpl.class);
private ServiceOffering getServiceOfferingFromVMProfile(VirtualMachineProfile virtualMachineProfile) {
ServiceOffering offering = virtualMachineProfile.getServiceOffering();
if (offering == null) {
throw new CloudRuntimeException("VM does not have an associated service offering");
}
return offering;
}
@Override
public boolean isDPDKvHostUserModeSettingOnServiceOffering(VirtualMachineProfile vm) {
ServiceOffering offering = getServiceOfferingFromVMProfile(vm);
ServiceOfferingDetailsVO detail = serviceOfferingDetailsDao.findDetail(offering.getId(), DPDK_VHOST_USER_MODE);
return detail != null;
}
@Override
public void setDpdkVhostUserMode(VirtualMachineTO to, VirtualMachineProfile vm) {
ServiceOffering offering = getServiceOfferingFromVMProfile(vm);
ServiceOfferingDetailsVO detail = serviceOfferingDetailsDao.findDetail(offering.getId(), DPDK_VHOST_USER_MODE);
if (detail != null) {
String mode = detail.getValue();
try {
VHostUserMode dpdKvHostUserMode = VHostUserMode.fromValue(mode);
to.addExtraConfig(DPDK_VHOST_USER_MODE, dpdKvHostUserMode.toString());
} catch (IllegalArgumentException e) {
s_logger.error(String.format("DPDK vHost User mode found as a detail for service offering: %s " +
"but value: %s is not supported. Supported values: %s, %s",
offering.getId(), mode,
VHostUserMode.CLIENT.toString(), VHostUserMode.SERVER.toString()));
}
}
}
}

19
server/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKHelper.java → server/src/main/java/com/cloud/hypervisor/kvm/dpdk/DpdkHelper.java
View File

@ -20,7 +20,7 @@ import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.vm.VirtualMachineProfile;
import org.apache.cloudstack.api.ApiConstants;
public interface DPDKHelper {
public interface DpdkHelper {
String DPDK_VHOST_USER_MODE = "DPDK-VHOSTUSER";
String DPDK_NUMA = ApiConstants.EXTRA_CONFIG + "-dpdk-numa";
@ -56,11 +56,26 @@ public interface DPDKHelper {
* True if the DPDK vHost user mode setting is part of the VM service offering details, false if not.
* @param vm
*/
boolean isDPDKvHostUserModeSettingOnServiceOffering(VirtualMachineProfile vm);
boolean isDpdkvHostUserModeSettingOnServiceOffering(VirtualMachineProfile vm);
/**
* Add DPDK vHost User Mode as extra configuration to the VM TO (if present on the VM service offering details)
*/
void setDpdkVhostUserMode(VirtualMachineTO to, VirtualMachineProfile vm);
/**
* True if VM is a guest DPDK enabled VM, false if not.
* It is determined by:
* - VM type is guest
* - VM details contains NUMA and Huge pages configurations for DPDK
* - VM host contains the DPDK capability
*/
boolean isVMDpdkEnabled(long vmId);
/**
* True if host is DPDK enabled, false if not.
* Host is DPDK enabled when:
* - 'dpdk' is part of the host capabilities
*/
boolean isHostDpdkEnabled(long hostId);
}

154
server/src/main/java/com/cloud/hypervisor/kvm/dpdk/DpdkHelperImpl.java Normal file
View File

@ -0,0 +1,154 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.hypervisor.kvm.dpdk;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingDetailsVO;
import com.cloud.service.dao.ServiceOfferingDetailsDao;
import com.cloud.utils.StringUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.UserVmDetailVO;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineProfile;
import com.cloud.vm.dao.UserVmDetailsDao;
import com.cloud.vm.dao.VMInstanceDao;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import java.util.List;
public class DpdkHelperImpl implements DpdkHelper {
@Inject
private ServiceOfferingDetailsDao serviceOfferingDetailsDao;
@Inject
private HostDao hostDao;
@Inject
private VMInstanceDao vmInstanceDao;
@Inject
private UserVmDetailsDao userVmDetailsDao;
public static final Logger s_logger = Logger.getLogger(DpdkHelperImpl.class);
private ServiceOffering getServiceOfferingFromVMProfile(VirtualMachineProfile virtualMachineProfile) {
ServiceOffering offering = virtualMachineProfile.getServiceOffering();
if (offering == null) {
throw new CloudRuntimeException("VM does not have an associated service offering");
}
return offering;
}
@Override
public boolean isDpdkvHostUserModeSettingOnServiceOffering(VirtualMachineProfile vm) {
ServiceOffering offering = getServiceOfferingFromVMProfile(vm);
ServiceOfferingDetailsVO detail = serviceOfferingDetailsDao.findDetail(offering.getId(), DPDK_VHOST_USER_MODE);
return detail != null;
}
@Override
public void setDpdkVhostUserMode(VirtualMachineTO to, VirtualMachineProfile vm) {
ServiceOffering offering = getServiceOfferingFromVMProfile(vm);
ServiceOfferingDetailsVO detail = serviceOfferingDetailsDao.findDetail(offering.getId(), DPDK_VHOST_USER_MODE);
if (detail != null) {
String mode = detail.getValue();
try {
VHostUserMode dpdKvHostUserMode = VHostUserMode.fromValue(mode);
to.addExtraConfig(DPDK_VHOST_USER_MODE, dpdKvHostUserMode.toString());
} catch (IllegalArgumentException e) {
s_logger.error(String.format("DPDK vHost User mode found as a detail for service offering: %s " +
"but value: %s is not supported. Supported values: %s, %s",
offering.getId(), mode,
VHostUserMode.CLIENT.toString(), VHostUserMode.SERVER.toString()));
}
}
}
@Override
public boolean isVMDpdkEnabled(long vmId) {
VMInstanceVO vm = vmInstanceDao.findById(vmId);
if (vm == null) {
throw new CloudRuntimeException("Could not find VM with id " + vmId);
}
if (vm.getType() != VirtualMachine.Type.User) {
return false;
}
List<UserVmDetailVO> details = userVmDetailsDao.listDetails(vm.getId());
List<ServiceOfferingDetailsVO> offeringDetails = serviceOfferingDetailsDao.listDetails(vm.getServiceOfferingId());
if (!hasRequiredDPDKConfigurations(details, offeringDetails)) {
return false;
}
return isHostDpdkEnabled(vm.getHostId());
}
/**
* True if VM is DPDK enabled. NUMA and HUGEPAGES configurations must be present on VM or service offering details
*/
private boolean hasRequiredDPDKConfigurations(List<UserVmDetailVO> details, List<ServiceOfferingDetailsVO> offeringDetails) {
if (CollectionUtils.isEmpty(details)) {
return hasValidDPDKConfigurationsOnServiceOffering(false, false, offeringDetails);
} else {
boolean isNumaSet = false;
boolean isHugePagesSet = false;
for (UserVmDetailVO detail : details) {
if (detail.getName().equals(DPDK_NUMA)) {
isNumaSet = true;
} else if (detail.getName().equals(DPDK_HUGE_PAGES)) {
isHugePagesSet = true;
}
}
boolean valid = isNumaSet && isHugePagesSet;
if (!valid) {
return hasValidDPDKConfigurationsOnServiceOffering(isNumaSet, isHugePagesSet, offeringDetails);
}
return true;
}
}
/**
* True if DPDK required configurations are set
*/
private boolean hasValidDPDKConfigurationsOnServiceOffering(boolean isNumaSet, boolean isHugePagesSet, List<ServiceOfferingDetailsVO> offeringDetails) {
if (!CollectionUtils.isEmpty(offeringDetails)) {
for (ServiceOfferingDetailsVO detail : offeringDetails) {
if (detail.getName().equals(DPDK_NUMA)) {
isNumaSet = true;
} else if (detail.getName().equals(DPDK_HUGE_PAGES)) {
isHugePagesSet = true;
}
}
}
return isNumaSet && isHugePagesSet;
}
@Override
public boolean isHostDpdkEnabled(long hostId) {
HostVO host = hostDao.findById(hostId);
if (host == null) {
throw new CloudRuntimeException("Could not find host with id " + hostId);
}
return StringUtils.isNotBlank(host.getCapabilities()) && host.getCapabilities().contains("dpdk");
}
}

24
server/src/main/java/com/cloud/server/ManagementServerImpl.java
View File

@ -37,6 +37,8 @@ import javax.crypto.spec.SecretKeySpec;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.storage.ScopeType;
import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.affinity.AffinityGroupProcessor;
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
@ -634,7 +636,6 @@ import com.cloud.storage.GuestOSHypervisor;
import com.cloud.storage.GuestOSHypervisorVO;
import com.cloud.storage.GuestOSVO;
import com.cloud.storage.GuestOsCategory;
import com.cloud.storage.ScopeType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.Volume;
@ -811,6 +812,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
private GuestOsDetailsDao _guestOsDetailsDao;
@Inject
private KeystoreManager _ksMgr;
@Inject
private DpdkHelper dpdkHelper;
private LockMasterListener _lockMasterListener;
private final ScheduledExecutorService _eventExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("EventChecker"));
@ -1302,6 +1305,10 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
final ExcludeList excludes = new ExcludeList();
excludes.addHost(srcHostId);
if (dpdkHelper.isVMDpdkEnabled(vm.getId())) {
excludeNonDPDKEnabledHosts(plan, excludes);
}
// call affinitygroup chain
final long vmGroupCount = _affinityGroupVMMapDao.countAffinityGroupsForVm(vm.getId());
@ -1334,6 +1341,21 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
return new Ternary<Pair<List<? extends Host>, Integer>, List<? extends Host>, Map<Host, Boolean>>(otherHosts, suitableHosts, requiresStorageMotion);
}
/**
* Add non DPDK enabled hosts to the avoid list
*/
private void excludeNonDPDKEnabledHosts(DataCenterDeployment plan, ExcludeList excludes) {
long dataCenterId = plan.getDataCenterId();
Long clusterId = plan.getClusterId();
Long podId = plan.getPodId();
List<HostVO> hosts = _hostDao.listAllUpAndEnabledNonHAHosts(Type.Routing, clusterId, podId, dataCenterId, null);
for (HostVO host : hosts) {
if (!dpdkHelper.isHostDpdkEnabled(host.getId())) {
excludes.addHost(host.getId());
}
}
}
private boolean hasSuitablePoolsForVolume(final VolumeVO volume, final Host host, final VirtualMachineProfile vmProfile) {
final DiskOfferingVO diskOffering = _diskOfferingDao.findById(volume.getDiskOfferingId());
final DiskProfile diskProfile = new DiskProfile(volume, diskOffering, vmProfile.getHypervisorType());

7
server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
View File

@ -96,6 +96,7 @@ import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command;
@ -472,6 +473,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
private TemplateApiService _tmplService;
@Inject
private ConfigurationDao _configDao;
@Inject
private DpdkHelper dpdkHelper;
private ScheduledExecutorService _executor = null;
private ScheduledExecutorService _vmIpFetchExecutor = null;
@ -5225,6 +5228,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
}
}
if (dpdkHelper.isVMDpdkEnabled(vm.getId()) && !dpdkHelper.isHostDpdkEnabled(destinationHost.getId())) {
throw new CloudRuntimeException("Cannot migrate VM, VM is DPDK enabled VM but destination host is not DPDK enabled");
}
checkHostsDedication(vm, srcHostId, destinationHost.getId());
// call to core process

2
server/src/main/resources/META-INF/cloudstack/core/spring-server-core-misc-context.xml
View File

@ -79,6 +79,6 @@
<property name="name" value="Basic" />
</bean>
<bean id="DPDKHelper" class="com.cloud.hypervisor.kvm.dpdk.DPDKHelperImpl" />
<bean id="DPDKHelper" class="com.cloud.hypervisor.kvm.dpdk.DpdkHelperImpl" />
</beans>

13
server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java
View File

@ -129,17 +129,4 @@ public class KVMGuruTest {
guru.setVmQuotaPercentage(vmTO, vmProfile);
Mockito.verify(vmTO).setCpuQuotaPercentage(1d);
}
@Test
public void testAddServiceOfferingExtraConfigurationDpdkDetails() {
guru.addServiceOfferingExtraConfiguration(vmTO, vmProfile);
Mockito.verify(vmTO).addExtraConfig(detail1Key, detail1Value);
}
@Test
public void testAddServiceOfferingExtraConfigurationEmptyDetails() {
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(null);
guru.addServiceOfferingExtraConfiguration(vmTO, vmProfile);
Mockito.verify(vmTO, Mockito.never()).addExtraConfig(Mockito.anyString(), Mockito.anyString());
}
}

135
server/src/test/java/com/cloud/hypervisor/kvm/dpdk/DPDKHelperImplTest.java
View File

@ -1,135 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.hypervisor.kvm.dpdk;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingDetailsVO;
import com.cloud.service.dao.ServiceOfferingDetailsDao;
import com.cloud.vm.VirtualMachineProfile;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.runners.MockitoJUnitRunner;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Arrays;
@RunWith(MockitoJUnitRunner.class)
public class DPDKHelperImplTest {
@Mock
ServiceOfferingDetailsDao serviceOfferingDetailsDao;
@Spy
@InjectMocks
private DPDKHelper dpdkHelper = new DPDKHelperImpl();
@Mock
VirtualMachineTO vmTO;
@Mock
VirtualMachineProfile vmProfile;
@Mock
ServiceOfferingDetailsVO dpdkVhostUserModeDetailVO;
@Mock
ServiceOfferingDetailsVO dpdkNumaDetailVO;
@Mock
ServiceOfferingDetailsVO dpdkHugePagesDetailVO;
@Mock
ServiceOffering serviceOffering;
private String dpdkVhostMode = DPDKHelper.VHostUserMode.SERVER.toString();
private static final String dpdkNumaConf =
"<cpu mode=\"host-passthrough\">\n" +
" <numa>\n" +
" <cell id=\"0\" cpus=\"0\" memory=\"9437184\" unit=\"KiB\" memAccess=\"shared\"/>\n" +
" </numa>\n" +
"</cpu>";
private static final String dpdkHugePagesConf =
"<memoryBacking>\n" +
" <hugePages/>\n" +
"</memoryBacking>";
private static String dpdkNumaValue;
private static String dpdkHugePagesValue;
private static final Long offeringId = 1L;
@Before
public void setup() throws UnsupportedEncodingException {
dpdkHugePagesValue = URLEncoder.encode(dpdkHugePagesConf, "UTF-8");
dpdkNumaValue = URLEncoder.encode(dpdkNumaConf, "UTF-8");
Mockito.when(dpdkVhostUserModeDetailVO.getName()).thenReturn(DPDKHelper.DPDK_VHOST_USER_MODE);
Mockito.when(dpdkVhostUserModeDetailVO.getValue()).thenReturn(dpdkVhostMode);
Mockito.when(dpdkVhostUserModeDetailVO.getResourceId()).thenReturn(offeringId);
Mockito.when(dpdkNumaDetailVO.getName()).thenReturn(DPDKHelper.DPDK_NUMA);
Mockito.when(dpdkNumaDetailVO.getResourceId()).thenReturn(offeringId);
Mockito.when(dpdkNumaDetailVO.getValue()).thenReturn(dpdkNumaValue);
Mockito.when(dpdkHugePagesDetailVO.getName()).thenReturn(DPDKHelper.DPDK_HUGE_PAGES);
Mockito.when(dpdkHugePagesDetailVO.getResourceId()).thenReturn(offeringId);
Mockito.when(dpdkHugePagesDetailVO.getValue()).thenReturn(dpdkHugePagesValue);
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(
Arrays.asList(dpdkNumaDetailVO, dpdkHugePagesDetailVO, dpdkVhostUserModeDetailVO));
Mockito.when(vmProfile.getServiceOffering()).thenReturn(serviceOffering);
Mockito.when(serviceOffering.getId()).thenReturn(offeringId);
}
@Test
public void testSetDpdkVhostUserModeValidDetail() {
Mockito.when(serviceOfferingDetailsDao.findDetail(offeringId, DPDKHelper.DPDK_VHOST_USER_MODE)).
thenReturn(dpdkVhostUserModeDetailVO);
dpdkHelper.setDpdkVhostUserMode(vmTO, vmProfile);
Mockito.verify(vmTO).addExtraConfig(DPDKHelper.DPDK_VHOST_USER_MODE, dpdkVhostMode);
}
@Test
public void testSetDpdkVhostUserModeInvalidDetail() {
Mockito.when(dpdkVhostUserModeDetailVO.getValue()).thenReturn("serverrrr");
Mockito.verify(vmTO, Mockito.never()).addExtraConfig(Mockito.anyString(), Mockito.anyString());
}
@Test
public void testSetDpdkVhostUserModeNotExistingDetail() {
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(
Arrays.asList(dpdkNumaDetailVO, dpdkHugePagesDetailVO));
Mockito.verify(vmTO, Mockito.never()).addExtraConfig(Mockito.anyString(), Mockito.anyString());
}
@Test
public void testDPDKvHostUserFromValueClient() {
DPDKHelper.VHostUserMode mode = DPDKHelper.VHostUserMode.fromValue("client");
Assert.assertEquals(DPDKHelper.VHostUserMode.CLIENT, mode);
}
@Test
public void testDPDKvHostUserFromValueServer() {
DPDKHelper.VHostUserMode mode = DPDKHelper.VHostUserMode.fromValue("server");
Assert.assertEquals(DPDKHelper.VHostUserMode.SERVER, mode);
}
@Test(expected = IllegalArgumentException.class)
public void testDPDKvHostUserFromValueServerInvalid() {
DPDKHelper.VHostUserMode.fromValue("serverrrr");
}
}

245
server/src/test/java/com/cloud/hypervisor/kvm/dpdk/DpdkHelperImplTest.java Normal file
View File

@ -0,0 +1,245 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.hypervisor.kvm.dpdk;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingDetailsVO;
import com.cloud.service.dao.ServiceOfferingDetailsDao;
import com.cloud.vm.UserVmDetailVO;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineProfile;
import com.cloud.vm.dao.UserVmDetailsDao;
import com.cloud.vm.dao.VMInstanceDao;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.runners.MockitoJUnitRunner;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@RunWith(MockitoJUnitRunner.class)
public class DpdkHelperImplTest {
@Mock
ServiceOfferingDetailsDao serviceOfferingDetailsDao;
@Mock
HostDao hostDao;
@Mock
VMInstanceDao vmInstanceDao;
@Mock
UserVmDetailsDao userVmDetailsDao;
@Spy
@InjectMocks
private DpdkHelper dpdkHelper = new DpdkHelperImpl();
@Mock
VirtualMachineTO vmTO;
@Mock
VirtualMachineProfile vmProfile;
@Mock
ServiceOfferingDetailsVO dpdkVhostUserModeDetailVO;
@Mock
ServiceOfferingDetailsVO dpdkNumaDetailVO;
@Mock
ServiceOfferingDetailsVO dpdkHugePagesDetailVO;
@Mock
ServiceOffering serviceOffering;
@Mock
UserVmDetailVO dpdkNumaVmDetail;
@Mock
UserVmDetailVO dpdkHugePagesVmDetail;
@Mock
HostVO hostVO;
@Mock
VMInstanceVO vmInstanceVO;
private String dpdkVhostMode = DpdkHelper.VHostUserMode.SERVER.toString();
private static final String dpdkNumaConf =
"<cpu mode=\"host-passthrough\">\n" +
" <numa>\n" +
" <cell id=\"0\" cpus=\"0\" memory=\"9437184\" unit=\"KiB\" memAccess=\"shared\"/>\n" +
" </numa>\n" +
"</cpu>";
private static final String dpdkHugePagesConf =
"<memoryBacking>\n" +
" <hugePages/>\n" +
"</memoryBacking>";
private static String dpdkNumaValue;
private static String dpdkHugePagesValue;
private static final Long offeringId = 1L;
private static final Long hostId = 1L;
private static final String hostCapabilities = "hvm,snapshot";
private static final Long vmId = 1L;
@Before
public void setup() throws UnsupportedEncodingException {
dpdkHugePagesValue = URLEncoder.encode(dpdkHugePagesConf, "UTF-8");
dpdkNumaValue = URLEncoder.encode(dpdkNumaConf, "UTF-8");
Mockito.when(dpdkVhostUserModeDetailVO.getName()).thenReturn(DpdkHelper.DPDK_VHOST_USER_MODE);
Mockito.when(dpdkVhostUserModeDetailVO.getValue()).thenReturn(dpdkVhostMode);
Mockito.when(dpdkVhostUserModeDetailVO.getResourceId()).thenReturn(offeringId);
Mockito.when(dpdkNumaDetailVO.getName()).thenReturn(DpdkHelper.DPDK_NUMA);
Mockito.when(dpdkNumaDetailVO.getResourceId()).thenReturn(offeringId);
Mockito.when(dpdkNumaDetailVO.getValue()).thenReturn(dpdkNumaValue);
Mockito.when(dpdkHugePagesDetailVO.getName()).thenReturn(DpdkHelper.DPDK_HUGE_PAGES);
Mockito.when(dpdkHugePagesDetailVO.getResourceId()).thenReturn(offeringId);
Mockito.when(dpdkHugePagesDetailVO.getValue()).thenReturn(dpdkHugePagesValue);
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(
Arrays.asList(dpdkNumaDetailVO, dpdkHugePagesDetailVO, dpdkVhostUserModeDetailVO));
Mockito.when(vmProfile.getServiceOffering()).thenReturn(serviceOffering);
Mockito.when(serviceOffering.getId()).thenReturn(offeringId);
Mockito.when(vmProfile.getServiceOffering()).thenReturn(serviceOffering);
Mockito.when(serviceOffering.getId()).thenReturn(offeringId);
Mockito.when(hostDao.findById(hostId)).thenReturn(hostVO);
Mockito.when(hostVO.getCapabilities()).thenReturn(hostCapabilities);
Mockito.when(vmInstanceDao.findById(vmId)).thenReturn(vmInstanceVO);
Mockito.when(vmInstanceVO.getType()).thenReturn(VirtualMachine.Type.User);
Mockito.when(vmInstanceVO.getHostId()).thenReturn(hostId);
Mockito.when(vmInstanceVO.getServiceOfferingId()).thenReturn(offeringId);
Mockito.when(vmInstanceVO.getId()).thenReturn(vmId);
Mockito.when(dpdkNumaVmDetail.getName()).thenReturn(DpdkHelper.DPDK_NUMA);
Mockito.when(dpdkNumaVmDetail.getValue()).thenReturn(dpdkNumaConf);
Mockito.when(dpdkHugePagesVmDetail.getName()).thenReturn(DpdkHelper.DPDK_HUGE_PAGES);
Mockito.when(dpdkHugePagesVmDetail.getValue()).thenReturn(dpdkHugePagesConf);
Mockito.when(userVmDetailsDao.listDetails(vmId)).thenReturn(Arrays.asList(dpdkNumaVmDetail, dpdkHugePagesVmDetail));
}
@Test
public void testSetDpdkVhostUserModeValidDetail() {
Mockito.when(serviceOfferingDetailsDao.findDetail(offeringId, DpdkHelper.DPDK_VHOST_USER_MODE)).
thenReturn(dpdkVhostUserModeDetailVO);
dpdkHelper.setDpdkVhostUserMode(vmTO, vmProfile);
Mockito.verify(vmTO).addExtraConfig(DpdkHelper.DPDK_VHOST_USER_MODE, dpdkVhostMode);
}
@Test
public void testSetDpdkVhostUserModeInvalidDetail() {
Mockito.when(dpdkVhostUserModeDetailVO.getValue()).thenReturn("serverrrr");
Mockito.verify(vmTO, Mockito.never()).addExtraConfig(Mockito.anyString(), Mockito.anyString());
}
@Test
public void testSetDpdkVhostUserModeNotExistingDetail() {
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(
Arrays.asList(dpdkNumaDetailVO, dpdkHugePagesDetailVO));
Mockito.verify(vmTO, Mockito.never()).addExtraConfig(Mockito.anyString(), Mockito.anyString());
}
@Test
public void testDpdkvHostUserFromValueClient() {
DpdkHelper.VHostUserMode mode = DpdkHelper.VHostUserMode.fromValue("client");
Assert.assertEquals(DpdkHelper.VHostUserMode.CLIENT, mode);
}
@Test
public void testDpdkvHostUserFromValueServer() {
DpdkHelper.VHostUserMode mode = DpdkHelper.VHostUserMode.fromValue("server");
Assert.assertEquals(DpdkHelper.VHostUserMode.SERVER, mode);
}
@Test(expected = IllegalArgumentException.class)
public void testDpdkvHostUserFromValueServerInvalid() {
DpdkHelper.VHostUserMode.fromValue("serverrrr");
}
@Test
public void testIsHostDpdkEnabledNonDPDKHost() {
Assert.assertFalse(dpdkHelper.isHostDpdkEnabled(hostId));
}
@Test
public void testIsHostDpdkEnabledDpdkHost() {
Mockito.when(hostVO.getCapabilities()).thenReturn(hostCapabilities + ",dpdk");
Assert.assertTrue(dpdkHelper.isHostDpdkEnabled(hostId));
}
@Test
public void testIsVMDPDKEnabledDPDKEnabledVM() {
Mockito.when(hostVO.getCapabilities()).thenReturn(hostCapabilities + ",dpdk");
Assert.assertTrue(dpdkHelper.isVMDpdkEnabled(vmId));
}
@Test
public void testIsVMDpdkEnabledGuestType() {
Mockito.when(vmInstanceVO.getType()).thenReturn(VirtualMachine.Type.SecondaryStorageVm);
Assert.assertFalse(dpdkHelper.isVMDpdkEnabled(vmId));
Mockito.verify(dpdkHelper, Mockito.never()).isHostDpdkEnabled(hostId);
}
@Test
public void testIsVMDpdkEnabledGuestTypeMissingConfigurationOnDetails() {
Mockito.when(userVmDetailsDao.listDetails(vmId)).thenReturn(Arrays.asList(dpdkNumaVmDetail));
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(new ArrayList<>());
Assert.assertFalse(dpdkHelper.isVMDpdkEnabled(vmId));
Mockito.verify(dpdkHelper, Mockito.never()).isHostDpdkEnabled(hostId);
}
@Test
public void testIsVMDpdkEnabledGuestTypeEmptyDetails() {
Mockito.when(userVmDetailsDao.listDetails(vmId)).thenReturn(new ArrayList<>());
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(new ArrayList<>());
Assert.assertFalse(dpdkHelper.isVMDpdkEnabled(vmId));
Mockito.verify(dpdkHelper, Mockito.never()).isHostDpdkEnabled(hostId);
}
@Test
public void testIsVMDpdkEnabledNonDPDKCapabilityOnHost() {
Assert.assertFalse(dpdkHelper.isVMDpdkEnabled(vmId));
}
@Test
public void testIsVMDpdkEnabledGuestTypeMissingConfigurationOnVmDetails() {
Mockito.when(userVmDetailsDao.listDetails(vmId)).thenReturn(Collections.singletonList(dpdkNumaVmDetail));
Mockito.when(hostVO.getCapabilities()).thenReturn(hostCapabilities + ",dpdk");
Assert.assertTrue(dpdkHelper.isVMDpdkEnabled(vmId));
}
@Test
public void testIsVMDpdkEnabledGuestTypeEmptyVmDetails() {
Mockito.when(userVmDetailsDao.listDetails(vmId)).thenReturn(new ArrayList<>());
Mockito.when(hostVO.getCapabilities()).thenReturn(hostCapabilities + ",dpdk");
Assert.assertTrue(dpdkHelper.isVMDpdkEnabled(vmId));
}
@Test
public void testIsVMDpdkEnabledGuestTypeMixedConfigurationOnDetails() {
Mockito.when(userVmDetailsDao.listDetails(vmId)).thenReturn(Collections.singletonList(dpdkNumaVmDetail));
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(Collections.singletonList(dpdkHugePagesDetailVO));
Mockito.when(hostVO.getCapabilities()).thenReturn(hostCapabilities + ",dpdk");
Assert.assertTrue(dpdkHelper.isVMDpdkEnabled(vmId));
}
}