Enable DPDK support on KVM (#2839)

* Enable DPDK support on KVM

* Allow DPDK deployments on user VMs only

* Fix port name ordering

agent/conf/agent.properties

@@ -108,6 +108,10 @@ domr.scripts.dir=scripts/network/domr/kvm
 # openvswitch = com.cloud.hypervisor.kvm.resource.OvsVifDriver
 #libvirt.vif.driver=com.cloud.hypervisor.kvm.resource.BridgeVifDriver
 
+# Set DPDK Support on OpenVswitch
+#openvswitch.dpdk.enabled=true
+#openvswitch.dpdk.ovs.path=/var/run/openvswitch
+
 # set the hypervisor type, values are: kvm, lxc
 hypervisor.type=kvm
 

api/src/main/java/com/cloud/agent/api/to/NicTO.java

@@ -30,6 +30,7 @@ public class NicTO extends NetworkTO {
     String nicUuid;
     List<String> nicSecIps;
     Map<NetworkOffering.Detail, String> details;
+    boolean dpdkDisabled;
 
     public NicTO() {
         super();
@@ -109,4 +110,12 @@ public class NicTO extends NetworkTO {
     public void setDetails(final Map<NetworkOffering.Detail, String> details) {
         this.details = details;
     }
+
+    public boolean isDpdkDisabled() {
+        return dpdkDisabled;
+    }
+
+    public void setDpdkDisabled(boolean dpdkDisabled) {
+        this.dpdkDisabled = dpdkDisabled;
+    }
 }

api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java

@@ -73,6 +73,7 @@ public class VirtualMachineTO {
     Double cpuQuotaPercentage = null;
 
     Map<String, String> guestOsDetails = new HashMap<String, String>();
+    Map<String, String> extraConfig = new HashMap<>();
 
     public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader,
             String os, boolean enableHA, boolean limitCpuUse, String vncPassword) {
@@ -350,4 +351,11 @@ public class VirtualMachineTO {
     public void setCpuQuotaPercentage(Double cpuQuotaPercentage) {
         this.cpuQuotaPercentage = cpuQuotaPercentage;
     }
+
+    public void addExtraConfig(String key, String value) {
+        extraConfig.put(key, value);
+    }
+    public Map<String, String> getExtraConfig() {
+        return extraConfig;
+    }
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -117,6 +117,7 @@ public class ApiConstants {
     public static final String END_PORT = "endport";
     public static final String ENTRY_TIME = "entrytime";
     public static final String EXPIRES = "expires";
+    public static final String EXTRA_CONFIG = "extraconfig";
     public static final String EXTRA_DHCP_OPTION = "extradhcpoption";
     public static final String EXTRA_DHCP_OPTION_NAME = "extradhcpoptionname";
     public static final String EXTRA_DHCP_OPTION_CODE = "extradhcpoptioncode";

api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java

@@ -200,6 +200,9 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
             " an optional parameter used to create additional data disks from datadisk templates; can't be specified with diskOfferingId parameter")
     private Map dataDiskTemplateToDiskOfferingList;
 
+    @Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", length = 5120)
+    private String extraConfig;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -482,6 +485,10 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
         return dataDiskTemplateToDiskOfferingMap;
     }
 
+    public String getExtraConfig() {
+        return extraConfig;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java

@@ -128,6 +128,8 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction
             + " Example: dhcpoptionsnetworklist[0].dhcp:114=url&dhcpoptionsetworklist[0].networkid=networkid&dhcpoptionsetworklist[0].dhcp:66=www.test.com")
     private Map dhcpOptionsNetworkList;
 
+    @Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", authorized = { RoleType.Admin }, length = 5120)
+    private String extraConfig;
 
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
@@ -221,6 +223,10 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction
         return dhcpOptionsMap;
     }
 
+    public String getExtraConfig() {
+        return extraConfig;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -39,6 +39,7 @@ import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
 import org.apache.cloudstack.ca.CAManager;
 import org.apache.cloudstack.context.CallContext;
@@ -1112,6 +1113,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
 
                     vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx);
 
+                    addExtraConfig(vmTO);
+
                     work = _workDao.findById(work.getId());
                     if (work == null || work.getStep() != Step.Prepare) {
                         throw new ConcurrentOperationException("Work steps have been changed: " + work);
@@ -1276,6 +1279,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         }
     }
 
+    private void addExtraConfig(VirtualMachineTO vmTO) {
+        Map<String, String> details = vmTO.getDetails();
+        for (String key : details.keySet()) {
+            if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
+                vmTO.addExtraConfig(key, details.get(key));
+            }
+        }
+    }
+
     // for managed storage on KVM, need to make sure the path field of the volume in question is populated with the IQN
     private void handlePath(final DiskTO[] disks, final HypervisorType hypervisorType) {
         if (hypervisorType != HypervisorType.KVM) {

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java

@@ -207,7 +207,7 @@ public class BridgeVifDriver extends VifDriverBase {
     }
 
     @Override
-    public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException {
+    public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
 
         if (s_logger.isDebugEnabled()) {
             s_logger.debug("nic=" + nic);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/DirectVifDriver.java

@@ -26,6 +26,8 @@ import com.cloud.agent.api.to.NicTO;
 import com.cloud.exception.InternalErrorException;
 import com.cloud.network.Networks;
 
+import java.util.Map;
+
 public class DirectVifDriver extends VifDriverBase {
 
     private static final Logger s_logger = Logger.getLogger(DirectVifDriver.class);
@@ -36,12 +38,13 @@ public class DirectVifDriver extends VifDriverBase {
      *
      * @param nic
      * @param guestOsType
+     * @param extraConfig
      * @return
      * @throws InternalErrorException
      * @throws LibvirtException
      */
     @Override
-    public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException {
+    public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
         LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef();
 
         if (nic.getType() == Networks.TrafficType.Guest) {

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/IvsVifDriver.java

@@ -77,7 +77,7 @@ public class IvsVifDriver extends VifDriverBase {
     }
 
     @Override
-    public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException {
+    public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
         LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef();
 
         String vNetId = null;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -48,6 +48,7 @@ import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
 import com.cloud.resource.RequestWrapper;
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
@@ -56,6 +57,7 @@ import org.apache.cloudstack.utils.linux.CPUStat;
 import org.apache.cloudstack.utils.linux.MemStat;
 import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
 import org.apache.cloudstack.utils.security.KeyStoreUtils;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.ArrayUtils;
@@ -521,6 +523,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
     protected StorageSubsystemCommandHandler storageHandler;
 
+    protected boolean dpdkSupport = false;
+    protected String dpdkOvsPath;
+    protected static final String DPDK_NUMA = ApiConstants.EXTRA_CONFIG + "-dpdk-numa";
+    protected static final String DPDK_HUGE_PAGES = ApiConstants.EXTRA_CONFIG + "-dpdk-hugepages";
+    protected static final String DPDK_INTERFACE_PREFIX = ApiConstants.EXTRA_CONFIG + "-dpdk-interface-";
+
     private String getEndIpFromStartIp(final String startIp, final int numIps) {
         final String[] tokens = startIp.split("[.]");
         assert tokens.length == 4;
@@ -637,6 +645,15 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
             _bridgeType = BridgeType.valueOf(bridgeType.toUpperCase());
         }
 
+        String dpdk = (String) params.get("openvswitch.dpdk.enabled");
+        if (_bridgeType == BridgeType.OPENVSWITCH && Boolean.parseBoolean(dpdk)) {
+            dpdkSupport = true;
+            dpdkOvsPath = (String) params.get("openvswitch.dpdk.ovs.path");
+            if (dpdkOvsPath != null && !dpdkOvsPath.endsWith("/")) {
+                dpdkOvsPath += "/";
+            }
+        }
+
         params.put("domr.scripts.dir", domrScriptsDir);
 
         _virtRouterResource = new VirtualRoutingResource(this);
@@ -1634,7 +1651,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         }
 
         final Domain vm = getDomain(conn, vmName);
-        vm.attachDevice(getVifDriver(nicTO.getType()).plug(nicTO, "Other PV", "").toString());
+        vm.attachDevice(getVifDriver(nicTO.getType()).plug(nicTO, "Other PV", "", null).toString());
     }
 
 
@@ -2039,6 +2056,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         vm.setDomDescription(vmTO.getOs());
         vm.setPlatformEmulator(vmTO.getPlatformEmulator());
 
+        Map<String, String> extraConfig = vmTO.getExtraConfig();
+        if (dpdkSupport && (!extraConfig.containsKey(DPDK_NUMA) || !extraConfig.containsKey(DPDK_HUGE_PAGES))) {
+            s_logger.info("DPDK is enabled but it needs extra configurations for CPU NUMA and Huge Pages for VM deployment");
+        }
+
         final GuestDef guest = new GuestDef();
 
         if (HypervisorType.LXC == _hypervisorType && VirtualMachine.Type.User == vmTO.getType()) {
@@ -2072,21 +2094,23 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         grd.setVcpuNum(vcpus);
         vm.addComp(grd);
 
-        final CpuModeDef cmd = new CpuModeDef();
+        if (!extraConfig.containsKey(DPDK_NUMA)) {
-        cmd.setMode(_guestCpuMode);
+            final CpuModeDef cmd = new CpuModeDef();
-        cmd.setModel(_guestCpuModel);
+            cmd.setMode(_guestCpuMode);
-        if (vmTO.getType() == VirtualMachine.Type.User) {
+            cmd.setModel(_guestCpuModel);
-            cmd.setFeatures(_cpuFeatures);
+            if (vmTO.getType() == VirtualMachine.Type.User) {
+                cmd.setFeatures(_cpuFeatures);
+            }
+            // multi cores per socket, for larger core configs
+            if (vcpus % 6 == 0) {
+                final int sockets = vcpus / 6;
+                cmd.setTopology(6, sockets);
+            } else if (vcpus % 4 == 0) {
+                final int sockets = vcpus / 4;
+                cmd.setTopology(4, sockets);
+            }
+            vm.addComp(cmd);
         }
-        // multi cores per socket, for larger core configs
-        if (vcpus % 6 == 0) {
-            final int sockets = vcpus / 6;
-            cmd.setTopology(6, sockets);
-        } else if (vcpus % 4 == 0) {
-            final int sockets = vcpus / 4;
-            cmd.setTopology(4, sockets);
-        }
-        vm.addComp(cmd);
 
         if (_hypervisorLibvirtVersion >= 9000) {
             final CpuTuneDef ctd = new CpuTuneDef();
@@ -2192,9 +2216,29 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
         vm.addComp(devices);
 
+        addExtraConfigComponent(extraConfig, vm);
+
         return vm;
     }
 
+    /**
+     * Add extra configurations (if any) as a String component to the domain XML
+     */
+    protected void addExtraConfigComponent(Map<String, String> extraConfig, LibvirtVMDef vm) {
+        if (MapUtils.isNotEmpty(extraConfig)) {
+            StringBuilder extraConfigBuilder = new StringBuilder();
+            for (String key : extraConfig.keySet()) {
+                if (!key.startsWith(DPDK_INTERFACE_PREFIX)) {
+                    extraConfigBuilder.append(extraConfig.get(key));
+                }
+            }
+            String comp = extraConfigBuilder.toString();
+            if (org.apache.commons.lang.StringUtils.isNotBlank(comp)) {
+                vm.addComp(comp);
+            }
+        }
+    }
+
     public void createVifs(final VirtualMachineTO vmSpec, final LibvirtVMDef vm) throws InternalErrorException, LibvirtException {
         final NicTO[] nics = vmSpec.getNics();
         final Map <String, String> params = vmSpec.getDetails();
@@ -2202,10 +2246,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         if (params != null && params.get("nicAdapter") != null && !params.get("nicAdapter").isEmpty()) {
             nicAdapter = params.get("nicAdapter");
         }
+        Map<String, String> extraConfig = vmSpec.getExtraConfig();
         for (int i = 0; i < nics.length; i++) {
             for (final NicTO nic : vmSpec.getNics()) {
                 if (nic.getDeviceId() == i) {
-                    createVif(vm, nic, nicAdapter);
+                    createVif(vm, nic, nicAdapter, extraConfig);
                 }
             }
         }
@@ -2400,7 +2445,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
     }
 
-    private void createVif(final LibvirtVMDef vm, final NicTO nic, final String nicAdapter) throws InternalErrorException, LibvirtException {
+    private void createVif(final LibvirtVMDef vm, final NicTO nic, final String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
 
         if (nic.getType().equals(TrafficType.Guest) && nic.getBroadcastType().equals(BroadcastDomainType.Vsp)) {
             String vrIp = nic.getBroadcastUri().getPath().substring(1);
@@ -2415,7 +2460,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
             s_logger.error("LibvirtVMDef object get devices with null result");
             throw new InternalErrorException("LibvirtVMDef object get devices with null result");
         }
-        vm.getDevices().addDevice(getVifDriver(nic.getType(), nic.getName()).plug(nic, vm.getPlatformEmulator(), nicAdapter));
+        vm.getDevices().addDevice(getVifDriver(nic.getType(), nic.getName()).plug(nic, vm.getPlatformEmulator(), nicAdapter, extraConfig));
     }
 
     public boolean cleanupDisk(Map<String, String> volumeToDisconnect) {

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java

@@ -182,6 +182,11 @@ public class LibvirtDomainXMLParser {
                 } else if (type.equalsIgnoreCase("ethernet")) {
                     String scriptPath = getAttrValue("script", "path", nic);
                     def.defEthernet(dev, mac, NicModel.valueOf(model.toUpperCase()), scriptPath, networkRateKBps);
+                } else if (type.equals("vhostuser")) {
+                    String sourcePort = getAttrValue("source", "path", nic);
+                    String[] sourcePathParts = sourcePort.split("/");
+                    String port = sourcePathParts[sourcePathParts.length - 1];
+                    def.setDpdkSourcePort(port);
                 }
 
                 if (StringUtils.isNotBlank(slot)) {

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java

@@ -23,6 +23,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 
 import com.google.common.collect.Maps;
@@ -900,7 +901,7 @@ public class LibvirtVMDef {
 
     public static class InterfaceDef {
         enum GuestNetType {
-            BRIDGE("bridge"), DIRECT("direct"), NETWORK("network"), USER("user"), ETHERNET("ethernet"), INTERNAL("internal");
+            BRIDGE("bridge"), DIRECT("direct"), NETWORK("network"), USER("user"), ETHERNET("ethernet"), INTERNAL("internal"), VHOSTUSER("vhostuser");
             String _type;
 
             GuestNetType(String type) {
@@ -933,7 +934,7 @@ public class LibvirtVMDef {
 
         private GuestNetType _netType; /*
          * bridge, ethernet, network, user,
-         * internal
+         * internal, vhostuser
          */
         private HostNicType _hostNetType; /* Only used by agent java code */
         private String _netSourceMode;
@@ -950,6 +951,9 @@ public class LibvirtVMDef {
         private boolean _pxeDisable = false;
         private boolean _linkStateUp = true;
         private Integer _slot;
+        private String _dpdkSourcePath;
+        private String _dpdkSourcePort;
+        private String _dpdkExtraLines;
 
         public void defBridgeNet(String brName, String targetBrName, String macAddr, NicModel model) {
             defBridgeNet(brName, targetBrName, macAddr, model, 0);
@@ -964,6 +968,16 @@ public class LibvirtVMDef {
             _networkRateKBps = networkRateKBps;
         }
 
+        public void defDpdkNet(String dpdkSourcePath, String dpdkPort, String macAddress, NicModel model, Integer networkRateKBps, String extra) {
+            _netType = GuestNetType.VHOSTUSER;
+            _dpdkSourcePath = dpdkSourcePath;
+            _dpdkSourcePort = dpdkPort;
+            _macAddr = macAddress;
+            _model = model;
+            _networkRateKBps = networkRateKBps;
+            _dpdkExtraLines = extra;
+        }
+
         public void defDirectNet(String sourceName, String targetName, String macAddr, NicModel model, String sourceMode) {
             defDirectNet(sourceName, targetName, macAddr, model, sourceMode, 0);
         }
@@ -1089,6 +1103,13 @@ public class LibvirtVMDef {
             return _linkStateUp;
         }
 
+        public String getDpdkSourcePort() {
+            return _dpdkSourcePort;
+        }
+        public void setDpdkSourcePort(String port) {
+            _dpdkSourcePort = port;
+        }
+
         @Override
         public String toString() {
             StringBuilder netBuilder = new StringBuilder();
@@ -1099,6 +1120,8 @@ public class LibvirtVMDef {
                 netBuilder.append("<source network='" + _sourceName + "'/>\n");
             } else if (_netType == GuestNetType.DIRECT) {
                 netBuilder.append("<source dev='" + _sourceName + "' mode='" + _netSourceMode + "'/>\n");
+            } else if (_netType == GuestNetType.VHOSTUSER) {
+                netBuilder.append("<source type='unix' path='"+ _dpdkSourcePath + _dpdkSourcePort + "' mode='client'/>\n");
             }
             if (_networkName != null) {
                 netBuilder.append("<target dev='" + _networkName + "'/>\n");
@@ -1132,7 +1155,13 @@ public class LibvirtVMDef {
                 netBuilder.append("<vlan trunk='no'>\n<tag id='" + _vlanTag + "'/>\n</vlan>");
             }
 
-            netBuilder.append("<link state='" + (_linkStateUp ? "up" : "down") +"'/>\n");
+            if (StringUtils.isNotBlank(_dpdkExtraLines)) {
+                netBuilder.append(_dpdkExtraLines);
+            }
+
+            if (_netType != GuestNetType.VHOSTUSER) {
+                netBuilder.append("<link state='" + (_linkStateUp ? "up" : "down") +"'/>\n");
+            }
 
             if (_slot  != null) {
                 netBuilder.append(String.format("<address type='pci' domain='0x0000' bus='0x00' slot='0x%02x' function='0x0'/>\n", _slot));

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java

@@ -24,6 +24,8 @@ import java.util.Map;
 
 import javax.naming.ConfigurationException;
 
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.libvirt.LibvirtException;
 
@@ -40,6 +42,8 @@ public class OvsVifDriver extends VifDriverBase {
     private static final Logger s_logger = Logger.getLogger(OvsVifDriver.class);
     private int _timeout;
 
+    protected static final String DPDK_PORT_PREFIX = "csdpdk-";
+
     @Override
     public void configure(Map<String, Object> params) throws ConfigurationException {
         super.configure(params);
@@ -76,12 +80,65 @@ public class OvsVifDriver extends VifDriverBase {
         s_logger.debug("done looking for pifs, no more bridges");
     }
 
+    /**
+     * Get the latest DPDK port number created on a DPDK enabled host
+     */
+    protected int getDpdkLatestPortNumberUsed() {
+        s_logger.debug("Checking the last DPDK port created");
+        String cmd = "ovs-vsctl show | grep Port | grep " + DPDK_PORT_PREFIX + " | " +
+                "awk '{ print $2 }' | sort -rV | head -1";
+        String port = Script.runSimpleBashScript(cmd);
+        int portNumber = 0;
+        if (StringUtils.isNotBlank(port)) {
+            String unquotedPort = port.replace("\"", "");
+            String dpdkPortNumber = unquotedPort.split(DPDK_PORT_PREFIX)[1];
+            portNumber = Integer.valueOf(dpdkPortNumber);
+        }
+        return portNumber;
+    }
+
+    /**
+     * Get the next DPDK port name to be created
+     */
+    protected String getNextDpdkPort() {
+        int portNumber = getDpdkLatestPortNumberUsed();
+        return DPDK_PORT_PREFIX + String.valueOf(portNumber + 1);
+    }
+
+    /**
+     * Add OVS port (if it does not exist) to bridge with DPDK support
+     */
+    protected void addDpdkPort(String bridgeName, String port, String vlan) {
+        String cmd = String.format("ovs-vsctl add-port %s %s " +
+                "vlan_mode=access tag=%s " +
+                "-- set Interface %s type=dpdkvhostuser", bridgeName, port, vlan, port);
+        s_logger.debug("DPDK property enabled, executing: " + cmd);
+        Script.runSimpleBashScript(cmd);
+    }
+
+    /**
+     * Check for additional extra 'dpdk-interface' configurations, return them appended
+     */
+    private String getExtraDpdkProperties(Map<String, String> extraConfig) {
+        StringBuilder stringBuilder = new StringBuilder();
+        for (String key : extraConfig.keySet()) {
+            if (key.startsWith(LibvirtComputingResource.DPDK_INTERFACE_PREFIX)) {
+                stringBuilder.append(extraConfig.get(key));
+            }
+        }
+        return stringBuilder.toString();
+    }
+
     @Override
-    public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException {
+    public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
         s_logger.debug("plugging nic=" + nic);
 
         LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef();
-        intf.setVirtualPortType("openvswitch");
+        if (!_libvirtComputingResource.dpdkSupport || nic.isDpdkDisabled()) {
+            // Let libvirt handle OVS ports creation when DPDK property is disabled or when it is enabled but disabled for the nic
+            // For DPDK support, libvirt does not handle ports creation, invoke 'addDpdkPort' method
+            intf.setVirtualPortType("openvswitch");
+        }
 
         String vlanId = null;
         String logicalSwitchUuid = null;
@@ -99,9 +156,19 @@ public class OvsVifDriver extends VifDriverBase {
             if ((nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan || nic.getBroadcastType() == Networks.BroadcastDomainType.Pvlan) &&
                     !vlanId.equalsIgnoreCase("untagged")) {
                 if (trafficLabel != null && !trafficLabel.isEmpty()) {
-                    s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel);
+                    if (_libvirtComputingResource.dpdkSupport && !nic.isDpdkDisabled()) {
-                    intf.defBridgeNet(_pifs.get(trafficLabel), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);
+                        s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
-                    intf.setVlanTag(Integer.parseInt(vlanId));
+                        if (StringUtils.isBlank(_libvirtComputingResource.dpdkOvsPath)) {
+                            throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
+                        }
+                        String port = getNextDpdkPort();
+                        addDpdkPort(_pifs.get(trafficLabel), port, vlanId);
+                        intf.defDpdkNet(_libvirtComputingResource.dpdkOvsPath, port, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), 0, getExtraDpdkProperties(extraConfig));
+                    } else {
+                        s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel);
+                        intf.defBridgeNet(_pifs.get(trafficLabel), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);
+                        intf.setVlanTag(Integer.parseInt(vlanId));
+                    }
                 } else {
                     intf.defBridgeNet(_pifs.get("private"), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);
                     intf.setVlanTag(Integer.parseInt(vlanId));
@@ -153,6 +220,13 @@ public class OvsVifDriver extends VifDriverBase {
     @Override
     public void unplug(InterfaceDef iface) {
         // Libvirt apparently takes care of this, see BridgeVifDriver unplug
+        if (_libvirtComputingResource.dpdkSupport) {
+            // If DPDK is enabled, we'll need to cleanup the port as libvirt won't
+            String dpdkPort = iface.getDpdkSourcePort();
+            String cmd = String.format("ovs-vsctl del-port %s", dpdkPort);
+            s_logger.debug("Removing DPDK port: " + dpdkPort);
+            Script.runSimpleBashScript(cmd);
+        }
     }
 
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java

@@ -32,7 +32,7 @@ public interface VifDriver {
 
     public void configure(Map<String, Object> params) throws ConfigurationException;
 
-    public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException;
+    public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException;
 
     public void unplug(LibvirtVMDef.InterfaceDef iface);
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriverBase.java

@@ -42,7 +42,7 @@ public abstract class VifDriverBase implements VifDriver {
     }
 
     @Override
-    public abstract LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter) throws InternalErrorException, LibvirtException;
+    public abstract LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException;
 
     @Override
     public abstract void unplug(LibvirtVMDef.InterfaceDef iface);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java

@@ -61,7 +61,7 @@ public final class LibvirtPlugNicCommandWrapper extends CommandWrapper<PlugNicCo
                 nicnum++;
             }
             final VifDriver vifDriver = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName());
-            final InterfaceDef interfaceDef = vifDriver.plug(nic, "Other PV", "");
+            final InterfaceDef interfaceDef = vifDriver.plug(nic, "Other PV", "", null);
             vm.attachDevice(interfaceDef.toString());
 
             return new PlugNicAnswer(command, true, "success");

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java

@@ -64,7 +64,7 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp
 
             final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vm.getName());
             for (final NicTO nic : nics) {
-                libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).plug(nic, null, "");
+                libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).plug(nic, null, "", null);
             }
 
             /* setup disks, e.g for iso */

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReplugNicCommandWrapper.java

@@ -65,7 +65,7 @@ public final class LibvirtReplugNicCommandWrapper extends CommandWrapper<ReplugN
             InterfaceDef oldPluggedNic = findPluggedNic(libvirtComputingResource, nic, vmName, conn);
 
             final VifDriver newVifDriver = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName());
-            final InterfaceDef interfaceDef = newVifDriver.plug(nic, "Other PV", oldPluggedNic.getModel().toString());
+            final InterfaceDef interfaceDef = newVifDriver.plug(nic, "Other PV", oldPluggedNic.getModel().toString(), null);
 
             interfaceDef.setSlot(oldPluggedNic.getSlot());
             interfaceDef.setDevName(oldPluggedNic.getDevName());

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java

@@ -69,6 +69,7 @@ public final class LibvirtStartCommandWrapper extends CommandWrapper<StartComman
             for (final NicTO nic : nics) {
                 if (vmSpec.getType() != VirtualMachine.Type.User) {
                     nic.setPxeDisable(true);
+                    nic.setDpdkDisabled(true);
                 }
             }
 

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java

@@ -26,7 +26,9 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.UUID;
 import java.util.Vector;
@@ -176,8 +178,10 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -189,6 +193,8 @@ public class LibvirtComputingResourceTest {
     private LibvirtComputingResource libvirtComputingResource;
     @Mock
     VirtualMachineTO vmTO;
+    @Mock
+    LibvirtVMDef vmDef;
 
     String hyperVisorType = "kvm";
     Random random = new Random();
@@ -3165,7 +3171,7 @@ public class LibvirtComputingResourceTest {
 
             when(libvirtComputingResource.getVifDriver(nic.getType(), nic.getName())).thenReturn(vifDriver);
 
-            when(vifDriver.plug(nic, "Other PV", "")).thenReturn(interfaceDef);
+            when(vifDriver.plug(nic, "Other PV", "", null)).thenReturn(interfaceDef);
             when(interfaceDef.toString()).thenReturn("Interface");
 
             final String interfaceDefStr = interfaceDef.toString();
@@ -3188,7 +3194,7 @@ public class LibvirtComputingResourceTest {
             verify(libvirtUtilitiesHelper, times(1)).getConnectionByVmName(command.getVmName());
             verify(libvirtComputingResource, times(1)).getDomain(conn, instanceName);
             verify(libvirtComputingResource, times(1)).getVifDriver(nic.getType(), nic.getName());
-            verify(vifDriver, times(1)).plug(nic, "Other PV", "");
+            verify(vifDriver, times(1)).plug(nic, "Other PV", "", null);
         } catch (final LibvirtException e) {
             fail(e.getMessage());
         } catch (final InternalErrorException e) {
@@ -3262,7 +3268,7 @@ public class LibvirtComputingResourceTest {
 
             when(libvirtComputingResource.getVifDriver(nic.getType(), nic.getName())).thenReturn(vifDriver);
 
-            when(vifDriver.plug(nic, "Other PV", "")).thenThrow(InternalErrorException.class);
+            when(vifDriver.plug(nic, "Other PV", "", null)).thenThrow(InternalErrorException.class);
 
         } catch (final LibvirtException e) {
             fail(e.getMessage());
@@ -3281,7 +3287,7 @@ public class LibvirtComputingResourceTest {
             verify(libvirtUtilitiesHelper, times(1)).getConnectionByVmName(command.getVmName());
             verify(libvirtComputingResource, times(1)).getDomain(conn, instanceName);
             verify(libvirtComputingResource, times(1)).getVifDriver(nic.getType(), nic.getName());
-            verify(vifDriver, times(1)).plug(nic, "Other PV", "");
+            verify(vifDriver, times(1)).plug(nic, "Other PV", "", null);
         } catch (final LibvirtException e) {
             fail(e.getMessage());
         } catch (final InternalErrorException e) {
@@ -5217,4 +5223,22 @@ public class LibvirtComputingResourceTest {
         assertFalse(ans instanceof UnsupportedAnswer);
         assertTrue(ans instanceof Answer);
     }
+
+    @Test
+    public void testAddExtraConfigComponentEmptyExtraConfig() {
+        libvirtComputingResource = new LibvirtComputingResource();
+        libvirtComputingResource.addExtraConfigComponent(new HashMap<>(), vmDef);
+        Mockito.verify(vmDef, never()).addComp(any());
+    }
+
+    @Test
+    public void testAddExtraConfigComponentNotEmptyExtraConfig() {
+        libvirtComputingResource = new LibvirtComputingResource();
+        Map<String, String> extraConfig = new HashMap<>();
+        extraConfig.put("extraconfig-1", "value1");
+        extraConfig.put("extraconfig-2", "value2");
+        extraConfig.put("extraconfig-3", "value3");
+        libvirtComputingResource.addExtraConfigComponent(extraConfig, vmDef);
+        Mockito.verify(vmDef, times(1)).addComp(any());
+    }
 }

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/OvsVifDriverTest.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.hypervisor.kvm.resource;
+
+import com.cloud.utils.script.Script;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@PrepareForTest({ Script.class })
+@RunWith(PowerMockRunner.class)
+public class OvsVifDriverTest {
+
+    private static final int dpdkPortNumber = 7;
+
+    private OvsVifDriver driver = new OvsVifDriver();
+
+    @Before
+    public void initMocks() {
+        MockitoAnnotations.initMocks(this);
+        PowerMockito.mockStatic(Script.class);
+        Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).thenReturn(null);
+    }
+
+    @Test
+    public void testGetDpdkLatestPortNumberUsedNoDpdkPorts() {
+        Assert.assertEquals(0, driver.getDpdkLatestPortNumberUsed());
+    }
+
+    @Test
+    public void testGetDpdkLatestPortNumberUsedExistingDpdkPorts() {
+        Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
+                thenReturn(OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
+        Assert.assertEquals(dpdkPortNumber, driver.getDpdkLatestPortNumberUsed());
+    }
+
+    @Test
+    public void testGetNextDpdkPortNoDpdkPorts() {
+        Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
+                thenReturn(null);
+        String expectedPortName = OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(1);
+        Assert.assertEquals(expectedPortName, driver.getNextDpdkPort());
+    }
+
+    @Test
+    public void testGetNextDpdkPortExistingDpdkPorts() {
+        Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
+                thenReturn(OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
+        String expectedPortName = OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber + 1);
+        Assert.assertEquals(expectedPortName, driver.getNextDpdkPort());
+    }
+}

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -44,6 +44,7 @@ import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 
+import com.cloud.user.AccountVO;
 import org.apache.cloudstack.acl.ControlledEntity.ACLType;
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
 import org.apache.cloudstack.affinity.AffinityGroupService;
@@ -514,6 +515,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     private static final ConfigKey<Boolean> AllowDeployVmIfGivenHostFails = new ConfigKey<Boolean>("Advanced", Boolean.class, "allow.deploy.vm.if.deploy.on.given.host.fails", "false",
             "allow vm to deploy on different host if vm fails to deploy on the given host ", true);
 
+    private static final ConfigKey<Boolean> EnableAdditionalVmConfig = new ConfigKey<>("Advanced", Boolean.class, "enable.additional.vm.configuration",
+            "false", "allow additional arbitrary configuration to vm", true, ConfigKey.Scope.Account);
 
     @Override
     public UserVmVO getVirtualMachine(long vmId) {
@@ -2388,8 +2391,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         Map<String,String> details = cmd.getDetails();
         List<Long> securityGroupIdList = getSecurityGroupIdList(cmd);
         boolean cleanupDetails = cmd.isCleanupDetails();
+        String extraConfig = cmd.getExtraConfig();
 
         UserVmVO vmInstance = _vmDao.findById(cmd.getId());
+        long accountId = vmInstance.getAccountId();
 
         if (isDisplayVm != null && isDisplayVm != vmInstance.isDisplay()) {
             updateDisplayVmFlag(isDisplayVm, id, vmInstance);
@@ -2397,9 +2402,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         if (cleanupDetails){
             userVmDetailsDao.removeDetails(id);
         }
-        else if (MapUtils.isNotEmpty(details)) {
+        else {
-            vmInstance.setDetails(details);
+            if (MapUtils.isNotEmpty(details)) {
-            _vmDao.saveDetails(vmInstance);
+                vmInstance.setDetails(details);
+                _vmDao.saveDetails(vmInstance);
+            }
+            if (StringUtils.isNotBlank(extraConfig) && EnableAdditionalVmConfig.valueIn(accountId)) {
+                AccountVO account = _accountDao.findById(accountId);
+                addExtraConfig(vmInstance, account, extraConfig);
+            }
         }
         return updateVirtualMachine(id, displayName, group, ha, isDisplayVm, osTypeId, userData, isDynamicallyScalable,
                 cmd.getHttpMethod(), cmd.getCustomId(), hostName, cmd.getInstanceName(), securityGroupIdList, cmd.getDhcpOptionsMap());
@@ -4847,9 +4858,75 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 _tmplService.attachIso(tmpl.getId(), vm.getId());
             }
         }
+
+        // Add extraConfig to user_vm_details table
+        Account caller = CallContext.current().getCallingAccount();
+        Long callerId = caller.getId();
+        String extraConfig = cmd.getExtraConfig();
+        if (StringUtils.isNotBlank(extraConfig) && EnableAdditionalVmConfig.valueIn(callerId) ) {
+            addExtraConfig(vm, caller, extraConfig);
+        }
+
         return vm;
     }
 
+    /**
+     * Persist extra configurations as details for VMware VMs
+     */
+    protected void persistExtraConfigVmware(String decodedUrl, UserVm vm) {
+        String[] configDataArr = decodedUrl.split("\\r?\\n");
+        for (String config: configDataArr) {
+            String[] keyValue = config.split("=");
+            try {
+                userVmDetailsDao.addDetail(vm.getId(), keyValue[0], keyValue[1], true);
+            } catch (ArrayIndexOutOfBoundsException e) {
+                throw new CloudRuntimeException("Issue occurred during parsing of:" + config);
+            }
+        }
+    }
+
+    /**
+     * Persist extra configurations as details for hypervisors except Vmware
+     */
+    protected void persistExtraConfigNonVmware(String decodedUrl, UserVm vm) {
+        String[] extraConfigs = decodedUrl.split("\n\n");
+        for (String cfg : extraConfigs) {
+            int i = 1;
+            String[] cfgParts = cfg.split("\n");
+            String extraConfigKey = ApiConstants.EXTRA_CONFIG;
+            String extraConfigValue;
+            if (cfgParts[0].matches("\\S+:$")) {
+                extraConfigKey += "-" + cfgParts[0].substring(0,cfgParts[0].length() - 1);
+                extraConfigValue = cfg.replace(cfgParts[0] + "\n", "");
+            } else {
+                extraConfigKey += "-" + String.valueOf(i);
+                extraConfigValue = cfg;
+            }
+            userVmDetailsDao.addDetail(vm.getId(), extraConfigKey, extraConfigValue, true);
+            i++;
+        }
+    }
+
+    protected void addExtraConfig(UserVm vm, Account caller, String extraConfig) {
+        String decodedUrl = decodeExtraConfig(extraConfig);
+        HypervisorType hypervisorType = vm.getHypervisorType();
+        if (hypervisorType == HypervisorType.VMware) {
+            persistExtraConfigVmware(decodedUrl, vm);
+        } else {
+            persistExtraConfigNonVmware(decodedUrl, vm);
+        }
+    }
+
+    protected String decodeExtraConfig(String encodeString) {
+        String decodedUrl;
+        try {
+            decodedUrl = URLDecoder.decode(encodeString, "UTF-8");
+        } catch (UnsupportedEncodingException e) {
+            throw new CloudRuntimeException("Failed to provided decode URL string: " + e.getMessage());
+        }
+        return decodedUrl;
+    }
+
     protected List<Long> getSecurityGroupIdList(SecurityGroupAction cmd) {
         if (cmd.getSecurityGroupNameList() != null && cmd.getSecurityGroupIdList() != null) {
             throw new InvalidParameterValueException("securitygroupids parameter is mutually exclusive with securitygroupnames parameter");
@@ -6394,7 +6471,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {EnableDynamicallyScaleVm, AllowUserExpungeRecoverVm, VmIpFetchWaitInterval, VmIpFetchTrialMax, VmIpFetchThreadPoolMax,
-            VmIpFetchTaskWorkers, AllowDeployVmIfGivenHostFails};
+            VmIpFetchTaskWorkers, AllowDeployVmIfGivenHostFails, EnableAdditionalVmConfig};
     }
 
     @Override