kvm: add vm setting for nic multiqueue number and packed virtqueues (#7333)

This PR adds two vm setting for user vms on KVM

- nic multiqueue number
- packed virtqueues enabled . optional are true and false (false by default). It requires qemu>=4.2.0 and libvirt >=6.3.0

Tested ok on ubuntu 22 and rocky 8.4

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -48,6 +48,10 @@ public interface VmDetailConstants {
     String IOTHREADS = "iothreads";
     String IO_POLICY = "io.policy";
 
+    // KVM specific, the number of queues for multiqueue NICs
+    String NIC_MULTIQUEUE_NUMBER = "nic.multiqueue.number";
+    String NIC_PACKED_VIRTQUEUES_ENABLED = "nic.packed.virtqueues.enabled";
+
     // Mac OSX guest specific (internal)
     String SMC_PRESENT = "smc.present";
     String FIRMWARE = "firmware";

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -299,6 +299,8 @@ public class ApiConstants {
     public static final String NIC = "nic";
     public static final String NIC_NETWORK_LIST = "nicnetworklist";
     public static final String NIC_IP_ADDRESS_LIST = "nicipaddresslist";
+    public static final String NIC_MULTIQUEUE_NUMBER = "nicmultiqueuenumber";
+    public static final String NIC_PACKED_VIRTQUEUES_ENABLED = "nicpackedvirtqueuesenabled";
     public static final String NEW_START_IP = "newstartip";
     public static final String NEW_END_IP = "newendip";
     public static final String NUM_RETRIES = "numretries";

api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java

@@ -264,6 +264,14 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
     @Parameter(name = ApiConstants.IO_DRIVER_POLICY, type = CommandType.STRING, description = "Controls specific policies on IO")
     private String ioDriverPolicy;
 
+    @Parameter(name = ApiConstants.NIC_MULTIQUEUE_NUMBER, type = CommandType.INTEGER, since = "4.18",
+            description = "The number of queues for multiqueue NICs.")
+    private Integer nicMultiqueueNumber;
+
+    @Parameter(name = ApiConstants.NIC_PACKED_VIRTQUEUES_ENABLED, type = CommandType.BOOLEAN, since = "4.18",
+            description = "Enable packed virtqueues or not.")
+    private Boolean nicPackedVirtQueues;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -338,6 +346,14 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
             customparameterMap.put(VmDetailConstants.IOTHREADS, BooleanUtils.toStringTrueFalse(iothreadsEnabled));
         }
 
+        if (nicMultiqueueNumber != null) {
+            customparameterMap.put(VmDetailConstants.NIC_MULTIQUEUE_NUMBER, nicMultiqueueNumber.toString());
+        }
+
+        if (BooleanUtils.toBoolean(nicPackedVirtQueues)) {
+            customparameterMap.put(VmDetailConstants.NIC_PACKED_VIRTQUEUES_ENABLED, BooleanUtils.toStringTrueFalse(nicPackedVirtQueues));
+        }
+
         return customparameterMap;
     }
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -2843,7 +2843,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         for (int i = 0; i < nics.length; i++) {
             for (final NicTO nic : vmSpec.getNics()) {
                 if (nic.getDeviceId() == i) {
-                    createVif(vm, nic, nicAdapter, extraConfig);
+                    createVif(vm, vmSpec, nic, nicAdapter, extraConfig);
                 }
             }
         }
@@ -3224,12 +3224,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         }
     }
 
-    private void createVif(final LibvirtVMDef vm, final NicTO nic, final String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
+    private void createVif(final LibvirtVMDef vm, final VirtualMachineTO vmSpec, final NicTO nic, final String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
         if (vm.getDevices() == null) {
             s_logger.error("LibvirtVMDef object get devices with null result");
             throw new InternalErrorException("LibvirtVMDef object get devices with null result");
         }
-        vm.getDevices().addDevice(getVifDriver(nic.getType(), nic.getName()).plug(nic, vm.getPlatformEmulator(), nicAdapter, extraConfig));
+        final InterfaceDef interfaceDef = getVifDriver(nic.getType(), nic.getName()).plug(nic, vm.getPlatformEmulator(), nicAdapter, extraConfig);
+        if (vmSpec.getDetails() != null) {
+            setInterfaceDefQueueSettings(vmSpec.getDetails(), vmSpec.getCpus(), interfaceDef);
+        }
+        vm.getDevices().addDevice(interfaceDef);
     }
 
     public boolean cleanupDisk(Map<String, String> volumeToDisconnect) {
@@ -5143,4 +5147,30 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
     public static String generateSecretUUIDFromString(String seed) {
         return UUID.nameUUIDFromBytes(seed.getBytes()).toString();
     }
+
+    public void setInterfaceDefQueueSettings(Map<String, String> details, Integer cpus, InterfaceDef interfaceDef) {
+        String nicMultiqueueNumber = details.get(VmDetailConstants.NIC_MULTIQUEUE_NUMBER);
+        if (nicMultiqueueNumber != null) {
+            try {
+                Integer nicMultiqueueNumberInteger = Integer.valueOf(nicMultiqueueNumber);
+                if (nicMultiqueueNumberInteger == InterfaceDef.MULTI_QUEUE_NUMBER_MEANS_CPU_CORES) {
+                    if (cpus != null) {
+                        interfaceDef.setMultiQueueNumber(cpus);
+                    }
+                } else {
+                    interfaceDef.setMultiQueueNumber(nicMultiqueueNumberInteger);
+                }
+            } catch (NumberFormatException ex) {
+                s_logger.warn(String.format("VM details %s is not a valid integer value %s", VmDetailConstants.NIC_MULTIQUEUE_NUMBER, nicMultiqueueNumber));
+            }
+        }
+        String nicPackedEnabled = details.get(VmDetailConstants.NIC_PACKED_VIRTQUEUES_ENABLED);
+        if (nicPackedEnabled != null) {
+            try {
+                interfaceDef.setPackedVirtQueues(Boolean.valueOf(nicPackedEnabled));
+            } catch (NumberFormatException ex) {
+                s_logger.warn(String.format("VM details %s is not a valid Boolean value %s", VmDetailConstants.NIC_PACKED_VIRTQUEUES_ENABLED, nicPackedEnabled));
+            }
+        }
+    }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java

@@ -249,6 +249,15 @@ public class LibvirtDomainXMLParser {
                     def.setDpdkOvsPath(ovsPath);
                     def.setInterfaceMode(mode);
                 }
+                String multiQueueNumber = getAttrValue("driver", "queues", nic);
+                if (StringUtils.isNotBlank(multiQueueNumber)) {
+                    def.setMultiQueueNumber(Integer.valueOf(multiQueueNumber));
+                }
+
+                String packedOn = getAttrValue("driver", "packed", nic);
+                if (StringUtils.isNotBlank(packedOn)) {
+                    def.setPackedVirtQueues("on".equalsIgnoreCase(packedOn));
+                }
 
                 if (StringUtils.isNotBlank(slot)) {
                     def.setSlot(Integer.parseInt(slot, 16));

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java

@@ -1280,6 +1280,8 @@ public class LibvirtVMDef {
             DIRECT_ATTACHED_WITHOUT_DHCP, DIRECT_ATTACHED_WITH_DHCP, VNET, VLAN;
         }
 
+        public static final int MULTI_QUEUE_NUMBER_MEANS_CPU_CORES = -1;
+
         private GuestNetType _netType; /*
          * bridge, ethernet, network, user,
          * internal, vhostuser
@@ -1305,6 +1307,8 @@ public class LibvirtVMDef {
         private String _interfaceMode;
         private String _userIp4Network;
         private Integer _userIp4Prefix;
+        private Integer _multiQueueNumber;
+        private Boolean _packedVirtQueues;
 
         public void defBridgeNet(String brName, String targetBrName, String macAddr, NicModel model) {
             defBridgeNet(brName, targetBrName, macAddr, model, 0);
@@ -1493,6 +1497,14 @@ public class LibvirtVMDef {
             _interfaceMode = mode;
         }
 
+        public void setMultiQueueNumber(Integer multiQueueNumber) {
+            this._multiQueueNumber = multiQueueNumber;
+        }
+
+        public void setPackedVirtQueues(Boolean packedVirtQueues) {
+            this._packedVirtQueues = packedVirtQueues;
+        }
+
         public String getContent() {
             StringBuilder netBuilder = new StringBuilder();
             if (_netType == GuestNetType.BRIDGE) {
@@ -1515,6 +1527,21 @@ public class LibvirtVMDef {
             if (_model != null) {
                 netBuilder.append("<model type='" + _model + "'/>\n");
             }
+            if (NicModel.VIRTIO.equals(_model)) {
+                boolean isMultiQueueNumberSpecified = _multiQueueNumber != null;
+                boolean isPackedVirtQueuesEnabled = _packedVirtQueues != null && _packedVirtQueues
+                        && s_qemuVersion >= 4200000 && s_libvirtVersion >= 6300000;
+                if (isMultiQueueNumberSpecified || isPackedVirtQueuesEnabled) {
+                    netBuilder.append("<driver");
+                    if (isMultiQueueNumberSpecified) {
+                        netBuilder.append(" queues='" + _multiQueueNumber + "'");
+                    }
+                    if (isPackedVirtQueuesEnabled) {
+                        netBuilder.append(" packed='on'");
+                    }
+                    netBuilder.append("/>\n");
+                }
+            }
             if ((s_libvirtVersion >= 9004) && (_networkRateKBps > 0)) { // supported from libvirt 0.9.4
                 netBuilder.append("<bandwidth>\n");
                 netBuilder.append("<inbound average='" + _networkRateKBps + "' peak='" + _networkRateKBps + "'/>\n");

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java

@@ -64,6 +64,9 @@ public final class LibvirtPlugNicCommandWrapper extends CommandWrapper<PlugNicCo
             }
             final VifDriver vifDriver = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName());
             final InterfaceDef interfaceDef = vifDriver.plug(nic, "Other PV", "", null);
+            if (command.getDetails() != null) {
+                libvirtComputingResource.setInterfaceDefQueueSettings(command.getDetails(), null, interfaceDef);
+            }
             vm.attachDevice(interfaceDef.toString());
 
             // apply default network rules on new nic

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java

@@ -80,6 +80,9 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp
 
             for (final NicTO nic : nics) {
                 LibvirtVMDef.InterfaceDef interfaceDef = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).plug(nic, null, "", vm.getExtraConfig());
+                if (vm.getDetails() != null) {
+                    libvirtComputingResource.setInterfaceDefQueueSettings(vm.getDetails(), vm.getCpus(), interfaceDef);
+                }
                 if (interfaceDef != null && interfaceDef.getNetType() == GuestNetType.VHOSTUSER) {
                     DpdkTO to = new DpdkTO(interfaceDef.getDpdkOvsPath(), interfaceDef.getDpdkSourcePort(), interfaceDef.getInterfaceMode());
                     dpdkInterfaceMapping.put(nic.getMac(), to);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReplugNicCommandWrapper.java

@@ -66,7 +66,9 @@ public final class LibvirtReplugNicCommandWrapper extends CommandWrapper<ReplugN
 
             final VifDriver newVifDriver = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName());
             final InterfaceDef interfaceDef = newVifDriver.plug(nic, "Other PV", oldPluggedNic.getModel().toString(), null);
-
+            if (command.getDetails() != null) {
+                libvirtComputingResource.setInterfaceDefQueueSettings(command.getDetails(), null, interfaceDef);
+            }
             interfaceDef.setSlot(oldPluggedNic.getSlot());
             interfaceDef.setDevName(oldPluggedNic.getDevName());
             interfaceDef.setLinkStateUp(false);

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java

@@ -150,6 +150,58 @@ public class LibvirtVMDefTest extends TestCase {
         assertEquals(expected, ifDef.toString());
     }
 
+    @Test
+    public void testInterfaceWithMultiQueueAndPacked() {
+        LibvirtVMDef.InterfaceDef ifDef = new LibvirtVMDef.InterfaceDef();
+        ifDef.defBridgeNet("targetDeviceName", null, "00:11:22:aa:bb:dd", LibvirtVMDef.InterfaceDef.NicModel.VIRTIO);
+        ifDef.setMultiQueueNumber(6);
+
+        LibvirtVMDef.setGlobalQemuVersion(5000000L);
+        LibvirtVMDef.setGlobalLibvirtVersion(6400000L);
+
+        String expected =
+                "<interface type='" + LibvirtVMDef.InterfaceDef.GuestNetType.BRIDGE + "'>\n"
+                        + "<source bridge='targetDeviceName'/>\n"
+                        + "<mac address='00:11:22:aa:bb:dd'/>\n"
+                        + "<model type='virtio'/>\n"
+                        + "<driver queues='6'/>\n"
+                        + "<link state='up'/>\n"
+                        + "</interface>\n";
+        assertEquals(expected, ifDef.toString());
+
+        ifDef.setPackedVirtQueues(true);
+        expected =
+                "<interface type='" + LibvirtVMDef.InterfaceDef.GuestNetType.BRIDGE + "'>\n"
+                        + "<source bridge='targetDeviceName'/>\n"
+                        + "<mac address='00:11:22:aa:bb:dd'/>\n"
+                        + "<model type='virtio'/>\n"
+                        + "<driver queues='6' packed='on'/>\n"
+                        + "<link state='up'/>\n"
+                        + "</interface>\n";
+        assertEquals(expected, ifDef.toString());
+
+        ifDef.setMultiQueueNumber(null);
+        expected =
+                "<interface type='" + LibvirtVMDef.InterfaceDef.GuestNetType.BRIDGE + "'>\n"
+                        + "<source bridge='targetDeviceName'/>\n"
+                        + "<mac address='00:11:22:aa:bb:dd'/>\n"
+                        + "<model type='virtio'/>\n"
+                        + "<driver packed='on'/>\n"
+                        + "<link state='up'/>\n"
+                        + "</interface>\n";
+        assertEquals(expected, ifDef.toString());
+
+        LibvirtVMDef.setGlobalLibvirtVersion(300000L);
+        expected =
+                "<interface type='" + LibvirtVMDef.InterfaceDef.GuestNetType.BRIDGE + "'>\n"
+                        + "<source bridge='targetDeviceName'/>\n"
+                        + "<mac address='00:11:22:aa:bb:dd'/>\n"
+                        + "<model type='virtio'/>\n"
+                        + "<link state='up'/>\n"
+                        + "</interface>\n";
+        assertEquals(expected, ifDef.toString());
+        }
+
     @Test
     public void testCpuModeDef() {
         LibvirtVMDef.CpuModeDef cpuModeDef = new LibvirtVMDef.CpuModeDef();

scripts/vm/network/tungsten/create_tap_device.sh

@@ -16,5 +16,5 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ip tuntap add dev $1 mode tap
+#ip tuntap add dev $1 mode tap multi_queue
-ip link set $1 up
+#ip link set $1 up

scripts/vm/network/tungsten/delete_tap_device.sh

@@ -16,4 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+ip tuntap del dev $1 mode tap multi_queue 2>/dev/null
+if [ $? -ne 0 ];then
   ip tuntap del dev $1 mode tap
+fi

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -4053,6 +4053,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
             options.put(VmDetailConstants.VIDEO_RAM, Collections.emptyList());
             options.put(VmDetailConstants.IO_POLICY, Arrays.asList("threads", "native", "io_uring", "storage_specific"));
             options.put(VmDetailConstants.IOTHREADS, Arrays.asList("enabled"));
+            options.put(VmDetailConstants.NIC_MULTIQUEUE_NUMBER, Collections.emptyList());
+            options.put(VmDetailConstants.NIC_PACKED_VIRTQUEUES_ENABLED, Arrays.asList("true", "false"));
         }
 
         if (HypervisorType.VMware.equals(hypervisorType)) {

ui/public/locales/en.json

@@ -1337,6 +1337,10 @@
 "label.nfsserver": "NFS server",
 "label.nic": "NIC",
 "label.nicadaptertype": "NIC adapter type",
+"label.nicmultiqueuenumber" : "NIC multiqueue number",
+"label.nicmultiqueuenumber.tooltip" : "NIC multiqueue number. This supports only KVM. The value \"-1\" indicates the NIC multiqueue number will be set to the vCPU number of the instance.",
+"label.nicpackedvirtqueuesenabled" : "NIC packed virtqueues enabled",
+"label.nicpackedvirtqueuesenabled.tooltip" : "Enable NIC packed virtqueues or not. This supports only KVM with QEMU >= 4.2.0 and Libvirt >=6.3.0.",
 "label.nics": "NICs",
 "label.no": "No",
 "label.no.data": "No data to show",

ui/src/views/compute/DeployVM.vue

@@ -697,6 +697,23 @@
                         @select-affinity-group-item="($event) => updateAffinityGroups($event)"
                         @handle-search-filter="($event) => handleSearchFilter('affinityGroups', $event)"/>
                     </a-form-item>
+                    <a-form-item name="nicmultiqueuenumber" ref="nicmultiqueuenumber" v-if="vm.templateid && ['KVM'].includes(hypervisor)">
+                      <template #label>
+                        <tooltip-label :title="$t('label.nicmultiqueuenumber')" :tooltip="$t('label.nicmultiqueuenumber.tooltip')"/>
+                      </template>
+                      <a-input-number
+                        style="width: 100%;"
+                        v-model:value="form.nicmultiqueuenumber" />
+                    </a-form-item>
+                    <a-form-item name="nicpackedvirtqueuesenabled" ref="nicpackedvirtqueuesenabled" v-if="vm.templateid && ['KVM'].includes(hypervisor)">
+                      <template #label>
+                        <tooltip-label :title="$t('label.nicpackedvirtqueuesenabled')" :tooltip="$t('label.nicpackedvirtqueuesenabled.tooltip')"/>
+                      </template>
+                      <a-switch
+                        v-model:checked="form.nicpackedvirtqueuesenabled"
+                        :checked="nicpackedvirtqueuesenabled"
+                        @change="val => { nicpackedvirtqueuesenabled = val }"/>
+                    </a-form-item>
                     <a-form-item name="iothreadsenabled" ref="iothreadsenabled" v-if="vm.templateid && ['KVM'].includes(hypervisor)">
                       <template #label>
                         <tooltip-label :title="$t('label.iothreadsenabled')" :tooltip="$t('label.iothreadsenabled.tooltip')"/>
@@ -1679,7 +1696,7 @@ export default {
       this.fetchInstaceGroups()
       this.fetchIoPolicyTypes()
       nextTick().then(() => {
-        ['name', 'keyboard', 'boottype', 'bootmode', 'userdata', 'iothreadsenabled', 'iodriverpolicy'].forEach(this.fillValue)
+        ['name', 'keyboard', 'boottype', 'bootmode', 'userdata', 'iothreadsenabled', 'iodriverpolicy', 'nicmultiqueuenumber', 'nicpackedvirtqueues'].forEach(this.fillValue)
         this.form.boottype = this.defaultBootType ? this.defaultBootType : this.options.bootTypes && this.options.bootTypes.length > 0 ? this.options.bootTypes[0].id : undefined
         this.form.bootmode = this.defaultBootMode ? this.defaultBootMode : this.options.bootModes && this.options.bootModes.length > 0 ? this.options.bootModes[0].id : undefined
         this.instanceConfig = toRaw(this.form)
@@ -1977,6 +1994,8 @@ export default {
         deployVmData.dynamicscalingenabled = values.dynamicscalingenabled
         deployVmData.iothreadsenabled = values.iothreadsenabled
         deployVmData.iodriverpolicy = values.iodriverpolicy
+        deployVmData.nicmultiqueuenumber = values.nicmultiqueuenumber
+        deployVmData.nicpackedvirtqueuesenabled = values.nicpackedvirtqueuesenabled
         if (values.userdata && values.userdata.length > 0) {
           deployVmData.userdata = encodeURIComponent(btoa(sanitizeReverse(values.userdata)))
         }