diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 9174da7fd7b..b88da3621cd 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -279,6 +279,11 @@ hypervisor.type=kvm # If this parameter is used, property host.overcommit.mem.mb must be set to 0. #host.reserved.mem.mb=1024 +# Number of CPU cores to subtract from advertised available cores. +# These are reserved for system activity, or otherwise share host CPU resources with +# CloudStack VM allocation. +# host.reserved.cpu.count = 0 + # The model of Watchdog timer to present to the Guest. # For all models refer to the libvirt documentation. #vm.watchdog.model=i6300esb diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 75248fb01bf..610c5be759f 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -502,6 +502,15 @@ public class AgentProperties{ */ public static final Property HOST_RESERVED_MEM_MB = new Property<>("host.reserved.mem.mb", 1024); + /** + * How many host CPUs to reserve for non-allocation.
+ * This can be used to set aside CPU cores on the host for other tasks, such as running hyperconverged storage
+ * processes, etc. + * Data type: Integer.
+ * Default value: 0 + */ + public static final Property HOST_RESERVED_CPU_CORE_COUNT = new Property<>("host.reserved.cpu.count", 0); + /** * The model of Watchdog timer to present to the Guest.
* For all models refer to the libvirt documentation.
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 16cccb0150b..6bbafcacef2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -460,6 +460,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private long dom0OvercommitMem; + private int dom0MinCpuCores; + protected int cmdsTimeout; protected int stopTimeout; protected CPUStat cpuStat = new CPUStat(); @@ -1063,6 +1065,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv // Reserve 1GB unless admin overrides dom0MinMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB)); + dom0MinCpuCores = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_CPU_CORE_COUNT); + // Support overcommit memory for host if host uses ZSWAP, KSM and other memory // compressing technologies dom0OvercommitMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_OVERCOMMIT_MEM_MB)); @@ -3540,7 +3544,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv @Override public StartupCommand[] initialize() { - final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed); + final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed, dom0MinCpuCores); String capabilities = String.join(",", info.getCapabilities()); if (dpdkSupport) { @@ -3548,7 +3552,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } final StartupRoutingCommand cmd = - new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, hypervisorType, + new StartupRoutingCommand(info.getAllocatableCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, hypervisorType, RouterPrivateIpStrategy.HostLocal); cmd.setCpuSockets(info.getCpuSockets()); fillNetworkInformation(cmd); diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java index 807b2541fd3..d160cbfac3b 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java @@ -48,7 +48,8 @@ public class KVMHostInfo { private static final Logger LOGGER = Logger.getLogger(KVMHostInfo.class); - private int cpus; + private int totalCpus; + private int allocatableCpus; private int cpusockets; private long cpuSpeed; private long totalMemory; @@ -58,16 +59,25 @@ public class KVMHostInfo { private static String cpuInfoFreqFileName = "/sys/devices/system/cpu/cpu0/cpufreq/base_frequency"; - public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed) { + public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed, int reservedCpus) { this.cpuSpeed = manualSpeed; this.reservedMemory = reservedMemory; this.overCommitMemory = overCommitMemory; this.getHostInfoFromLibvirt(); this.totalMemory = new MemStat(this.getReservedMemory(), this.getOverCommitMemory()).getTotal(); + this.allocatableCpus = totalCpus - reservedCpus; + if (allocatableCpus < 1) { + LOGGER.warn(String.format("Aggressive reserved CPU config leaves no usable CPUs for VMs! Total system CPUs: %d, Reserved: %d, Allocatable: %d", totalCpus, reservedCpus, allocatableCpus)); + allocatableCpus = 0; + } } - public int getCpus() { - return this.cpus; + public int getTotalCpus() { + return this.totalCpus; + } + + public int getAllocatableCpus() { + return this.allocatableCpus; } public int getCpuSockets() { @@ -189,7 +199,7 @@ public class KVMHostInfo { if (hosts.nodes > 0) { this.cpusockets = hosts.sockets * hosts.nodes; } - this.cpus = hosts.cpus; + this.totalCpus = hosts.cpus; final LibvirtCapXMLParser parser = new LibvirtCapXMLParser(); parser.parseCapabilitiesXML(capabilities); diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java index 92cece6ae06..3b5422f564c 100644 --- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java @@ -73,8 +73,36 @@ public class KVMHostInfoTest { Mockito.when(conn.close()).thenReturn(0); int manualSpeed = 500; - KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed); + KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed, 0); Assert.assertEquals(kvmHostInfo.getCpuSpeed(), manualSpeed); } } + + @Test + public void reservedCpuCoresTest() throws Exception { + if (!System.getProperty("os.name").equals("Linux")) { + return; + } + try (MockedStatic ignored = Mockito.mockStatic(LibvirtConnection.class)) { + Connect conn = Mockito.mock(Connect.class); + NodeInfo nodeInfo = Mockito.mock(NodeInfo.class); + nodeInfo.cpus = 10; + String capabilitiesXml = ""; + + Mockito.when(LibvirtConnection.getConnection()).thenReturn(conn); + Mockito.when(conn.nodeInfo()).thenReturn(nodeInfo); + Mockito.when(conn.getCapabilities()).thenReturn(capabilitiesXml); + Mockito.when(conn.close()).thenReturn(0); + int manualSpeed = 500; + + KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, 100, 2); + Assert.assertEquals("reserve two CPU cores", 8, kvmHostInfo.getAllocatableCpus()); + + kvmHostInfo = new KVMHostInfo(10, 10, 100, 0); + Assert.assertEquals("no reserve CPU core setting", 10, kvmHostInfo.getAllocatableCpus()); + + kvmHostInfo = new KVMHostInfo(10, 10, 100, 12); + Assert.assertEquals("Misconfigured/too large CPU reserve", 0, kvmHostInfo.getAllocatableCpus()); + } + } }