mirror of
https://github.com/apache/cloudstack.git
synced 2025-10-26 08:42:29 +01:00
kvm: Add support for cgroupv2 (#8252)
1. Problem description
In Apache CloudStack (ACS), when a VM is deployed in a host with the KVM hypervisor, an XML file is created in the assigned host, which has a property shares that defines the weight of the VM to access the host CPU. The value of this property has no unit, and it is a relative measure to calculate how much CPU a given VM will have in the host. However, this value has a limit, which depends on the version of cgroup utilized by the host's kernel. The problem lies at the range value of shares that varies between both versions: [2, 264144] for cgroups version 1; and [1, 10000] for cgroups version 2. Currently, ACS calculates the value of shares using Equation 1, presented below, where CPU is the number of cores and speed is the CPU frequency; both specified in the VM's compute offering. Therefore, if a compute offering has, for example, 6 cores at 2 GHz, the shares value will be 12000 and an exception will be thrown by libvirt if the host utilizes cgroup v2. The second version is becoming the default one in current Linux distributions; thus, it is necessary to address this limitation.
Equation 1
shares = CPU * speed
Fixes: #6744
2. Proposed changes
To address the problem described, we propose to apply a scale conversion considering the max shares of the host. Using the same formula currently utilized by ACS, it is possible to calculate the maximum shares of a VM for a given host. In other words, using the number of cores and the nominal speed of the host's CPU as the upper limit of shares allowed to a VM. Then, this value will be scaled to the allowed interval of [1, 10000] of cgroup v2 by using a linear scale conversion.
The VM shares would be calculated as Equation 2, presented below, where VM requested shares is the requested shares value calculated using Equation 1, cgroup upper limit is fixed with a value of 10000 (cgroups v2 upper limit), and host max shares is the maximum shares value of the host, calculated using Equation 1. Using Equation 2, the only case where a VM passes the cgroup v2 limit is when the user requests more resources than the host has, which is not possible with the current implementation of ACS.
Equation 2
shares = (VM requested shares * cgroup upper limit)/host max shares
To implement the proposal, the following APIs will be updated: deployVirtualMachine, migrateVirtualMachine and scaleVirtualMachine. When a VM is being deployed, a new verification will be added to find a suitable host. The max shares of each host will be calculated, and the VM calculated shares will be verified if it does not surpass the host's value. Likewise, the migration of VMs will have a similar new verification. Lastly, the scale of VMs will also have the same verification for the VM's host.
To determine the max shares of a given host, we will use the same equation currently used in ACS for calculating the shares of VMs, presented in Section 1. When Equation 1 is used to determine the maximum shares of a host, CPU is the number of cores of the host, and speed is the nominal CPU speed, i.e., considering the CPU's base frequency.
It is important to note that these changes are only for hosts with the KVM hypervisor using cgroup v2 for now.
This commit is contained in:
Bryan Lima
GitHub
parent
3ce7c39bef
commit
3bb318bab9
@ -40,6 +40,9 @@ public class MigrateCommand extends Command {
|
||||
private boolean executeInSequence = false;
|
||||
private List<MigrateDiskInfo> migrateDiskInfoList = new ArrayList<>();
|
||||
private Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
|
||||
|
||||
private int newVmCpuShares;
|
||||
|
||||
Map<String, Boolean> vlanToPersistenceMap = new HashMap<>();
|
||||
|
||||
public Map<String, DpdkTO> getDpdkInterfaceMapping() {
|
||||
@ -138,6 +141,14 @@ public class MigrateCommand extends Command {
|
||||
this.migrateDiskInfoList = migrateDiskInfoList;
|
||||
}
|
||||
|
||||
public int getNewVmCpuShares() {
|
||||
return newVmCpuShares;
|
||||
}
|
||||
|
||||
public void setNewVmCpuShares(int newVmCpuShares) {
|
||||
this.newVmCpuShares = newVmCpuShares;
|
||||
}
|
||||
|
||||
public static class MigrateDiskInfo {
|
||||
public enum DiskType {
|
||||
FILE, BLOCK;
|
||||
|
||||
@ -28,6 +28,8 @@ public class PrepareForMigrationAnswer extends Answer {
|
||||
|
||||
private Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
|
||||
|
||||
private Integer newVmCpuShares = null;
|
||||
|
||||
protected PrepareForMigrationAnswer() {
|
||||
}
|
||||
|
||||
@ -50,4 +52,12 @@ public class PrepareForMigrationAnswer extends Answer {
|
||||
public Map<String, DpdkTO> getDpdkInterfaceMapping() {
|
||||
return this.dpdkInterfaceMapping;
|
||||
}
|
||||
|
||||
public Integer getNewVmCpuShares() {
|
||||
return newVmCpuShares;
|
||||
}
|
||||
|
||||
public void setNewVmCpuShares(Integer newVmCpuShares) {
|
||||
this.newVmCpuShares = newVmCpuShares;
|
||||
}
|
||||
}
|
||||
|
||||
@ -48,6 +48,7 @@ import javax.naming.ConfigurationException;
|
||||
import javax.persistence.EntityExistsException;
|
||||
|
||||
import com.cloud.event.ActionEventUtils;
|
||||
import com.google.gson.Gson;
|
||||
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
|
||||
import org.apache.cloudstack.annotation.AnnotationService;
|
||||
import org.apache.cloudstack.annotation.dao.AnnotationDao;
|
||||
@ -2790,23 +2791,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
|
||||
}
|
||||
|
||||
boolean migrated = false;
|
||||
Map<String, DpdkTO> dpdkInterfaceMapping = null;
|
||||
Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
|
||||
try {
|
||||
final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
|
||||
Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
|
||||
final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
|
||||
if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
|
||||
mc.setVlanToPersistenceMap(vlanToPersistenceMap);
|
||||
}
|
||||
|
||||
boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value();
|
||||
mc.setAutoConvergence(kvmAutoConvergence);
|
||||
mc.setHostGuid(dest.getHost().getGuid());
|
||||
|
||||
dpdkInterfaceMapping = ((PrepareForMigrationAnswer) pfma).getDpdkInterfaceMapping();
|
||||
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
|
||||
mc.setDpdkInterfaceMapping(dpdkInterfaceMapping);
|
||||
}
|
||||
final MigrateCommand mc = buildMigrateCommand(vm, to, dest, pfma, dpdkInterfaceMapping);
|
||||
|
||||
try {
|
||||
final Answer ma = _agentMgr.send(vm.getLastHostId(), mc);
|
||||
@ -2878,6 +2865,43 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create and set parameters for the {@link MigrateCommand} used in the migration and scaling of VMs.
|
||||
*/
|
||||
protected MigrateCommand buildMigrateCommand(VMInstanceVO vmInstance, VirtualMachineTO virtualMachineTO, DeployDestination destination, Answer answer,
|
||||
Map<String, DpdkTO> dpdkInterfaceMapping) {
|
||||
final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vmInstance.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
|
||||
final MigrateCommand migrateCommand = new MigrateCommand(vmInstance.getInstanceName(), destination.getHost().getPrivateIpAddress(), isWindows, virtualMachineTO,
|
||||
getExecuteInSequence(vmInstance.getHypervisorType()));
|
||||
|
||||
Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vmInstance.getId());
|
||||
if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
|
||||
s_logger.debug(String.format("Setting VLAN persistence to [%s] as part of migrate command for VM [%s].", new Gson().toJson(vlanToPersistenceMap), virtualMachineTO));
|
||||
migrateCommand.setVlanToPersistenceMap(vlanToPersistenceMap);
|
||||
}
|
||||
|
||||
migrateCommand.setAutoConvergence(StorageManager.KvmAutoConvergence.value());
|
||||
migrateCommand.setHostGuid(destination.getHost().getGuid());
|
||||
|
||||
PrepareForMigrationAnswer prepareForMigrationAnswer = (PrepareForMigrationAnswer) answer;
|
||||
|
||||
Map<String, DpdkTO> answerDpdkInterfaceMapping = prepareForMigrationAnswer.getDpdkInterfaceMapping();
|
||||
if (MapUtils.isNotEmpty(answerDpdkInterfaceMapping) && dpdkInterfaceMapping != null) {
|
||||
s_logger.debug(String.format("Setting DPDK interface mapping to [%s] as part of migrate command for VM [%s].", new Gson().toJson(vlanToPersistenceMap),
|
||||
virtualMachineTO));
|
||||
dpdkInterfaceMapping.putAll(answerDpdkInterfaceMapping);
|
||||
migrateCommand.setDpdkInterfaceMapping(dpdkInterfaceMapping);
|
||||
}
|
||||
|
||||
Integer newVmCpuShares = prepareForMigrationAnswer.getNewVmCpuShares();
|
||||
if (newVmCpuShares != null) {
|
||||
s_logger.debug(String.format("Setting CPU shares to [%d] as part of migrate command for VM [%s].", newVmCpuShares, virtualMachineTO));
|
||||
migrateCommand.setNewVmCpuShares(newVmCpuShares);
|
||||
}
|
||||
|
||||
return migrateCommand;
|
||||
}
|
||||
|
||||
private void updateVmPod(VMInstanceVO vm, long dstHostId) {
|
||||
// update the VMs pod
|
||||
HostVO host = _hostDao.findById(dstHostId);
|
||||
@ -4395,16 +4419,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
|
||||
|
||||
boolean migrated = false;
|
||||
try {
|
||||
Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
|
||||
final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
|
||||
final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
|
||||
if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
|
||||
mc.setVlanToPersistenceMap(vlanToPersistenceMap);
|
||||
}
|
||||
|
||||
boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value();
|
||||
mc.setAutoConvergence(kvmAutoConvergence);
|
||||
mc.setHostGuid(dest.getHost().getGuid());
|
||||
final MigrateCommand mc = buildMigrateCommand(vm, to, dest, pfma, null);
|
||||
|
||||
try {
|
||||
final Answer ma = _agentMgr.send(vm.getLastHostId(), mc);
|
||||
|
||||
@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import com.cloud.agent.api.PrepareForMigrationAnswer;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
|
||||
@ -1884,9 +1885,10 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
|
||||
}
|
||||
|
||||
PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(vmTO);
|
||||
Answer pfma;
|
||||
|
||||
try {
|
||||
Answer pfma = agentManager.send(destHost.getId(), pfmc);
|
||||
pfma = agentManager.send(destHost.getId(), pfmc);
|
||||
|
||||
if (pfma == null || !pfma.getResult()) {
|
||||
String details = pfma != null ? pfma.getDetails() : "null answer returned";
|
||||
@ -1894,8 +1896,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
|
||||
|
||||
throw new AgentUnavailableException(msg, destHost.getId());
|
||||
}
|
||||
}
|
||||
catch (final OperationTimedoutException e) {
|
||||
} catch (final OperationTimedoutException e) {
|
||||
throw new AgentUnavailableException("Operation timed out", destHost.getId());
|
||||
}
|
||||
|
||||
@ -1911,6 +1912,12 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
|
||||
migrateCommand.setMigrateStorageManaged(managedStorageDestination);
|
||||
migrateCommand.setMigrateNonSharedInc(migrateNonSharedInc);
|
||||
|
||||
Integer newVmCpuShares = ((PrepareForMigrationAnswer) pfma).getNewVmCpuShares();
|
||||
if (newVmCpuShares != null) {
|
||||
LOGGER.debug(String.format("Setting CPU shares to [%d] as part of migrate VM with volumes command for VM [%s].", newVmCpuShares, vmTO));
|
||||
migrateCommand.setNewVmCpuShares(newVmCpuShares);
|
||||
}
|
||||
|
||||
boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value();
|
||||
migrateCommand.setAutoConvergence(kvmAutoConvergence);
|
||||
|
||||
|
||||
@ -72,6 +72,7 @@ import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
import org.apache.commons.lang.BooleanUtils;
|
||||
import org.apache.commons.lang.math.NumberUtils;
|
||||
import org.apache.commons.lang3.ObjectUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.apache.xerces.impl.xpath.regex.Match;
|
||||
@ -472,6 +473,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
|
||||
*/
|
||||
private static final String COMMAND_SET_MEM_BALLOON_STATS_PERIOD = "virsh dommemstat %s --period %s --live";
|
||||
|
||||
private static int hostCpuMaxCapacity = 0;
|
||||
|
||||
private static final int CGROUP_V2_UPPER_LIMIT = 10000;
|
||||
|
||||
private static final String COMMAND_GET_CGROUP_HOST_VERSION = "stat -fc %T /sys/fs/cgroup/";
|
||||
|
||||
public static final String CGROUP_V2 = "cgroup2fs";
|
||||
|
||||
protected long getHypervisorLibvirtVersion() {
|
||||
return _hypervisorLibvirtVersion;
|
||||
}
|
||||
@ -547,6 +556,18 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
|
||||
return new ExecutionResult(true, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the host CPU max capacity according to the method {@link LibvirtComputingResource#calculateHostCpuMaxCapacity(int, Long)}; if the host utilizes cgroup v1, this
|
||||
* value is 0.
|
||||
*/
|
||||
public int getHostCpuMaxCapacity() {
|
||||
return hostCpuMaxCapacity;
|
||||
}
|
||||
|
||||
public void setHostCpuMaxCapacity(int hostCpuMaxCapacity) {
|
||||
LibvirtComputingResource.hostCpuMaxCapacity = hostCpuMaxCapacity;
|
||||
}
|
||||
|
||||
public LibvirtKvmAgentHook getTransformer() throws IOException {
|
||||
return new LibvirtKvmAgentHook(_agentHooksBasedir, _agentHooksLibvirtXmlScript, _agentHooksLibvirtXmlMethod);
|
||||
}
|
||||
@ -2673,12 +2694,41 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
|
||||
*/
|
||||
protected CpuTuneDef createCpuTuneDef(VirtualMachineTO vmTO) {
|
||||
CpuTuneDef ctd = new CpuTuneDef();
|
||||
int shares = vmTO.getCpus() * (vmTO.getMinSpeed() != null ? vmTO.getMinSpeed() : vmTO.getSpeed());
|
||||
ctd.setShares(shares);
|
||||
ctd.setShares(calculateCpuShares(vmTO));
|
||||
setQuotaAndPeriod(vmTO, ctd);
|
||||
return ctd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the VM CPU shares considering the cgroup version of the host.
|
||||
* <ul>
|
||||
* <li>
|
||||
* If the host utilize cgroup v1, then, the CPU shares is calculated as <b>VM CPU shares = CPU cores * CPU frequency</b>.
|
||||
* </li>
|
||||
* <li>
|
||||
* If the host utilize cgroup v2, the CPU shares calculation considers the cgroup v2 upper limit of <b>10,000</b>, and a linear scale conversion is applied
|
||||
* considering the maximum host CPU shares (i.e. using the number of CPU cores and CPU nominal frequency of the host). Therefore, the VM CPU shares is calculated as
|
||||
* <b>VM CPU shares = (VM requested shares * cgroup upper limit) / host max shares</b>.
|
||||
* </li>
|
||||
* </ul>
|
||||
*/
|
||||
public int calculateCpuShares(VirtualMachineTO vmTO) {
|
||||
int vCpus = vmTO.getCpus();
|
||||
int cpuSpeed = ObjectUtils.defaultIfNull(vmTO.getMinSpeed(), vmTO.getSpeed());
|
||||
int requestedCpuShares = vCpus * cpuSpeed;
|
||||
int hostCpuMaxCapacity = getHostCpuMaxCapacity();
|
||||
|
||||
if (hostCpuMaxCapacity > 0) {
|
||||
int updatedCpuShares = (int) Math.ceil((requestedCpuShares * CGROUP_V2_UPPER_LIMIT) / (double) hostCpuMaxCapacity);
|
||||
s_logger.debug(String.format("This host utilizes cgroupv2 (as the max shares value is [%s]), thus, the VM requested shares of [%s] will be converted to " +
|
||||
"consider the host limits; the new CPU shares value is [%s].", hostCpuMaxCapacity, requestedCpuShares, updatedCpuShares));
|
||||
return updatedCpuShares;
|
||||
}
|
||||
s_logger.debug(String.format("This host does not have a maximum CPU shares set; therefore, this host utilizes cgroupv1 and the VM requested CPU shares [%s] will not be " +
|
||||
"converted.", requestedCpuShares));
|
||||
return requestedCpuShares;
|
||||
}
|
||||
|
||||
private CpuModeDef createCpuModeDef(VirtualMachineTO vmTO, int vcpus) {
|
||||
final CpuModeDef cmd = new CpuModeDef();
|
||||
cmd.setMode(_guestCpuMode);
|
||||
@ -3469,8 +3519,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
|
||||
|
||||
@Override
|
||||
public StartupCommand[] initialize() {
|
||||
|
||||
final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem, _manualCpuSpeed);
|
||||
calculateHostCpuMaxCapacity(info.getCpus(), info.getCpuSpeed());
|
||||
|
||||
String capabilities = String.join(",", info.getCapabilities());
|
||||
if (dpdkSupport) {
|
||||
@ -3514,6 +3564,32 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
|
||||
return startupCommandsArray;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates and sets the host CPU max capacity according to the cgroup version of the host.
|
||||
* <ul>
|
||||
* <li>
|
||||
* <b>cgroup v1</b>: the max CPU capacity for the host is set to <b>0</b>.
|
||||
* </li>
|
||||
* <li>
|
||||
* <b>cgroup v2</b>: the max CPU capacity for the host is the value of <b>cpuCores * cpuSpeed</b>.
|
||||
* </li>
|
||||
* </ul>
|
||||
*/
|
||||
protected void calculateHostCpuMaxCapacity(int cpuCores, Long cpuSpeed) {
|
||||
String output = Script.runSimpleBashScript(COMMAND_GET_CGROUP_HOST_VERSION);
|
||||
s_logger.info(String.format("Host uses control group [%s].", output));
|
||||
|
||||
if (!CGROUP_V2.equals(output)) {
|
||||
s_logger.info(String.format("Setting host CPU max capacity to 0, as it uses cgroup v1.", getHostCpuMaxCapacity()));
|
||||
setHostCpuMaxCapacity(0);
|
||||
return;
|
||||
}
|
||||
|
||||
s_logger.info(String.format("Calculating the max shares of the host."));
|
||||
setHostCpuMaxCapacity(cpuCores * cpuSpeed.intValue());
|
||||
s_logger.info(String.format("The max shares of the host is [%d].", getHostCpuMaxCapacity()));
|
||||
}
|
||||
|
||||
private StartupStorageCommand createLocalStoragePool(String localStoragePath, String localStorageUUID, StartupRoutingCommand cmd) {
|
||||
StartupStorageCommand sscmd = null;
|
||||
try {
|
||||
|
||||
@ -23,6 +23,7 @@ import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
@ -211,6 +212,8 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
|
||||
}
|
||||
}
|
||||
|
||||
xmlDesc = updateVmSharesIfNeeded(command, xmlDesc, libvirtComputingResource);
|
||||
|
||||
dconn = libvirtUtilitiesHelper.retrieveQemuConnection(destinationUri);
|
||||
|
||||
if (to.getType() == VirtualMachine.Type.User) {
|
||||
@ -362,6 +365,44 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
|
||||
return new MigrateAnswer(command, result == null, result, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the CPU shares are equal in the source host and destination host.
|
||||
* <ul>
|
||||
* <li>
|
||||
* If both hosts utilize cgroup v1; then, the shares value of the VM is equal in both hosts, and there is no need to update the VM CPU shares value for the
|
||||
* migration.</li>
|
||||
* <li>
|
||||
* If, at least, one of the hosts utilize cgroup v2, the VM CPU shares must be recalculated for the migration, accordingly to
|
||||
* method {@link LibvirtComputingResource#calculateCpuShares(VirtualMachineTO)}.
|
||||
* </li>
|
||||
* </ul>
|
||||
*/
|
||||
protected String updateVmSharesIfNeeded(MigrateCommand migrateCommand, String xmlDesc, LibvirtComputingResource libvirtComputingResource)
|
||||
throws ParserConfigurationException, IOException, SAXException, TransformerException {
|
||||
Integer newVmCpuShares = migrateCommand.getNewVmCpuShares();
|
||||
int currentCpuShares = libvirtComputingResource.calculateCpuShares(migrateCommand.getVirtualMachine());
|
||||
|
||||
if (newVmCpuShares == currentCpuShares) {
|
||||
s_logger.info(String.format("Current CPU shares [%s] is equal in both hosts; therefore, there is no need to update the CPU shares for the new host.",
|
||||
currentCpuShares));
|
||||
return xmlDesc;
|
||||
}
|
||||
|
||||
InputStream inputStream = IOUtils.toInputStream(xmlDesc, StandardCharsets.UTF_8);
|
||||
DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory();
|
||||
DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
|
||||
Document document = docBuilder.parse(inputStream);
|
||||
|
||||
Element root = document.getDocumentElement();
|
||||
Node sharesNode = root.getElementsByTagName("shares").item(0);
|
||||
String currentShares = sharesNode.getTextContent();
|
||||
|
||||
s_logger.info(String.format("VM [%s] will have CPU shares altered from [%s] to [%s] as part of migration because the cgroups version differs between hosts.",
|
||||
migrateCommand.getVmName(), currentShares, newVmCpuShares));
|
||||
sharesNode.setTextContent(String.valueOf(newVmCpuShares));
|
||||
return getXml(document);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace DPDK source path and target before migrations
|
||||
*/
|
||||
|
||||
@ -122,11 +122,7 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp
|
||||
return new PrepareForMigrationAnswer(command, "failed to connect physical disks to host");
|
||||
}
|
||||
|
||||
PrepareForMigrationAnswer answer = new PrepareForMigrationAnswer(command);
|
||||
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
|
||||
answer.setDpdkInterfaceMapping(dpdkInterfaceMapping);
|
||||
}
|
||||
return answer;
|
||||
return createPrepareForMigrationAnswer(command, dpdkInterfaceMapping, libvirtComputingResource, vm);
|
||||
} catch (final LibvirtException | CloudRuntimeException | InternalErrorException | URISyntaxException e) {
|
||||
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
|
||||
for (DpdkTO to : dpdkInterfaceMapping.values()) {
|
||||
@ -143,6 +139,22 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp
|
||||
}
|
||||
}
|
||||
|
||||
protected PrepareForMigrationAnswer createPrepareForMigrationAnswer(PrepareForMigrationCommand command, Map<String, DpdkTO> dpdkInterfaceMapping,
|
||||
LibvirtComputingResource libvirtComputingResource, VirtualMachineTO vm) {
|
||||
PrepareForMigrationAnswer answer = new PrepareForMigrationAnswer(command);
|
||||
|
||||
if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
|
||||
s_logger.debug(String.format("Setting DPDK interface for the migration of VM [%s].", vm));
|
||||
answer.setDpdkInterfaceMapping(dpdkInterfaceMapping);
|
||||
}
|
||||
|
||||
int newCpuShares = libvirtComputingResource.calculateCpuShares(vm);
|
||||
s_logger.debug(String.format("Setting CPU shares to [%s] for the migration of VM [%s].", newCpuShares, vm));
|
||||
answer.setNewVmCpuShares(newCpuShares);
|
||||
|
||||
return answer;
|
||||
}
|
||||
|
||||
private Answer handleRollback(PrepareForMigrationCommand command, LibvirtComputingResource libvirtComputingResource) {
|
||||
KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
|
||||
VirtualMachineTO vmTO = command.getVirtualMachine();
|
||||
|
||||
@ -39,8 +39,7 @@ public class LibvirtScaleVmCommandWrapper extends CommandWrapper<ScaleVmCommand,
|
||||
|
||||
long newMemory = ByteScaleUtils.bytesToKibibytes(vmSpec.getMaxRam());
|
||||
int newVcpus = vmSpec.getCpus();
|
||||
int newCpuSpeed = vmSpec.getMinSpeed() != null ? vmSpec.getMinSpeed() : vmSpec.getSpeed();
|
||||
int newCpuShares = newVcpus * newCpuSpeed;
|
||||
int newCpuShares = libvirtComputingResource.calculateCpuShares(vmSpec);
|
||||
String vmDefinition = vmSpec.toString();
|
||||
String scalingDetails = String.format("%s memory to [%s KiB], CPU cores to [%s] and cpu_shares to [%s]", vmDefinition, newMemory, newVcpus, newCpuShares);
|
||||
|
||||
|
||||
@ -6270,4 +6270,97 @@ public class LibvirtComputingResourceTest {
|
||||
Mockito.verify(loggerMock).debug("Skipping the memory balloon stats period setting for the VM (Libvirt Domain) with ID [1] and name [fake-VM-name] because this"
|
||||
+ " VM has no memory balloon.");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void calculateCpuSharesTestMinSpeedNullAndHostCgroupV1ShouldNotConsiderCgroupLimit() {
|
||||
int cpuCores = 2;
|
||||
int cpuSpeed = 2000;
|
||||
int maxCpuShares = 0;
|
||||
int expectedCpuShares = 4000;
|
||||
|
||||
Mockito.doReturn(cpuCores).when(vmTO).getCpus();
|
||||
Mockito.doReturn(null).when(vmTO).getMinSpeed();
|
||||
Mockito.doReturn(cpuSpeed).when(vmTO).getSpeed();
|
||||
Mockito.doReturn(maxCpuShares).when(libvirtComputingResourceSpy).getHostCpuMaxCapacity();
|
||||
int calculatedCpuShares = libvirtComputingResourceSpy.calculateCpuShares(vmTO);
|
||||
|
||||
Assert.assertEquals(expectedCpuShares, calculatedCpuShares);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void calculateCpuSharesTestMinSpeedNotNullAndHostCgroupV1ShouldNotConsiderCgroupLimit() {
|
||||
int cpuCores = 2;
|
||||
int cpuSpeed = 2000;
|
||||
int maxCpuShares = 0;
|
||||
int expectedCpuShares = 4000;
|
||||
|
||||
Mockito.doReturn(cpuCores).when(vmTO).getCpus();
|
||||
Mockito.doReturn(cpuSpeed).when(vmTO).getMinSpeed();
|
||||
Mockito.doReturn(maxCpuShares).when(libvirtComputingResourceSpy).getHostCpuMaxCapacity();
|
||||
int calculatedCpuShares = libvirtComputingResourceSpy.calculateCpuShares(vmTO);
|
||||
|
||||
Assert.assertEquals(expectedCpuShares, calculatedCpuShares);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void calculateCpuSharesTestMinSpeedNullAndHostCgroupV2ShouldConsiderCgroupLimit() {
|
||||
int cpuCores = 2;
|
||||
int cpuSpeed = 2000;
|
||||
int maxCpuShares = 5000;
|
||||
int expectedCpuShares = 8000;
|
||||
|
||||
Mockito.doReturn(cpuCores).when(vmTO).getCpus();
|
||||
Mockito.doReturn(null).when(vmTO).getMinSpeed();
|
||||
Mockito.doReturn(cpuSpeed).when(vmTO).getSpeed();
|
||||
Mockito.doReturn(maxCpuShares).when(libvirtComputingResourceSpy).getHostCpuMaxCapacity();
|
||||
int calculatedCpuShares = libvirtComputingResourceSpy.calculateCpuShares(vmTO);
|
||||
|
||||
Assert.assertEquals(expectedCpuShares, calculatedCpuShares);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void calculateCpuSharesTestMinSpeedNotNullAndHostCgroupV2ShouldConsiderCgroupLimit() {
|
||||
int cpuCores = 2;
|
||||
int cpuSpeed = 2000;
|
||||
int maxCpuShares = 5000;
|
||||
int expectedCpuShares = 8000;
|
||||
|
||||
Mockito.doReturn(cpuCores).when(vmTO).getCpus();
|
||||
Mockito.doReturn(cpuSpeed).when(vmTO).getMinSpeed();
|
||||
Mockito.doReturn(maxCpuShares).when(libvirtComputingResourceSpy).getHostCpuMaxCapacity();
|
||||
int calculatedCpuShares = libvirtComputingResourceSpy.calculateCpuShares(vmTO);
|
||||
|
||||
Assert.assertEquals(expectedCpuShares, calculatedCpuShares);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void setMaxHostCpuSharesIfCGroupV2TestShouldCalculateMaxCpuCapacityIfHostUtilizesCgroupV2() {
|
||||
int cpuCores = 2;
|
||||
long cpuSpeed = 2500L;
|
||||
int expectedShares = 5000;
|
||||
|
||||
String hostCgroupVersion = LibvirtComputingResource.CGROUP_V2;
|
||||
PowerMockito.mockStatic(Script.class);
|
||||
Mockito.when(Script.runSimpleBashScript(Mockito.anyString())).thenReturn(hostCgroupVersion);
|
||||
|
||||
libvirtComputingResourceSpy.calculateHostCpuMaxCapacity(cpuCores, cpuSpeed);
|
||||
|
||||
Assert.assertEquals(expectedShares, libvirtComputingResourceSpy.getHostCpuMaxCapacity());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void setMaxHostCpuSharesIfCGroupV2TestShouldNotCalculateMaxCpuCapacityIfHostDoesNotUtilizesCgroupV2() {
|
||||
int cpuCores = 2;
|
||||
long cpuSpeed = 2500L;
|
||||
int expectedShares = 0;
|
||||
|
||||
String hostCgroupVersion = "tmpfs";
|
||||
PowerMockito.mockStatic(Script.class);
|
||||
Mockito.when(Script.runSimpleBashScript(Mockito.anyString())).thenReturn(hostCgroupVersion);
|
||||
|
||||
libvirtComputingResourceSpy.calculateHostCpuMaxCapacity(cpuCores, cpuSpeed);
|
||||
|
||||
Assert.assertEquals(expectedShares, libvirtComputingResourceSpy.getHostCpuMaxCapacity());
|
||||
}
|
||||
}
|
||||
|
||||
@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
@ -38,6 +39,8 @@ import javax.xml.xpath.XPathExpressionException;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
|
||||
import org.apache.cloudstack.utils.linux.MemStat;
|
||||
import com.cloud.agent.api.to.VirtualMachineTO;
|
||||
import org.apache.cloudstack.utils.security.ParserUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
@ -51,7 +54,11 @@ import org.powermock.api.mockito.PowerMockito;
|
||||
import org.powermock.core.classloader.annotations.PowerMockIgnore;
|
||||
import org.powermock.core.classloader.annotations.PrepareForTest;
|
||||
import org.powermock.modules.junit4.PowerMockRunner;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.Node;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import com.cloud.agent.api.MigrateCommand;
|
||||
@ -444,6 +451,16 @@ public class LibvirtMigrateCommandWrapperTest {
|
||||
" </seclabel>\n" +
|
||||
"</domain>";
|
||||
|
||||
@Mock
|
||||
MigrateCommand migrateCommandMock;
|
||||
|
||||
@Mock
|
||||
LibvirtComputingResource libvirtComputingResourceMock;
|
||||
|
||||
@Mock
|
||||
VirtualMachineTO virtualMachineTOMock;
|
||||
|
||||
@Spy
|
||||
LibvirtMigrateCommandWrapper libvirtMigrateCmdWrapper = new LibvirtMigrateCommandWrapper();
|
||||
|
||||
final String memInfo = "MemTotal: 5830236 kB\n" +
|
||||
@ -871,4 +888,67 @@ public class LibvirtMigrateCommandWrapperTest {
|
||||
Assert.assertTrue(replaced.contains("csdpdk-7"));
|
||||
Assert.assertFalse(replaced.contains("csdpdk-1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void updateVmSharesIfNeededTestNewCpuSharesEqualCurrentSharesShouldNotUpdateVmShares() throws ParserConfigurationException, IOException, TransformerException,
|
||||
SAXException {
|
||||
int newVmCpuShares = 1000;
|
||||
int currentVmCpuShares = 1000;
|
||||
|
||||
Mockito.doReturn(newVmCpuShares).when(migrateCommandMock).getNewVmCpuShares();
|
||||
Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine();
|
||||
Mockito.doReturn(currentVmCpuShares).when(libvirtComputingResourceMock).calculateCpuShares(virtualMachineTOMock);
|
||||
|
||||
String finalXml = libvirtMigrateCmdWrapper.updateVmSharesIfNeeded(migrateCommandMock, fullfile, libvirtComputingResourceMock);
|
||||
|
||||
Assert.assertEquals(finalXml, fullfile);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void updateVmSharesIfNeededTestNewCpuSharesHigherThanCurrentSharesShouldUpdateVmShares() throws ParserConfigurationException, IOException, TransformerException,
|
||||
SAXException {
|
||||
int newVmCpuShares = 2000;
|
||||
int currentVmCpuShares = 1000;
|
||||
|
||||
Mockito.doReturn(newVmCpuShares).when(migrateCommandMock).getNewVmCpuShares();
|
||||
Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine();
|
||||
Mockito.doReturn(currentVmCpuShares).when(libvirtComputingResourceMock).calculateCpuShares(virtualMachineTOMock);
|
||||
|
||||
String finalXml = libvirtMigrateCmdWrapper.updateVmSharesIfNeeded(migrateCommandMock, fullfile, libvirtComputingResourceMock);
|
||||
|
||||
InputStream inputStream = IOUtils.toInputStream(finalXml, StandardCharsets.UTF_8);
|
||||
DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory();
|
||||
DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
|
||||
Document document = docBuilder.parse(inputStream);
|
||||
|
||||
Element root = document.getDocumentElement();
|
||||
Node sharesNode = root.getElementsByTagName("shares").item(0);
|
||||
int updateShares = Integer.parseInt(sharesNode.getTextContent());
|
||||
|
||||
Assert.assertEquals(updateShares, newVmCpuShares);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void updateVmSharesIfNeededTestNewCpuSharesLowerThanCurrentSharesShouldUpdateVmShares() throws ParserConfigurationException, IOException, TransformerException,
|
||||
SAXException {
|
||||
int newVmCpuShares = 500;
|
||||
int currentVmCpuShares = 1000;
|
||||
|
||||
Mockito.doReturn(newVmCpuShares).when(migrateCommandMock).getNewVmCpuShares();
|
||||
Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine();
|
||||
Mockito.doReturn(currentVmCpuShares).when(libvirtComputingResourceMock).calculateCpuShares(virtualMachineTOMock);
|
||||
|
||||
String finalXml = libvirtMigrateCmdWrapper.updateVmSharesIfNeeded(migrateCommandMock, fullfile, libvirtComputingResourceMock);
|
||||
|
||||
InputStream inputStream = IOUtils.toInputStream(finalXml, StandardCharsets.UTF_8);
|
||||
DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory();
|
||||
DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
|
||||
Document document = docBuilder.parse(inputStream);
|
||||
|
||||
Element root = document.getDocumentElement();
|
||||
Node sharesNode = root.getElementsByTagName("shares").item(0);
|
||||
int updateShares = Integer.parseInt(sharesNode.getTextContent());
|
||||
|
||||
Assert.assertEquals(updateShares, newVmCpuShares);
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,75 @@
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
//
|
||||
|
||||
package com.cloud.hypervisor.kvm.resource.wrapper;
|
||||
|
||||
import com.cloud.agent.api.PrepareForMigrationAnswer;
|
||||
import com.cloud.agent.api.PrepareForMigrationCommand;
|
||||
import com.cloud.agent.api.to.DpdkTO;
|
||||
import com.cloud.agent.api.to.VirtualMachineTO;
|
||||
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.Spy;
|
||||
import org.powermock.core.classloader.annotations.PrepareForTest;
|
||||
import org.powermock.modules.junit4.PowerMockRunner;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@RunWith(PowerMockRunner.class)
|
||||
@PrepareForTest(value = {LibvirtPrepareForMigrationCommandWrapper.class})
|
||||
public class LibvirtPrepareForMigrationCommandWrapperTest {
|
||||
|
||||
@Mock
|
||||
LibvirtComputingResource libvirtComputingResourceMock;
|
||||
|
||||
@Mock
|
||||
PrepareForMigrationCommand prepareForMigrationCommandMock;
|
||||
|
||||
@Mock
|
||||
VirtualMachineTO virtualMachineTOMock;
|
||||
|
||||
@Spy
|
||||
LibvirtPrepareForMigrationCommandWrapper libvirtPrepareForMigrationCommandWrapperSpy = new LibvirtPrepareForMigrationCommandWrapper();
|
||||
|
||||
@Test
|
||||
public void createPrepareForMigrationAnswerTestDpdkInterfaceNotEmptyShouldSetParamOnAnswer() {
|
||||
Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
|
||||
dpdkInterfaceMapping.put("Interface", new DpdkTO());
|
||||
|
||||
PrepareForMigrationAnswer prepareForMigrationAnswer = libvirtPrepareForMigrationCommandWrapperSpy.createPrepareForMigrationAnswer(prepareForMigrationCommandMock, dpdkInterfaceMapping, libvirtComputingResourceMock,
|
||||
virtualMachineTOMock);
|
||||
|
||||
Assert.assertEquals(prepareForMigrationAnswer.getDpdkInterfaceMapping(), dpdkInterfaceMapping);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void createPrepareForMigrationAnswerTestVerifyThatCpuSharesIsSet() {
|
||||
int cpuShares = 1000;
|
||||
Mockito.doReturn(cpuShares).when(libvirtComputingResourceMock).calculateCpuShares(virtualMachineTOMock);
|
||||
PrepareForMigrationAnswer prepareForMigrationAnswer = libvirtPrepareForMigrationCommandWrapperSpy.createPrepareForMigrationAnswer(prepareForMigrationCommandMock,null,
|
||||
libvirtComputingResourceMock, virtualMachineTOMock);
|
||||
|
||||
Assert.assertEquals(cpuShares, prepareForMigrationAnswer.getNewVmCpuShares().intValue());
|
||||
}
|
||||
}
|
||||
@ -207,9 +207,11 @@ public class LibvirtScaleVmCommandWrapperTest extends TestCase {
|
||||
@Test
|
||||
public void validateExecuteHandleLibvirtException() throws LibvirtException {
|
||||
String errorMessage = "";
|
||||
int shares = vmTo.getCpus() * vmTo.getSpeed();
|
||||
|
||||
Mockito.doReturn(vmTo).when(scaleVmCommandMock).getVirtualMachine();
|
||||
Mockito.doReturn(libvirtUtilitiesHelperMock).when(libvirtComputingResourceMock).getLibvirtUtilitiesHelper();
|
||||
Mockito.doReturn(shares).when(libvirtComputingResourceMock).calculateCpuShares(vmTo);
|
||||
Mockito.doThrow(libvirtException).when(libvirtUtilitiesHelperMock).getConnectionByVmName(Mockito.anyString());
|
||||
Mockito.doReturn(errorMessage).when(libvirtException).getMessage();
|
||||
|
||||
@ -222,9 +224,12 @@ public class LibvirtScaleVmCommandWrapperTest extends TestCase {
|
||||
|
||||
@Test
|
||||
public void validateExecuteSuccessfully() throws LibvirtException {
|
||||
int shares = vmTo.getCpus() * vmTo.getSpeed();
|
||||
|
||||
Mockito.doReturn(vmTo).when(scaleVmCommandMock).getVirtualMachine();
|
||||
Mockito.doReturn(libvirtUtilitiesHelperMock).when(libvirtComputingResourceMock).getLibvirtUtilitiesHelper();
|
||||
Mockito.doReturn(connectMock).when(libvirtUtilitiesHelperMock).getConnectionByVmName(Mockito.anyString());
|
||||
Mockito.doReturn(shares).when(libvirtComputingResourceMock).calculateCpuShares(vmTo);
|
||||
Mockito.doReturn(domainMock).when(connectMock).domainLookupByName(Mockito.anyString());
|
||||
Mockito.doNothing().when(libvirtScaleVmCommandWrapperSpy).scaleMemory(Mockito.any(), Mockito.anyLong(), Mockito.anyString());
|
||||
Mockito.doNothing().when(libvirtScaleVmCommandWrapperSpy).scaleVcpus(Mockito.any(), Mockito.anyInt(), Mockito.anyString());
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user