apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

IoT/ARM64 support: allow cloudstack-agent on Raspberry Pi 4 (armv8) to use kvm acceleration (#3644)

Browse Source 
KVM is supported on arm64 Linux (https://www.linux-kvm.org/page/Processor_support#ARM:).
For a small (IoT) platform such as the new Raspberry Pi 4 that uses armv8 processor
(cortex-a72) it's possible to run Linux host with `/dev/kvm`
accleration. This adds support for IoT IaaS in CloudStack.

This PR is from a fun weekend project where:
- I set up a Raspberry Pi 4 - 4GB RAM model with 4 CPU cores @ 1.5Ghz, 128GB SD samsung evo plus card
- Installed Ubuntu 19.10 raspi3 base image: http://cdimage.ubuntu.com/releases/19.10/release/ubuntu-19.10-preinstalled-server-arm64+raspi3.img.xz
- Build a custom Linux 5.3 kernel with KVM enabled, deb here: http://dl.rohityadav.cloud/cloudstack-rpi/kernel-19.10/ and install the linux-image and linux-module
- Then install/setup CloudStack on it (fix some issues around jna, by manually installing newer libjna-java to /usr/share/cloudstack-agent/lib)
- Since the host processor is not x86_64, I had to build a new arm64 (or aarch64) systemvmtemplate: http://dl.rohityadav.cloud/cloudstack-rpi/systemvmtemplate/

I could finally get a 4.13 CloudStack + Adv zone/networking to run on it
and deployed a KVM based Ubuntu 19.10 environment and NFS storage.
Deployed a test vm with isolated network, VR works as expected. Console
proxy works as well, for this tested against arm64 openstack Debian 9/10
templates.

I raised the issue of enabling KVM in upstream Ubuntu arm64 build: https://bugs.launchpad.net/ubuntu/+source/linux-raspi2/+bug/1783961
Ubuntu kernel team has come back and future arm64 releases may have 
KVM enabled by default.

Limitation: on my aarch64 env, it did not support IDE, therefore all
default bus type for volumes are SCSI by default. With VIRTIO it fails
sometimes.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
This commit is contained in:
Rohit Yadav 2019-11-11 22:01:05 +05:30 committed by GitHub
parent b0e3fbec3a
commit 524b995083
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 84 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
agent/conf/agent.properties
View File

@ -146,6 +146,12 @@ hypervisor.type=kvm
# on,run virsh capabilities for more details.
# guest.cpu.model=
#
# This param will set the CPU architecture for the domain override what
# the management server would send
# In case of arm64 (aarch64), this will change the machine type to 'virt' and
# adds a SCSI and a USB controller in the domain xml.
# guest.cpu.arch=x86_64|aarch64
#
# This param will require CPU features on the <cpu> section
# guest.cpu.features=vmx vme
#

3
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtCapXMLParser.java
View File

@ -80,7 +80,8 @@ public class LibvirtCapXMLParser extends LibvirtXMLParser {
}
} else if (qName.equalsIgnoreCase("arch")) {
for (int i = 0; i < attributes.getLength(); i++) {
if (attributes.getQName(i).equalsIgnoreCase("name") && attributes.getValue(i).equalsIgnoreCase("x86_64")) {
if (attributes.getQName(i).equalsIgnoreCase("name") &&
(attributes.getValue(i).equalsIgnoreCase("x86_64") || attributes.getValue(i).equalsIgnoreCase("aarch64"))) {
_archTypex8664 = true;
}
}

35
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -46,8 +46,6 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
import com.cloud.resource.RequestWrapper;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
@ -112,6 +110,7 @@ import com.cloud.dc.Vlan;
import com.cloud.exception.InternalErrorException;
import com.cloud.host.Host.Type;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ClockDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ConsoleDef;
@ -148,6 +147,7 @@ import com.cloud.hypervisor.kvm.storage.KVMStorageProcessor;
import com.cloud.network.Networks.BroadcastDomainType;
import com.cloud.network.Networks.RouterPrivateIpStrategy;
import com.cloud.network.Networks.TrafficType;
import com.cloud.resource.RequestWrapper;
import com.cloud.resource.ServerResource;
import com.cloud.resource.ServerResourceBase;
import com.cloud.storage.JavaStorageLayer;
@ -260,6 +260,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
protected String _localStoragePath;
protected String _localStorageUUID;
protected boolean _noMemBalloon = false;
protected String _guestCpuArch;
protected String _guestCpuMode;
protected String _guestCpuModel;
protected boolean _noKvmClock;
@ -955,6 +956,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
s_logger.trace("Ignoring libvirt error.", e);
}
final String cpuArchOverride = (String)params.get("guest.cpu.arch");
if (!Strings.isNullOrEmpty(cpuArchOverride)) {
_guestCpuArch = cpuArchOverride;
s_logger.info("Using guest CPU architecture: " + _guestCpuArch);
}
_guestCpuMode = (String)params.get("guest.cpu.mode");
if (_guestCpuMode != null) {
_guestCpuModel = (String)params.get("guest.cpu.model");
@ -2086,8 +2093,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
vm.setLibvirtVersion(_hypervisorLibvirtVersion);
vm.setQemuVersion(_hypervisorQemuVersion);
}
guest.setGuestArch(vmTO.getArch());
guest.setMachineType("pc");
guest.setGuestArch(_guestCpuArch != null ? _guestCpuArch : vmTO.getArch());
guest.setMachineType(_guestCpuArch != null && _guestCpuArch.equals("aarch64") ? "virt" : "pc");
guest.setUuid(uuid);
guest.setBootOrder(GuestDef.BootOrder.CDROM);
guest.setBootOrder(GuestDef.BootOrder.HARDISK);
@ -2208,6 +2215,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
final InputDef input = new InputDef("tablet", "usb");
devices.addDevice(input);
// Add an explicit USB devices for ARM64
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
devices.addDevice(new InputDef("keyboard", "usb"));
devices.addDevice(new InputDef("mouse", "usb"));
devices.addDevice(new LibvirtVMDef.USBDef((short)0, 0, 5, 0, 0));
}
DiskDef.DiskBus busT = getDiskModelFromVMDetail(vmTO);
@ -2353,6 +2366,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
} else {
disk.defISODisk(volPath, devId);
}
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
disk.setBusType(DiskDef.DiskBus.SCSI);
}
} else {
if (diskBusType == DiskDef.DiskBus.SCSI ) {
disk.setQemuDriver(true);
@ -2411,6 +2427,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
if (_sysvmISOPath != null) {
final DiskDef iso = new DiskDef();
iso.defISODisk(_sysvmISOPath);
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
iso.setBusType(DiskDef.DiskBus.SCSI);
}
vm.getDevices().addDevice(iso);
}
}
@ -3183,6 +3202,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
return null;
}
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
return DiskDef.DiskBus.SCSI;
}
final String rootDiskController = details.get(VmDetailConstants.ROOT_DISK_CONTROLLER);
if (StringUtils.isNotBlank(rootDiskController)) {
s_logger.debug("Passed custom disk bus " + rootDiskController);
@ -3197,6 +3220,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
}
private DiskDef.DiskBus getGuestDiskModel(final String platformEmulator) {
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
return DiskDef.DiskBus.SCSI;
}
if (platformEmulator == null) {
return DiskDef.DiskBus.IDE;
} else if (platformEmulator.startsWith("Other PV Virtio-SCSI")) {

40
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java
View File

@ -125,12 +125,17 @@ public class LibvirtVMDef {
guestDef.append(" machine='" + _machine + "'");
}
guestDef.append(">hvm</type>\n");
if (_arch != null && _arch.equals("aarch64")) {
guestDef.append("<loader readonly='yes' type='pflash'>/usr/share/AAVMF/AAVMF_CODE.fd</loader>\n");
}
if (!_bootdevs.isEmpty()) {
for (BootOrder bo : _bootdevs) {
guestDef.append("<boot dev='" + bo + "'/>\n");
}
}
if (_arch == null || !_arch.equals("aarch64")) {
guestDef.append("<smbios mode='sysinfo'/>\n");
}
guestDef.append("</os>\n");
return guestDef.toString();
} else if (_type == GuestType.LXC) {
@ -782,6 +787,10 @@ public class LibvirtVMDef {
return _bus;
}
public void setBusType(DiskBus busType) {
_bus = busType;
}
public DiskFmtType getDiskFormatType() {
return _diskFmtType;
}
@ -1624,6 +1633,37 @@ public class LibvirtVMDef {
}
}
public static class USBDef {
private short index = 0;
private int domain = 0;
private int bus = 0;
private int slot = 9;
private int function = 0;
public USBDef(short index, int domain, int bus, int slot, int function) {
this.index = index;
this.domain = domain;
this.bus = bus;
this.slot = slot;
this.function = function;
}
public USBDef() {
}
@Override
public String toString() {
StringBuilder scsiBuilder = new StringBuilder();
scsiBuilder.append(String.format("<controller type='usb' index='%d' model='qemu-xhci'>\n", this.index));
scsiBuilder.append("<alias name='usb'/>");
scsiBuilder.append(String.format("<address type='pci' domain='0x%04X' bus='0x%02X' slot='0x%02X' function='0x%01X'/>\n",
this.domain, this.bus, this.slot, this.function ) );
scsiBuilder.append("</controller>\n");
return scsiBuilder.toString();
}
}
public static class InputDef {
private final String _type; /* tablet, mouse */
private final String _bus; /* ps2, usb, xen */

3
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java
View File

@ -243,6 +243,7 @@ public class LibvirtComputingResourceTest {
final VirtualMachineTO to = new VirtualMachineTO(id, name, VirtualMachine.Type.User, cpus, speed, minRam, maxRam, BootloaderType.HVM, os, false, false, vncPassword);
to.setVncAddr(vncAddr);
to.setArch("x86_64");
to.setUuid("b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9");
final LibvirtVMDef vm = lcr.createVMFromSpec(to);
@ -275,6 +276,7 @@ public class LibvirtComputingResourceTest {
final VirtualMachineTO to = new VirtualMachineTO(id, name, VirtualMachine.Type.User, cpus, minSpeed, maxSpeed, minRam, maxRam, BootloaderType.HVM, os, false, false, vncPassword);
to.setVncAddr(vncAddr);
to.setArch("x86_64");
to.setUuid("b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9");
final LibvirtVMDef vm = lcr.createVMFromSpec(to);
@ -344,6 +346,7 @@ public class LibvirtComputingResourceTest {
final VirtualMachineTO to =
new VirtualMachineTO(id, name, VirtualMachine.Type.User, cpus, minSpeed, maxSpeed, minRam, maxRam, BootloaderType.HVM, os, false, false, vncPassword);
to.setVncAddr(vncAddr);
to.setArch("x86_64");
to.setUuid("b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9");
final LibvirtVMDef vm = lcr.createVMFromSpec(to);

2
server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java
View File

@ -251,7 +251,7 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
throw new DiscoveredWithErrorException("Authentication error");
}
if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "lsmod|grep kvm")) {
if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "ls /dev/kvm")) {
s_logger.debug("It's not a KVM enabled machine");
return null;
}