From 524b995083653bac1ceec334b6193500c0d3b906 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 11 Nov 2019 22:01:05 +0530 Subject: [PATCH] IoT/ARM64 support: allow cloudstack-agent on Raspberry Pi 4 (armv8) to use kvm acceleration (#3644) KVM is supported on arm64 Linux (https://www.linux-kvm.org/page/Processor_support#ARM:). For a small (IoT) platform such as the new Raspberry Pi 4 that uses armv8 processor (cortex-a72) it's possible to run Linux host with `/dev/kvm` accleration. This adds support for IoT IaaS in CloudStack. This PR is from a fun weekend project where: - I set up a Raspberry Pi 4 - 4GB RAM model with 4 CPU cores @ 1.5Ghz, 128GB SD samsung evo plus card - Installed Ubuntu 19.10 raspi3 base image: http://cdimage.ubuntu.com/releases/19.10/release/ubuntu-19.10-preinstalled-server-arm64+raspi3.img.xz - Build a custom Linux 5.3 kernel with KVM enabled, deb here: http://dl.rohityadav.cloud/cloudstack-rpi/kernel-19.10/ and install the linux-image and linux-module - Then install/setup CloudStack on it (fix some issues around jna, by manually installing newer libjna-java to /usr/share/cloudstack-agent/lib) - Since the host processor is not x86_64, I had to build a new arm64 (or aarch64) systemvmtemplate: http://dl.rohityadav.cloud/cloudstack-rpi/systemvmtemplate/ I could finally get a 4.13 CloudStack + Adv zone/networking to run on it and deployed a KVM based Ubuntu 19.10 environment and NFS storage. Deployed a test vm with isolated network, VR works as expected. Console proxy works as well, for this tested against arm64 openstack Debian 9/10 templates. I raised the issue of enabling KVM in upstream Ubuntu arm64 build: https://bugs.launchpad.net/ubuntu/+source/linux-raspi2/+bug/1783961 Ubuntu kernel team has come back and future arm64 releases may have KVM enabled by default. Limitation: on my aarch64 env, it did not support IDE, therefore all default bus type for volumes are SCSI by default. With VIRTIO it fails sometimes. Signed-off-by: Rohit Yadav --- agent/conf/agent.properties | 6 +++ .../kvm/resource/LibvirtCapXMLParser.java | 3 +- .../resource/LibvirtComputingResource.java | 35 ++++++++++++++-- .../hypervisor/kvm/resource/LibvirtVMDef.java | 42 ++++++++++++++++++- .../LibvirtComputingResourceTest.java | 3 ++ .../discoverer/LibvirtServerDiscoverer.java | 2 +- 6 files changed, 84 insertions(+), 7 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index b45526aa6af..7472e392227 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -146,6 +146,12 @@ hypervisor.type=kvm # on,run virsh capabilities for more details. # guest.cpu.model= # +# This param will set the CPU architecture for the domain override what +# the management server would send +# In case of arm64 (aarch64), this will change the machine type to 'virt' and +# adds a SCSI and a USB controller in the domain xml. +# guest.cpu.arch=x86_64|aarch64 +# # This param will require CPU features on the section # guest.cpu.features=vmx vme # diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtCapXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtCapXMLParser.java index 2f12d21eb24..358fafae4b5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtCapXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtCapXMLParser.java @@ -80,7 +80,8 @@ public class LibvirtCapXMLParser extends LibvirtXMLParser { } } else if (qName.equalsIgnoreCase("arch")) { for (int i = 0; i < attributes.getLength(); i++) { - if (attributes.getQName(i).equalsIgnoreCase("name") && attributes.getValue(i).equalsIgnoreCase("x86_64")) { + if (attributes.getQName(i).equalsIgnoreCase("name") && + (attributes.getValue(i).equalsIgnoreCase("x86_64") || attributes.getValue(i).equalsIgnoreCase("aarch64"))) { _archTypex8664 = true; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index b20f1a58b94..ce71a50d5e9 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -46,8 +46,6 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import com.cloud.hypervisor.kvm.dpdk.DpdkHelper; -import com.cloud.resource.RequestWrapper; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; @@ -112,6 +110,7 @@ import com.cloud.dc.Vlan; import com.cloud.exception.InternalErrorException; import com.cloud.host.Host.Type; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.hypervisor.kvm.dpdk.DpdkHelper; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ClockDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ConsoleDef; @@ -148,6 +147,7 @@ import com.cloud.hypervisor.kvm.storage.KVMStorageProcessor; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.RouterPrivateIpStrategy; import com.cloud.network.Networks.TrafficType; +import com.cloud.resource.RequestWrapper; import com.cloud.resource.ServerResource; import com.cloud.resource.ServerResourceBase; import com.cloud.storage.JavaStorageLayer; @@ -260,6 +260,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected String _localStoragePath; protected String _localStorageUUID; protected boolean _noMemBalloon = false; + protected String _guestCpuArch; protected String _guestCpuMode; protected String _guestCpuModel; protected boolean _noKvmClock; @@ -955,6 +956,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv s_logger.trace("Ignoring libvirt error.", e); } + final String cpuArchOverride = (String)params.get("guest.cpu.arch"); + if (!Strings.isNullOrEmpty(cpuArchOverride)) { + _guestCpuArch = cpuArchOverride; + s_logger.info("Using guest CPU architecture: " + _guestCpuArch); + } + _guestCpuMode = (String)params.get("guest.cpu.mode"); if (_guestCpuMode != null) { _guestCpuModel = (String)params.get("guest.cpu.model"); @@ -2086,8 +2093,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv vm.setLibvirtVersion(_hypervisorLibvirtVersion); vm.setQemuVersion(_hypervisorQemuVersion); } - guest.setGuestArch(vmTO.getArch()); - guest.setMachineType("pc"); + guest.setGuestArch(_guestCpuArch != null ? _guestCpuArch : vmTO.getArch()); + guest.setMachineType(_guestCpuArch != null && _guestCpuArch.equals("aarch64") ? "virt" : "pc"); guest.setUuid(uuid); guest.setBootOrder(GuestDef.BootOrder.CDROM); guest.setBootOrder(GuestDef.BootOrder.HARDISK); @@ -2208,6 +2215,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv final InputDef input = new InputDef("tablet", "usb"); devices.addDevice(input); + // Add an explicit USB devices for ARM64 + if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) { + devices.addDevice(new InputDef("keyboard", "usb")); + devices.addDevice(new InputDef("mouse", "usb")); + devices.addDevice(new LibvirtVMDef.USBDef((short)0, 0, 5, 0, 0)); + } DiskDef.DiskBus busT = getDiskModelFromVMDetail(vmTO); @@ -2353,6 +2366,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } else { disk.defISODisk(volPath, devId); } + if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) { + disk.setBusType(DiskDef.DiskBus.SCSI); + } } else { if (diskBusType == DiskDef.DiskBus.SCSI ) { disk.setQemuDriver(true); @@ -2411,6 +2427,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv if (_sysvmISOPath != null) { final DiskDef iso = new DiskDef(); iso.defISODisk(_sysvmISOPath); + if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) { + iso.setBusType(DiskDef.DiskBus.SCSI); + } vm.getDevices().addDevice(iso); } } @@ -3183,6 +3202,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return null; } + if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) { + return DiskDef.DiskBus.SCSI; + } + final String rootDiskController = details.get(VmDetailConstants.ROOT_DISK_CONTROLLER); if (StringUtils.isNotBlank(rootDiskController)) { s_logger.debug("Passed custom disk bus " + rootDiskController); @@ -3197,6 +3220,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } private DiskDef.DiskBus getGuestDiskModel(final String platformEmulator) { + if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) { + return DiskDef.DiskBus.SCSI; + } + if (platformEmulator == null) { return DiskDef.DiskBus.IDE; } else if (platformEmulator.startsWith("Other PV Virtio-SCSI")) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index dab1af510bf..bfbb4d5d1bc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -125,12 +125,17 @@ public class LibvirtVMDef { guestDef.append(" machine='" + _machine + "'"); } guestDef.append(">hvm\n"); + if (_arch != null && _arch.equals("aarch64")) { + guestDef.append("/usr/share/AAVMF/AAVMF_CODE.fd\n"); + } if (!_bootdevs.isEmpty()) { for (BootOrder bo : _bootdevs) { guestDef.append("\n"); } } - guestDef.append("\n"); + if (_arch == null || !_arch.equals("aarch64")) { + guestDef.append("\n"); + } guestDef.append("\n"); return guestDef.toString(); } else if (_type == GuestType.LXC) { @@ -782,6 +787,10 @@ public class LibvirtVMDef { return _bus; } + public void setBusType(DiskBus busType) { + _bus = busType; + } + public DiskFmtType getDiskFormatType() { return _diskFmtType; } @@ -1624,6 +1633,37 @@ public class LibvirtVMDef { } } + public static class USBDef { + private short index = 0; + private int domain = 0; + private int bus = 0; + private int slot = 9; + private int function = 0; + + public USBDef(short index, int domain, int bus, int slot, int function) { + this.index = index; + this.domain = domain; + this.bus = bus; + this.slot = slot; + this.function = function; + } + + public USBDef() { + } + + @Override + public String toString() { + StringBuilder scsiBuilder = new StringBuilder(); + + scsiBuilder.append(String.format("\n", this.index)); + scsiBuilder.append(""); + scsiBuilder.append(String.format("
\n", + this.domain, this.bus, this.slot, this.function ) ); + scsiBuilder.append("\n"); + return scsiBuilder.toString(); + } + } + public static class InputDef { private final String _type; /* tablet, mouse */ private final String _bus; /* ps2, usb, xen */ diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index d9f8edca814..58d5ee64f63 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -243,6 +243,7 @@ public class LibvirtComputingResourceTest { final VirtualMachineTO to = new VirtualMachineTO(id, name, VirtualMachine.Type.User, cpus, speed, minRam, maxRam, BootloaderType.HVM, os, false, false, vncPassword); to.setVncAddr(vncAddr); + to.setArch("x86_64"); to.setUuid("b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9"); final LibvirtVMDef vm = lcr.createVMFromSpec(to); @@ -275,6 +276,7 @@ public class LibvirtComputingResourceTest { final VirtualMachineTO to = new VirtualMachineTO(id, name, VirtualMachine.Type.User, cpus, minSpeed, maxSpeed, minRam, maxRam, BootloaderType.HVM, os, false, false, vncPassword); to.setVncAddr(vncAddr); + to.setArch("x86_64"); to.setUuid("b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9"); final LibvirtVMDef vm = lcr.createVMFromSpec(to); @@ -344,6 +346,7 @@ public class LibvirtComputingResourceTest { final VirtualMachineTO to = new VirtualMachineTO(id, name, VirtualMachine.Type.User, cpus, minSpeed, maxSpeed, minRam, maxRam, BootloaderType.HVM, os, false, false, vncPassword); to.setVncAddr(vncAddr); + to.setArch("x86_64"); to.setUuid("b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9"); final LibvirtVMDef vm = lcr.createVMFromSpec(to); diff --git a/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index 6c9bcace90f..e232d6a6415 100644 --- a/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -251,7 +251,7 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements throw new DiscoveredWithErrorException("Authentication error"); } - if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "lsmod|grep kvm")) { + if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "ls /dev/kvm")) { s_logger.debug("It's not a KVM enabled machine"); return null; }