systemvm: new qemu-guest-agent based patching for KVM (#3278)

This introduces a new patching script for patching systemvms on KVM
using qemu-guest-agent that runs inside the systemvm on startup. This
also removes the vport device which was previously used by the legacy
patching script and instead uses the modern and new uniform guest
agent vport for host-guest communication.

Also updates the sytemvmtemplate build config to use the latest Debian
9.9.0 iso.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
This commit is contained in:
Rohit Yadav 2019-05-10 23:42:19 +05:30 committed by GitHub
parent f9b61bc737
commit 9ff819da2c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 123 additions and 302 deletions

View File

@ -47,7 +47,6 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import com.cloud.resource.RequestWrapper;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
@ -146,6 +145,7 @@ import com.cloud.hypervisor.kvm.storage.KVMStorageProcessor;
import com.cloud.network.Networks.BroadcastDomainType;
import com.cloud.network.Networks.RouterPrivateIpStrategy;
import com.cloud.network.Networks.TrafficType;
import com.cloud.resource.RequestWrapper;
import com.cloud.resource.ServerResource;
import com.cloud.resource.ServerResourceBase;
import com.cloud.storage.JavaStorageLayer;
@ -199,7 +199,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
private String _modifyVlanPath;
private String _versionstringpath;
private String _patchViaSocketPath;
private String _patchScriptPath;
private String _createvmPath;
private String _manageSnapshotPath;
private String _resizeVolumePath;
@ -682,9 +682,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
throw new ConfigurationException("Unable to find versions.sh");
}
_patchViaSocketPath = Script.findScript(kvmScriptsDir + "/patch/", "patchviasocket.py");
if (_patchViaSocketPath == null) {
throw new ConfigurationException("Unable to find patchviasocket.py");
_patchScriptPath = Script.findScript(kvmScriptsDir, "patch.sh");
if (_patchScriptPath == null) {
throw new ConfigurationException("Unable to find patch.sh");
}
_heartBeatPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh");
@ -1362,13 +1362,13 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
}
public boolean passCmdLine(final String vmName, final String cmdLine) throws InternalErrorException {
final Script command = new Script(_patchViaSocketPath, 5 * 1000, s_logger);
final Script command = new Script(_patchScriptPath, 30 * 1000, s_logger);
String result;
command.add("-n", vmName);
command.add("-p", cmdLine.replaceAll(" ", "%"));
command.add("-c", cmdLine);
result = command.execute();
if (result != null) {
s_logger.error("passcmd failed:" + result);
s_logger.error("Passing cmdline failed:" + result);
return false;
}
return true;
@ -2141,12 +2141,6 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
final SerialDef serial = new SerialDef("pty", null, (short)0);
devices.addDevice(serial);
/* Add a VirtIO channel for SystemVMs for communication and provisioning */
if (vmTO.getType() != VirtualMachine.Type.User) {
devices.addDevice(new ChannelDef(vmTO.getName() + ".vport", ChannelDef.ChannelType.UNIX,
new File(_qemuSocketsPath + "/" + vmTO.getName() + ".agent")));
}
if (_rngEnable) {
final RngDef rngDevice = new RngDef(_rngPath, _rngBackendModel, _rngRateBytes, _rngRatePeriod);
devices.addDevice(rngDevice);

View File

@ -115,7 +115,6 @@ public final class LibvirtStartCommandWrapper extends CommandWrapper<StartComman
// pass cmdline info to system vms
if (vmSpec.getType() != VirtualMachine.Type.User) {
//wait and try passCmdLine for 5 minutes at most for CLOUDSTACK-2823
String controlIp = null;
for (final NicTO nic : nics) {
if (nic.getType() == TrafficType.Control) {
@ -123,9 +122,11 @@ public final class LibvirtStartCommandWrapper extends CommandWrapper<StartComman
break;
}
}
for (int count = 0; count < 30; count++) {
// try to patch and SSH into the systemvm for up to 5 minutes
for (int count = 0; count < 10; count++) {
// wait and try passCmdLine for 30 seconds at most for CLOUDSTACK-2823
libvirtComputingResource.passCmdLine(vmName, vmSpec.getBootArgs());
//check router is up?
// check router is up?
final VirtualRoutingResource virtRouterResource = libvirtComputingResource.getVirtRouterResource();
final boolean result = virtRouterResource.connect(controlIp, 1, 5000);
if (result) {

View File

@ -22,14 +22,14 @@ package com.cloud.hypervisor.kvm.resource;
import java.io.File;
import java.util.List;
import junit.framework.TestCase;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.WatchDogDef;
import junit.framework.TestCase;
public class LibvirtDomainXMLParserTest extends TestCase {
public void testDomainXMLParser() {
@ -45,9 +45,6 @@ public class LibvirtDomainXMLParserTest extends TestCase {
InterfaceDef.GuestNetType ifType = InterfaceDef.GuestNetType.BRIDGE;
ChannelDef.ChannelType channelType = ChannelDef.ChannelType.UNIX;
ChannelDef.ChannelState channelState = ChannelDef.ChannelState.DISCONNECTED;
String ssvmAgentPath = "/var/lib/libvirt/qemu/s-2970-VM.agent";
String ssvmAgentName = "s-2970-VM.vport";
String guestAgentPath = "/var/lib/libvirt/qemu/guest-agent.org.qemu.guest_agent.0";
String guestAgentName = "org.qemu.guest_agent.0";
@ -155,12 +152,6 @@ public class LibvirtDomainXMLParserTest extends TestCase {
"<target type='serial' port='0'/>" +
"<alias name='serial0'/>" +
"</console>" +
"<channel type='unix'>" +
"<source mode='bind' path='/var/lib/libvirt/qemu/s-2970-VM.agent'/>" +
"<target type='virtio' name='s-2970-VM.vport' state='disconnected'/>" +
"<alias name='channel0'/>" +
"<address type='virtio-serial' controller='0' bus='0' port='1'/>" +
"</channel>" +
"<input type='tablet' bus='usb'>" +
"<alias name='input0'/>" +
"</input>" +
@ -215,14 +206,9 @@ public class LibvirtDomainXMLParserTest extends TestCase {
assertEquals(channelType, channels.get(i).getChannelType());
}
/* SSVM provisioning port/channel */
assertEquals(channelState, channels.get(0).getChannelState());
assertEquals(new File(ssvmAgentPath), channels.get(0).getPath());
assertEquals(ssvmAgentName, channels.get(0).getName());
/* Qemu Guest Agent port/channel */
assertEquals(new File(guestAgentPath), channels.get(1).getPath());
assertEquals(guestAgentName, channels.get(1).getName());
assertEquals(new File(guestAgentPath), channels.get(0).getPath());
assertEquals(guestAgentName, channels.get(0).getName());
List<InterfaceDef> ifs = parser.getInterfaces();
for (int i = 0; i < ifs.size(); i++) {

View File

@ -21,13 +21,13 @@ package com.cloud.hypervisor.kvm.resource;
import java.io.File;
import junit.framework.TestCase;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.SCSIDef;
import com.cloud.utils.Pair;
import junit.framework.TestCase;
public class LibvirtVMDefTest extends TestCase {
public void testInterfaceEtehrnet() {
@ -180,7 +180,7 @@ public class LibvirtVMDefTest extends TestCase {
public void testChannelDef() {
ChannelDef.ChannelType type = ChannelDef.ChannelType.UNIX;
ChannelDef.ChannelState state = ChannelDef.ChannelState.CONNECTED;
String name = "v-136-VM.vport";
String name = "v-136-VM.org.qemu.guest_agent.0";
File path = new File("/var/lib/libvirt/qemu/" + name);
ChannelDef channelDef = new ChannelDef(name, type, state, path);

View File

@ -1046,7 +1046,7 @@
<File Id="fil47212822DDAFFCD71C79615CF4583BAF" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\kvmheartbeat.sh" />
</Component>
<Component Id="cmpF2BBDD336FEC0B34B3C744ACF1E4B959" Guid="{56D8ECF7-49F8-4B26-A8F4-662252C0A647}">
<File Id="filD809C7F728AC5D1BD36E7DB403BFA141" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\patchviasocket.py" />
<File Id="filD809C7F728AC5D1BD36E7DB403BFA141" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\patch.sh" />
</Component>
<Component Id="cmp2F4D4D81563D153E86B0A652A83D363A" Guid="{7BFC7637-E33D-4BC4-8B25-3CDEA601110C}">
<File Id="fil349420D6088A01C9F63E27634623F5BE" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\setup_agent.sh" />

View File

@ -0,0 +1,80 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -e
# Get the VM name and cmdline
while getopts "n:c:h" opt; do
case ${opt} in
n )
name=$OPTARG
;;
c )
cmdline=$(echo $OPTARG | base64 -w 0)
;;
h )
echo "Usage: $0 -n [VM name] -c [command line]"
exit 0
;;
esac
done
SSHKEY_FILE="/root/.ssh/id_rsa.pub.cloud"
if [ ! -e $SSHKEY_FILE ]; then
echo "SSH public key file $SSHKEY_FILE not found!"
exit 1
fi
if ! which virsh > /dev/null; then
echo "Libvirt CLI 'virsh' not found"
exit 1
fi
# Read the SSH public key
sshkey=$(cat $SSHKEY_FILE | base64 -w 0)
# Method to send and write payload inside the VM
send_file() {
local name=${1}
local path=${2}
local content=${@:3}
local fd=$(virsh qemu-agent-command $name "{\"execute\":\"guest-file-open\", \"arguments\":{\"path\":\"$path\",\"mode\":\"w+\"}}" | sed 's/[^:]*:\([^}]*\).*/\1/')
virsh qemu-agent-command $name "{\"execute\":\"guest-file-write\", \"arguments\":{\"handle\":$fd,\"buf-b64\":\"$content\"}}" > /dev/null
virsh qemu-agent-command $name "{\"execute\":\"guest-file-close\", \"arguments\":{\"handle\":$fd}}" > /dev/null
}
# Wait for the guest agent to come online
while ! virsh qemu-agent-command $name '{"execute":"guest-ping"}' >/dev/null 2>&1
do
sleep 0.1
done
# Test guest agent sanity
while [ "$(virsh qemu-agent-command $name '{"execute":"guest-sync","arguments":{"id":1234567890}}' 2>/dev/null)" != '{"return":1234567890}' ]
do
sleep 0.1
done
# Write ssh public key
send_file $name "/root/.ssh/authorized_keys" $sshkey
# Fix ssh public key permission
virsh qemu-agent-command $name '{"execute":"guest-exec","arguments":{"path":"chmod","arg":["go-rwx","/root/.ssh/authorized_keys"]}}' > /dev/null
# Write cmdline payload
send_file $name "/var/cache/cloud/cmdline" $cmdline

View File

@ -1,76 +0,0 @@
#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This script connects to the system vm socket and writes the
# authorized_keys and cmdline data to it. The system VM then
# reads it from /dev/vport0p1 in cloud_early_config
#
import argparse
import os
import socket
SOCK_FILE = "/var/lib/libvirt/qemu/{name}.agent"
PUB_KEY_FILE = "/root/.ssh/id_rsa.pub.cloud"
MESSAGE = "pubkey:{key}\ncmdline:{cmdline}\n"
def send_to_socket(sock_file, key_file, cmdline):
if not os.path.exists(key_file):
print("ERROR: ssh public key not found on host at {0}".format(key_file))
return 1
try:
with open(key_file, "r") as f:
pub_key = f.read()
except IOError as e:
print("ERROR: unable to open {0} - {1}".format(key_file, e.strerror))
return 1
# Keep old substitution from perl code:
cmdline = cmdline.replace("%", " ")
msg = MESSAGE.format(key=pub_key, cmdline=cmdline)
if not os.path.exists(sock_file):
print("ERROR: {0} socket not found".format(sock_file))
return 1
try:
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect(sock_file)
s.sendall(msg)
s.close()
except IOError as e:
print("ERROR: unable to connect to {0} - {1}".format(sock_file, e.strerror))
return 1
return 0 # Success
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Send configuration to system VM socket")
parser.add_argument("-n", "--name", required=True, help="Name of VM")
parser.add_argument("-p", "--cmdline", required=True, help="Command line")
arguments = parser.parse_args()
socket_file = SOCK_FILE.format(name=arguments.name)
exit(send_to_socket(socket_file, PUB_KEY_FILE, arguments.cmdline))

View File

@ -1,144 +0,0 @@
#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import patchviasocket
import getpass
import os
import socket
import tempfile
import time
import threading
import unittest
KEY_DATA = "I luv\nCloudStack\n"
CMD_DATA = "/run/this-for-me --please=TRUE! very%quickly"
NON_EXISTING_FILE = "must-not-exist"
def write_key_file():
_, tmpfile = tempfile.mkstemp(".sck")
with open(tmpfile, "w") as f:
f.write(KEY_DATA)
return tmpfile
class SocketThread(threading.Thread):
def __init__(self):
super(SocketThread, self).__init__()
self._data = ""
self._folder = tempfile.mkdtemp(".sck")
self._file = os.path.join(self._folder, "socket")
self._ready = False
def data(self):
return self._data
def file(self):
return self._file
def wait_until_ready(self):
while not self._ready:
time.sleep(0.050)
def run(self):
TIMEOUT = 0.314 # Very short time for tests that don't write to socket.
MAX_SIZE = 10 * 1024
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
s.bind(self._file)
s.listen(1)
s.settimeout(TIMEOUT)
try:
self._ready = True
client, address = s.accept()
self._data = client.recv(MAX_SIZE)
client.close()
except socket.timeout:
pass
finally:
s.close()
os.remove(self._file)
os.rmdir(self._folder)
class TestPatchViaSocket(unittest.TestCase):
def setUp(self):
self._key_file = write_key_file()
self._unreadable = write_key_file()
os.chmod(self._unreadable, 0)
self.assertFalse(os.path.exists(NON_EXISTING_FILE))
self.assertNotEqual("root", getpass.getuser(), "must be non-root user (to test access denied errors)")
def tearDown(self):
os.remove(self._key_file)
os.remove(self._unreadable)
def test_write_to_socket(self):
reader = SocketThread()
reader.start()
reader.wait_until_ready()
self.assertEquals(0, patchviasocket.send_to_socket(reader.file(), self._key_file, CMD_DATA))
reader.join()
data = reader.data()
self.assertIn(KEY_DATA, data)
self.assertIn(CMD_DATA.replace("%", " "), data)
self.assertNotIn("LUV", data)
self.assertNotIn("very%quickly", data) # Testing substitution
def test_host_key_error(self):
reader = SocketThread()
reader.start()
reader.wait_until_ready()
self.assertEquals(1, patchviasocket.send_to_socket(reader.file(), NON_EXISTING_FILE, CMD_DATA))
reader.join() # timeout
def test_host_key_access_denied(self):
reader = SocketThread()
reader.start()
reader.wait_until_ready()
self.assertEquals(1, patchviasocket.send_to_socket(reader.file(), self._unreadable, CMD_DATA))
reader.join() # timeout
def test_nonexistant_socket_error(self):
reader = SocketThread()
reader.start()
reader.wait_until_ready()
self.assertEquals(1, patchviasocket.send_to_socket(NON_EXISTING_FILE, self._key_file, CMD_DATA))
reader.join() # timeout
def test_invalid_socket_error(self):
reader = SocketThread()
reader.start()
reader.wait_until_ready()
self.assertEquals(1, patchviasocket.send_to_socket(self._key_file, self._key_file, CMD_DATA))
reader.join() # timeout
def test_access_denied_socket_error(self):
reader = SocketThread()
reader.start()
reader.wait_until_ready()
self.assertEquals(1, patchviasocket.send_to_socket(self._unreadable, self._key_file, CMD_DATA))
reader.join() # timeout
if __name__ == '__main__':
unittest.main()

View File

@ -90,41 +90,29 @@ get_boot_params() {
sed -i "s/%/ /g" $CMDLINE
;;
kvm)
VPORT=$(find /dev/virtio-ports -type l -name '*.vport' 2>/dev/null|head -1)
if [ -z "$VPORT" ]; then
log_it "No suitable VirtIO port was found in /dev/virtio-ports" && exit 2
# Use any old cmdline as backup only
if [ -s $CMDLINE ]; then
mv $CMDLINE $CMDLINE.old
log_it "Found a non-empty old cmdline file"
fi
if [ ! -e "$VPORT" ]; then
log_it "${VPORT} not loaded, perhaps guest kernel is too old." && exit 2
fi
local factor=2
local progress=1
for i in {1..5}
do
while read line; do
if [[ $line == cmdline:* ]]; then
cmd=${line//cmdline:/}
echo $cmd > $CMDLINE
elif [[ $line == pubkey:* ]]; then
pubkey=${line//pubkey:/}
echo $pubkey > /var/cache/cloud/authorized_keys
echo $pubkey > /root/.ssh/authorized_keys
fi
done < $VPORT
# In case of reboot we do not send the boot args again.
# So, no need to wait for them, as the boot args are already set at startup
if [ -s $CMDLINE ]
then
log_it "Found a non empty cmdline file. Will now exit the loop and proceed with configuration."
break;
systemctl enable --now qemu-guest-agent
# Wait for $CMDLINE file to be written by the qemu-guest-agent
for i in {1..60}; do
if [ -s $CMDLINE ]; then
log_it "Received a new non-empty cmdline file from qemu-guest-agent"
break
fi
sleep ${progress}s
progress=$[ progress * factor ]
sleep 1
done
chmod go-rwx /root/.ssh/authorized_keys
# Use any old cmdline only when new cmdline is not received
if [ -e $CMDLINE.old ]; then
if [ -s $CMDLINE ]; then
rm $CMDLINE.old
else
mv $CMDLINE.old $CMDLINE
log_it "Using old cmdline file, VM was perhaps rebooted."
fi
fi
;;
vmware)
vmtoolsd --cmd 'machine.id.get' > $CMDLINE

View File

@ -1 +0,0 @@
ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2RIE3hgSAD8zULuyE7KDW9EKh2oVbNGY7iSL/VI5xHLISKh4e8ksTshWjlGBtrUCnuzR7y2BUxZ65RI8XkB1fEDxcOU4/0lVPvJYDSsGveXoOgpLwOtKRoGLgjFUGzBQlj2s6YaYQxoNTqtBVkDIH6ekPNq0Q38hRrFcsVIk1sFo5ejuvFxt2wx6APcFIQtHSNezEDO0GVUScDU1N1YEMMv1PU3M/SrcezkXrGl/efF3kWtY9L5xm7sojHMCCqsI38r8ogof67F7JdWRXM6Nl3VzkdCBzWGcyAl+cYfjzgOiBGXyAyYBk8qqzJjKwUOtdjfRvCyowA/0xBwMW1T7PQ==

View File

@ -87,12 +87,6 @@
<include>agent.zip</include>
</includes>
</resource>
<resource>
<directory>debian/root/.ssh</directory>
<includes>
<include>authorized_keys</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
@ -167,7 +161,6 @@
<argument>systemvm.iso</argument>
<argument>agent.zip</argument>
<argument>cloud-scripts.tgz</argument>
<argument>authorized_keys</argument>
</arguments>
</configuration>
</plugin>

View File

@ -19,7 +19,7 @@
set -e
set -x
CLOUDSTACK_RELEASE=4.11.2
CLOUDSTACK_RELEASE=4.11.3
function configure_apache2() {
# Enable ssl, rewrite and auth

View File

@ -38,8 +38,8 @@
"disk_interface": "virtio",
"net_device": "virtio-net",
"iso_url": "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-9.6.0-amd64-netinst.iso",
"iso_checksum": "fcd77acbd46f33e0a266faf284acc1179ab0a3719e4b8abebac555307aa978aa242d7052c8d41e1a5fc6d1b30bc6ca6d62269e71526b71c9d5199b13339f0e25",
"iso_url": "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-9.9.0-amd64-netinst.iso",
"iso_checksum": "42d9818abc4a08681dc0638f07e7aeb35d0c44646ab1e5b05a31a71d76c99da52b6192db9a3e852171ac78c2ba6b110b337c0b562c7be3d32e86a105023a6a0c",
"iso_checksum_type": "sha512",
"vm_name": "systemvmtemplate",