apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-11-04 00:02:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

Summary: KVM - use virtio socket to communicate config to system vms

Browse Source 
Detail: This gets rid of the patchdisk method of passing cmdline and
authorized_keys to KVM system VMs. It instead passes them to a virtio socket,
which the KVM guest reads from the character device /dev/vport0p1 during
cloud-early-config. Tested to work on CentOS 6.3 and Ubuntu 12.04. Should
work with even older versions of libvirt.

Signed-off-by: Marcus Sorensen <marcus@betterservers.com> 1362691685 -0700
This commit is contained in:
Marcus Sorensen 2013-03-07 14:28:05 -07:00
parent c9ee05517d
commit 9ad54a082c
7 changed files with 87 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
debian/control vendored
View File

@ -22,7 +22,7 @@ Description: CloudStack server library
Package: cloudstack-agent
Architecture: all
Depends: openjdk-6-jre | openjdk-7-jre, cloudstack-common (= ${source:Version}), lsb-base (>= 3.2), libcommons-daemon-java, libjna-java, openssh-client, libvirt0, sysvinit-utils, chkconfig, qemu-kvm, libvirt-bin, uuid-runtime, rsync, grep, iproute, ebtables, vlan, wget, jsvc
Depends: openjdk-6-jre | openjdk-7-jre, cloudstack-common (= ${source:Version}), lsb-base (>= 3.2), libcommons-daemon-java, libjna-java, openssh-client, libvirt0, sysvinit-utils, chkconfig, qemu-kvm, libvirt-bin, uuid-runtime, rsync, grep, iproute, perl-base, perl-modules, ebtables, vlan, wget, jsvc
Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
Description: CloudStack agent
The CloudStack agent is in charge of managing shared computing resources in

1
packaging/centos63/cloud.spec
View File

@ -116,6 +116,7 @@ Requires: ebtables
Requires: jsvc
Requires: jakarta-commons-daemon
Requires: jakarta-commons-daemon-jsvc
Requires: perl
Provides: cloud-agent
Obsoletes: cloud-agent < 4.1.0
Obsoletes: cloud-test < 4.1.0

19
patches/systemvm/debian/config/etc/init.d/cloud-early-config
View File

@ -108,14 +108,17 @@ get_boot_params() {
sed -i "s/%/ /g" /var/cache/cloud/cmdline
;;
kvm)
# KVM needs to mount another disk, to get cmdline
mkdir -p $EXTRA_MOUNT
mount /dev/vdb $EXTRA_MOUNT
cp -f $EXTRA_MOUNT/cmdline /var/cache/cloud/cmdline
cp -f $EXTRA_MOUNT/authorized_keys /var/cache/cloud/authorized_keys
privkey=/var/cache/cloud/authorized_keys
umount $EXTRA_MOUNT
cp -f $privkey /root/.ssh/ && chmod go-rwx /root/.ssh/authorized_keys
while read line; do
if [[ $line == cmdline:* ]]; then
cmd=${line//cmdline:/}
echo $cmd > /var/cache/cloud/cmdline
elif [[ $line == pubkey:* ]]; then
pubkey=${line//pubkey:/}
echo $pubkey > /var/cache/cloud/authorized_keys
echo $pubkey > /root/.ssh/authorized_keys
fi
done < /dev/vport0p1
chmod go-rwx /root/.ssh/authorized_keys
;;
vmware)
vmtoolsd --cmd 'machine.id.get' > /var/cache/cloud/cmdline

103
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -255,7 +255,7 @@ ServerResource {
private String _modifyVlanPath;
private String _versionstringpath;
private String _patchdomrPath;
private String _patchViaSocketPath;
private String _createvmPath;
private String _manageSnapshotPath;
private String _resizeVolumePath;
@ -521,10 +521,10 @@ ServerResource {
throw new ConfigurationException("Unable to find versions.sh");
}
_patchdomrPath = Script.findScript(kvmScriptsDir + "/patch/",
"rundomrpre.sh");
if (_patchdomrPath == null) {
throw new ConfigurationException("Unable to find rundomrpre.sh");
_patchViaSocketPath = Script.findScript(kvmScriptsDir + "/patch/",
"patchviasocket.pl");
if (_patchViaSocketPath == null) {
throw new ConfigurationException("Unable to find patchviasocket.pl");
}
_heartBeatPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh");
@ -1014,13 +1014,11 @@ ServerResource {
return vnetId;
}
private void patchSystemVm(String cmdLine, String dataDiskPath,
String vmName) throws InternalErrorException {
private void passCmdLine(String vmName, String cmdLine)
throws InternalErrorException {
final Script command = new Script(_patchViaSocketPath, _timeout, s_logger);
String result;
final Script command = new Script(_patchdomrPath, _timeout, s_logger);
command.add("-l", vmName);
command.add("-t", "all");
command.add("-d", dataDiskPath);
command.add("-n",vmName);
command.add("-p", cmdLine.replaceAll(" ", "%"));
result = command.execute();
if (result != null) {
@ -1460,24 +1458,6 @@ ServerResource {
pool.deletePhysicalDisk(vol.getPath());
String vmName = cmd.getVmName();
String poolPath = pool.getLocalPath();
/* if vol is a root disk for a system vm, try to remove accompanying patch disk as well
this is a bit tricky since the patchdisk is only a LibvirtComputingResource construct
and not tracked anywhere in cloudstack */
if (vol.getType() == Volume.Type.ROOT && vmName.matches("^[rsv]-\\d+-.+$")) {
File patchVbd = new File(poolPath + File.separator + vmName + "-patchdisk");
if(patchVbd.exists()){
try {
_storagePoolMgr.deleteVbdByPath(vol.getPoolType(),patchVbd.getAbsolutePath());
} catch(CloudRuntimeException e) {
s_logger.warn("unable to destroy patch disk '" + patchVbd.getAbsolutePath() +
"' while removing root disk for " + vmName + " : " + e);
}
} else {
s_logger.debug("file '" +patchVbd.getAbsolutePath()+ "' not found");
}
}
return new Answer(cmd, true, "Success");
} catch (CloudRuntimeException e) {
s_logger.debug("Failed to delete volume: " + e.toString());
@ -3121,6 +3101,11 @@ ServerResource {
}
}
// pass cmdline info to system vms
if (vmSpec.getType() != VirtualMachine.Type.User) {
passCmdLine(vmName, vmSpec.getBootArgs() );
}
state = State.Running;
return new StartAnswer(cmd);
} catch (LibvirtException e) {
@ -3248,8 +3233,6 @@ ServerResource {
iso.defISODisk(_sysvmISOPath);
vm.getDevices().addDevice(iso);
}
createPatchVbd(conn, vmName, vm, vmSpec);
}
}
@ -3263,64 +3246,6 @@ ServerResource {
return null;
}
private void createPatchVbd(Connect conn, String vmName, LibvirtVMDef vm,
VirtualMachineTO vmSpec) throws LibvirtException,
InternalErrorException {
List<DiskDef> disks = vm.getDevices().getDisks();
DiskDef rootDisk = disks.get(0);
VolumeTO rootVol = getVolume(vmSpec, Volume.Type.ROOT);
String patchName = vmName + "-patchdisk";
KVMStoragePool pool = _storagePoolMgr.getStoragePool(
rootVol.getPoolType(),
rootVol.getPoolUuid());
String patchDiskPath = pool.getLocalPath() + "/" + patchName;
List<KVMPhysicalDisk> phyDisks = pool.listPhysicalDisks();
boolean foundDisk = false;
for (KVMPhysicalDisk phyDisk : phyDisks) {
if (phyDisk.getPath().equals(patchDiskPath)) {
foundDisk = true;
break;
}
}
if (!foundDisk) {
s_logger.debug("generating new patch disk for " + vmName + " since none was found");
KVMPhysicalDisk disk = pool.createPhysicalDisk(patchName, KVMPhysicalDisk.PhysicalDiskFormat.RAW,
10L * 1024 * 1024);
} else {
s_logger.debug("found existing patch disk at " + patchDiskPath + " using it for " + vmName);
}
/* Format/create fs on this disk */
final Script command = new Script(_createvmPath, _timeout, s_logger);
command.add("-f", patchDiskPath);
String result = command.execute();
if (result != null) {
s_logger.debug("Failed to create data disk: " + result);
throw new InternalErrorException("Failed to create data disk: "
+ result);
}
/* add patch disk */
DiskDef patchDisk = new DiskDef();
if (pool.getType() == StoragePoolType.CLVM) {
patchDisk.defBlockBasedDisk(patchDiskPath, 1, rootDisk.getBusType());
} else {
patchDisk.defFileBasedDisk(patchDiskPath, 1, rootDisk.getBusType(),
DiskDef.diskFmtType.RAW);
}
disks.add(patchDisk);
String bootArgs = vmSpec.getBootArgs();
patchSystemVm(bootArgs, patchDiskPath, vmName);
}
private void createVif(LibvirtVMDef vm, NicTO nic)
throws InternalErrorException, LibvirtException {
vm.getDevices().addDevice(

4
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java
View File

@ -864,8 +864,8 @@ public class LibvirtVMDef {
virtioSerialBuilder.append("<channel type='unix'>\n");
virtioSerialBuilder.append("<source mode='bind' path='" + _path
+ "/" + _name + ".agent'/>\n");
virtioSerialBuilder.append("<target type='virtio' name='org.qemu.guest_agent.0'/>\n");
virtioSerialBuilder.append("<address type='virtio-serial' controller='0' bus='0' port='1'/>\n");
virtioSerialBuilder.append("<target type='virtio' name='" + _name + ".vport'/>\n");
virtioSerialBuilder.append("<address type='virtio-serial'/>\n");
virtioSerialBuilder.append("</channel>\n");
return virtioSerialBuilder.toString();
}

58
scripts/vm/hypervisor/kvm/patchviasocket.pl Normal file
View File

@ -0,0 +1,58 @@
#!/usr/bin/perl -w
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#############################################################
# This script connects to the system vm socket and writes the
# authorized_keys and cmdline data to it. The system VM then
# reads it from /dev/vport0p1 in cloud_early_config
#############################################################
use strict;
use Getopt::Std;
use IO::Socket;
$|=1;
my $opts = {};
getopt('pn',$opts);
my $name = $opts->{n};
my $cmdline = $opts->{p};
my $sockfile = "/var/lib/libvirt/qemu/$name.agent";
my $pubkeyfile = "/root/.ssh/id_rsa.pub.cloud";
if (! -S $sockfile) {
print "ERROR: $sockfile socket not found\n";
exit 1;
}
if (! -f $pubkeyfile) {
print "ERROR: ssh public key not found on host at $pubkeyfile\n";
exit 1;
}
open(FILE,$pubkeyfile) or die "ERROR: unable to open $pubkeyfile - $^E";
my $key = <FILE>;
close FILE;
$cmdline =~ s/%/ /g;
my $msg = "pubkey:" . $key . "\ncmdline:" . $cmdline;
my $socket = IO::Socket::UNIX->new(Peer=>$sockfile,Type=>SOCK_STREAM)
or die "ERROR: unable to connect to $sockfile - $^E\n";
print $socket "$msg\r\n";
close $socket;

147
scripts/vm/hypervisor/kvm/rundomrpre.sh
View File

@ -1,147 +0,0 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# $Id: rundomrpre.sh 10427 2010-07-09 03:30:48Z edison $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/vm/hypervisor/kvm/rundomrpre.sh $
set -x
pubKey="/root/.ssh/id_rsa.pub.cloud"
mntpath() {
local vmname=$1
if [ ! -d /mnt/$vmname ]
then
mkdir -p /mnt/$vmname
fi
echo "/mnt/$vmname"
}
mount_raw_disk() {
local vmname=$1
local datadisk=$2
local path=$(mntpath $vmname)
if [ ! -f $datadisk -a ! -b $datadisk ]
then
printf "$datadisk doesn't exist" >&2
return 2
fi
retry=10
while [ $retry -gt 0 ]
do
if [ -b $datadisk ]; then
mount $datadisk $path &>/dev/null
ret=$?
else
mount $datadisk $path -o loop &>/dev/null
ret=$?
fi
sleep 10
if [ $ret -gt 0 ]
then
sleep 5
else
break
fi
retry=$(($retry-1))
done
return 0
}
umount_raw_disk() {
local vmname=$1
local datadisk=$2
local path=$(mntpath $vmname)
retry=10
sync
while [ $retry -gt 0 ]
do
umount -d $path &>/dev/null
if [ $? -gt 0 ]
then
sleep 5
else
rm -rf $path
break
fi
retry=$(($retry-1))
done
return $?
}
patch_all() {
local vmname=$1
local cmdline=$2
local datadisk=$3
local path=$(mntpath $vmname)
if [ -f $pubKey ]
then
cp $pubKey $path/authorized_keys
fi
echo $cmdline > $path/cmdline
sed -i "s/%/\ /g" $path/cmdline
return 0
}
lflag=
dflag=
while getopts 't:v:i:m:e:E:a:A:g:l:n:d:b:B:p:I:N:Mx:X:' OPTION
do
case $OPTION in
l) lflag=1
vmname="$OPTARG"
;;
t) tflag=1
vmtype="$OPTARG"
;;
d) dflag=1
rootdisk="$OPTARG"
;;
p) pflag=1
cmdline="$OPTARG"
;;
*) ;;
esac
done
if [ "$lflag$tflag$dflag" != "111" ]
then
printf "Error: No enough parameter\n" >&2
exit 1
fi
if [ "$vmtype" = "all" ]
then
mount_raw_disk $vmname $rootdisk
if [ $? -gt 0 ]
then
printf "Failed to mount $rootdisk"
exit $?
fi
patch_all $vmname $cmdline $rootdisk
umount_raw_disk $vmname $rootdisk
exit $?
fi
exit $?