mirror of
https://github.com/apache/cloudstack.git
synced 2025-10-26 08:42:29 +01:00
Summary: Get away from dozens of ssh/scp calls for KVM vm_data push
Detail: userdata and vm metadata take a long time to program on KVM routers. This does it all in one go, processed on the router. BUG-ID: CLOUDSTACK-3163 Tested-by: Wido Signed-off-by: Marcus Sorensen <marcus@betterservers.com> 1374695897 -0600
This commit is contained in:
Marcus Sorensen
parent
11dce48855
commit
28855b4987
@ -16,6 +16,8 @@
|
||||
// under the License.
|
||||
package com.cloud.agent.resource.virtualnetwork;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
|
||||
import com.cloud.agent.api.Answer;
|
||||
import com.cloud.agent.api.BumpUpPriorityCommand;
|
||||
import com.cloud.agent.api.CheckRouterAnswer;
|
||||
@ -85,6 +87,7 @@ import java.net.InetSocketAddress;
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
import java.nio.channels.SocketChannel;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@ -463,61 +466,21 @@ public class VirtualRoutingResource implements Manager {
|
||||
|
||||
protected Answer execute(VmDataCommand cmd) {
|
||||
List<String[]> vmData = cmd.getVmData();
|
||||
String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
|
||||
Map<String, List<String[]>> data = new HashMap<String, List<String[]>>();
|
||||
data.put(cmd.getVmIpAddress(), cmd.getVmData());
|
||||
|
||||
String json = new Gson().toJson(data);
|
||||
s_logger.debug("JSON IS:" + json);
|
||||
|
||||
for (String[] vmDataEntry : vmData) {
|
||||
String folder = vmDataEntry[0];
|
||||
String file = vmDataEntry[1];
|
||||
String data = vmDataEntry[2];
|
||||
File tmpFile = null;
|
||||
json = Base64.encodeBase64String(json.getBytes());
|
||||
|
||||
byte[] dataBytes = null;
|
||||
if (data != null) {
|
||||
if (folder.equals("userdata")) {
|
||||
dataBytes = Base64.decodeBase64(data);//userdata is supplied in url-safe unchunked mode
|
||||
} else {
|
||||
dataBytes = data.getBytes();
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
tmpFile = File.createTempFile("vmdata_", null);
|
||||
FileOutputStream outStream = new FileOutputStream(tmpFile);
|
||||
if (dataBytes != null)
|
||||
outStream.write(dataBytes);
|
||||
outStream.close();
|
||||
} catch (IOException e) {
|
||||
String tmpDir = System.getProperty("java.io.tmpdir");
|
||||
s_logger.warn("Failed to create temporary file: is " + tmpDir + " full?", e);
|
||||
return new Answer(cmd, false, "Failed to create or write to temporary file: is " + tmpDir + " full? " + e.getMessage() );
|
||||
}
|
||||
|
||||
|
||||
final Script command = new Script(_vmDataPath, _timeout, s_logger);
|
||||
command.add("-r", cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP));
|
||||
command.add("-v", cmd.getVmIpAddress());
|
||||
command.add("-F", folder);
|
||||
command.add("-f", file);
|
||||
|
||||
if (tmpFile != null) {
|
||||
command.add("-d", tmpFile.getAbsolutePath());
|
||||
}
|
||||
|
||||
final String result = command.execute();
|
||||
|
||||
if (tmpFile != null) {
|
||||
boolean deleted = tmpFile.delete();
|
||||
if (!deleted) {
|
||||
s_logger.warn("Failed to clean up temp file after sending vmdata");
|
||||
tmpFile.deleteOnExit();
|
||||
}
|
||||
}
|
||||
|
||||
if (result != null) {
|
||||
return new Answer(cmd, false, result);
|
||||
}
|
||||
String args = "-d " + json;
|
||||
|
||||
final String result = routerProxy("vmdata_kvm.py", routerIp, args);
|
||||
if (result != null) {
|
||||
return new Answer(cmd, false, "VmDataCommand failed, check agent logs");
|
||||
}
|
||||
|
||||
return new Answer(cmd);
|
||||
}
|
||||
|
||||
@ -1192,11 +1155,6 @@ public class VirtualRoutingResource implements Manager {
|
||||
throw new ConfigurationException("Unable to find dhcp_entry.sh");
|
||||
}
|
||||
|
||||
_vmDataPath = findScript("vm_data.sh");
|
||||
if(_vmDataPath == null) {
|
||||
throw new ConfigurationException("Unable to find user_data.sh");
|
||||
}
|
||||
|
||||
_publicEthIf = (String)params.get("public.network.device");
|
||||
if (_publicEthIf == null) {
|
||||
_publicEthIf = "xenbr1";
|
||||
|
||||
140
patches/systemvm/debian/config/opt/cloud/bin/vmdata_kvm.py
Normal file
140
patches/systemvm/debian/config/opt/cloud/bin/vmdata_kvm.py
Normal file
@ -0,0 +1,140 @@
|
||||
#!/usr/bin/python
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sys, getopt, json, os, base64
|
||||
|
||||
def main(argv):
|
||||
fpath = ''
|
||||
b64data = ''
|
||||
|
||||
try:
|
||||
opts, args = getopt.getopt(argv,"f:d:")
|
||||
except getopt.GetoptError:
|
||||
print 'params: -f <filename> -d <b64jsondata>'
|
||||
sys.exit(2)
|
||||
for opt, arg in opts:
|
||||
if opt == '-f':
|
||||
fpath = arg
|
||||
elif opt == '-d':
|
||||
b64data = arg
|
||||
|
||||
json_data = ''
|
||||
if fpath != '':
|
||||
fh = open(fpath, 'r')
|
||||
json_data = json.loads(fh.read())
|
||||
elif b64data != '':
|
||||
json_data = json.loads(base64.b64decode(b64data))
|
||||
else:
|
||||
print '-f <filename> or -d <b64jsondata> required'
|
||||
sys.exit(2)
|
||||
|
||||
for ip in json_data:
|
||||
for item in json_data[ip]:
|
||||
folder = item[0]
|
||||
file = item[1]
|
||||
data = item[2]
|
||||
|
||||
# process only valid data
|
||||
if folder != "userdata" and folder != "metadata":
|
||||
continue
|
||||
|
||||
if file == "":
|
||||
continue
|
||||
|
||||
htaccess(ip, folder, file)
|
||||
|
||||
if data == "":
|
||||
deletefile(ip, folder, file)
|
||||
else:
|
||||
createfile(ip, folder, file, data)
|
||||
|
||||
if fpath != '':
|
||||
fh.close()
|
||||
os.remove(fpath)
|
||||
|
||||
def deletefile(ip, folder, file):
|
||||
datafile = "/var/www/html/" + folder + "/" + ip + "/" + file
|
||||
|
||||
if os.path.exists(datafile):
|
||||
os.remove(datafile)
|
||||
|
||||
def createfile(ip, folder, file, data):
|
||||
dest = "/var/www/html/" + folder + "/" + ip + "/" + file
|
||||
metamanifestdir = "/var/www/html/" + folder + "/" + ip
|
||||
metamanifest = metamanifestdir + "/meta-data"
|
||||
|
||||
# base64 decode userdata
|
||||
if folder == "userdata" or folder == "user-data":
|
||||
if data is not None:
|
||||
data = base64.b64decode(data)
|
||||
|
||||
if data is not None:
|
||||
open(dest, "w").write(data)
|
||||
else:
|
||||
open(dest, "w").write("")
|
||||
os.chmod(dest, 0644)
|
||||
|
||||
if folder == "metadata" or folder == "meta-data":
|
||||
if not os.path.exists(metamanifestdir):
|
||||
os.makedirs(metamanifestdir, 0755)
|
||||
if os.path.exists(metamanifest):
|
||||
if not file in open(metamanifest).read():
|
||||
open(metamanifest, "a").write(file + '\n')
|
||||
else:
|
||||
open(metamanifest, "w").write(file + '\n')
|
||||
|
||||
if os.path.exists(metamanifest):
|
||||
os.chmod(metamanifest, 0644)
|
||||
|
||||
def htaccess(ip, folder, file):
|
||||
entry = "RewriteRule ^" + file + "$ ../" + folder + "/%{REMOTE_ADDR}/" + file + " [L,NC,QSA]"
|
||||
htaccessFolder = "/var/www/html/latest"
|
||||
htaccessFile = htaccessFolder + "/.htaccess"
|
||||
|
||||
if not os.path.exists(htaccessFolder):
|
||||
os.mkdir(htaccessFolder,0755)
|
||||
|
||||
if os.path.exists(htaccessFile):
|
||||
if not entry in open(htaccessFile).read():
|
||||
open(htaccessFile, "a").write(entry + '\n')
|
||||
|
||||
entry="Options -Indexes\nOrder Deny,Allow\nDeny from all\nAllow from " + ip
|
||||
htaccessFolder = "/var/www/html/" + folder + "/" + ip
|
||||
htaccessFile = htaccessFolder+"/.htaccess"
|
||||
|
||||
if not os.path.exists(htaccessFolder):
|
||||
os.makedirs(htaccessFolder,0755)
|
||||
|
||||
open(htaccessFile, "w").write(entry + '\n')
|
||||
|
||||
if folder == "metadata" or folder == "meta-data":
|
||||
entry="RewriteRule ^meta-data/(.+)$ ../" + folder + "/%{REMOTE_ADDR}/$1 [L,NC,QSA]"
|
||||
htaccessFolder = "/var/www/html/latest"
|
||||
htaccessFile = htaccessFolder + "/.htaccess"
|
||||
|
||||
if not entry in open(htaccessFile).read():
|
||||
open(htaccessFile, "a").write(entry + '\n')
|
||||
|
||||
entry="RewriteRule ^meta-data/$ ../" + folder + "/%{REMOTE_ADDR}/meta-data [L,NC,QSA]"
|
||||
|
||||
if not entry in open(htaccessFile).read():
|
||||
open(htaccessFile, "a").write(entry + '\n')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@ -1,147 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
# $Id: vm_data.sh 9307 2010-06-08 00:43:08Z chiradeep $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/vm/hypervisor/xenserver/patch/vm_data.sh $
|
||||
# @VERSION@
|
||||
|
||||
usage() {
|
||||
printf "Usage: %s: -r <domr-ip> -v <vm ip> -F <vm data folder> -f <vm data file> -d <data to put in file> \n" $(basename $0) >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
set -x
|
||||
cert="/root/.ssh/id_rsa.cloud"
|
||||
PORT=3922
|
||||
|
||||
create_htaccess() {
|
||||
local domrIp=$1
|
||||
local vmIp=$2
|
||||
local folder=$3
|
||||
local file=$4
|
||||
|
||||
local result=0
|
||||
#rewrite rule in top level /latest folder to redirect
|
||||
#to vm specific folder based on source ip
|
||||
entry="RewriteRule ^$file$ ../$folder/%{REMOTE_ADDR}/$file [L,NC,QSA]"
|
||||
htaccessFolder="/var/www/html/latest"
|
||||
htaccessFile=$htaccessFolder/.htaccess
|
||||
ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "mkdir -p $htaccessFolder; touch $htaccessFile; grep -F \"$entry\" $htaccessFile; if [ \$? -gt 0 ]; then echo -e \"$entry\" >> $htaccessFile; fi" >/dev/null
|
||||
result=$?
|
||||
|
||||
if [ $result -eq 0 ]
|
||||
then
|
||||
#ensure that vm specific folder cannot be listed and that only
|
||||
#the vm that owns the data can access the items in this directory
|
||||
entry="Options -Indexes\\nOrder Deny,Allow\\nDeny from all\\nAllow from $vmIp"
|
||||
htaccessFolder="/var/www/html/$folder/$vmIp"
|
||||
htaccessFile=$htaccessFolder/.htaccess
|
||||
ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "mkdir -p $htaccessFolder; echo -e \"$entry\" > $htaccessFile" >/dev/null
|
||||
result=$?
|
||||
fi
|
||||
|
||||
#support access by http://<dhcp server>/latest/<metadata key> (legacy, see above) also
|
||||
# http://<dhcp server>/latest/meta-data/<metadata key> (correct)
|
||||
if [ "$folder" == "metadata" ] || [ "$folder" == "meta-data" ]
|
||||
then
|
||||
entry="RewriteRule ^meta-data/(.+)$ ../$folder/%{REMOTE_ADDR}/\\\$1 [L,NC,QSA]"
|
||||
htaccessFolder="/var/www/html/latest"
|
||||
htaccessFile=$htaccessFolder/.htaccess
|
||||
ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "grep -F \"$entry\" $htaccessFile; if [ \$? -gt 0 ]; then echo -e \"$entry\" >> $htaccessFile; fi" >/dev/null
|
||||
entry="RewriteRule ^meta-data/$ ../$folder/%{REMOTE_ADDR}/meta-data [L,NC,QSA]"
|
||||
ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "grep -F \"$entry\" $htaccessFile; if [ \$? -gt 0 ]; then echo -e \"$entry\" >> $htaccessFile; fi" >/dev/null
|
||||
result=$?
|
||||
fi
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
copy_vm_data_file() {
|
||||
local domrIp=$1
|
||||
local vmIp=$2
|
||||
local folder=$3
|
||||
local file=$4
|
||||
local dataFile=$5
|
||||
|
||||
dest=/var/www/html/$folder/$vmIp/$file
|
||||
metamanifest=/var/www/html/$folder/$vmIp/meta-data
|
||||
scp -P $PORT -o StrictHostKeyChecking=no -i $cert $dataFile root@$domrIp:$dest >/dev/null
|
||||
ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "chmod 644 $dest" > /dev/null
|
||||
ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "touch $metamanifest; chmod 644 $metamanifest" > /dev/null
|
||||
if [ "$folder" == "metadata" ] || [ "$folder" == "meta-data" ]
|
||||
then
|
||||
ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "sed -i '/$file/d' $metamanifest; echo $file >> $metamanifest" > /dev/null
|
||||
fi
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
delete_vm_data_file() {
|
||||
local domrIp=$1
|
||||
local vmIp=$2
|
||||
local folder=$3
|
||||
local file=$4
|
||||
|
||||
vmDataFilePath="/var/www/html/$folder/$vmIp/$file"
|
||||
ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "if [ -f $vmDataFilePath ]; then rm -rf $vmDataFilePath; fi" >/dev/null
|
||||
return $?
|
||||
}
|
||||
|
||||
domrIp=
|
||||
vmIp=
|
||||
folder=
|
||||
file=
|
||||
dataFile=
|
||||
|
||||
while getopts 'r:v:F:f:d:' OPTION
|
||||
do
|
||||
case $OPTION in
|
||||
r) domrIp="$OPTARG"
|
||||
;;
|
||||
v) vmIp="$OPTARG"
|
||||
;;
|
||||
F) folder="$OPTARG"
|
||||
;;
|
||||
f) file="$OPTARG"
|
||||
;;
|
||||
d) dataFile="$OPTARG"
|
||||
;;
|
||||
?) usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ "$domrIp" == "" ] || [ "$vmIp" == "" ] || [ "$folder" == "" ] || [ "$file" == "" ] && usage
|
||||
[ "$folder" != "userdata" ] && [ "$folder" != "metadata" ] && usage
|
||||
|
||||
if [ "$dataFile" != "" ]
|
||||
then
|
||||
create_htaccess $domrIp $vmIp $folder $file
|
||||
|
||||
if [ $? -gt 0 ]
|
||||
then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
copy_vm_data_file $domrIp $vmIp $folder $file $dataFile
|
||||
else
|
||||
delete_vm_data_file $domrIp $vmIp $folder $file
|
||||
fi
|
||||
|
||||
exit $?
|
||||
Loading…
x
Reference in New Issue
Block a user