Summary: Get away from dozens of ssh/scp calls for KVM vm_data push

Detail: userdata and vm metadata take a long time to program on KVM routers.
This does it all in one go, processed on the router.

BUG-ID: CLOUDSTACK-3163
Tested-by: Wido
Signed-off-by: Marcus Sorensen <marcus@betterservers.com> 1374695897 -0600

core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.agent.resource.virtualnetwork;
 
+import com.google.gson.Gson;
+
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.BumpUpPriorityCommand;
 import com.cloud.agent.api.CheckRouterAnswer;
@@ -85,6 +87,7 @@ import java.net.InetSocketAddress;
 import java.net.URL;
 import java.net.URLConnection;
 import java.nio.channels.SocketChannel;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -463,61 +466,21 @@ public class VirtualRoutingResource implements Manager {
 
     protected Answer execute(VmDataCommand cmd) {
         List<String[]> vmData = cmd.getVmData();
+        String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
+        Map<String, List<String[]>> data = new HashMap<String, List<String[]>>();
+        data.put(cmd.getVmIpAddress(), cmd.getVmData());
  
-        for (String[] vmDataEntry : vmData) {
+        String json = new Gson().toJson(data);
-            String folder = vmDataEntry[0];
+        s_logger.debug("JSON IS:" + json);
-            String file = vmDataEntry[1];
-            String data = vmDataEntry[2];
-            File tmpFile = null;
 
-            byte[] dataBytes = null;
+        json = Base64.encodeBase64String(json.getBytes());
-            if (data != null) {
-                if (folder.equals("userdata")) {
-                    dataBytes = Base64.decodeBase64(data);//userdata is supplied in url-safe unchunked mode
-                } else {
-                    dataBytes = data.getBytes();
-                }
-            }
 
-            try {
+        String args = "-d " + json;
-                tmpFile = File.createTempFile("vmdata_", null);
-                FileOutputStream outStream = new FileOutputStream(tmpFile);
-                if (dataBytes != null)
-                    outStream.write(dataBytes); 
-                outStream.close();
-            } catch (IOException e) {
-                String tmpDir = System.getProperty("java.io.tmpdir");
-                s_logger.warn("Failed to create temporary file: is " + tmpDir + " full?", e);
-                return new Answer(cmd, false, "Failed to create or write to temporary file: is " + tmpDir + " full? " + e.getMessage() );
-            }
-       
-
-            final Script command  = new Script(_vmDataPath, _timeout, s_logger);
-            command.add("-r", cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP));
-            command.add("-v", cmd.getVmIpAddress());
-            command.add("-F", folder);
-            command.add("-f", file);
-
-            if (tmpFile != null) {
-                command.add("-d", tmpFile.getAbsolutePath());
-            }
-
-            final String result = command.execute();
-
-            if (tmpFile != null) {
-                boolean deleted = tmpFile.delete();
-                if (!deleted) {
-                    s_logger.warn("Failed to clean up temp file after sending vmdata");
-                    tmpFile.deleteOnExit();
-                }
-            }
 
+        final String result = routerProxy("vmdata_kvm.py", routerIp, args);
         if (result != null) {
-                return new Answer(cmd, false, result);        	
+            return new Answer(cmd, false, "VmDataCommand failed, check agent logs");
         }
-
-        }
-
         return new Answer(cmd);
     }
 
@@ -1192,11 +1155,6 @@ public class VirtualRoutingResource implements Manager {
             throw new ConfigurationException("Unable to find dhcp_entry.sh");
         }
 
-        _vmDataPath = findScript("vm_data.sh");
-        if(_vmDataPath == null) {
-            throw new ConfigurationException("Unable to find user_data.sh");
-        }
-
         _publicEthIf = (String)params.get("public.network.device");
         if (_publicEthIf == null) {
             _publicEthIf = "xenbr1";

patches/systemvm/debian/config/opt/cloud/bin/vmdata_kvm.py

@@ -0,0 +1,140 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys, getopt, json, os, base64
+
+def main(argv):
+    fpath =  ''
+    b64data = ''
+
+    try:
+        opts, args = getopt.getopt(argv,"f:d:")
+    except getopt.GetoptError:
+        print 'params: -f <filename> -d <b64jsondata>'
+        sys.exit(2)
+    for opt, arg in opts:
+        if opt == '-f':
+            fpath = arg
+        elif opt == '-d':
+            b64data = arg
+
+    json_data = ''
+    if fpath != '':
+        fh = open(fpath, 'r')
+        json_data = json.loads(fh.read())
+    elif b64data != '':
+        json_data = json.loads(base64.b64decode(b64data))
+    else:
+        print '-f <filename> or -d <b64jsondata> required'
+        sys.exit(2)
+
+    for ip in json_data:
+        for item in json_data[ip]:
+            folder = item[0]
+            file   = item[1]
+            data   = item[2]
+
+            # process only valid data
+            if folder != "userdata" and folder != "metadata":
+                continue
+
+            if file == "":
+                continue
+
+            htaccess(ip, folder, file)
+
+            if data == "":
+                deletefile(ip, folder, file)
+            else:
+                createfile(ip, folder, file, data)
+    
+    if fpath != '':
+        fh.close()
+        os.remove(fpath)
+
+def deletefile(ip, folder, file):
+    datafile = "/var/www/html/" + folder + "/" + ip + "/" + file
+
+    if os.path.exists(datafile):
+        os.remove(datafile)
+
+def createfile(ip, folder, file, data):
+    dest = "/var/www/html/" + folder + "/" + ip + "/" + file
+    metamanifestdir = "/var/www/html/" + folder + "/" + ip 
+    metamanifest =  metamanifestdir + "/meta-data"
+
+    # base64 decode userdata
+    if folder == "userdata" or folder == "user-data":
+        if data is not None:
+            data = base64.b64decode(data)
+
+    if data is not None:
+        open(dest, "w").write(data)
+    else:
+        open(dest, "w").write("")
+    os.chmod(dest, 0644)
+
+    if folder == "metadata" or folder == "meta-data":
+        if not os.path.exists(metamanifestdir):
+            os.makedirs(metamanifestdir, 0755)
+        if os.path.exists(metamanifest):
+            if not file in open(metamanifest).read():
+                open(metamanifest, "a").write(file + '\n')
+        else:
+            open(metamanifest, "w").write(file + '\n')
+
+    if os.path.exists(metamanifest):
+        os.chmod(metamanifest, 0644)
+
+def htaccess(ip, folder, file):
+    entry = "RewriteRule ^" + file + "$  ../" + folder + "/%{REMOTE_ADDR}/" + file + " [L,NC,QSA]"
+    htaccessFolder = "/var/www/html/latest"
+    htaccessFile = htaccessFolder + "/.htaccess"
+
+    if not os.path.exists(htaccessFolder):
+        os.mkdir(htaccessFolder,0755)
+
+    if os.path.exists(htaccessFile):
+        if not entry in open(htaccessFile).read():
+            open(htaccessFile, "a").write(entry + '\n')
+
+    entry="Options -Indexes\nOrder Deny,Allow\nDeny from all\nAllow from " + ip
+    htaccessFolder = "/var/www/html/" + folder + "/" + ip
+    htaccessFile = htaccessFolder+"/.htaccess"
+
+    if not os.path.exists(htaccessFolder):
+        os.makedirs(htaccessFolder,0755)
+
+    open(htaccessFile, "w").write(entry + '\n')
+
+    if folder == "metadata" or folder == "meta-data":
+        entry="RewriteRule ^meta-data/(.+)$  ../" + folder + "/%{REMOTE_ADDR}/$1 [L,NC,QSA]"
+        htaccessFolder = "/var/www/html/latest"
+        htaccessFile = htaccessFolder + "/.htaccess"
+
+        if not entry in open(htaccessFile).read():
+            open(htaccessFile, "a").write(entry + '\n')
+
+        entry="RewriteRule ^meta-data/$  ../" + folder + "/%{REMOTE_ADDR}/meta-data [L,NC,QSA]"
+
+        if not entry in open(htaccessFile).read():
+            open(htaccessFile, "a").write(entry + '\n')
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

scripts/network/domr/vm_data.sh

@@ -1,147 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-# $Id: vm_data.sh 9307 2010-06-08 00:43:08Z chiradeep $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/vm/hypervisor/xenserver/patch/vm_data.sh $
-# @VERSION@
-
-usage() {
-  printf "Usage: %s: -r <domr-ip> -v <vm ip> -F <vm data folder> -f <vm data file> -d <data to put in file> \n" $(basename $0) >&2
-  exit 2
-}
-
-set -x
-cert="/root/.ssh/id_rsa.cloud"
-PORT=3922
-
-create_htaccess() {
-  local domrIp=$1
-  local vmIp=$2
-  local folder=$3
-  local file=$4
-  
-  local result=0
-  #rewrite rule in top level /latest folder to redirect 
-  #to vm specific folder based on source ip
-  entry="RewriteRule ^$file$  ../$folder/%{REMOTE_ADDR}/$file [L,NC,QSA]"
-  htaccessFolder="/var/www/html/latest"
-  htaccessFile=$htaccessFolder/.htaccess
-  ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "mkdir -p $htaccessFolder; touch $htaccessFile; grep -F \"$entry\" $htaccessFile; if [ \$? -gt 0 ]; then echo -e \"$entry\" >> $htaccessFile; fi" >/dev/null
-  result=$?
-  
-  if [ $result -eq 0 ]
-  then
-    #ensure that vm specific folder cannot be listed and that only 
-    #the vm that owns the data can access the items in this directory
-    entry="Options -Indexes\\nOrder Deny,Allow\\nDeny from all\\nAllow from $vmIp"
-    htaccessFolder="/var/www/html/$folder/$vmIp"
-    htaccessFile=$htaccessFolder/.htaccess
-    ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "mkdir -p $htaccessFolder; echo -e \"$entry\" > $htaccessFile" >/dev/null
-    result=$?
-  fi
-  
-  #support access by http://<dhcp server>/latest/<metadata key> (legacy, see above) also
-  # http://<dhcp server>/latest/meta-data/<metadata key> (correct)
-  if [ "$folder" == "metadata" ] || [ "$folder" == "meta-data" ]
-  then
-    entry="RewriteRule ^meta-data/(.+)$  ../$folder/%{REMOTE_ADDR}/\\\$1 [L,NC,QSA]"
-    htaccessFolder="/var/www/html/latest"
-    htaccessFile=$htaccessFolder/.htaccess
-    ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "grep -F \"$entry\" $htaccessFile; if [ \$? -gt 0 ]; then echo -e \"$entry\" >> $htaccessFile; fi" >/dev/null
-    entry="RewriteRule ^meta-data/$  ../$folder/%{REMOTE_ADDR}/meta-data [L,NC,QSA]"
-    ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "grep -F \"$entry\" $htaccessFile; if [ \$? -gt 0 ]; then echo -e \"$entry\" >> $htaccessFile; fi" >/dev/null
-    result=$?
-  fi
-  
-  return $result  
-}
-
-copy_vm_data_file() {
-  local domrIp=$1
-  local vmIp=$2
-  local folder=$3
-  local file=$4
-  local dataFile=$5        
-  
-  dest=/var/www/html/$folder/$vmIp/$file
-  metamanifest=/var/www/html/$folder/$vmIp/meta-data
-  scp -P $PORT -o StrictHostKeyChecking=no -i $cert $dataFile root@$domrIp:$dest >/dev/null
-  ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "chmod 644 $dest" > /dev/null
-  ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "touch $metamanifest; chmod 644 $metamanifest" > /dev/null
-  if [ "$folder" == "metadata" ] || [ "$folder" == "meta-data" ]
-  then
-    ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "sed -i '/$file/d' $metamanifest; echo $file >> $metamanifest" > /dev/null
-  fi
-  
-  return $?
-}
-
-delete_vm_data_file() {
-  local domrIp=$1
-  local vmIp=$2
-  local folder=$3
-  local file=$4
-  
-  vmDataFilePath="/var/www/html/$folder/$vmIp/$file"
-  ssh -p $PORT -o StrictHostKeyChecking=no -i $cert root@$domrIp "if [ -f $vmDataFilePath ]; then rm -rf $vmDataFilePath; fi" >/dev/null
-  return $?
-}
-
-domrIp=
-vmIp=
-folder=
-file=
-dataFile=
-
-while getopts 'r:v:F:f:d:' OPTION
-do
-  case $OPTION in
-  r)	domrIp="$OPTARG"
-		;;
-  v)	vmIp="$OPTARG"
-		;;
-  F)	folder="$OPTARG"
-  		;;
-  f)	file="$OPTARG"
-  		;;
-  d)	dataFile="$OPTARG"
-  		;;
-  ?)    usage
-		exit 1
-		;;
-  esac
-done
-
-[ "$domrIp" == "" ] || [ "$vmIp" == "" ]  || [ "$folder" == "" ] || [ "$file" == "" ] && usage 
-[ "$folder" != "userdata" ] && [ "$folder" != "metadata" ] && usage
-
-if [ "$dataFile" != "" ]
-then
-  create_htaccess $domrIp $vmIp $folder $file
-  
-  if [ $? -gt 0 ]
-  then
-    exit 1
-  fi
-  
-  copy_vm_data_file $domrIp $vmIp $folder $file $dataFile
-else
-  delete_vm_data_file $domrIp $vmIp $folder $file
-fi
-
-exit $?