From 631b6fd46a4371425991127cf19379feea16f869 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 1 Feb 2013 14:34:49 -0800 Subject: [PATCH] CLOUDSTACK-1037: Make cloudmonkey awesome-er Signed-off-by: Rohit Yadav --- tools/cli/cloudmonkey/__init__.py | 2 +- tools/cli/cloudmonkey/cachegen.py | 97 ------- tools/cli/cloudmonkey/cachemaker.py | 145 ++++++++++ tools/cli/cloudmonkey/cloudmonkey.py | 252 +++++------------- .../cli/cloudmonkey/{common.py => config.py} | 58 +++- .../cli/cloudmonkey/{lexer.py => printer.py} | 18 +- tools/cli/cloudmonkey/requester.py | 153 +++++++++++ tools/cli/pom.xml | 4 +- 8 files changed, 437 insertions(+), 292 deletions(-) delete mode 100644 tools/cli/cloudmonkey/cachegen.py create mode 100644 tools/cli/cloudmonkey/cachemaker.py rename tools/cli/cloudmonkey/{common.py => config.py} (54%) rename tools/cli/cloudmonkey/{lexer.py => printer.py} (91%) create mode 100644 tools/cli/cloudmonkey/requester.py diff --git a/tools/cli/cloudmonkey/__init__.py b/tools/cli/cloudmonkey/__init__.py index 967477794bf..e4c4e6d24f1 100644 --- a/tools/cli/cloudmonkey/__init__.py +++ b/tools/cli/cloudmonkey/__init__.py @@ -16,6 +16,6 @@ # under the License. try: - from common import __version__ + from config import __version__ except ImportError, e: print e diff --git a/tools/cli/cloudmonkey/cachegen.py b/tools/cli/cloudmonkey/cachegen.py deleted file mode 100644 index 509c0c68587..00000000000 --- a/tools/cli/cloudmonkey/cachegen.py +++ /dev/null @@ -1,97 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -try: - import re - from marvin.cloudstackAPI import * - from marvin import cloudstackAPI -except ImportError, e: - import sys - print "ImportError", e - sys.exit(1) - -completions = cloudstackAPI.__all__ - - -def get_api_module(api_name, api_class_strs=[]): - try: - api_mod = __import__("marvin.cloudstackAPI.%s" % api_name, - globals(), locals(), api_class_strs, -1) - except ImportError, e: - print "Error: API not found", e - return None - return api_mod - - -def main(): - """ - cachegen.py creates a precached dictionary for all the available verbs in - the predefined grammar of cloudmonkey, it dumps the dictionary in an - importable python module. This way we cheat on the runtime overhead of - completing commands and help docs. This reduces the overall search and - cache_miss (computation) complexity from O(n) to O(1) for any valid cmd. - """ - pattern = re.compile("[A-Z]") - verbs = list(set([x[:pattern.search(x).start()] for x in completions - if pattern.search(x) is not None]).difference(['cloudstack'])) - # datastructure {'verb': {cmd': ['api', [params], doc, required=[]]}} - cache_verbs = {} - for verb in verbs: - completions_found = filter(lambda x: x.startswith(verb), completions) - cache_verbs[verb] = {} - for api_name in completions_found: - api_cmd_str = "%sCmd" % api_name - api_mod = get_api_module(api_name, [api_cmd_str]) - if api_mod is None: - continue - try: - api_cmd = getattr(api_mod, api_cmd_str)() - required = api_cmd.required - doc = api_mod.__doc__ - except AttributeError, e: - print "Error: API attribute %s not found!" % e - params = filter(lambda x: '__' not in x and 'required' not in x, - dir(api_cmd)) - if len(required) > 0: - doc += "\nRequired args: %s" % " ".join(required) - doc += "\nArgs: %s" % " ".join(params) - api_name_lower = api_name.replace(verb, '').lower() - cache_verbs[verb][api_name_lower] = [api_name, params, doc, - required] - f = open("precache.py", "w") - f.write("""# Auto-generated code by cachegen.py -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License.""") - f.write("\nprecached_verbs = %s" % cache_verbs) - f.close() - -if __name__ == "__main__": - main() diff --git a/tools/cli/cloudmonkey/cachemaker.py b/tools/cli/cloudmonkey/cachemaker.py new file mode 100644 index 00000000000..264fa600af4 --- /dev/null +++ b/tools/cli/cloudmonkey/cachemaker.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +try: + import json + import os + import re + + from requester import monkeyrequest +except ImportError, e: + import sys + print "ImportError", e + sys.exit(1) + + +def getvalue(dictionary, key): + if key in dictionary: + return dictionary[key] + else: + return None + + +def csv_str_as_list(string): + if string is not None: + return filter(lambda x: x.strip() != '', string.split(',')) + else: + return [] + + +def cachegen_from_file(json_file): + f = open(json_file, 'r') + data = f.read() + f.close() + try: + apis = json.loads(data) + except ValueError, e: + print "Error processing json in cachegen()", e + return cachegen(apis) + + +def cachegen(apis): + pattern = re.compile("[A-Z]") + responsekey = filter(lambda x: 'response' in x, apis.keys()) + + if len(responsekey) == 0: + print "[cachegen] Invalid dictionary, has no response" + return None + if len(responsekey) != 1: + print "[cachegen] Multiple responsekeys, chosing first one" + + responsekey = responsekey[0] + verbs = set() + cache = {} + cache['count'] = getvalue(apis[responsekey], 'count') + + for api in getvalue(apis[responsekey], 'api'): + name = getvalue(api, 'name') + response = getvalue(api, 'response') + + idx = pattern.search(name).start() + verb = name[:idx] + subject = name[idx:] + + apidict = {} + apidict['name'] = name + apidict['description'] = getvalue(api, 'description') + apidict['isasync'] = getvalue(api, 'isasync') + apidict['related'] = csv_str_as_list(getvalue(api, 'related')) + + required = [] + apiparams = [] + for param in getvalue(api, 'params'): + apiparam = {} + apiparam['name'] = getvalue(param, 'name') + apiparam['description'] = getvalue(param, 'description') + apiparam['required'] = (getvalue(param, 'required') is True) + apiparam['length'] = int(getvalue(param, 'length')) + apiparam['type'] = getvalue(param, 'type') + apiparam['related'] = csv_str_as_list(getvalue(param, 'related')) + if apiparam['required']: + required.append(apiparam['name']) + apiparams.append(apiparam) + + apidict['requiredparams'] = required + apidict['params'] = apiparams + apidict['response'] = getvalue(api, 'response') + cache[verb] = {subject: apidict} + verbs.add(verb) + + cache['verbs'] = list(verbs) + return cache + + +def main(json_file): + """ + cachegen.py creates a precache datastore of all available apis of + CloudStack and dumps the precache dictionary in an + importable python module. This way we cheat on the runtime overhead of + completing commands and help docs. This reduces the overall search and + cache_miss (computation) complexity from O(n) to O(1) for any valid cmd. + """ + f = open("precache.py", "w") + f.write("""# -*- coding: utf-8 -*- +# Auto-generated code by cachegen.py +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License.""") + f.write("\nprecache = %s" % cachegen_from_file(json_file)) + f.close() + +if __name__ == "__main__": + json_file = 'listapis.json' + if os.path.exists(json_file): + main(json_file) + else: + pass + #print "[ERROR] cli:cachegen is unable to locate %s" % json_file diff --git a/tools/cli/cloudmonkey/cloudmonkey.py b/tools/cli/cloudmonkey/cloudmonkey.py index 646ad409993..49f3ee958e1 100644 --- a/tools/cli/cloudmonkey/cloudmonkey.py +++ b/tools/cli/cloudmonkey/cloudmonkey.py @@ -20,7 +20,6 @@ try: import atexit import cmd - import codecs import json import logging import os @@ -28,18 +27,16 @@ try: import re import shlex import sys - import time - import types - from ConfigParser import ConfigParser, SafeConfigParser from urllib2 import HTTPError, URLError from httplib import BadStatusLine - from prettytable import PrettyTable - from common import __version__, config_dir, config_file, config_fields - from common import precached_verbs - from lexer import monkeyprint + from config import __version__, config_file + from config import precached_verbs, read_config, write_config + from printer import monkeyprint + from requester import monkeyrequest + from prettytable import PrettyTable from marvin.cloudstackConnection import cloudConnection from marvin.cloudstackException import cloudstackAPIException from marvin.cloudstackAPI import * @@ -70,8 +67,7 @@ class CloudMonkeyShell(cmd.Cmd, object): intro = ("☁ Apache CloudStack 🐵 cloudmonkey " + __version__ + ". Type help or ? to list commands.\n") ruler = "=" - config_dir = config_dir - config_file = config_file + apicache = {} # datastructure {'verb': {cmd': ['api', [params], doc, required=[]]}} cache_verbs = precached_verbs config_options = [] @@ -79,31 +75,8 @@ class CloudMonkeyShell(cmd.Cmd, object): def __init__(self, pname, verbs): self.program_name = pname self.verbs = verbs - global config_fields - first_time = False - if not os.path.exists(self.config_dir): - os.makedirs(self.config_dir) - if os.path.exists(self.config_file): - config = self.read_config() - else: - first_time = True - config = self.write_config(first_time) - - for section in config_fields.keys(): - for key in config_fields[section].keys(): - try: - self.config_options.append(key) - setattr(self, key, config.get(section, key)) - except Exception: - print "Please fix `%s` in %s" % (key, self.config_file) - sys.exit() - - if first_time: - print "Welcome! Using `set` configure the necessary settings:" - print " ".join(sorted(self.config_options)) - print "Config file:", self.config_file - print "For debugging, tail -f", self.log_file, "\n" + self.config_options = read_config(self.get_attr, self.set_attr) self.prompt = self.prompt.strip() + " " # Cosmetic fix for prompt logging.basicConfig(filename=self.log_file, @@ -111,40 +84,20 @@ class CloudMonkeyShell(cmd.Cmd, object): logger.debug("Loaded config fields:\n%s" % map(lambda x: "%s=%s" % (x, getattr(self, x)), self.config_options)) - cmd.Cmd.__init__(self) - if not os.path.exists(self.config_file): - config = self.write_config() try: if os.path.exists(self.history_file): readline.read_history_file(self.history_file) atexit.register(readline.write_history_file, self.history_file) except IOError: - print("Error: history support") + monkeyprint("Error: history support") - def read_config(self): - config = ConfigParser() - try: - with open(self.config_file, 'r') as cfg: - config.readfp(cfg) - except IOError, e: - self.print_shell("Error: config_file not found", e) - return config + def get_attr(self, field): + return getattr(self, field) - def write_config(self, first_time=False): - global config_fields - config = ConfigParser() - for section in config_fields.keys(): - config.add_section(section) - for key in config_fields[section].keys(): - if first_time: - config.set(section, key, config_fields[section][key]) - else: - config.set(section, key, getattr(self, key)) - with open(self.config_file, 'w') as cfg: - config.write(cfg) - return config + def set_attr(self, field, value): + return setattr(self, field, value) def emptyline(self): pass @@ -158,20 +111,8 @@ class CloudMonkeyShell(cmd.Cmd, object): except KeyboardInterrupt: print("^C") - def print_shell(self, *args): - output = "" - try: - for arg in args: - arg = str(arg) - if isinstance(type(args), types.NoneType): - continue - output += arg - if self.color == 'true': - monkeyprint(output) - else: - print output - except Exception, e: - self.print_shell("Error: " + e) + def monkeyprint(self, *args): + monkeyprint((self.color == 'true'), *args) def print_result(self, result, result_filter=None): if result is None or len(result) == 0: @@ -179,7 +120,7 @@ class CloudMonkeyShell(cmd.Cmd, object): def printer_helper(printer, toprow): if printer: - self.print_shell(printer) + self.monkeyprint(printer) return PrettyTable(toprow) def print_result_tabular(result, result_filter=None): @@ -200,16 +141,16 @@ class CloudMonkeyShell(cmd.Cmd, object): if printer and row: printer.add_row(row) if printer: - self.print_shell(printer) + self.monkeyprint(printer) def print_result_as_dict(result, result_filter=None): for key in sorted(result.keys(), key=lambda x: x not in ['id', 'count', 'name'] and x): if not (isinstance(result[key], list) or isinstance(result[key], dict)): - self.print_shell("%s = %s" % (key, result[key])) + self.monkeyprint("%s = %s" % (key, result[key])) else: - self.print_shell(key + ":") + self.monkeyprint(key + ":") self.print_result(result[key], result_filter) def print_result_as_list(result, result_filter=None): @@ -220,7 +161,7 @@ class CloudMonkeyShell(cmd.Cmd, object): break self.print_result(node) if len(result) > 1: - self.print_shell(self.ruler * 80) + self.monkeyprint(self.ruler * 80) if isinstance(result, dict): print_result_as_dict(result, result_filter) @@ -229,92 +170,18 @@ class CloudMonkeyShell(cmd.Cmd, object): elif isinstance(result, str): print result elif not (str(result) is None): - self.print_shell(result) - - def make_request(self, command, requests={}, isAsync=False): - conn = cloudConnection(self.host, port=int(self.port), - apiKey=self.apikey, securityKey=self.secretkey, - asyncTimeout=self.timeout, logging=logger, - protocol=self.protocol, path=self.path) - response = None - logger.debug("====START Request====") - logger.debug("Requesting command=%s, args=%s" % (command, requests)) - try: - response = conn.make_request_with_auth(command, requests) - except cloudstackAPIException, e: - self.print_shell("API Error:", e) - except HTTPError, e: - self.print_shell(e) - except (URLError, BadStatusLine), e: - self.print_shell("Connection Error:", e) - logger.debug("====END Request====\n") - - def process_json(response): - try: - response = json.loads(str(response)) - except ValueError, e: - pass - return response - - response = process_json(response) - if response is None: - return - - isAsync = isAsync and (self.asyncblock == "true") - responsekey = filter(lambda x: 'response' in x, response.keys())[0] - if isAsync and 'jobid' in response[responsekey]: - jobId = response[responsekey]['jobid'] - command = "queryAsyncJobResult" - requests = {'jobid': jobId} - timeout = int(self.timeout) - pollperiod = 3 - progress = 1 - while timeout > 0: - print '\r' + '.' * progress, - sys.stdout.flush() - response = process_json(conn.make_request_with_auth(command, - requests)) - responsekeys = filter(lambda x: 'response' in x, - response.keys()) - if len(responsekeys) < 1: - continue - result = response[responsekeys[0]] - jobstatus = result['jobstatus'] - if jobstatus == 2: - jobresult = result["jobresult"] - self.print_shell("\rAsync query failed for jobid", - jobId, "\nError", jobresult["errorcode"], - jobresult["errortext"]) - return - elif jobstatus == 1: - print '\r', - return response - time.sleep(pollperiod) - timeout = timeout - pollperiod - progress += 1 - logger.debug("job: %s to timeout in %ds" % (jobId, timeout)) - self.print_shell("Error:", "Async query timeout for jobid", jobId) + self.monkeyprint(result) + def make_request(self, command, args={}, isasync=False): + response, error = monkeyrequest(command, args, isasync, + self.asyncblock, logger, + self.host, self.port, + self.apikey, self.secretkey, + self.timeout, self.protocol, self.path) + if error is not None: + self.monkeyprint(error) return response - def get_api_module(self, api_name, api_class_strs=[]): - try: - api_mod = __import__("marvin.cloudstackAPI.%s" % api_name, - globals(), locals(), api_class_strs, -1) - except ImportError, e: - self.print_shell("Error: API not found", e) - return None - return api_mod - - def pipe_runner(self, args): - if args.find(' |') > -1: - pname = self.program_name - if '.py' in pname: - pname = "python " + pname - self.do_shell("%s %s" % (pname, args)) - return True - return False - def default(self, args): if self.pipe_runner(args): return @@ -340,31 +207,20 @@ class CloudMonkeyShell(cmd.Cmd, object): map(lambda x: x.strip(), args_dict.pop('filter').split(','))) - api_cmd_str = "%sCmd" % api_name - api_mod = self.get_api_module(api_name, [api_cmd_str]) - if api_mod is None: - return - - try: - api_cmd = getattr(api_mod, api_cmd_str) - except AttributeError, e: - self.print_shell("Error: API attribute %s not found!" % e) - return - for attribute in args_dict.keys(): setattr(api_cmd, attribute, args_dict[attribute]) - command = api_cmd() - missing_args = filter(lambda x: x not in args_dict.keys(), - command.required) + #command = api_cmd() + #missing_args = filter(lambda x: x not in args_dict.keys(), + # command.required) - if len(missing_args) > 0: - self.print_shell("Missing arguments: ", ' '.join(missing_args)) - return + #if len(missing_args) > 0: + # self.monkeyprint("Missing arguments: ", ' '.join(missing_args)) + # return isAsync = False - if "isAsync" in dir(command): - isAsync = (command.isAsync == "true") + #if "isAsync" in dir(command): + # isAsync = (command.isAsync == "true") result = self.make_request(api_name, args_dict, isAsync) if result is None: @@ -375,7 +231,7 @@ class CloudMonkeyShell(cmd.Cmd, object): self.print_result(result[responsekey], field_filter) print except Exception as e: - self.print_shell("🙈 Error on parsing and printing", e) + self.monkeyprint("🙈 Error on parsing and printing", e) def completedefault(self, text, line, begidx, endidx): partitions = line.partition(" ") @@ -403,6 +259,17 @@ class CloudMonkeyShell(cmd.Cmd, object): autocompletions.append("filter=") return [s for s in autocompletions if s.startswith(search_string)] + def do_sync(self, args): + """ + Asks cloudmonkey to discovery and sync apis available on user specified + CloudStack host server which has the API discovery plugin, on failure + it rollbacks last datastore or api precached datastore. + """ + response = self.make_request("listApis") + f = open('test.json', "w") + f.write(json.dumps(response)) + f.close() + def do_api(self, args): """ Make raw api calls. Syntax: api =. @@ -413,7 +280,7 @@ class CloudMonkeyShell(cmd.Cmd, object): if len(args) > 0: return self.default(args) else: - self.print_shell("Please use a valid syntax") + self.monkeyprint("Please use a valid syntax") def complete_api(self, text, line, begidx, endidx): mline = line.partition(" ")[2] @@ -435,7 +302,7 @@ class CloudMonkeyShell(cmd.Cmd, object): key, value = (args[0], args[2]) setattr(self, key, value) # keys and attributes should have same names self.prompt = self.prompt.strip() + " " # prompt fix - self.write_config() + write_config(self.get_attr) def complete_set(self, text, line, begidx, endidx): mline = line.partition(" ")[2] @@ -443,6 +310,15 @@ class CloudMonkeyShell(cmd.Cmd, object): return [s[offs:] for s in self.config_options if s.startswith(mline)] + def pipe_runner(self, args): + if args.find(' |') > -1: + pname = self.program_name + if '.py' in pname: + pname = "python " + pname + self.do_shell("%s %s" % (pname, args)) + return True + return False + def do_shell(self, args): """ Execute shell commands using shell or ! @@ -474,9 +350,9 @@ class CloudMonkeyShell(cmd.Cmd, object): subject = fields[2].partition(" ")[0] if subject in self.cache_verbs[verb]: - self.print_shell(self.cache_verbs[verb][subject][2]) + self.monkeyprint(self.cache_verbs[verb][subject][2]) else: - self.print_shell("Error: no such api (%s) on %s" % + self.monkeyprint("Error: no such api (%s) on %s" % (subject, verb)) def complete_help(self, text, line, begidx, endidx): @@ -500,7 +376,7 @@ class CloudMonkeyShell(cmd.Cmd, object): """ Quit CloudMonkey CLI """ - self.print_shell("Bye!") + self.monkeyprint("Bye!") return self.do_EOF(args) def do_EOF(self, args): @@ -526,10 +402,10 @@ def main(): helpdoc = res[2] args = args_partition[2] except KeyError, e: - self.print_shell("Error: invalid %s api arg" % verb, e) + self.monkeyprint("Error: invalid %s api arg" % verb, e) return if ' --help' in args or ' -h' in args: - self.print_shell(helpdoc) + self.monkeyprint(helpdoc) return self.default("%s %s" % (cmd, args)) return grammar_closure diff --git a/tools/cli/cloudmonkey/common.py b/tools/cli/cloudmonkey/config.py similarity index 54% rename from tools/cli/cloudmonkey/common.py rename to tools/cli/cloudmonkey/config.py index 05767a57cb8..b6de6418ca8 100644 --- a/tools/cli/cloudmonkey/common.py +++ b/tools/cli/cloudmonkey/config.py @@ -21,8 +21,11 @@ __version__ = "4.1.0-0" try: - from os.path import expanduser import os + import sys + + from ConfigParser import ConfigParser, SafeConfigParser + from os.path import expanduser from precache import precached_verbs except ImportError, e: precached_verbs = {} @@ -30,6 +33,8 @@ except ImportError, e: param_type = ['boolean', 'date', 'float', 'integer', 'short', 'list', 'long', 'object', 'map', 'string', 'tzdate', 'uuid'] +iterable_type = ['set', 'list', 'object'] + config_dir = expanduser('~/.cloudmonkey') config_file = expanduser(config_dir + '/config') @@ -57,3 +62,54 @@ config_fields['server']['timeout'] = '3600' # user config_fields['user']['apikey'] = '' config_fields['user']['secretkey'] = '' + + +def write_config(get_attr, first_time=False): + global config_fields, config_file + config = ConfigParser() + for section in config_fields.keys(): + config.add_section(section) + for key in config_fields[section].keys(): + if first_time: + config.set(section, key, config_fields[section][key]) + else: + config.set(section, key, get_attr(key)) + with open(config_file, 'w') as cfg: + config.write(cfg) + return config + + +def read_config(get_attr, set_attr): + global config_fields, config_dir, config_file + if not os.path.exists(config_dir): + os.makedirs(config_dir) + + config_options = reduce(lambda x, y: x + y, map(lambda x: + config_fields[x].keys(), config_fields.keys())) + + if os.path.exists(config_file): + config = ConfigParser() + try: + with open(config_file, 'r') as cfg: + config.readfp(cfg) + except IOError, e: + print "Error: config_file not found", e + else: + config = write_config(get_attr, True) + print "Welcome! Using `set` configure the necessary settings:" + print " ".join(sorted(config_options)) + print "Config file:", config_file + + missing_keys = [] + for section in config_fields.keys(): + for key in config_fields[section].keys(): + try: + set_attr(key, config.get(section, key)) + except Exception: + missing_keys.appned(key) + + if len(missing_keys) > 0: + print "Please fix `%s` in %s" % (key, config_file) + sys.exit() + + return config_options diff --git a/tools/cli/cloudmonkey/lexer.py b/tools/cli/cloudmonkey/printer.py similarity index 91% rename from tools/cli/cloudmonkey/lexer.py rename to tools/cli/cloudmonkey/printer.py index 373c9f27469..40f84732fd0 100644 --- a/tools/cli/cloudmonkey/lexer.py +++ b/tools/cli/cloudmonkey/printer.py @@ -25,6 +25,7 @@ try: from pygments.token import * import sys + import types except ImportError, e: print e @@ -92,7 +93,6 @@ class MonkeyFormatter(Formatter): self.colorscheme = get_colorscheme() def format(self, tokensource, outfile): - self.encoding = outfile.encoding return Formatter.format(self, tokensource, outfile) def format_unencoded(self, tokensource, outfile): @@ -113,9 +113,21 @@ class MonkeyFormatter(Formatter): outfile.write(value) -def monkeyprint(text): +def monkeyprint(color=True, *args): fmter = MonkeyFormatter() lexer = MonkeyLexer() lexer.encoding = 'utf-8' fmter.encoding = 'utf-8' - highlight(text, lexer, fmter, sys.stdout) + output = "" + try: + for arg in args: + if isinstance(type(arg), types.NoneType): + continue + output += str(arg) + except Exception, e: + print e + + if color: + highlight(output, lexer, fmter, sys.stdout) + else: + print output diff --git a/tools/cli/cloudmonkey/requester.py b/tools/cli/cloudmonkey/requester.py new file mode 100644 index 00000000000..5c4cd1e36fa --- /dev/null +++ b/tools/cli/cloudmonkey/requester.py @@ -0,0 +1,153 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +try: + import base64 + import hashlib + import hmac + import httplib + import json + import os + import pdb + import re + import shlex + import sys + import time + import types + import urllib + import urllib2 + +except ImportError, e: + print "Import error in %s : %s" % (__name__, e) + import sys + sys.exit() + + +def logger_debug(logger, message): + if logger is not None: + logger.debug(message) + + +def make_request(command, args, logger, host, port, + apikey, secretkey, protocol, path): + response = None + error = None + + if protocol != 'http' and protocol != 'https': + error = "Protocol must be 'http' or 'https'" + return None, error + + if args is None: + args = {} + + args["command"] = command + args["apiKey"] = apikey + args["response"] = "json" + request = zip(args.keys(), args.values()) + request.sort(key=lambda x: str.lower(x[0])) + + request_url = "&".join(["=".join([r[0], urllib.quote_plus(str(r[1]))]) + for r in request]) + hashStr = "&".join(["=".join([str.lower(r[0]), + str.lower(urllib.quote_plus(str(r[1]))).replace("+", + "%20")]) for r in request]) + + sig = urllib.quote_plus(base64.encodestring(hmac.new(secretkey, hashStr, + hashlib.sha1).digest()).strip()) + request_url += "&signature=%s" % sig + request_url = "%s://%s:%s%s?%s" % (protocol, host, port, path, request_url) + + try: + logger_debug(logger, "Request sent: %s" % request_url) + connection = urllib2.urlopen(request_url) + response = connection.read() + except Exception, e: + error = str(e) + + logger_debug(logger, "Response received: %s" % response) + if error is not None: + logger_debug(logger, error) + + return response, error + + +def monkeyrequest(command, args, isasync, asyncblock, logger, host, port, + apikey, secretkey, timeout, protocol, path): + + response = None + error = None + logger_debug(logger, "======== START Request ========") + logger_debug(logger, "Requesting command=%s, args=%s" % (command, args)) + response, error = make_request(command, args, logger, host, port, + apikey, secretkey, protocol, path) + logger_debug(logger, "======== END Request ========\n") + + if error is not None: + return response, error + + def process_json(response): + try: + response = json.loads(str(response)) + except ValueError, e: + error = "Error processing json response, %s" % e + logger_debug(logger, "Error processing json", e) + return response + + response = process_json(response) + if response is None: + return response, error + + isasync = isasync and (asyncblock == "true") + responsekey = filter(lambda x: 'response' in x, response.keys())[0] + + if isasync and 'jobid' in response[responsekey]: + jobid = response[responsekey]['jobid'] + command = "queryAsyncJobResult" + request = {'jobid': jobid} + timeout = int(timeout) + pollperiod = 3 + progress = 1 + while timeout > 0: + print '\r' + '.' * progress, + time.sleep(pollperiod) + timeout = timeout - pollperiod + progress += 1 + logger_debug(logger, "Job %s to timeout in %ds" % (jobid, timeout)) + sys.stdout.flush() + response, error = monkeyrequest(command, request, isasync, + asyncblock, logger, + host, port, apikey, secretkey, + timeout, protocol, path) + response = process_json(response) + responsekeys = filter(lambda x: 'response' in x, response.keys()) + if len(responsekeys) < 1: + continue + result = response[responsekeys[0]] + jobstatus = result['jobstatus'] + if jobstatus == 2: + jobresult = result["jobresult"] + error = "\rAsync job %s failed\nError %s, %s" % (jobid, + jobresult["errorcode"], jobresult["errortext"]) + return response, error + elif jobstatus == 1: + print '\r', + return response, error + error = "Error: Async query timeout occurred for jobid %s" % jobid + + return response, error diff --git a/tools/cli/pom.xml b/tools/cli/pom.xml index aba5ec3c2a5..582cc5701f2 100644 --- a/tools/cli/pom.xml +++ b/tools/cli/pom.xml @@ -72,7 +72,7 @@ - cachegen + cachemaker compile exec @@ -81,7 +81,7 @@ ${basedir}/cloudmonkey python - cachegen.py + cachemaker.py