CLOUDSTACK-499: Fix cloudmonkey's request and response methods

The old way of parsing involved complex object loading and traversing, in
new implementation json is used directly and authenticated requests via port
8080 making use of api and secret keys are recommended.

All args on shell are created in {"key": "value"} pairs which are used to create
url with &key=value. This way maps can be passed as args, example:

create networkoffering name=unique-name guestiptype=Isolated traffictype=GUEST
supportedservices=Dhcp,Dns serviceproviderlist[0].service=Dhcp
serviceproviderlist[0].provider=VirtualRouter serviceproviderlist[1].service=Dns
serviceproviderlist[1].provider=VirtualRouter

Added new conf variables:
- timeout: the no. of seconds after which an async job query timeouts
  default value of 3600 seconds
- asycnblock: if user wants their async commands to be tracked
  default value is 'true', if set to false, user can query using jobid which is
  returned by the async command and queried using:
     query asyncjobresult jobid=<jobid>

BUG-ID : CLOUDSTACK-499
Reviewed-by: Rohit Yadav <bhaisaab@apache.org>
Reported-by: Dave Cahill

Signed-off-by: Rohit Yadav <bhaisaab@apache.org>

tools/cli/cloudmonkey/cloudmonkey.py

@@ -22,11 +22,14 @@ try:
     import cmd
     import clint
     import codecs
+    import json
     import logging
     import os
     import pdb
+    import sets
     import shlex
     import sys
+    import time
     import types
 
     from clint.textui import colored
@@ -63,16 +66,18 @@ completions = cloudstackAPI.__all__
 class CloudStackShell(cmd.Cmd):
     intro = ("☁ Apache CloudStack 🐵 cloudmonkey " + __version__ +
              ". Type help or ? to list commands.\n")
-    ruler = "-"
+    ruler = "="
     config_file = os.path.expanduser('~/.cloudmonkey_config')
     grammar = []
 
-    # datastructure {'list': {'users': ['listUsers', [params], docstring]}}
+    # datastructure {'list': {'users': ['listUsers', [params], docstring,
+    #                                   required=[]]}}
     cache_verbs = {}
 
     def __init__(self):
         self.config_fields = {'host': 'localhost', 'port': '8080',
                               'apikey': '', 'secretkey': '',
+                              'timeout': '3600', 'asyncblock': 'true',
                               'prompt': '🐵 cloudmonkey>', 'color': 'true',
                               'log_file':
                               os.path.expanduser('~/.cloudmonkey_log'),
@@ -88,12 +93,15 @@ class CloudStackShell(cmd.Cmd):
                   " log_file, history_file using the set command!")
 
         for key in self.config_fields.keys():
+            try:
                 setattr(self, key, config.get('CLI', key))
+            except Exception:
+                print "Please fix `%s` config in %s" % (key, self.config_file)
+                sys.exit()
 
         self.prompt += " "  # Cosmetic fix for prompt
         logging.basicConfig(filename=self.log_file,
                             level=logging.DEBUG, format=log_fmt)
-        self.logger = logging.getLogger(self.__class__.__name__)
 
         cmd.Cmd.__init__(self)
         # Update config if config_file does not exist
@@ -147,7 +155,7 @@ class CloudStackShell(cmd.Cmd):
                         print colored.green(arg),
                     elif 'type' in arg:
                         print colored.green(arg),
-                    elif 'state' in arg:
+                    elif 'state' in arg or 'count' in arg:
                         print colored.yellow(arg),
                     elif 'id =' in arg:
                         print colored.cyan(arg),
@@ -155,6 +163,8 @@ class CloudStackShell(cmd.Cmd):
                         print colored.magenta(arg),
                     elif 'Error' in arg:
                         print colored.red(arg),
+                    elif ":\n=" in arg:
+                        print colored.red(arg),
                     elif ':' in arg:
                         print colored.blue(arg),
                     else:
@@ -165,52 +175,41 @@ class CloudStackShell(cmd.Cmd):
         except Exception, e:
             print colored.red("Error: "), e
 
-    # FIXME: Fix result processing and printing
+    def print_result(self, result):
-    def print_result(self, result, response, api_mod):
+        if result is None or len(result) == 0:
-        def print_result_as_list():
-            if result is None:
             return
-            for node in result:
-                print_result_as_instance(node)
 
-        def print_result_as_instance(node):
+        def print_result_as_dict(result):
-            for attribute in dir(response):
+            for key in result.keys():
-                if "__" not in attribute:
+                if not (isinstance(result[key], list) or
-                    attribute_value = getattr(node, attribute)
+                        isinstance(result[key], dict)):
-                    if isinstance(attribute_value, list):
+                    self.print_shell("%s = %s" % (key, result[key]))
-                        self.print_shell("\n%s:" % attribute)
+                else:
-                        try:
+                    self.print_shell(key + ":\n" + len(key) * "=")
-                            self.print_result(attribute_value,
+                    self.print_result(result[key])
-                                              getattr(api_mod, attribute)(),
+
-                                              api_mod)
+        def print_result_as_list(result):
-                        except AttributeError, e:
+            for node in result:
-                            pass
+                self.print_result(node)
-                    elif attribute_value is not None:
+                if len(result) > 1:
-                        self.print_shell("%s = %s" %
-                                         (attribute, attribute_value))
                     self.print_shell(self.ruler * 80)
 
-        if result is None:
+        if isinstance(result, dict):
-            return
+            print_result_as_dict(result)
-
-        if type(result) is types.InstanceType:
-            print_result_as_instance(result)
         elif isinstance(result, list):
-            print_result_as_list()
+            print_result_as_list(result)
         elif isinstance(result, str):
             print result
-        elif isinstance(type(result), types.NoneType):
-            print_result_as_instance(result)
         elif not (str(result) is None):
             self.print_shell(result)
 
-    def make_request(self, command, requests={}):
+    def make_request(self, command, requests={}, isAsync=False):
         conn = cloudConnection(self.host, port=int(self.port),
                                apiKey=self.apikey, securityKey=self.secretkey,
-                               logging=logging.getLogger("cloudConnection"))
+                               asyncTimeout=self.timeout, logging=logger)
+        response = None
         try:
-            response = conn.make_request(command, requests)
+            response = conn.make_request_with_auth(command, requests)
-            return response
         except cloudstackAPIException, e:
             self.print_shell("API Error:", e)
         except HTTPError, e:
@@ -218,6 +217,44 @@ class CloudStackShell(cmd.Cmd):
         except URLError, e:
             self.print_shell("Connection Error:", e)
 
+        def process_json(response):
+            try:
+                response = json.loads(str(response))
+            except ValueError, e:
+                pass
+
+            if response is None:
+                return {'error': 'Error: json error'}
+            return response
+
+        response = process_json(response)
+
+        isAsync = isAsync and (self.asyncblock == "true")
+        if isAsync and 'jobid' in response[response.keys()[0]]:
+            jobId = response[response.keys()[0]]['jobid']
+            command = "queryAsyncJobResult"
+            requests = {'jobid': jobId}
+            timeout = int(self.timeout)
+            while timeout > 0:
+                response = process_json(conn.make_request_with_auth(command,
+                                                                    requests))
+                result = response[response.keys()[0]]
+                jobstatus = result['jobstatus']
+                if jobstatus == 2:
+                    jobresult = result["jobresult"]
+                    self.print_shell("Async query failed for jobid=",
+                                     jobId, "\nError", jobresult["errorcode"],
+                                     jobresult["errortext"])
+                    return
+                elif jobstatus == 1:
+                    return response
+                time.sleep(4)
+                timeout = timeout - 4
+                logger.debug("job: %s to timeout in %ds" % (jobId, timeout))
+            self.print_shell("Error:", "Async query timeout for jobid=", jobId)
+
+        return response
+
     def get_api_module(self, api_name, api_class_strs=[]):
         try:
             api_mod = __import__("marvin.cloudstackAPI.%s" % api_name,
@@ -234,6 +271,10 @@ class CloudStackShell(cmd.Cmd):
         args = list(lexp)
         api_name = args[0]
 
+        args_dict = dict(map(lambda x: [x.partition("=")[0],
+                                        x.partition("=")[2]],
+                             args[1:])[x] for x in range(len(args) - 1))
+
         try:
             api_cmd_str = "%sCmd" % api_name
             api_rsp_str = "%sResponse" % api_name
@@ -244,19 +285,30 @@ class CloudStackShell(cmd.Cmd):
             self.print_shell("Error: API %s not found!" % e)
             return
 
+        for attribute in args_dict.keys():
+#            if attribute in args_dict:
+            setattr(api_cmd, attribute, args_dict[attribute])
+
         command = api_cmd()
         response = api_rsp()
-        args_dict = dict(map(lambda x: [x.partition("=")[0],
-                                        x.partition("=")[2]],
-                             args[1:])[x] for x in range(len(args) - 1))
 
-        for attribute in dir(command):
+        missing_args = list(sets.Set(command.required).difference(
-            if attribute in args_dict:
+                            sets.Set(args_dict.keys())))
-                setattr(command, attribute, args_dict[attribute])
 
-        result = self.make_request(command, response)
+        if len(missing_args) > 0:
+            self.print_shell("Missing arguments:", ' '.join(missing_args))
+            return
+
+        isAsync = False
+        if "isAsync" in dir(command):
+            isAsync = (command.isAsync == "true")
+
+        result = self.make_request(api_name, args_dict, isAsync)
+        if result is None:
+            return
         try:
-            self.print_result(result, response, api_mod)
+            self.print_result(result.values())
+            print
         except Exception as e:
             self.print_shell("🙈  Error on parsing and printing", e)
 
@@ -268,16 +320,18 @@ class CloudStackShell(cmd.Cmd):
                 api_cmd_str = "%sCmd" % api_name
                 api_mod = self.get_api_module(api_name, [api_cmd_str])
                 api_cmd = getattr(api_mod, api_cmd_str)()
+                required = api_cmd.required
                 doc = api_mod.__doc__
             except AttributeError, e:
                 self.print_shell("Error: API attribute %s not found!" % e)
             params = filter(lambda x: '__' not in x and 'required' not in x,
                             dir(api_cmd))
-            if len(api_cmd.required) > 0:
+            if len(required) > 0:
-                doc += "\nRequired args: %s" % " ".join(api_cmd.required)
+                doc += "\nRequired args: %s" % " ".join(required)
             doc += "\nArgs: %s" % " ".join(params)
             api_name_lower = api_name.replace(verb, '').lower()
-            self.cache_verbs[verb][api_name_lower] = [api_name, params, doc]
+            self.cache_verbs[verb][api_name_lower] = [api_name, params, doc,
+                                                      required]
 
     def completedefault(self, text, line, begidx, endidx):
         partitions = line.partition(" ")
@@ -344,7 +398,8 @@ class CloudStackShell(cmd.Cmd):
         mline = line.partition(" ")[2]
         offs = len(mline) - len(text)
         return [s[offs:] for s in
-               ['host', 'port', 'apikey', 'secretkey', 'prompt', 'color',
+               ['host', 'port', 'apikey', 'secretkey',
+                'prompt', 'color', 'timeout', 'asyncblock',
                 'log_file', 'history_file'] if s.startswith(mline)]
 
     def do_shell(self, args):
@@ -418,14 +473,14 @@ def main():
                'start', 'restart', 'reboot', 'stop', 'reconnect',
                'cancel', 'destroy', 'revoke',
                'copy', 'extract', 'migrate', 'restore',
-               'get', 'prepare', 'deploy', 'upload']
+               'get', 'query', 'prepare', 'deploy', 'upload']
 
     # Create handlers on the fly using closures
     self = CloudStackShell
     for rule in grammar:
         def add_grammar(rule):
             def grammar_closure(self, args):
-                if '|' in args:  #FIXME: Consider parsing issues
+                if '|' in args:  # FIXME: Consider parsing issues
                     prog_name = sys.argv[0]
                     if '.py' in prog_name:
                         prog_name = "python " + prog_name
@@ -436,6 +491,7 @@ def main():
                 try:
                     args_partition = args.partition(" ")
                     res = self.cache_verbs[rule][args_partition[0]]
+
                 except KeyError, e:
                     self.print_shell("Error: no such command on %s" % rule)
                     return