CLOUDSTACK-4832. Added support for https to marvin.

advanced.cfg: Contains three additional flags "useHttps,certCAPath,certPath"
for https usage in marvin for establishing cs connection. We will use the
configuraiton under advanced.cfg provided by user to establish connection over
https. If establishing the connection over https failed, then the default certs
will be used.  or else raise the exception, the existing http will work as it
is when useHttps flag set to "False"

Signed-off-by: Santhosh Edukulla <Santhosh.Edukulla@citrix.com>
Signed-off-by: Prasanna Santhanam <tsp@apache.org>

setup/dev/advanced.cfg

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
 {
     "zones": [
         {
@@ -220,7 +219,10 @@
             "passwd": "password",
             "user": "root",
             "port": 8096,
-            "hypervisor" : "simulator"
+            "hypervisor": "simulator",
+       "useHttps": "True",
+       "certCAPath":  "NA",
+       "certPath":  "NA"
         }
     ]
 }

tools/marvin/marvin/cloudstackConnection.py

@@ -35,29 +35,27 @@ class cloudConnection(object):
 
     """ Connections to make API calls to the cloudstack management server
     """
-
+    def __init__(self, mgmtDet,asyncTimeout=3600, logging=None, scheme='http',path='client/api'):
-    def __init__(self, mgtSvr, port=8096, user=None, passwd=None,
-                 apiKey=None, securityKey=None,
-                 asyncTimeout=3600, logging=None, scheme='http',
-                 path='client/api'):
         self.loglevel()  # Turn off requests logs
-        self.apiKey = apiKey
+        self.apiKey = mgmtDet.apiKey
-        self.securityKey = securityKey
+        self.securityKey = mgmtDet.securityKey
-        self.mgtSvr = mgtSvr
+        self.mgtSvr = mgmtDet.mgtSvrIp
-        self.port = port
+        self.port = mgmtDet.port
-        self.user = user
+        self.user = mgmtDet.user
-        self.passwd = passwd
+        self.passwd = mgmtDet.passwd
+   self.certCAPath = mgmtDet.certCAPath
+   self.certPath = mgmtDet.certPath
         self.logging = logging
         self.path = path
         self.retries = 5
+   self.protocol = "http"
         self.asyncTimeout = asyncTimeout
         self.auth = True
-        if port == 8096 or \
+        if self.port == 8096 or \
            (self.apiKey is None and self.securityKey is None):
             self.auth = False
-        if scheme not in ['http', 'https']:
+        if mgmtDet.useHttps == "True":
-                raise RequestException("Protocol must be HTTP")
+           self.protocol = "https"
-        self.protocol = scheme
         self.baseurl = "%s://%s:%d/%s"\
                        % (self.protocol, self.mgtSvr, self.port, self.path)
 
@@ -145,15 +143,52 @@ class cloudConnection(object):
             payload["signature"] = signature
 
         try:
-            if method == 'POST':
+       '''
-                response = requests.post(
+       https_flag : Signifies whether to verify connection over http or https, if set to true uses https otherwise http
-                    self.baseurl, params=payload, verify=False)
+       cert_path  : Signifies ca and cert path required by requests library for the connection
+       '''
+       https_flag  = False
+       cert_path    = ()
+       if self.protocol == "https":
+       https_flag = True
+           if self.certCAPath != "NA" and self.certPath  != "NA":
+           cert_path = ( self.certCAPath,self.certPath )
+
+       '''
+       Verify whether protocol is "http", then call the request over http
+            '''
+       if self.protocol == "http":
+               if method == 'POST':
+                   response = requests.post(self.baseurl, params=payload, verify=https_flag)
+               else:
+                   response = requests.get(self.baseurl, params=payload,  verify=https_flag)
             else:
-                response = requests.get(
+       exception_check = False
-                    self.baseurl, params=payload, verify=False)
+       exception_info  = None
+           '''
+       If protocol is https, then request the url with user provided certificates provided as part of cert
+       '''
+           try:
+                   if method == 'POST':
+                       response = requests.post(self.baseurl, params=payload, cert=cert_path, verify=https_flag)
+                   else:
+                       response = requests.get(self.baseurl, params=payload,  cert=cert_path, verify=https_flag)
+           except Exception,e:
+           '''
+           If an exception occurs with current CA certs, then try with default certs path, we dont need to mention here the cert path
+           '''
+           self.logging.debug( "Creating CS connection over https didnt worked with user provided certs %s"%e )
+           exception_check  = True
+           exception_info = e
+           if method == 'POST':
+                               response = requests.post(self.baseurl, params=payload, verify=https_flag)
+                   else:
+                               response = requests.get(self.baseurl, params=payload,  verify=https_flag)
+       finally:
+           if exception_check == True and exception_info is not None:
+               raise exception_info
         except ConnectionError, c:
-            self.logging.debug("Connection refused. Reason: %s : %s" %
+            self.logging.debug("Connection refused. Reason: %s : %s" %(self.baseurl, c))
-                               (self.baseurl, c))
             raise c
         except HTTPError, h:
             self.logging.debug("Server returned error code: %s" % h)

tools/marvin/marvin/cloudstackTestClient.py

@@ -25,13 +25,9 @@ import hashlib
 
 
 class cloudstackTestClient(object):
-    def __init__(self, mgtSvr=None, port=8096, user=None, passwd=None,
+    def __init__(self,mgmtDetails,asyncTimeout=3600,defaultWorkerThreads=10, logging=None):
-                 apiKey=None, securityKey=None, asyncTimeout=3600,
-                 defaultWorkerThreads=10, logging=None):
         self.connection = \
-            cloudstackConnection.cloudConnection(mgtSvr, port, user, passwd,
+            cloudstackConnection.cloudConnection( mgmtDetails,asyncTimeout,logging)
-                                                 apiKey, securityKey,
-                                                 asyncTimeout, logging)
         self.apiClient =\
             cloudstackAPIClient.CloudStackAPIClient(self.connection)
         self.dbConnection = None

tools/marvin/marvin/configGenerator.py

@@ -27,6 +27,9 @@ class managementServer(object):
         self.port = 8096
         self.apiKey = None
         self.securityKey = None
+   self.useHttps = None
+   self.certCAPath = None
+   self.certPath = None
 
 
 class dbServer(object):
@@ -830,7 +833,7 @@ def generate_setup_config(config, file=None):
 def get_setup_config(file):
     if not os.path.exists(file):
         raise IOError("config file %s not found. \
-please specify a valid config file" % file)
+                      please specify a valid config file" % file)
     config = cloudstackConfiguration()
     configLines = []
     with open(file, 'r') as fp:
@@ -838,7 +841,9 @@ please specify a valid config file" % file)
             ws = line.strip()
             if not ws.startswith("#"):
                 configLines.append(ws)
-    config = json.loads("\n".join(configLines))
+    k = json.loads("\n".join(configLines))
+    #config = json.loads("\n".join(configLines))
+    config = k
     return jsonHelper.jsonLoader(config)
 
 if __name__ == "__main__":

tools/marvin/marvin/deployDataCenter.py

@@ -512,11 +512,9 @@ specify a valid config file" % cfgFile)
         try:
             self.config = configGenerator.get_setup_config(self.configFile)
         except:
-            raise cloudstackException.InvalidParameterException(
+            raise cloudstackException.InvalidParameterException("Failed to load config %s" % self.configFile)
-                "Failed to load config %s" % self.configFile)
-
-        mgt = self.config.mgtSvr[0]
 
+        mgtDetails = self.config.mgtSvr[0]
         loggers = self.config.logger
         testClientLogFile = None
         self.testCaseLogFile = None
@@ -535,38 +533,35 @@ specify a valid config file" % cfgFile)
             testClientLogger = logging.getLogger("testclient.testengine.run")
             fh = logging.FileHandler(testClientLogFile)
             fh.setFormatter(logging.
-                            Formatter("%(asctime)s - %(levelname)s - %(name)s\
+            Formatter("%(asctime)s - %(levelname)s - %(name)s\
- - %(message)s"))
+           - %(message)s"))
             testClientLogger.addHandler(fh)
             testClientLogger.setLevel(logging.INFO)
         self.testClientLogger = testClientLogger
 
         self.testClient = \
             cloudstackTestClient.\
-            cloudstackTestClient(mgt.mgtSvrIp, mgt.port, mgt.user, mgt.passwd,
+            cloudstackTestClient( mgtDetails,logging=self.testClientLogger)
-                                 mgt.apiKey, mgt.securityKey,
+
-                                 logging=self.testClientLogger)
+        if mgtDetails.apiKey is None:
-        if mgt.apiKey is None:
+           mgtDetails.apiKey,mgtDetails.securityKey = self.registerApiKey()
-            apiKey, securityKey = self.registerApiKey()
+           mgtDetails.port  = 8080
-            self.testClient = cloudstackTestClient.cloudstackTestClient(
+                self.testClient = cloudstackTestClient.cloudstackTestClient( mgtDetails,logging=self.testClientLogger)
-                mgt.mgtSvrIp, 8080,
-                mgt.user, mgt.passwd,
-                apiKey, securityKey,
-                logging=self.testClientLogger)
 
         """config database"""
         dbSvr = self.config.dbSvr
         if dbSvr is not None:
-            self.testClient.dbConfigure(dbSvr.dbSvr, dbSvr.port, dbSvr.user,
+            self.testClient.dbConfigure(dbSvr.dbSvr, dbSvr.port, dbSvr.user, \
                                         dbSvr.passwd, dbSvr.db)
 
         self.apiClient = self.testClient.getApiClient()
         """set hypervisor"""
-        if mgt.hypervisor:
+        if mgtDetails.hypervisor:
-            self.apiClient.hypervisor = mgt.hypervisor
+            self.apiClient.hypervisor = mgtDetails.hypervisor
         else:
             self.apiClient.hypervisor = "XenServer"  # Defaults to Xenserver
 
+
     def updateConfiguration(self, globalCfg):
         if globalCfg is None:
             return None
@@ -578,7 +573,6 @@ specify a valid config file" % cfgFile)
             self.apiClient.updateConfiguration(updateCfg)
 
     def copyAttributesToCommand(self, source, command):
-
         map(lambda attr: setattr(command, attr, getattr(source, attr, None)),
             filter(lambda attr: not attr.startswith("__") and attr not in
                    ["required", "isAsync"], dir(command)))
@@ -586,7 +580,6 @@ specify a valid config file" % cfgFile)
     def configureS3(self, s3):
         if s3 is None:
             return
-
         command = addS3.addS3Cmd()
         self.copyAttributesToCommand(s3, command)
         self.apiClient.addS3(command)