Merge latest testClient fixes to marvin

reviewed-by: unittest

tools/marvin/marvin/TestCaseExecuteEngine.py

@@ -1,3 +1,15 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 try:
     import unittest2 as unittest
 except ImportError:
@@ -8,8 +20,6 @@ import os
 import sys
 import logging
 
-module_logger = "testclient.testcase"
-
 def testCaseLogger(message, logger=None):
     if logger is not None:
         logger.debug(message)
@@ -18,18 +28,19 @@ class TestCaseExecuteEngine(object):
     def __init__(self, testclient, testCaseFolder, testcaseLogFile=None, testResultLogFile=None):
         self.testclient = testclient
         self.testCaseFolder = testCaseFolder
-        self.logger = None
+        self.logformat = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s")
+
         if testcaseLogFile is not None:
-            logger = logging.getLogger("testclient.testcase.TestCaseExecuteEngine")
+            self.logfile = testcaseLogFile
-            fh = logging.FileHandler(testcaseLogFile)
+            self.logger = logging.getLogger("TestCaseExecuteEngine")
-            fh.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s"))
+            fh = logging.FileHandler(self.logfile) 
-            logger.addHandler(fh)
+            fh.setFormatter(self.logformat)
-            logger.setLevel(logging.DEBUG)
+            self.logger.addHandler(fh)
-            self.logger = logger
+            self.logger.setLevel(logging.DEBUG)
         if testResultLogFile is not None:
             ch = logging.StreamHandler()
             ch.setLevel(logging.ERROR)
-            ch.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s"))
+            ch.setFormatter(self.logformat)
             self.logger.addHandler(ch)
             fp = open(testResultLogFile, "w")
             self.testResultLogFile = fp
@@ -41,8 +52,20 @@ class TestCaseExecuteEngine(object):
             if isinstance(test, unittest.BaseTestSuite):
                 self.injectTestCase(test)
             else:
+                #logger bears the name of the test class
+                testcaselogger = logging.getLogger("testclient.testcase.%s"%test.__class__.__name__)
+                fh = logging.FileHandler(self.logfile) 
+                fh.setFormatter(self.logformat)
+                testcaselogger.addHandler(fh)
+                testcaselogger.setLevel(logging.DEBUG)
+                
+                #inject testclient and logger into each unittest 
                 setattr(test, "testClient", self.testclient)
-                setattr(test, "debug", partial(testCaseLogger, logger=self.logger)) 
+                setattr(test, "debug", partial(testCaseLogger, logger=testcaselogger))
+                setattr(test.__class__, "clstestclient", self.testclient)
+                if hasattr(test, "UserName"):
+                    self.testclient.createNewApiClient(test.UserName, test.DomainName, test.AcctType)
+
     def run(self):
         loader = unittest.loader.TestLoader()
         suite = loader.discover(self.testCaseFolder)

tools/marvin/marvin/asyncJobMgr.py

@@ -1,3 +1,15 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 import threading
 import cloudstackException
 import time
@@ -20,12 +32,14 @@ class jobStatus(object):
         self.duration = None
         self.jobId = None
         self.responsecls = None
+    def __str__(self):
+        return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v) in self.__dict__.iteritems()))
 class workThread(threading.Thread):
     def __init__(self, in_queue, outqueue, apiClient, db=None, lock=None):
         threading.Thread.__init__(self)
         self.inqueue = in_queue
         self.output = outqueue
-        self.connection = apiClient.connection
+        self.connection = apiClient.connection.__copy__()
         self.db = None
         self.lock = lock
         

tools/marvin/marvin/cloudstackConnection.py

@@ -1,7 +1,18 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 import urllib2
 import urllib
 import base64
-import copy
 import hmac
 import hashlib
 import json
@@ -52,14 +63,14 @@ class cloudConnection(object):
 
         try:
             self.connection = urllib2.urlopen("http://%s:%d/client/api?%s"%(self.mgtSvr, self.port, requestUrl))
-            self.logging.debug("sending request: %s"%requestUrl)
+            self.logging.debug("sending GET request: %s"%requestUrl)
             response = self.connection.read()
-            self.logging.debug("got response: %s"%response)
+            self.logging.info("got response: %s"%response)
         except IOError, e:
             if hasattr(e, 'reason'):
-                self.logging.debug("failed to reach %s because of %s"%(self.mgtSvr, e.reason))
+                self.logging.critical("failed to reach %s because of %s"%(self.mgtSvr, e.reason))
             elif hasattr(e, 'code'):
-                self.logging.debug("server returned %d error code"%e.code)
+                self.logging.critical("server returned %d error code"%e.code)
         except HTTPException, h:
             self.logging.debug("encountered http Exception %s"%h.args)
             if self.retries > 0:
@@ -78,16 +89,17 @@ class cloudConnection(object):
         requestUrl = "&".join(["=".join([request[0], urllib.quote_plus(str(request[1]))]) for request in requests])
 
         self.connection = urllib2.urlopen("http://%s:%d/client/api?%s"%(self.mgtSvr, self.port, requestUrl))
-        self.logging.debug("sending request without auth: %s"%requestUrl)
+        self.logging.debug("sending GET request without auth: %s"%requestUrl)
         response = self.connection.read()
-        self.logging.debug("got response: %s"%response)
+        self.logging.info("got response: %s"%response)
         return response
     
     def pollAsyncJob(self, jobId, response):
         cmd = queryAsyncJobResult.queryAsyncJobResultCmd()
         cmd.jobid = jobId
+        timeout = self.asyncTimeout
         
-        while self.asyncTimeout > 0:
+        while timeout > 0:
             asyncResonse = self.make_request(cmd, response, True)
             
             if asyncResonse.jobstatus == 2:
@@ -96,9 +108,10 @@ class cloudConnection(object):
                 return asyncResonse
             
             time.sleep(5)
-            self.asyncTimeout = self.asyncTimeout - 5
+            self.logging.debug("job: %s still processing, will timeout in %ds"%(jobId, timeout))
+            timeout = timeout - 5
             
-        raise cloudstackException.cloudstackAPIException("asyncquery", "Async job timeout")
+        raise cloudstackException.cloudstackAPIException("asyncquery", "Async job timeout %s"%jobId)
     
     def make_request(self, cmd, response = None, raw=False):
         commandName = cmd.__class__.__name__.replace("Cmd", "")
@@ -136,7 +149,7 @@ class cloudConnection(object):
                             i = i + 1
         
         if self.logging is not None:
-            self.logging.debug("sending command: %s %s"%(commandName, str(requests)))
+            self.logging.info("sending command: %s %s"%(commandName, str(requests)))
         result = None
         if self.auth:
             result = self.make_request_with_auth(commandName, requests)
@@ -145,8 +158,6 @@ class cloudConnection(object):
         
         if result is None:
             return None
-        if self.logging is not None:
-            self.logging.debug("got result: "  + result)
         
         result = jsonHelper.getResultObj(result, response)
         if raw or isAsync == "false":

tools/marvin/marvin/cloudstackException.py

@@ -1,3 +1,15 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 
 class cloudstackAPIException(Exception):
     def __init__(self, cmd = "", result = ""):

tools/marvin/marvin/cloudstackTestCase.py

@@ -1,3 +1,15 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 from cloudstackAPI import *
 try:
     import unittest2 as unittest
@@ -5,7 +17,37 @@ except ImportError:
     import unittest
 import cloudstackTestClient
 
+#class UserName(object):
+#    def __init__(self, account, domain, type=0):
+#        self.account = account
+#        self.domain = domain
+#        self.accounttype = type
+#    def __call__(self, cls):
+#        class Wrapped(cls):
+#            cls.UserName = self.account
+#            cls.DomainName = self.domain
+#            cls.AcctType = self.accounttype
+#        return Wrapped
+
+def UserName(Name, DomainName, AcctType):
+    def wrapper(cls):
+        orig_init = cls.__init__
+        def __init__(self, *args, **kws):
+            cls.UserName = Name
+            cls.DomainName = DomainName
+            cls.AcctType = AcctType
+            orig_init(self, *args, **kws)
+        cls.__init__ = __init__
+        return cls
+    return wrapper
+
 class cloudstackTestCase(unittest.case.TestCase):
+    clstestclient = None
+    
     def __init__(self, args):
         unittest.case.TestCase.__init__(self, args)
         self.testClient = cloudstackTestClient.cloudstackTestClient()
+        
+    @classmethod
+    def getClsTestClient(cls):
+        return cls.clstestclient

tools/marvin/marvin/cloudstackTestClient.py

@@ -1,7 +1,21 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 import cloudstackConnection
 import asyncJobMgr
 import dbConnection
 from cloudstackAPI import * 
+import random
+import string
 
 class cloudstackTestClient(object):
     def __init__(self, mgtSvr=None, port=8096, apiKey = None, securityKey = None, asyncTimeout=3600, defaultWorkerThreads=10, logging=None):
@@ -16,6 +30,82 @@ class cloudstackTestClient(object):
     def dbConfigure(self, host="localhost", port=3306, user='cloud', passwd='cloud', db='cloud'):
         self.dbConnection = dbConnection.dbConnection(host, port, user, passwd, db)
     
+    def isAdminContext(self):
+        """
+        A user is a regular user if he fails to listDomains;
+        if he is a domain-admin, he can list only domains that are non-ROOT;
+        if he is an admin, he can list the ROOT domain successfully
+        """
+        try:
+            listdom = listDomains.listDomainsCmd()
+            listdom.name = 'ROOT'
+            listdomres = self.apiClient.listDomains(listdom)
+            rootdom = listdomres[0].name
+            if rootdom == 'ROOT':
+                return 1 #admin
+            else:
+                return 2 #domain-admin
+        except:
+            return 0 #user
+    
+    def random_gen(self, size=6, chars=string.ascii_uppercase + string.digits):
+        """Generate Random Strings of variable length"""
+        return ''.join(random.choice(chars) for x in range(size))
+
+    def createNewApiClient(self, UserName, DomainName, acctType=0):
+        if not self.isAdminContext():
+            return self.apiClient
+        
+        listDomain = listDomains.listDomainsCmd()
+        listDomain.listall = True
+        listDomain.name = DomainName
+        try:
+            domains = self.apiClient.listDomains(listDomain)
+            domId = domains[0].id
+        except:
+            cdomain = createDomain.createDomainCmd()
+            cdomain.name = DomainName
+            domain = self.apiClient.createDomain(cdomain)
+            domId = domain.id
+        
+        cmd = listAccounts.listAccountsCmd()
+        cmd.name = UserName
+        cmd.domainid = domId
+        try:
+            accounts = self.apiClient.listAccounts(cmd)
+            acctId = accounts[0].id
+        except:
+            createAcctCmd = createAccount.createAccountCmd()
+            createAcctCmd.accounttype = acctType
+            createAcctCmd.domainid = domId
+            createAcctCmd.email = "test-" + self.random_gen() + "@citrix.com"
+            createAcctCmd.firstname = UserName
+            createAcctCmd.lastname = UserName
+            createAcctCmd.password = "password"
+            createAcctCmd.username = UserName
+            acct = self.apiClient.createAccount(createAcctCmd)
+            acctId = acct.id
+        
+        listuser = listUsers.listUsersCmd() 
+        listuser.username = UserName
+        
+        listuserRes = self.apiClient.listUsers(listuser)
+        userId = listuserRes[0].id
+        apiKey = listuserRes[0].apikey
+        securityKey = listuserRes[0].secretkey
+     
+        if apiKey is None:
+            registerUser = registerUserKeys.registerUserKeysCmd()
+            registerUser.id = userId
+            registerUserRes = self.apiClient.registerUserKeys(registerUser)
+            apiKey = registerUserRes.apikey
+            securityKey = registerUserRes.secretkey
+        
+        nConnection = cloudstackConnection.cloudConnection(self.connection.mgtSvr, self.connection.port, apiKey, securityKey, self.connection.asyncTimeout, self.connection.logging)
+        self.connection.close()
+        self.connection = nConnection
+        self.apiClient = cloudstackAPIClient.CloudStackAPIClient(self.connection)
+        
     def close(self):
         if self.connection is not None:
             self.connection.close()

tools/marvin/marvin/codegenerator.py

@@ -1,3 +1,15 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 import xml.dom.minidom
 from optparse import OptionParser
 import os

tools/marvin/marvin/configGenerator.py

@@ -1,3 +1,15 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 import json
 import os
 from optparse import OptionParser
@@ -45,16 +57,27 @@ class zone():
         '''Basic or Advanced'''
         self.networktype = None
         self.dns2 = None
-        self.guestcidraddress = None
         self.internaldns2 = None
         self.securitygroupenabled = None
+        ''' Guest Vlan range - only advanced zone'''
         self.vlan = None
         '''default public network, in advanced mode'''
         self.ipranges = []
-        '''tagged network, in advanced mode'''
         self.networks = []
         self.pods = []
         self.secondaryStorages = []
+        '''enable default virtual router provider'''
+        vrouter = provider()
+        vrouter.name = 'VirtualRouter'
+        self.providers = [vrouter]
+        
+class provider():
+    def __init__(self):
+        self.name = None
+        self.state = None
+        self.broadcastdomainrange = 'ZONE'
+        self.zoneid = None
+        self.servicelist = []
 
 class pod():
     def __init__(self):
@@ -102,10 +125,8 @@ class network():
         self.displaytext = None
         self.name = None
         self.zoneid = None
-        self.account = None
+        self.acltype = None
         self.domainid = None
-        self.isdefault = None
-        self.isshared = None
         self.networkdomain = None
         self.networkofferingid = None
         self.ipranges = []
@@ -145,34 +166,33 @@ def describe_setup_in_basic_mode():
         z.networktype = 'Basic'
     
         '''create 10 pods'''
-        for i in range(300):
+        for i in range(2):
             p = pod()
             p.name = "test" +str(l) + str(i)
-            p.gateway = "192.%d.%d.1"%((i/255)+168,i%255)
+            p.gateway = "192.168.%d.1"%i
             p.netmask = "255.255.255.0"
-            
+            p.startip = "192.168.%d.150"%i
-            p.startip = "192.%d.%d.150"%((i/255)+168,i%255)
+            p.endip = "192.168.%d.220"%i
-            p.endip = "192.%d.%d.220"%((i/255)+168,i%255)
         
             '''add two pod guest ip ranges'''
-            for j in range(1):
+            for j in range(2):
                 ip = iprange()
                 ip.gateway = p.gateway
                 ip.netmask = p.netmask
-                ip.startip = "192.%d.%d.%d"%(((i/255)+168), i%255,j*20)
+                ip.startip = "192.168.%d.%d"%(i,j*20)
-                ip.endip = "192.%d.%d.%d"%((i/255)+168,i%255,j*20+10)
+                ip.endip = "192.168.%d.%d"%(i,j*20+10)
             
                 p.guestIpRanges.append(ip)
         
             '''add 10 clusters'''
-            for j in range(10):
+            for j in range(2):
                 c = cluster()
                 c.clustername = "test"+str(l)+str(i) + str(j)
                 c.clustertype = "CloudManaged"
                 c.hypervisor = "Simulator"
             
                 '''add 10 hosts'''
-                for k in range(1):
+                for k in range(2):
                     h = host()
                     h.username = "root"
                     h.password = "password"
@@ -183,7 +203,6 @@ def describe_setup_in_basic_mode():
                     c.hosts.append(h)
                 
                 '''add 2 primary storages'''
-                '''
                 for m in range(2):
                     primary = primaryStorage()
                     size=1*1024*1024*1024*1024
@@ -191,7 +210,6 @@ def describe_setup_in_basic_mode():
                     #primary.url = "nfs://localhost/path%s/size=%d"%(str(l) + str(i) + str(j) + str(m), size)
                     primary.url = "nfs://localhost/path%s"%(str(l) + str(i) + str(j) + str(m))
                     c.primaryStorages.append(primary)
-                '''
         
                 p.clusters.append(c)
             

tools/marvin/marvin/dbConnection.py

@@ -1,3 +1,15 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 import pymysql
 import cloudstackException
 import sys
@@ -33,6 +45,8 @@ class dbConnection(object):
         resultRow = []
         cursor = None
         try:
+            # commit to restart the transaction, else we don't get fresh data
+            self.db.commit()
             cursor = self.db.cursor()
             cursor.execute(sql)
         
@@ -42,7 +56,7 @@ class dbConnection(object):
                     resultRow.append(r)
             return resultRow
         except pymysql.MySQLError, e:
-            raise cloudstackException.dbException("db Exception:%s"%e[1]) 
+            raise cloudstackException.dbException("db Exception:%s"%e) 
         except:
             raise cloudstackException.internalError(sys.exc_info())
         finally:

tools/marvin/marvin/deployAndRun.py

@@ -1,3 +1,15 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 import deployDataCenter
 import TestCaseExecuteEngine
 from optparse import OptionParser

tools/marvin/marvin/deployDataCenter.py

@@ -1,4 +1,16 @@
-'''Deploy datacenters according to a json configuration file'''
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
+"""Deploy datacenters according to a json configuration file"""
 import configGenerator
 import cloudstackException
 import cloudstackTestClient
@@ -7,10 +19,8 @@ import logging
 from cloudstackAPI import *
 from optparse import OptionParser
 
-module_logger = "testclient.deploy"
-
-
 class deployDataCenters():
+
     def __init__(self, cfgFile):
         self.configFile = cfgFile
 
@@ -51,8 +61,10 @@ class deployDataCenters():
             clusterresponse = self.apiClient.addCluster(clustercmd)
             clusterId = clusterresponse[0].id
 
-            self.addHosts(cluster.hosts, zoneId, podId, clusterId, cluster.hypervisor)
+            self.addHosts(cluster.hosts, zoneId, podId, clusterId,\
-            self.createPrimaryStorages(cluster.primaryStorages, zoneId, podId, clusterId)
+                          cluster.hypervisor)
+            self.createPrimaryStorages(cluster.primaryStorages, zoneId, podId,\
+                                       clusterId)
 
     def createPrimaryStorages(self, primaryStorages, zoneId, podId, clusterId):
         if primaryStorages is None:
@@ -83,12 +95,13 @@ class deployDataCenters():
             podId = createpodResponse.id
 
             if pod.guestIpRanges is not None:
-                self.createVlanIpRanges("Basic", pod.guestIpRanges, zoneId, podId)
+                self.createVlanIpRanges("Basic", pod.guestIpRanges, zoneId,\
+                                        podId)
 
             self.createClusters(pod.clusters, zoneId, podId)
 
-            
+    def createVlanIpRanges(self, mode, ipranges, zoneId, podId=None,\
-    def createVlanIpRanges(self, mode, ipranges, zoneId, podId=None, networkId=None):
+                           networkId=None):
         if ipranges is None:
             return
         for iprange in ipranges:
@@ -119,35 +132,105 @@ class deployDataCenters():
             secondarycmd.zoneid = zoneId
             self.apiClient.addSecondaryStorage(secondarycmd)
 
-    def createnetworks(self, networks, zoneId):
+    def createnetworks(self, networks, zoneId, mode):
         if networks is None:
             return
         for network in networks:
-            ipranges = network.ipranges
-            if ipranges is None:
-                continue
-            iprange = ipranges.pop()
             networkcmd = createNetwork.createNetworkCmd()
-            networkcmd.account = network.account
             networkcmd.displaytext = network.displaytext
-            networkcmd.domainid = network.domainid
+            networkcmd.name = network.name
+            networkcmd.networkofferingid = network.networkofferingid
+            networkcmd.zoneid = zoneId
+
+            ipranges = network.ipranges
+            if ipranges:
+                iprange = ipranges.pop()
+                networkcmd.startip = iprange.startip
                 networkcmd.endip = iprange.endip
                 networkcmd.gateway = iprange.gateway
-            networkcmd.isdefault = network.isdefault
-            networkcmd.isshared = network.isshared
-            networkcmd.name = network.name
                 networkcmd.netmask = iprange.netmask
-            networkcmd.networkdomain = network.networkdomain
+
-            networkcmd.networkofferingid = network.networkofferingid
             networkcmdresponse = self.apiClient.createNetwork(networkcmd)
             networkId = networkcmdresponse.id
-            self.createVlanIpRanges("Advanced", ipranges, zoneId, networkId=networkId)
+
+            self.createVlanIpRanges(mode, ipranges, zoneId, networkId)
+
+    def createPhysicalNetwork(self, name, zoneid, vlan=None):
+        phynet = createPhysicalNetwork.createPhysicalNetworkCmd()
+        phynet.zoneid = zoneid
+        phynet.name = name
+        if vlan:
+            phynet.vlan = vlan
+        return self.apiClient.createPhysicalNetwork(phynet)
+
+    def updatePhysicalNetwork(self, networkid, state="Enabled", vlan=None):
+        upnet = updatePhysicalNetwork.updatePhysicalNetworkCmd()
+        upnet.id = networkid
+        upnet.state = state
+        if vlan:
+            upnet.vlan = vlan
+        return self.apiClient.updatePhysicalNetwork(upnet)
+
+    def configureProviders(self, phynetwrk, zone):
+        pnetprov = listNetworkServiceProviders.listNetworkServiceProvidersCmd()
+        pnetprov.physicalnetworkid = phynetwrk.id
+        pnetprov.state = "Disabled"
+        pnetprov.name = "VirtualRouter"
+        pnetprovres = self.apiClient.listNetworkServiceProviders(pnetprov)
+
+        vrprov = listVirtualRouterElements.listVirtualRouterElementsCmd()
+        vrprov.nspid = pnetprovres[0].id
+        vrprovresponse = self.apiClient.listVirtualRouterElements(vrprov)
+        vrprovid = vrprovresponse[0].id
+
+        vrconfig = \
+        configureVirtualRouterElement.configureVirtualRouterElementCmd()
+        vrconfig.enabled = "true"
+        vrconfig.id = vrprovid
+        vrconfigresponse = \
+        self.apiClient.configureVirtualRouterElement(vrconfig)
+
+        if zone.networktype == "Basic" and zone.securitygroupenabled:
+            sgprovider = configGenerator.provider()
+            sgprovider.name = "SecurityGroupProvider"
+            zone.providers.append(sgprovider)
+
+        for prov in zone.providers:
+            pnetprov = \
+            listNetworkServiceProviders.listNetworkServiceProvidersCmd()
+            pnetprov.physicalnetworkid = phynetwrk.id
+            pnetprov.name = prov.name
+            pnetprov.state = "Disabled"
+            pnetprovs = self.apiClient.listNetworkServiceProviders(pnetprov)
+
+            upnetprov = \
+            updateNetworkServiceProvider.updateNetworkServiceProviderCmd()
+            upnetprov.id = pnetprovs[0].id
+            upnetprov.state = "Enabled"
+            upnetprovresponse = \
+            self.apiClient.updateNetworkServiceProvider(upnetprov)
+
+    def addTrafficTypes(self, physical_network_id, traffictypes=None, \
+                        network_labels=None):
+        [self.addTrafficType(physical_network_id, traffictype) for \
+         traffictype in traffictypes]
+
+    def addTrafficType(self, physical_network_id, traffictype, \
+                       network_label=None):
+        traffic_type = addTrafficType.addTrafficTypeCmd()
+        traffic_type.physicalnetworkid = physical_network_id
+        traffic_type.traffictype = traffictype
+        return self.apiClient.addTrafficType(traffic_type)
+
+    def enableZone(self, zoneid, allocation_state="Enabled"):
+        zoneCmd = updateZone.updateZoneCmd()
+        zoneCmd.id = zoneid
+        zoneCmd.allocationstate = allocation_state
+        return self.apiClient.updateZone(zoneCmd)
 
     def createZones(self, zones):
         for zone in zones:
-            '''create a zone'''
             createzone = createZone.createZoneCmd()
-            createzone.guestcidraddress = zone.guestcidraddress
             createzone.dns1 = zone.dns1
             createzone.dns2 = zone.dns2
             createzone.internaldns1 = zone.internaldns1
@@ -155,21 +238,46 @@ class deployDataCenters():
             createzone.name = zone.name
             createzone.securitygroupenabled = zone.securitygroupenabled
             createzone.networktype = zone.networktype
-            createzone.vlan = zone.vlan
+            createzone.guestcidraddress = zone.guestcidraddress
 
             zoneresponse = self.apiClient.createZone(createzone)
             zoneId = zoneresponse.id
 
-            '''create pods'''
+            phynetwrk = self.createPhysicalNetwork(zone.name + "-pnet", \
+                                                   zoneId)
+
+            self.addTrafficTypes(phynetwrk.id, ["Guest", "Public", \
+                                                    "Management"])
+
+            self.configureProviders(phynetwrk, zone)
+            self.updatePhysicalNetwork(phynetwrk.id, "Enabled", vlan=zone.vlan)
+
+            if zone.networktype == "Basic":
+                listnetworkoffering = \
+                listNetworkOfferings.listNetworkOfferingsCmd()
+
+                listnetworkoffering.name = \
+                "DefaultSharedNetworkOfferingWithSGService"
+
+                listnetworkofferingresponse = \
+                self.apiClient.listNetworkOfferings(listnetworkoffering)
+
+                guestntwrk = configGenerator.network()
+                guestntwrk.displaytext = "guestNetworkForBasicZone"
+                guestntwrk.name = "guestNetworkForBasicZone"
+                guestntwrk.zoneid = zoneId
+                guestntwrk.networkofferingid = \
+                        listnetworkofferingresponse[0].id
+                self.createnetworks([guestntwrk], zoneId, zone.networktype)
+
             self.createpods(zone.pods, zone, zoneId)
 
             if zone.networktype == "Advanced":
-                '''create pubic network'''
+                self.createVlanIpRanges(zone.networktype, zone.ipranges, \
-                self.createVlanIpRanges(zone.networktype, zone.ipranges, zoneId)
+                                        zoneId)
 
-            self.createnetworks(zone.networks, zoneId)
-            '''create secondary storage'''
             self.createSecondaryStorages(zone.secondaryStorages, zoneId)
+            self.enableZone(zoneId, "Enabled")
         return
 
     def registerApiKey(self):
@@ -182,7 +290,9 @@ class deployDataCenters():
         if apiKey is None:
             registerUser = registerUserKeys.registerUserKeysCmd()
             registerUser.id = userId
-            registerUserRes = self.testClient.getApiClient().registerUserKeys(registerUser)
+            registerUserRes = \
+            self.testClient.getApiClient().registerUserKeys(registerUser)
+
             apiKey = registerUserRes.apikey
             securityKey = registerUserRes.secretkey
 
@@ -196,7 +306,7 @@ class deployDataCenters():
             self.config = configGenerator.get_setup_config(self.configFile)
         except:
             raise cloudstackException.InvalidParameterException( \
-                            "Failed to load config" + sys.exc_info())
+                            "Failed to load config %s" %sys.exc_info())
 
         mgt = self.config.mgtSvr[0]
 
@@ -215,22 +325,30 @@ class deployDataCenters():
 
         testClientLogger = None
         if testClientLogFile is not None:
-            testClientLogger = logging.getLogger("testclient.deploy.deployDataCenters")
+            testClientLogger = logging.getLogger("testclient.testengine.run")
             fh = logging.FileHandler(testClientLogFile)
             fh.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s"))
             testClientLogger.addHandler(fh)
-            testClientLogger.setLevel(logging.DEBUG)
+            testClientLogger.setLevel(logging.INFO)
         self.testClientLogger = testClientLogger
 
-        self.testClient = cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, mgt.port, mgt.apiKey, mgt.securityKey, logging=self.testClientLogger)
+        self.testClient = \
+        cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, mgt.port, \
+                                                  mgt.apiKey, \
+                                                  mgt.securityKey, \
+                                            logging=self.testClientLogger)
         if mgt.apiKey is None:
             apiKey, securityKey = self.registerApiKey()
             self.testClient.close()
-            self.testClient = cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, 8080, apiKey, securityKey, logging=self.testClientLogger)
+            self.testClient = \
+            cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, 8080, \
+                                                      apiKey, securityKey, \
+                                             logging=self.testClientLogger)
 
-        '''config database'''
+        """config database"""
         dbSvr = self.config.dbSvr
-        self.testClient.dbConfigure(dbSvr.dbSvr, dbSvr.port, dbSvr.user, dbSvr.passwd, dbSvr.db)
+        self.testClient.dbConfigure(dbSvr.dbSvr, dbSvr.port, dbSvr.user, \
+                                    dbSvr.passwd, dbSvr.db)
         self.apiClient = self.testClient.getApiClient()
 
     def updateConfiguration(self, globalCfg):
@@ -253,14 +371,17 @@ if __name__ == "__main__":
 
     parser = OptionParser()
 
-    parser.add_option("-i", "--intput", action="store", default="./datacenterCfg", dest="input", help="the path where the json config file generated, by default is ./datacenterCfg")
+    parser.add_option("-i", "--intput", action="store", \
+                      default="./datacenterCfg", dest="input", help="the path \
+                      where the json config file generated, by default is \
+                      ./datacenterCfg")
 
     (options, args) = parser.parse_args()
 
     deploy = deployDataCenters(options.input)
     deploy.deploy()
 
-    '''
+    """
     create = createStoragePool.createStoragePoolCmd()
     create.clusterid = 1
     create.podid = 2
@@ -271,4 +392,4 @@ if __name__ == "__main__":
     deploy = deployDataCenters("./datacenterCfg")
     deploy.loadCfg()
     deploy.apiClient.createStoragePool(create)
-    '''
+    """

tools/marvin/marvin/jsonHelper.py

@@ -1,3 +1,15 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 import cloudstackException
 import json
 import inspect
@@ -12,7 +24,10 @@ class jsonLoader:
             if isinstance(v, dict):
                 setattr(self, k, jsonLoader(v))
             elif isinstance(v, (list, tuple)):
+                if len(v) > 0 and isinstance(v[0], dict):
                     setattr(self, k, [jsonLoader(elem) for elem in v])
+                else:
+                    setattr(self, k, v)
             else:
                 setattr(self,k,v)
     def __getattr__(self, val):
@@ -125,6 +140,7 @@ if __name__ == "__main__":
 
     result = '{ "listnetworkserviceprovidersresponse" : { "count":1 ,"networkserviceprovider" : [ {"name":"VirtualRouter","physicalnetworkid":"ad2948fc-1054-46c7-b1c7-61d990b86710","destinationphysicalnetworkid":"0","state":"Disabled","id":"d827cae4-4998-4037-95a2-55b92b6318b1","servicelist":["Vpn","Dhcp","Dns","Gateway","Firewall","Lb","SourceNat","StaticNat","PortForwarding","UserData"]} ] } }'
     nsp = getResultObj(result)
+    print nsp[0].id
     
     result = '{ "listzonesresponse" : { "count":1 ,"zone" : [  {"id":1,"name":"test0","dns1":"8.8.8.8","dns2":"4.4.4.4","internaldns1":"192.168.110.254","internaldns2":"192.168.110.253","networktype":"Basic","securitygroupsenabled":true,"allocationstate":"Enabled","zonetoken":"5e818a11-6b00-3429-9a07-e27511d3169a","dhcpprovider":"DhcpServer"} ] } }'
     zones = getResultObj(result)
@@ -165,6 +181,8 @@ if __name__ == "__main__":
     result = '{ "queryasyncjobresultresponse" : {"jobid":34,"jobstatus":2,"jobprocstatus":0,"jobresultcode":530,"jobresulttype":"object","jobresult":{"errorcode":431,"errortext":"Please provide either a volume id, or a tuple(device id, instance id)"}} }'
     print getResultObj(result, listTemplates.listTemplatesResponse())
     result = '{ "queryasyncjobresultresponse" : {"jobid":41,"jobstatus":1,"jobprocstatus":0,"jobresultcode":0,"jobresulttype":"object","jobresult":{"virtualmachine":{"id":37,"name":"i-2-37-TEST","displayname":"i-2-37-TEST","account":"admin","domainid":1,"domain":"ROOT","created":"2011-08-25T11:13:42-0700","state":"Running","haenable":false,"zoneid":1,"zonename":"test0","hostid":5,"hostname":"SimulatedAgent.1e629060-f547-40dd-b792-57cdc4b7d611","templateid":10,"templatename":"CentOS 5.3(64-bit) no GUI (Simulator)","templatedisplaytext":"CentOS 5.3(64-bit) no GUI (Simulator)","passwordenabled":false,"serviceofferingid":7,"serviceofferingname":"Small Instance","cpunumber":1,"cpuspeed":500,"memory":512,"guestosid":11,"rootdeviceid":0,"rootdevicetype":"NetworkFilesystem","securitygroup":[{"id":1,"name":"default","description":"Default Security Group"}],"nic":[{"id":43,"networkid":204,"netmask":"255.255.255.0","gateway":"192.168.1.1","ipaddress":"192.168.1.27","isolationuri":"ec2://untagged","broadcasturi":"vlan://untagged","traffictype":"Guest","type":"Direct","isdefault":true,"macaddress":"06:56:b8:00:00:53"}],"hypervisor":"Simulator"}}} }'
+    
+    result='{ "queryasyncjobresultresponse" : {"accountid":"30910093-22e4-4d3c-a464-8b36b60c8001","userid":"cb0aeca3-42ee-47c4-838a-2cd9053441f2","cmd":"com.cloud.api.commands.DeployVMCmd","jobstatus":1,"jobprocstatus":0,"jobresultcode":0,"jobresulttype":"object","jobresult":{"virtualmachine":{"id":"d2e4d724-e089-4e59-be8e-647674059016","name":"i-2-14-TEST","displayname":"i-2-14-TEST","account":"admin","domainid":"8cfafe79-81eb-445e-8608-c5b7c31fc3a5","domain":"ROOT","created":"2012-01-15T18:30:11+0530","state":"Running","haenable":false,"zoneid":"30a397e2-1c85-40c0-8463-70278952b046","zonename":"Sandbox-simulator","hostid":"cc0105aa-a2a9-427a-8ad7-4d835483b8a9","hostname":"SimulatedAgent.9fee20cc-95ca-48b1-8268-5513d6e83a1b","templateid":"d92570fa-bf40-44db-9dff-45cc7042604d","templatename":"CentOS 5.3(64-bit) no GUI (Simulator)","templatedisplaytext":"CentOS 5.3(64-bit) no GUI (Simulator)","passwordenabled":false,"serviceofferingid":"3734d632-797b-4f1d-ac62-33f9cf70d005","serviceofferingname":"Sample SO","cpunumber":1,"cpuspeed":100,"memory":128,"guestosid":"1e36f523-23e5-4e90-869b-a1b5e9ba674d","rootdeviceid":0,"rootdevicetype":"NetworkFilesystem","nic":[{"id":"4d3ab903-f511-4dab-8a6d-c2a3b51de7e0","networkid":"faeb7f24-a4b9-447d-bec6-c4956c4ab0f6","netmask":"255.255.240.0","gateway":"10.6.240.1","ipaddress":"10.6.253.89","isolationuri":"vlan://211","broadcasturi":"vlan://211","traffictype":"Guest","type":"Isolated","isdefault":true,"macaddress":"02:00:04:74:00:09"}],"hypervisor":"Simulator"}},"created":"2012-01-15T18:30:11+0530","jobid":"f4a13f28-fcd6-4d7f-b9cd-ba7eb5a5701f"} }'
     vm = getResultObj(result, deployVirtualMachine.deployVirtualMachineResponse())
     print vm.jobresult.id
     

tools/marvin/marvin/remoteSSHClient.py

@@ -1,7 +1,19 @@
+# Copyright 2012 Citrix Systems, Inc. Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License.  Citrix Systems, Inc.
+# reserves all rights not expressly granted by the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# Automatically generated by addcopyright.py at 04/03/2012
 import paramiko 
 import cloudstackException
 class remoteSSHClient(object):
-    def __init__(self, host, port, user, passwd, timeout=120):
+    def __init__(self, host, port, user, passwd):
         self.host = host
         self.port = port
         self.user = user
@@ -9,7 +21,7 @@ class remoteSSHClient(object):
         self.ssh = paramiko.SSHClient()
         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
         try:
-            self.ssh.connect(str(host),int(port), user, passwd, timeout=timeout)
+            self.ssh.connect(str(host),int(port), user, passwd)
         except paramiko.SSHException, sshex:
             raise cloudstackException.InvalidParameterException(repr(sshex))