diff --git a/tools/marvin/marvin/TestCaseExecuteEngine.py b/tools/marvin/marvin/TestCaseExecuteEngine.py index cceed9ca8bd..77ca95b694c 100644 --- a/tools/marvin/marvin/TestCaseExecuteEngine.py +++ b/tools/marvin/marvin/TestCaseExecuteEngine.py @@ -1,3 +1,15 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 try: import unittest2 as unittest except ImportError: @@ -8,8 +20,6 @@ import os import sys import logging -module_logger = "testclient.testcase" - def testCaseLogger(message, logger=None): if logger is not None: logger.debug(message) @@ -18,18 +28,19 @@ class TestCaseExecuteEngine(object): def __init__(self, testclient, testCaseFolder, testcaseLogFile=None, testResultLogFile=None): self.testclient = testclient self.testCaseFolder = testCaseFolder - self.logger = None + self.logformat = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s") + if testcaseLogFile is not None: - logger = logging.getLogger("testclient.testcase.TestCaseExecuteEngine") - fh = logging.FileHandler(testcaseLogFile) - fh.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s")) - logger.addHandler(fh) - logger.setLevel(logging.DEBUG) - self.logger = logger + self.logfile = testcaseLogFile + self.logger = logging.getLogger("TestCaseExecuteEngine") + fh = logging.FileHandler(self.logfile) + fh.setFormatter(self.logformat) + self.logger.addHandler(fh) + self.logger.setLevel(logging.DEBUG) if testResultLogFile is not None: ch = logging.StreamHandler() ch.setLevel(logging.ERROR) - ch.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s")) + ch.setFormatter(self.logformat) self.logger.addHandler(ch) fp = open(testResultLogFile, "w") self.testResultLogFile = fp @@ -41,8 +52,20 @@ class TestCaseExecuteEngine(object): if isinstance(test, unittest.BaseTestSuite): self.injectTestCase(test) else: + #logger bears the name of the test class + testcaselogger = logging.getLogger("testclient.testcase.%s"%test.__class__.__name__) + fh = logging.FileHandler(self.logfile) + fh.setFormatter(self.logformat) + testcaselogger.addHandler(fh) + testcaselogger.setLevel(logging.DEBUG) + + #inject testclient and logger into each unittest setattr(test, "testClient", self.testclient) - setattr(test, "debug", partial(testCaseLogger, logger=self.logger)) + setattr(test, "debug", partial(testCaseLogger, logger=testcaselogger)) + setattr(test.__class__, "clstestclient", self.testclient) + if hasattr(test, "UserName"): + self.testclient.createNewApiClient(test.UserName, test.DomainName, test.AcctType) + def run(self): loader = unittest.loader.TestLoader() suite = loader.discover(self.testCaseFolder) diff --git a/tools/marvin/marvin/asyncJobMgr.py b/tools/marvin/marvin/asyncJobMgr.py index d843d05ec6e..0b4499bb645 100644 --- a/tools/marvin/marvin/asyncJobMgr.py +++ b/tools/marvin/marvin/asyncJobMgr.py @@ -1,3 +1,15 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 import threading import cloudstackException import time @@ -20,12 +32,14 @@ class jobStatus(object): self.duration = None self.jobId = None self.responsecls = None + def __str__(self): + return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v) in self.__dict__.iteritems())) class workThread(threading.Thread): def __init__(self, in_queue, outqueue, apiClient, db=None, lock=None): threading.Thread.__init__(self) self.inqueue = in_queue self.output = outqueue - self.connection = apiClient.connection + self.connection = apiClient.connection.__copy__() self.db = None self.lock = lock @@ -213,4 +227,4 @@ class asyncJobMgr(object): for i in range(nums_threads): work = jobThread(inqueue1, interval) work.start() - inqueue1.join() \ No newline at end of file + inqueue1.join() diff --git a/tools/marvin/marvin/cloudstackConnection.py b/tools/marvin/marvin/cloudstackConnection.py index d3a57263df7..b72e54c3c10 100644 --- a/tools/marvin/marvin/cloudstackConnection.py +++ b/tools/marvin/marvin/cloudstackConnection.py @@ -1,7 +1,18 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 import urllib2 import urllib import base64 -import copy import hmac import hashlib import json @@ -52,14 +63,14 @@ class cloudConnection(object): try: self.connection = urllib2.urlopen("http://%s:%d/client/api?%s"%(self.mgtSvr, self.port, requestUrl)) - self.logging.debug("sending request: %s"%requestUrl) + self.logging.debug("sending GET request: %s"%requestUrl) response = self.connection.read() - self.logging.debug("got response: %s"%response) + self.logging.info("got response: %s"%response) except IOError, e: if hasattr(e, 'reason'): - self.logging.debug("failed to reach %s because of %s"%(self.mgtSvr, e.reason)) + self.logging.critical("failed to reach %s because of %s"%(self.mgtSvr, e.reason)) elif hasattr(e, 'code'): - self.logging.debug("server returned %d error code"%e.code) + self.logging.critical("server returned %d error code"%e.code) except HTTPException, h: self.logging.debug("encountered http Exception %s"%h.args) if self.retries > 0: @@ -78,16 +89,17 @@ class cloudConnection(object): requestUrl = "&".join(["=".join([request[0], urllib.quote_plus(str(request[1]))]) for request in requests]) self.connection = urllib2.urlopen("http://%s:%d/client/api?%s"%(self.mgtSvr, self.port, requestUrl)) - self.logging.debug("sending request without auth: %s"%requestUrl) + self.logging.debug("sending GET request without auth: %s"%requestUrl) response = self.connection.read() - self.logging.debug("got response: %s"%response) + self.logging.info("got response: %s"%response) return response def pollAsyncJob(self, jobId, response): cmd = queryAsyncJobResult.queryAsyncJobResultCmd() cmd.jobid = jobId + timeout = self.asyncTimeout - while self.asyncTimeout > 0: + while timeout > 0: asyncResonse = self.make_request(cmd, response, True) if asyncResonse.jobstatus == 2: @@ -96,9 +108,10 @@ class cloudConnection(object): return asyncResonse time.sleep(5) - self.asyncTimeout = self.asyncTimeout - 5 + self.logging.debug("job: %s still processing, will timeout in %ds"%(jobId, timeout)) + timeout = timeout - 5 - raise cloudstackException.cloudstackAPIException("asyncquery", "Async job timeout") + raise cloudstackException.cloudstackAPIException("asyncquery", "Async job timeout %s"%jobId) def make_request(self, cmd, response = None, raw=False): commandName = cmd.__class__.__name__.replace("Cmd", "") @@ -136,7 +149,7 @@ class cloudConnection(object): i = i + 1 if self.logging is not None: - self.logging.debug("sending command: %s %s"%(commandName, str(requests))) + self.logging.info("sending command: %s %s"%(commandName, str(requests))) result = None if self.auth: result = self.make_request_with_auth(commandName, requests) @@ -145,8 +158,6 @@ class cloudConnection(object): if result is None: return None - if self.logging is not None: - self.logging.debug("got result: " + result) result = jsonHelper.getResultObj(result, response) if raw or isAsync == "false": diff --git a/tools/marvin/marvin/cloudstackException.py b/tools/marvin/marvin/cloudstackException.py index f731be383c7..c8640975481 100644 --- a/tools/marvin/marvin/cloudstackException.py +++ b/tools/marvin/marvin/cloudstackException.py @@ -1,3 +1,15 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 class cloudstackAPIException(Exception): def __init__(self, cmd = "", result = ""): diff --git a/tools/marvin/marvin/cloudstackTestCase.py b/tools/marvin/marvin/cloudstackTestCase.py index 4595aefaa42..cf822adc39e 100644 --- a/tools/marvin/marvin/cloudstackTestCase.py +++ b/tools/marvin/marvin/cloudstackTestCase.py @@ -1,3 +1,15 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 from cloudstackAPI import * try: import unittest2 as unittest @@ -5,7 +17,37 @@ except ImportError: import unittest import cloudstackTestClient +#class UserName(object): +# def __init__(self, account, domain, type=0): +# self.account = account +# self.domain = domain +# self.accounttype = type +# def __call__(self, cls): +# class Wrapped(cls): +# cls.UserName = self.account +# cls.DomainName = self.domain +# cls.AcctType = self.accounttype +# return Wrapped + +def UserName(Name, DomainName, AcctType): + def wrapper(cls): + orig_init = cls.__init__ + def __init__(self, *args, **kws): + cls.UserName = Name + cls.DomainName = DomainName + cls.AcctType = AcctType + orig_init(self, *args, **kws) + cls.__init__ = __init__ + return cls + return wrapper + class cloudstackTestCase(unittest.case.TestCase): + clstestclient = None + def __init__(self, args): unittest.case.TestCase.__init__(self, args) self.testClient = cloudstackTestClient.cloudstackTestClient() + + @classmethod + def getClsTestClient(cls): + return cls.clstestclient diff --git a/tools/marvin/marvin/cloudstackTestClient.py b/tools/marvin/marvin/cloudstackTestClient.py index 0d8f8108ea9..29765fac2f9 100644 --- a/tools/marvin/marvin/cloudstackTestClient.py +++ b/tools/marvin/marvin/cloudstackTestClient.py @@ -1,7 +1,21 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 import cloudstackConnection import asyncJobMgr import dbConnection from cloudstackAPI import * +import random +import string class cloudstackTestClient(object): def __init__(self, mgtSvr=None, port=8096, apiKey = None, securityKey = None, asyncTimeout=3600, defaultWorkerThreads=10, logging=None): @@ -15,6 +29,82 @@ class cloudstackTestClient(object): def dbConfigure(self, host="localhost", port=3306, user='cloud', passwd='cloud', db='cloud'): self.dbConnection = dbConnection.dbConnection(host, port, user, passwd, db) + + def isAdminContext(self): + """ + A user is a regular user if he fails to listDomains; + if he is a domain-admin, he can list only domains that are non-ROOT; + if he is an admin, he can list the ROOT domain successfully + """ + try: + listdom = listDomains.listDomainsCmd() + listdom.name = 'ROOT' + listdomres = self.apiClient.listDomains(listdom) + rootdom = listdomres[0].name + if rootdom == 'ROOT': + return 1 #admin + else: + return 2 #domain-admin + except: + return 0 #user + + def random_gen(self, size=6, chars=string.ascii_uppercase + string.digits): + """Generate Random Strings of variable length""" + return ''.join(random.choice(chars) for x in range(size)) + + def createNewApiClient(self, UserName, DomainName, acctType=0): + if not self.isAdminContext(): + return self.apiClient + + listDomain = listDomains.listDomainsCmd() + listDomain.listall = True + listDomain.name = DomainName + try: + domains = self.apiClient.listDomains(listDomain) + domId = domains[0].id + except: + cdomain = createDomain.createDomainCmd() + cdomain.name = DomainName + domain = self.apiClient.createDomain(cdomain) + domId = domain.id + + cmd = listAccounts.listAccountsCmd() + cmd.name = UserName + cmd.domainid = domId + try: + accounts = self.apiClient.listAccounts(cmd) + acctId = accounts[0].id + except: + createAcctCmd = createAccount.createAccountCmd() + createAcctCmd.accounttype = acctType + createAcctCmd.domainid = domId + createAcctCmd.email = "test-" + self.random_gen() + "@citrix.com" + createAcctCmd.firstname = UserName + createAcctCmd.lastname = UserName + createAcctCmd.password = "password" + createAcctCmd.username = UserName + acct = self.apiClient.createAccount(createAcctCmd) + acctId = acct.id + + listuser = listUsers.listUsersCmd() + listuser.username = UserName + + listuserRes = self.apiClient.listUsers(listuser) + userId = listuserRes[0].id + apiKey = listuserRes[0].apikey + securityKey = listuserRes[0].secretkey + + if apiKey is None: + registerUser = registerUserKeys.registerUserKeysCmd() + registerUser.id = userId + registerUserRes = self.apiClient.registerUserKeys(registerUser) + apiKey = registerUserRes.apikey + securityKey = registerUserRes.secretkey + + nConnection = cloudstackConnection.cloudConnection(self.connection.mgtSvr, self.connection.port, apiKey, securityKey, self.connection.asyncTimeout, self.connection.logging) + self.connection.close() + self.connection = nConnection + self.apiClient = cloudstackAPIClient.CloudStackAPIClient(self.connection) def close(self): if self.connection is not None: diff --git a/tools/marvin/marvin/codegenerator.py b/tools/marvin/marvin/codegenerator.py index aea53b8854b..c7190577096 100644 --- a/tools/marvin/marvin/codegenerator.py +++ b/tools/marvin/marvin/codegenerator.py @@ -1,3 +1,15 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 import xml.dom.minidom from optparse import OptionParser import os diff --git a/tools/marvin/marvin/configGenerator.py b/tools/marvin/marvin/configGenerator.py index 3baa3edcf52..ba639ff9882 100644 --- a/tools/marvin/marvin/configGenerator.py +++ b/tools/marvin/marvin/configGenerator.py @@ -1,3 +1,15 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 import json import os from optparse import OptionParser @@ -45,17 +57,28 @@ class zone(): '''Basic or Advanced''' self.networktype = None self.dns2 = None - self.guestcidraddress = None self.internaldns2 = None self.securitygroupenabled = None + ''' Guest Vlan range - only advanced zone''' self.vlan = None '''default public network, in advanced mode''' self.ipranges = [] - '''tagged network, in advanced mode''' self.networks = [] self.pods = [] self.secondaryStorages = [] + '''enable default virtual router provider''' + vrouter = provider() + vrouter.name = 'VirtualRouter' + self.providers = [vrouter] +class provider(): + def __init__(self): + self.name = None + self.state = None + self.broadcastdomainrange = 'ZONE' + self.zoneid = None + self.servicelist = [] + class pod(): def __init__(self): self.gateway = None @@ -102,10 +125,8 @@ class network(): self.displaytext = None self.name = None self.zoneid = None - self.account = None + self.acltype = None self.domainid = None - self.isdefault = None - self.isshared = None self.networkdomain = None self.networkofferingid = None self.ipranges = [] @@ -145,34 +166,33 @@ def describe_setup_in_basic_mode(): z.networktype = 'Basic' '''create 10 pods''' - for i in range(300): + for i in range(2): p = pod() p.name = "test" +str(l) + str(i) - p.gateway = "192.%d.%d.1"%((i/255)+168,i%255) + p.gateway = "192.168.%d.1"%i p.netmask = "255.255.255.0" - - p.startip = "192.%d.%d.150"%((i/255)+168,i%255) - p.endip = "192.%d.%d.220"%((i/255)+168,i%255) + p.startip = "192.168.%d.150"%i + p.endip = "192.168.%d.220"%i '''add two pod guest ip ranges''' - for j in range(1): + for j in range(2): ip = iprange() ip.gateway = p.gateway ip.netmask = p.netmask - ip.startip = "192.%d.%d.%d"%(((i/255)+168), i%255,j*20) - ip.endip = "192.%d.%d.%d"%((i/255)+168,i%255,j*20+10) + ip.startip = "192.168.%d.%d"%(i,j*20) + ip.endip = "192.168.%d.%d"%(i,j*20+10) p.guestIpRanges.append(ip) '''add 10 clusters''' - for j in range(10): + for j in range(2): c = cluster() c.clustername = "test"+str(l)+str(i) + str(j) c.clustertype = "CloudManaged" c.hypervisor = "Simulator" '''add 10 hosts''' - for k in range(1): + for k in range(2): h = host() h.username = "root" h.password = "password" @@ -183,7 +203,6 @@ def describe_setup_in_basic_mode(): c.hosts.append(h) '''add 2 primary storages''' - ''' for m in range(2): primary = primaryStorage() size=1*1024*1024*1024*1024 @@ -191,7 +210,6 @@ def describe_setup_in_basic_mode(): #primary.url = "nfs://localhost/path%s/size=%d"%(str(l) + str(i) + str(j) + str(m), size) primary.url = "nfs://localhost/path%s"%(str(l) + str(i) + str(j) + str(m)) c.primaryStorages.append(primary) - ''' p.clusters.append(c) diff --git a/tools/marvin/marvin/dbConnection.py b/tools/marvin/marvin/dbConnection.py index e6135edf00f..afe5be58695 100644 --- a/tools/marvin/marvin/dbConnection.py +++ b/tools/marvin/marvin/dbConnection.py @@ -1,3 +1,15 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 import pymysql import cloudstackException import sys @@ -33,6 +45,8 @@ class dbConnection(object): resultRow = [] cursor = None try: + # commit to restart the transaction, else we don't get fresh data + self.db.commit() cursor = self.db.cursor() cursor.execute(sql) @@ -42,7 +56,7 @@ class dbConnection(object): resultRow.append(r) return resultRow except pymysql.MySQLError, e: - raise cloudstackException.dbException("db Exception:%s"%e[1]) + raise cloudstackException.dbException("db Exception:%s"%e) except: raise cloudstackException.internalError(sys.exc_info()) finally: diff --git a/tools/marvin/marvin/deployAndRun.py b/tools/marvin/marvin/deployAndRun.py index 084eb995418..b91453c22a9 100644 --- a/tools/marvin/marvin/deployAndRun.py +++ b/tools/marvin/marvin/deployAndRun.py @@ -1,3 +1,15 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 import deployDataCenter import TestCaseExecuteEngine from optparse import OptionParser diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py index e47dc726676..aa40b00fba8 100644 --- a/tools/marvin/marvin/deployDataCenter.py +++ b/tools/marvin/marvin/deployDataCenter.py @@ -1,19 +1,29 @@ -'''Deploy datacenters according to a json configuration file''' +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 +"""Deploy datacenters according to a json configuration file""" import configGenerator import cloudstackException import cloudstackTestClient import sys import logging -from cloudstackAPI import * +from cloudstackAPI import * from optparse import OptionParser -module_logger = "testclient.deploy" - - class deployDataCenters(): + def __init__(self, cfgFile): self.configFile = cfgFile - + def addHosts(self, hosts, zoneId, podId, clusterId, hypervisor): if hosts is None: return @@ -33,11 +43,11 @@ class deployDataCenters(): hostcmd.zoneid = zoneId hostcmd.hypervisor = hypervisor self.apiClient.addHost(hostcmd) - + def createClusters(self, clusters, zoneId, podId): if clusters is None: return - + for cluster in clusters: clustercmd = addCluster.addClusterCmd() clustercmd.clustername = cluster.clustername @@ -50,9 +60,11 @@ class deployDataCenters(): clustercmd.zoneid = zoneId clusterresponse = self.apiClient.addCluster(clustercmd) clusterId = clusterresponse[0].id - - self.addHosts(cluster.hosts, zoneId, podId, clusterId, cluster.hypervisor) - self.createPrimaryStorages(cluster.primaryStorages, zoneId, podId, clusterId) + + self.addHosts(cluster.hosts, zoneId, podId, clusterId,\ + cluster.hypervisor) + self.createPrimaryStorages(cluster.primaryStorages, zoneId, podId,\ + clusterId) def createPrimaryStorages(self, primaryStorages, zoneId, podId, clusterId): if primaryStorages is None: @@ -67,7 +79,7 @@ class deployDataCenters(): primarycmd.zoneid = zoneId primarycmd.clusterid = clusterId self.apiClient.createStoragePool(primarycmd) - + def createpods(self, pods, zone, zoneId): if pods is None: return @@ -81,14 +93,15 @@ class deployDataCenters(): createpod.zoneid = zoneId createpodResponse = self.apiClient.createPod(createpod) podId = createpodResponse.id - + if pod.guestIpRanges is not None: - self.createVlanIpRanges("Basic", pod.guestIpRanges, zoneId, podId) - + self.createVlanIpRanges("Basic", pod.guestIpRanges, zoneId,\ + podId) + self.createClusters(pod.clusters, zoneId, podId) - - - def createVlanIpRanges(self, mode, ipranges, zoneId, podId=None, networkId=None): + + def createVlanIpRanges(self, mode, ipranges, zoneId, podId=None,\ + networkId=None): if ipranges is None: return for iprange in ipranges: @@ -107,9 +120,9 @@ class deployDataCenters(): vlanipcmd.forvirtualnetwork = "false" else: vlanipcmd.forvirtualnetwork = "true" - + self.apiClient.createVlanIpRange(vlanipcmd) - + def createSecondaryStorages(self, secondaryStorages, zoneId): if secondaryStorages is None: return @@ -118,36 +131,106 @@ class deployDataCenters(): secondarycmd.url = secondary.url secondarycmd.zoneid = zoneId self.apiClient.addSecondaryStorage(secondarycmd) - - def createnetworks(self, networks, zoneId): + + def createnetworks(self, networks, zoneId, mode): if networks is None: return for network in networks: - ipranges = network.ipranges - if ipranges is None: - continue - iprange = ipranges.pop() networkcmd = createNetwork.createNetworkCmd() - networkcmd.account = network.account networkcmd.displaytext = network.displaytext - networkcmd.domainid = network.domainid - networkcmd.endip = iprange.endip - networkcmd.gateway = iprange.gateway - networkcmd.isdefault = network.isdefault - networkcmd.isshared = network.isshared networkcmd.name = network.name - networkcmd.netmask = iprange.netmask - networkcmd.networkdomain = network.networkdomain networkcmd.networkofferingid = network.networkofferingid + networkcmd.zoneid = zoneId + + ipranges = network.ipranges + if ipranges: + iprange = ipranges.pop() + networkcmd.startip = iprange.startip + networkcmd.endip = iprange.endip + networkcmd.gateway = iprange.gateway + networkcmd.netmask = iprange.netmask + networkcmdresponse = self.apiClient.createNetwork(networkcmd) networkId = networkcmdresponse.id - self.createVlanIpRanges("Advanced", ipranges, zoneId, networkId=networkId) + + self.createVlanIpRanges(mode, ipranges, zoneId, networkId) + + def createPhysicalNetwork(self, name, zoneid, vlan=None): + phynet = createPhysicalNetwork.createPhysicalNetworkCmd() + phynet.zoneid = zoneid + phynet.name = name + if vlan: + phynet.vlan = vlan + return self.apiClient.createPhysicalNetwork(phynet) + + def updatePhysicalNetwork(self, networkid, state="Enabled", vlan=None): + upnet = updatePhysicalNetwork.updatePhysicalNetworkCmd() + upnet.id = networkid + upnet.state = state + if vlan: + upnet.vlan = vlan + return self.apiClient.updatePhysicalNetwork(upnet) + + def configureProviders(self, phynetwrk, zone): + pnetprov = listNetworkServiceProviders.listNetworkServiceProvidersCmd() + pnetprov.physicalnetworkid = phynetwrk.id + pnetprov.state = "Disabled" + pnetprov.name = "VirtualRouter" + pnetprovres = self.apiClient.listNetworkServiceProviders(pnetprov) + + vrprov = listVirtualRouterElements.listVirtualRouterElementsCmd() + vrprov.nspid = pnetprovres[0].id + vrprovresponse = self.apiClient.listVirtualRouterElements(vrprov) + vrprovid = vrprovresponse[0].id + + vrconfig = \ + configureVirtualRouterElement.configureVirtualRouterElementCmd() + vrconfig.enabled = "true" + vrconfig.id = vrprovid + vrconfigresponse = \ + self.apiClient.configureVirtualRouterElement(vrconfig) + + if zone.networktype == "Basic" and zone.securitygroupenabled: + sgprovider = configGenerator.provider() + sgprovider.name = "SecurityGroupProvider" + zone.providers.append(sgprovider) + + for prov in zone.providers: + pnetprov = \ + listNetworkServiceProviders.listNetworkServiceProvidersCmd() + pnetprov.physicalnetworkid = phynetwrk.id + pnetprov.name = prov.name + pnetprov.state = "Disabled" + pnetprovs = self.apiClient.listNetworkServiceProviders(pnetprov) + + upnetprov = \ + updateNetworkServiceProvider.updateNetworkServiceProviderCmd() + upnetprov.id = pnetprovs[0].id + upnetprov.state = "Enabled" + upnetprovresponse = \ + self.apiClient.updateNetworkServiceProvider(upnetprov) + + def addTrafficTypes(self, physical_network_id, traffictypes=None, \ + network_labels=None): + [self.addTrafficType(physical_network_id, traffictype) for \ + traffictype in traffictypes] + + def addTrafficType(self, physical_network_id, traffictype, \ + network_label=None): + traffic_type = addTrafficType.addTrafficTypeCmd() + traffic_type.physicalnetworkid = physical_network_id + traffic_type.traffictype = traffictype + return self.apiClient.addTrafficType(traffic_type) + + def enableZone(self, zoneid, allocation_state="Enabled"): + zoneCmd = updateZone.updateZoneCmd() + zoneCmd.id = zoneid + zoneCmd.allocationstate = allocation_state + return self.apiClient.updateZone(zoneCmd) def createZones(self, zones): for zone in zones: - '''create a zone''' createzone = createZone.createZoneCmd() - createzone.guestcidraddress = zone.guestcidraddress createzone.dns1 = zone.dns1 createzone.dns2 = zone.dns2 createzone.internaldns1 = zone.internaldns1 @@ -155,21 +238,46 @@ class deployDataCenters(): createzone.name = zone.name createzone.securitygroupenabled = zone.securitygroupenabled createzone.networktype = zone.networktype - createzone.vlan = zone.vlan - + createzone.guestcidraddress = zone.guestcidraddress + zoneresponse = self.apiClient.createZone(createzone) zoneId = zoneresponse.id - - '''create pods''' + + phynetwrk = self.createPhysicalNetwork(zone.name + "-pnet", \ + zoneId) + + self.addTrafficTypes(phynetwrk.id, ["Guest", "Public", \ + "Management"]) + + self.configureProviders(phynetwrk, zone) + self.updatePhysicalNetwork(phynetwrk.id, "Enabled", vlan=zone.vlan) + + if zone.networktype == "Basic": + listnetworkoffering = \ + listNetworkOfferings.listNetworkOfferingsCmd() + + listnetworkoffering.name = \ + "DefaultSharedNetworkOfferingWithSGService" + + listnetworkofferingresponse = \ + self.apiClient.listNetworkOfferings(listnetworkoffering) + + guestntwrk = configGenerator.network() + guestntwrk.displaytext = "guestNetworkForBasicZone" + guestntwrk.name = "guestNetworkForBasicZone" + guestntwrk.zoneid = zoneId + guestntwrk.networkofferingid = \ + listnetworkofferingresponse[0].id + self.createnetworks([guestntwrk], zoneId, zone.networktype) + self.createpods(zone.pods, zone, zoneId) - + if zone.networktype == "Advanced": - '''create pubic network''' - self.createVlanIpRanges(zone.networktype, zone.ipranges, zoneId) - - self.createnetworks(zone.networks, zoneId) - '''create secondary storage''' + self.createVlanIpRanges(zone.networktype, zone.ipranges, \ + zoneId) + self.createSecondaryStorages(zone.secondaryStorages, zoneId) + self.enableZone(zoneId, "Enabled") return def registerApiKey(self): @@ -182,24 +290,26 @@ class deployDataCenters(): if apiKey is None: registerUser = registerUserKeys.registerUserKeysCmd() registerUser.id = userId - registerUserRes = self.testClient.getApiClient().registerUserKeys(registerUser) + registerUserRes = \ + self.testClient.getApiClient().registerUserKeys(registerUser) + apiKey = registerUserRes.apikey securityKey = registerUserRes.secretkey - + self.config.mgtSvr[0].port = 8080 self.config.mgtSvr[0].apiKey = apiKey self.config.mgtSvr[0].securityKey = securityKey return apiKey, securityKey - + def loadCfg(self): try: - self.config = configGenerator.get_setup_config(self.configFile) + self.config = configGenerator.get_setup_config(self.configFile) except: raise cloudstackException.InvalidParameterException( \ - "Failed to load config" + sys.exc_info()) + "Failed to load config %s" %sys.exc_info()) mgt = self.config.mgtSvr[0] - + loggers = self.config.logger testClientLogFile = None self.testCaseLogFile = None @@ -212,63 +322,74 @@ class deployDataCenters(): self.testCaseLogFile = log.file elif log.name == "TestResult": self.testResultLogFile = log.file - + testClientLogger = None if testClientLogFile is not None: - testClientLogger = logging.getLogger("testclient.deploy.deployDataCenters") + testClientLogger = logging.getLogger("testclient.testengine.run") fh = logging.FileHandler(testClientLogFile) fh.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s")) testClientLogger.addHandler(fh) - testClientLogger.setLevel(logging.DEBUG) + testClientLogger.setLevel(logging.INFO) self.testClientLogger = testClientLogger - - self.testClient = cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, mgt.port, mgt.apiKey, mgt.securityKey, logging=self.testClientLogger) + + self.testClient = \ + cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, mgt.port, \ + mgt.apiKey, \ + mgt.securityKey, \ + logging=self.testClientLogger) if mgt.apiKey is None: apiKey, securityKey = self.registerApiKey() self.testClient.close() - self.testClient = cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, 8080, apiKey, securityKey, logging=self.testClientLogger) - - '''config database''' + self.testClient = \ + cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, 8080, \ + apiKey, securityKey, \ + logging=self.testClientLogger) + + """config database""" dbSvr = self.config.dbSvr - self.testClient.dbConfigure(dbSvr.dbSvr, dbSvr.port, dbSvr.user, dbSvr.passwd, dbSvr.db) + self.testClient.dbConfigure(dbSvr.dbSvr, dbSvr.port, dbSvr.user, \ + dbSvr.passwd, dbSvr.db) self.apiClient = self.testClient.getApiClient() - + def updateConfiguration(self, globalCfg): if globalCfg is None: return None - + for config in globalCfg: updateCfg = updateConfiguration.updateConfigurationCmd() updateCfg.name = config.name updateCfg.value = config.value self.apiClient.updateConfiguration(updateCfg) - + def deploy(self): self.loadCfg() self.createZones(self.config.zones) self.updateConfiguration(self.config.globalConfig) - - + + if __name__ == "__main__": - + parser = OptionParser() - - parser.add_option("-i", "--intput", action="store", default="./datacenterCfg", dest="input", help="the path where the json config file generated, by default is ./datacenterCfg") - + + parser.add_option("-i", "--intput", action="store", \ + default="./datacenterCfg", dest="input", help="the path \ + where the json config file generated, by default is \ + ./datacenterCfg") + (options, args) = parser.parse_args() - - deploy = deployDataCenters(options.input) - deploy.deploy() - - ''' + + deploy = deployDataCenters(options.input) + deploy.deploy() + + """ create = createStoragePool.createStoragePoolCmd() create.clusterid = 1 create.podid = 2 create.name = "fdffdf" create.url = "nfs://jfkdjf/fdkjfkd" create.zoneid = 2 - + deploy = deployDataCenters("./datacenterCfg") deploy.loadCfg() deploy.apiClient.createStoragePool(create) - ''' + """ diff --git a/tools/marvin/marvin/jsonHelper.py b/tools/marvin/marvin/jsonHelper.py index 6bf6d056f1f..aa817b7883b 100644 --- a/tools/marvin/marvin/jsonHelper.py +++ b/tools/marvin/marvin/jsonHelper.py @@ -1,7 +1,19 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 import cloudstackException import json import inspect -from cloudstackAPI import * +from cloudstackAPI import * import pdb class jsonLoader: @@ -12,7 +24,10 @@ class jsonLoader: if isinstance(v, dict): setattr(self, k, jsonLoader(v)) elif isinstance(v, (list, tuple)): - setattr(self, k, [jsonLoader(elem) for elem in v]) + if len(v) > 0 and isinstance(v[0], dict): + setattr(self, k, [jsonLoader(elem) for elem in v]) + else: + setattr(self, k, v) else: setattr(self,k,v) def __getattr__(self, val): @@ -125,6 +140,7 @@ if __name__ == "__main__": result = '{ "listnetworkserviceprovidersresponse" : { "count":1 ,"networkserviceprovider" : [ {"name":"VirtualRouter","physicalnetworkid":"ad2948fc-1054-46c7-b1c7-61d990b86710","destinationphysicalnetworkid":"0","state":"Disabled","id":"d827cae4-4998-4037-95a2-55b92b6318b1","servicelist":["Vpn","Dhcp","Dns","Gateway","Firewall","Lb","SourceNat","StaticNat","PortForwarding","UserData"]} ] } }' nsp = getResultObj(result) + print nsp[0].id result = '{ "listzonesresponse" : { "count":1 ,"zone" : [ {"id":1,"name":"test0","dns1":"8.8.8.8","dns2":"4.4.4.4","internaldns1":"192.168.110.254","internaldns2":"192.168.110.253","networktype":"Basic","securitygroupsenabled":true,"allocationstate":"Enabled","zonetoken":"5e818a11-6b00-3429-9a07-e27511d3169a","dhcpprovider":"DhcpServer"} ] } }' zones = getResultObj(result) @@ -165,6 +181,8 @@ if __name__ == "__main__": result = '{ "queryasyncjobresultresponse" : {"jobid":34,"jobstatus":2,"jobprocstatus":0,"jobresultcode":530,"jobresulttype":"object","jobresult":{"errorcode":431,"errortext":"Please provide either a volume id, or a tuple(device id, instance id)"}} }' print getResultObj(result, listTemplates.listTemplatesResponse()) result = '{ "queryasyncjobresultresponse" : {"jobid":41,"jobstatus":1,"jobprocstatus":0,"jobresultcode":0,"jobresulttype":"object","jobresult":{"virtualmachine":{"id":37,"name":"i-2-37-TEST","displayname":"i-2-37-TEST","account":"admin","domainid":1,"domain":"ROOT","created":"2011-08-25T11:13:42-0700","state":"Running","haenable":false,"zoneid":1,"zonename":"test0","hostid":5,"hostname":"SimulatedAgent.1e629060-f547-40dd-b792-57cdc4b7d611","templateid":10,"templatename":"CentOS 5.3(64-bit) no GUI (Simulator)","templatedisplaytext":"CentOS 5.3(64-bit) no GUI (Simulator)","passwordenabled":false,"serviceofferingid":7,"serviceofferingname":"Small Instance","cpunumber":1,"cpuspeed":500,"memory":512,"guestosid":11,"rootdeviceid":0,"rootdevicetype":"NetworkFilesystem","securitygroup":[{"id":1,"name":"default","description":"Default Security Group"}],"nic":[{"id":43,"networkid":204,"netmask":"255.255.255.0","gateway":"192.168.1.1","ipaddress":"192.168.1.27","isolationuri":"ec2://untagged","broadcasturi":"vlan://untagged","traffictype":"Guest","type":"Direct","isdefault":true,"macaddress":"06:56:b8:00:00:53"}],"hypervisor":"Simulator"}}} }' + + result='{ "queryasyncjobresultresponse" : {"accountid":"30910093-22e4-4d3c-a464-8b36b60c8001","userid":"cb0aeca3-42ee-47c4-838a-2cd9053441f2","cmd":"com.cloud.api.commands.DeployVMCmd","jobstatus":1,"jobprocstatus":0,"jobresultcode":0,"jobresulttype":"object","jobresult":{"virtualmachine":{"id":"d2e4d724-e089-4e59-be8e-647674059016","name":"i-2-14-TEST","displayname":"i-2-14-TEST","account":"admin","domainid":"8cfafe79-81eb-445e-8608-c5b7c31fc3a5","domain":"ROOT","created":"2012-01-15T18:30:11+0530","state":"Running","haenable":false,"zoneid":"30a397e2-1c85-40c0-8463-70278952b046","zonename":"Sandbox-simulator","hostid":"cc0105aa-a2a9-427a-8ad7-4d835483b8a9","hostname":"SimulatedAgent.9fee20cc-95ca-48b1-8268-5513d6e83a1b","templateid":"d92570fa-bf40-44db-9dff-45cc7042604d","templatename":"CentOS 5.3(64-bit) no GUI (Simulator)","templatedisplaytext":"CentOS 5.3(64-bit) no GUI (Simulator)","passwordenabled":false,"serviceofferingid":"3734d632-797b-4f1d-ac62-33f9cf70d005","serviceofferingname":"Sample SO","cpunumber":1,"cpuspeed":100,"memory":128,"guestosid":"1e36f523-23e5-4e90-869b-a1b5e9ba674d","rootdeviceid":0,"rootdevicetype":"NetworkFilesystem","nic":[{"id":"4d3ab903-f511-4dab-8a6d-c2a3b51de7e0","networkid":"faeb7f24-a4b9-447d-bec6-c4956c4ab0f6","netmask":"255.255.240.0","gateway":"10.6.240.1","ipaddress":"10.6.253.89","isolationuri":"vlan://211","broadcasturi":"vlan://211","traffictype":"Guest","type":"Isolated","isdefault":true,"macaddress":"02:00:04:74:00:09"}],"hypervisor":"Simulator"}},"created":"2012-01-15T18:30:11+0530","jobid":"f4a13f28-fcd6-4d7f-b9cd-ba7eb5a5701f"} }' vm = getResultObj(result, deployVirtualMachine.deployVirtualMachineResponse()) print vm.jobresult.id diff --git a/tools/marvin/marvin/remoteSSHClient.py b/tools/marvin/marvin/remoteSSHClient.py index 806de7a9eb8..99d298be5ae 100644 --- a/tools/marvin/marvin/remoteSSHClient.py +++ b/tools/marvin/marvin/remoteSSHClient.py @@ -1,7 +1,19 @@ +# Copyright 2012 Citrix Systems, Inc. Licensed under the +# Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. Citrix Systems, Inc. +# reserves all rights not expressly granted by the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Automatically generated by addcopyright.py at 04/03/2012 import paramiko import cloudstackException class remoteSSHClient(object): - def __init__(self, host, port, user, passwd, timeout=120): + def __init__(self, host, port, user, passwd): self.host = host self.port = port self.user = user @@ -9,7 +21,7 @@ class remoteSSHClient(object): self.ssh = paramiko.SSHClient() self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: - self.ssh.connect(str(host),int(port), user, passwd, timeout=timeout) + self.ssh.connect(str(host),int(port), user, passwd) except paramiko.SSHException, sshex: raise cloudstackException.InvalidParameterException(repr(sshex)) @@ -33,4 +45,4 @@ class remoteSSHClient(object): if __name__ == "__main__": ssh = remoteSSHClient("192.168.137.2", 22, "root", "password") print ssh.execute("ls -l") - print ssh.execute("rm x") + print ssh.execute("rm x") \ No newline at end of file diff --git a/tools/marvin/marvin/sandbox/__init__.py b/tools/marvin/marvin/sandbox/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/marvin/marvin/sandbox/advanced/__init__.py b/tools/marvin/marvin/sandbox/advanced/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/marvin/marvin/sandbox/basic/__init__.py b/tools/marvin/marvin/sandbox/basic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/marvin/marvin/sandbox/demo/__init__.py b/tools/marvin/marvin/sandbox/demo/__init__.py new file mode 100644 index 00000000000..e69de29bb2d