packaging: Adding Centos8, Ubuntu 20.04, XCPNG8.1 Support (#4068)

* DB : Add support for MySQL 8

- Splits commands to create user and grant access on database, the old
statement is no longer supported by MySQL 8.x
- `NO_AUTO_CREATE_USER` is no longer supported by MySQL 8.x so remove
that from db.properties conn parameters

For mysql-server 8.x setup the following changes were added/tested to
make it work with CloudStack in /etc/mysql/mysql.conf.d/mysqld.cnf and
then restart the mysql-server process:

    server_id = 1
    sql-mode="STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION,ERROR_FOR_DIVISION_BY_ZERO,NO_ZERO_DATE,NO_ZERO_IN_DATE,NO_ENGINE_SUBSTITUTION"
    innodb_rollback_on_timeout=1
    innodb_lock_wait_timeout=600
    max_connections=1000
    log-bin=mysql-bin
    binlog-format = 'ROW'

    default-authentication-plugin=mysql_native_password

Notice the last line above, this is to reset the old password based
authentication used by MySQL 5.x.

Developers can set empty password as follows:

    > sudo mysql -u root
    ALTER USER 'root'@'localhost' IDENTIFIED BY '';

In libvirt repository, there are two related commits

2019-08-23 13:13 Daniel P. Berrangé            ● rpm: don't enable socket activation in upgrade if --listen present
2019-08-22 14:52 Daniel P. Berrangé            ● remote: forbid the --listen arg when systemd socket activation

In libvirt.spec.in

        /bin/systemctl mask libvirtd.socket >/dev/null 2>&1 || :
        /bin/systemctl mask libvirtd-ro.socket >/dev/null 2>&1 || :
        /bin/systemctl mask libvirtd-admin.socket >/dev/null 2>&1 || :
        /bin/systemctl mask libvirtd-tls.socket >/dev/null 2>&1 || :
        /bin/systemctl mask libvirtd-tcp.socket >/dev/null 2>&1 || :

Co-authored-by: Wei Zhou <w.zhou@global.leaseweb.com>
Co-authored-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
Co-authored-by: Rohit Yadav <rohit.yadav@shapeblue.com>

agent/bindir/cloud-guest-tool.in

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -78,7 +78,7 @@ class GuestCommand:
         info['network'] = 'guest-network-get-interfaces'
 
         result = dict()
-        for key, cmd in info.items():
+        for key, cmd in list(info.items()):
             result[key] = self.virt.agent_command(self.dom, cmd)
 
         return result, 0

agent/bindir/cloud-setup-agent.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -31,34 +31,34 @@ from cloudutils.serviceConfig import configureLibvirtConfig
 from optparse import OptionParser
 
 def getUserInputs():
-    print "Welcome to the CloudStack Agent Setup:"
+    print("Welcome to the CloudStack Agent Setup:")
 
     cfo = configFileOps("@AGENTSYSCONFDIR@/agent.properties")
     oldMgt = cfo.getEntry("host")
 
-    mgtSvr = raw_input("Please input the Management Server Hostname/IP-Address:[%s]"%oldMgt)
+    mgtSvr = input("Please input the Management Server Hostname/IP-Address:[%s]"%oldMgt)
     if mgtSvr == "":
         mgtSvr = oldMgt
     try:
         socket.getaddrinfo(mgtSvr, 443)
     except:
-        print "Failed to resolve %s. Please input a valid hostname or IP-Address."%mgtSvr
+        print("Failed to resolve %s. Please input a valid hostname or IP-Address."%mgtSvr)
         exit(1)
 
     oldToken = cfo.getEntry("zone")
-    zoneToken = raw_input("Please input the Zone Id:[%s]"%oldToken)
+    zoneToken = input("Please input the Zone Id:[%s]"%oldToken)
 
     if zoneToken == "":
         zoneToken = oldToken
 
     oldPod = cfo.getEntry("pod")
-    podId = raw_input("Please input the Pod Id:[%s]"%oldPod)
+    podId = input("Please input the Pod Id:[%s]"%oldPod)
 
     if podId == "":
        podId  = oldToken
 
     oldCluster = cfo.getEntry("cluster")
-    clusterId = raw_input("Please input the Cluster Id:[%s]"%oldCluster)
+    clusterId = input("Please input the Cluster Id:[%s]"%oldCluster)
     if clusterId == "":
         clusterId = oldCluster
 
@@ -66,21 +66,21 @@ def getUserInputs():
     if oldHypervisor == "":
         oldHypervisor = "kvm"
 
-    hypervisor = raw_input("Please input the Hypervisor type kvm/lxc:[%s]"%oldHypervisor)
+    hypervisor = input("Please input the Hypervisor type kvm/lxc:[%s]"%oldHypervisor)
     if hypervisor == "":
         hypervisor = oldHypervisor
 
     try:
         defaultNic = networkConfig.getDefaultNetwork()
     except:
-        print "Failed to get default route. Please configure your network to have a default route"
+        print("Failed to get default route. Please configure your network to have a default route")
         exit(1)
 
     defNic = defaultNic.name
-    network = raw_input("Please choose which network used to create VM:[%s]"%defNic)
+    network = input("Please choose which network used to create VM:[%s]"%defNic)
     if network == "":
         if defNic == "":
-            print "You need to specifiy one of Nic or bridge on your system"
+            print("You need to specifiy one of Nic or bridge on your system")
             exit(1)
         elif network == "":
             network = defNic
@@ -115,7 +115,7 @@ if __name__ == '__main__':
 
     if not options.auto and options.secure:
         configureLibvirtConfig(True)
-        print "Libvirtd with TLS configured"
+        print("Libvirtd with TLS configured")
         sys.exit(0)
 
     if options.auto is None:
@@ -131,10 +131,10 @@ if __name__ == '__main__':
         if glbEnv.uuid == "":
             glbEnv.uuid = bash("uuidgen").getStdout()
     else:
-        for para, value in options.__dict__.items():
+        for para, value in list(options.__dict__.items()):
             if value is None:
-                print "Missing operand:%s"%para
+                print("Missing operand:%s"%para)
-                print "Try %s --help for more information"%sys.argv[0]
+                print("Try %s --help for more information"%sys.argv[0])
                 sys.exit(1)
 
         glbEnv.uuid = options.guid
@@ -149,14 +149,14 @@ if __name__ == '__main__':
 
     glbEnv.secure = options.secure
 
-    print "Starting to configure your system:"
+    print("Starting to configure your system:")
     syscfg = sysConfigFactory.getSysConfigFactory(glbEnv)
     try:
         syscfg.config()
-        print "CloudStack Agent setup is done!"
+        print("CloudStack Agent setup is done!")
-    except (CloudRuntimeException,CloudInternalException), e:
+    except (CloudRuntimeException,CloudInternalException) as e:
-        print e
+        print(e)
-        print "Try to restore your system:"
+        print("Try to restore your system:")
         try:
             syscfg.restore()
         except:

agent/bindir/cloudstack-agent-upgrade.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -32,7 +32,7 @@ def upgradeBridgeName(brName, enslavedDev):
     print("find physical device %s"%phyDev)
     newBrName = "br" + phyDev + "-" + vlanId
     print("new bridge name %s"%newBrName)
-    bash("ip link set %s down"%brName) 
+    bash("ip link set %s down"%brName)
     bash("ip link set %s name %s"%(brName, newBrName))
     bash("ip link set %s up" %newBrName)
     cmd = "iptables-save | grep FORWARD | grep -w " +  brName
@@ -47,16 +47,16 @@ def upgradeBridgeName(brName, enslavedDev):
         except:
             logging.exception("Ignoring failure to update rules for rule " + rule + " on bridge " + brName)
 if __name__ == '__main__':
-   netlib = networkConfig() 
+   netlib = networkConfig()
    bridges = netlib.listNetworks()
-   bridges = filter(isOldStyleBridge, bridges)
+   bridges = list(filter(isOldStyleBridge, bridges))
    for br in bridges:
        enslavedDev = netlib.getEnslavedDev(br, 1)
        if enslavedDev is not None:
            upgradeBridgeName(br, enslavedDev)
-   
+
    bridges = netlib.listNetworks()
-   bridges = filter(isOldStyleBridge, bridges)
+   bridges = list(filter(isOldStyleBridge, bridges))
    if len(bridges) > 0:
        print("Warning: upgrade is not finished, still some bridges have the old style name:" + str(bridges))
    else:

agent/bindir/libvirtqemuhook.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -32,7 +32,7 @@ logging.basicConfig(filename='/var/log/libvirt/qemu-hook.log',
 logger = logging.getLogger('qemu-hook')
 
 customDir = "/etc/libvirt/hooks/custom"
-customDirPermissions = 0744
+customDirPermissions = 0o744
 timeoutSeconds = 10 * 60
 validQemuActions = ['prepare', 'start', 'started', 'stopped', 'release', 'migrate', 'restore', 'reconnect', 'attach']
 
@@ -128,9 +128,8 @@ def terminateProcess(process, scriptName):
 
 
 def getCustomScriptsFromDirectory():
-    return sorted(filter(lambda fileName: (fileName is not None) & (fileName != "") & ('_' in fileName) &
+    return sorted([fileName for fileName in os.listdir(customDir) if (fileName is not None) & (fileName != "") & ('_' in fileName) &
-                                          (fileName.startswith((action + '_')) | fileName.startswith(('all' + '_'))),
+                                          (fileName.startswith((action + '_')) | fileName.startswith(('all' + '_')))], key=lambda fileName: substringAfter(fileName, '_'))
-                         os.listdir(customDir)), key=lambda fileName: substringAfter(fileName, '_'))
 
 
 def substringAfter(s, delimiter):

agent/bindir/rolling-maintenance.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -43,7 +43,7 @@ def execute_script(stage, script, payload, timeout):
 
         success = True if exitStatus == 0 or exitStatus == AVOID_MAINTENANCE_EXIT_STATUS else False
         avoid_maintenance = True if exitStatus == AVOID_MAINTENANCE_EXIT_STATUS else False
-        return {"success": success, "message": stdout.strip(), "avoidmaintenance": avoid_maintenance}
+        return {"success": success, "message": stdout.decode('utf-8').strip(), "avoidmaintenance": avoid_maintenance}
     except Exception as e:
         logger.error("Error in stage %s: %s" % (script, e))
         sys.exit(1)

client/bindir/cloud-setup-management.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -35,26 +35,26 @@ if __name__ == '__main__':
     if options.https:
         glbEnv.svrMode = "HttpsServer"
     if options.tomcat7:
-        print "The --tomcat7 option is deprecated, CloudStack now uses embedded Jetty server."
+        print("The --tomcat7 option is deprecated, CloudStack now uses embedded Jetty server.")
     if options.nostart:
         glbEnv.noStart = True
 
     glbEnv.mode = "Server"
 
-    print "Starting to configure CloudStack Management Server:"
+    print("Starting to configure CloudStack Management Server:")
     try:
         syscfg = sysConfigFactory.getSysConfigFactory(glbEnv)
     except UnknownSystemException:
-        print >>sys.stderr, ("Error: CloudStack failed to detect your "
+        print(("Error: CloudStack failed to detect your "
-                "operating system. Exiting.")
+                "operating system. Exiting."), file=sys.stderr)
         sys.exit(1)
     try:
-    	syscfg.registerService(cloudManagementConfig)
+        syscfg.registerService(cloudManagementConfig)
         syscfg.config()
-        print "CloudStack Management Server setup is Done!"
+        print("CloudStack Management Server setup is Done!")
-    except (CloudRuntimeException, CloudInternalException), e:
+    except (CloudRuntimeException, CloudInternalException) as e:
-        print e
+        print(e)
-        print "Try to restore your system:"
+        print("Try to restore your system:")
         try:
             syscfg.restore()
         except:

client/bindir/cloud-update-xenserver-licenses.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python -W ignore::DeprecationWarning
+#!/usr/bin/python3 -W ignore::DeprecationWarning
 # -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -65,7 +65,7 @@ def getknownhosts(host,username,password):
 	usernames = dict(cur.fetchall())
 	cur.execute("SELECT h.private_ip_address,d.value FROM cloud.host h inner join cloud.host_details d on (h.id = d.host_id) where d.name = 'password' and setup = 1")
 	passwords = dict(cur.fetchall())
-	creds = dict( [ [x,(usernames[x],passwords[x])] for x in usernames.keys() ] )
+	creds = dict( [ [x,(usernames[x],passwords[x])] for x in list(usernames.keys()) ] )
 	cur.close()
 	conn.close()
 	return creds
@@ -121,7 +121,7 @@ class XenServerConfigurator(Thread):
 			if self.retval != 0: self.state = 'failed'
 			else: self.state = 'finished'
 			
-		except Exception,e:
+		except Exception as e:
 			self.state = 'failed'
 			self.retval = e
 			#raise
@@ -144,7 +144,7 @@ if options.all:
 	if len(args) != 0: e("IP addresses cannot be specified if -a is specified")
 	config = read_properties(cfg)
 	creds = getknownhosts(config["db.cloud.host"],config["db.cloud.username"],config["db.cloud.password"])
-	hosts = creds.keys()
+	hosts = list(creds.keys())
 else:
 	if not args: e("You must specify at least one IP address, or -a")
 	hosts = args
@@ -152,31 +152,31 @@ else:
 
 try:
 	keyfiledata = file(licensefile).read(-1)
-except OSError,e:
+except OSError as e:
 	sys.stderr.write("The file %s cannot be opened"%licensefile)
 	sys.exit(1)
 
 configurators = []
-for host,(user,password) in creds.items():
+for host,(user,password) in list(creds.items()):
 	configurators.append ( XenServerConfigurator(host,user,password,keyfiledata ) )
 
 
 for c in configurators: c.start()
 	
 for c in configurators:
-	print c.host + "...",
+	print(c.host + "...", end=' ')
 	c.join()
 	if c.state == 'failed':
 		if c.retval:
 			msg = "failed with return code %s: %s%s"%(c.retval,c.stdout,c.stderr)
 			msg = msg.strip()
-			print msg
+			print(msg)
-		else: print "failed: %s"%c.retval
+		else: print("failed: %s"%c.retval)
 	else:
-		print "done"
+		print("done")
 
 successes = len( [ a for a in configurators if not a.state == 'failed' ] )
 failures = len( [ a for a in configurators if a.state == 'failed' ] )
 
-print "%3s successes"%successes
+print("%3s successes"%successes)
-print "%3s failures"%failures
+print("%3s failures"%failures)

client/conf/db.properties.in

@@ -39,7 +39,7 @@ db.cloud.testWhileIdle=true
 db.cloud.timeBetweenEvictionRunsMillis=40000
 db.cloud.minEvictableIdleTimeMillis=240000
 db.cloud.poolPreparedStatements=false
-db.cloud.url.params=prepStmtCacheSize=517&cachePrepStmts=true&sessionVariables=sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'&serverTimezone=UTC
+db.cloud.url.params=prepStmtCacheSize=517&cachePrepStmts=true&sessionVariables=sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'&serverTimezone=UTC
 
 # CloudStack database SSL settings
 db.cloud.useSSL=false

cloud-cli/bindir/cloud-tool

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

cloud-cli/cloudapis/cloud.py

@@ -21,15 +21,15 @@
 
 
 from cloudtool.utils import describe
-import urllib
+import urllib.request, urllib.parse, urllib.error
-import urllib2
+import urllib.request, urllib.error, urllib.parse
 import os
 import xml.dom.minidom
 import re
 import base64
 import hmac
 import hashlib
-import httplib
+import http.client
 
 class CloudAPI:
 
@@ -51,30 +51,30 @@ class CloudAPI:
         requests["command"] = command
         requests["apiKey"] = self.apiKey
         requests["response"] = "xml"
-        requests = zip(requests.keys(), requests.values())
+        requests = list(zip(list(requests.keys()), list(requests.values())))
         requests.sort(key=lambda x: str.lower(x[0]))
 
-        requestUrl = "&".join(["=".join([request[0], urllib.quote_plus(str(request[1]))]) for request in requests])
+        requestUrl = "&".join(["=".join([request[0], urllib.parse.quote_plus(str(request[1]))]) for request in requests])
-        hashStr = "&".join(["=".join([str.lower(request[0]), urllib.quote_plus(str.lower(str(request[1])))]) for request in requests])
+        hashStr = "&".join(["=".join([str.lower(request[0]), urllib.parse.quote_plus(str.lower(str(request[1])))]) for request in requests])
 
-        sig = urllib.quote_plus(base64.encodestring(hmac.new(self.securityKey, hashStr, hashlib.sha1).digest()).strip())
+        sig = urllib.parse.quote_plus(base64.encodestring(hmac.new(self.securityKey, hashStr, hashlib.sha1).digest()).strip())
 
         requestUrl += "&signature=%s"%sig
         return requestUrl
 
 
     def _make_request_with_auth(self, command, requests):
-        self.connection = httplib.HTTPConnection("%s"%(self.server))
+        self.connection = http.client.HTTPConnection("%s"%(self.server))
         requests["command"] = command
         requests["apiKey"] = self.apiKey
         requests["response"] = self.responseformat
-        requests = zip(requests.keys(), requests.values())
+        requests = list(zip(list(requests.keys()), list(requests.values())))
         requests.sort(key=lambda x: str.lower(x[0]))
 
-        requestUrl = "&".join(["=".join([request[0], urllib.quote(str(request[1],""))]) for request in requests])
+        requestUrl = "&".join(["=".join([request[0], urllib.parse.quote(str(request[1],""))]) for request in requests])
-        hashStr = "&".join(["=".join([str.lower(request[0]), urllib.quote(str.lower(str(request[1])),"")]) for request in requests])
+        hashStr = "&".join(["=".join([str.lower(request[0]), urllib.parse.quote(str.lower(str(request[1])),"")]) for request in requests])
 
-        sig = urllib.quote_plus(base64.encodestring(hmac.new(self.securityKey, str.lower(hashStr), hashlib.sha1).digest()).strip())
+        sig = urllib.parse.quote_plus(base64.encodestring(hmac.new(self.securityKey, str.lower(hashStr), hashlib.sha1).digest()).strip())
 
         requestUrl += "&signature=%s"%sig
 
@@ -99,11 +99,11 @@ class CloudAPI:
         else:
             parameters["command"] = command
             parameters["response"] = self.responseformat
-            querystring = urllib.urlencode(parameters)
+            querystring = urllib.parse.urlencode(parameters)
 
         url += querystring
 
-        f = urllib2.urlopen(url)
+        f = urllib.request.urlopen(url)
         data = f.read()
         if self.stripxml == "true":
             data=re.sub("<\?.*\?>", "\n", data);
@@ -149,7 +149,7 @@ def load_dynamic_methods():
             required = getText(param.getElementsByTagName('required')[0].childNodes).strip()
             if required == 'true': required = True
             elif required == 'false': required = False
-            else: raise AssertionError, "Not reached"
+            else: raise AssertionError("Not reached")
             if required: arguments.append(argname)
             options.append(argname)
 
@@ -179,7 +179,7 @@ def load_dynamic_methods():
         """%(name,funcparams,description,arguments,name)
 
         namespace = {}
-        exec code.strip() in namespace
+        exec(code.strip(), namespace)
 
         func = namespace[name]
         for argname,description in descriptions:

cloud-cli/cloudtool/__init__.py

@@ -55,7 +55,7 @@ def main(argv=None):
     
     try:
         api = apis(**api_optionsdict)
-    except utils.OptParseError,e:
+    except utils.OptParseError as e:
         parser.error(str(e))
     
     command = utils.lookup_command_in_api(api,args[0])
@@ -64,7 +64,7 @@ def main(argv=None):
     args = args[2:]
 
     try: return command(*args,**cmd_optionsdict)
-    except TypeError,e: parser.error(str(e))
+    except TypeError as e: parser.error(str(e))
 
 
 if __name__ == '__main__':

cloud-cli/cloudtool/utils.py

@@ -109,7 +109,7 @@ def get_parser(api_callable=None,cmd_callable=None): # this should probably be t
         assert len(api_arguments) is 0 # no mandatory arguments for class initializers
         
     if cmd_callable:
-        cmd_name = cmd_callable.func_name.replace("_","-")
+        cmd_name = cmd_callable.__name__.replace("_","-")
         cmd_arguments,cmd_options = get_arguments_and_options(cmd_callable)
         if cmd_arguments:
             arguments   = " " + " ".join( [ s[0].upper() for s in cmd_arguments ] )

debian/cloudstack-common.postinst

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+CLOUDUTILS_DIR="/usr/share/pyshared/"
+DIST_DIR=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")
+
+if which pycompile >/dev/null 2>&1; then
+    pycompile -p cloudstack-common
+fi
+
+if which pycompile >/dev/null 2>&1; then
+    pycompile -p cloudstack-common /usr/share/cloudstack-common
+fi
+
+cp $CLOUDUTILS_DIR/cloud_utils.py $DIST_DIR
+cp -R $CLOUDUTILS_DIR/cloudutils $DIST_DIR
+python3 -m py_compile $DIST_DIR/cloud_utils.py
+python3 -m compileall $DIST_DIR/cloudutils

debian/control

@@ -9,20 +9,20 @@ Homepage: http://www.cloudstack.org/
 
 Package: cloudstack-common
 Architecture: all
-Depends: ${misc:Depends}, ${python:Depends}, genisoimage, nfs-common
+Depends: ${misc:Depends}, ${python3:Depends}, genisoimage, nfs-common, python3-pip, python3-distutils, python3-netaddr
 Conflicts: cloud-scripts, cloud-utils, cloud-system-iso, cloud-console-proxy, cloud-daemonize, cloud-deps, cloud-python, cloud-setup
 Description: A common package which contains files which are shared by several CloudStack packages
 
 Package: cloudstack-management
 Architecture: all
-Depends: ${python:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), sudo, python-mysql.connector, augeas-tools, mysql-client | mariadb-client, adduser, bzip2, ipmitool, file, gawk, iproute2, qemu-utils, python-dnspython, lsb-release, init-system-helpers (>= 1.14~)
+Depends: ${python3:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), sudo, python3-mysql.connector, augeas-tools, mysql-client | mariadb-client, adduser, bzip2, ipmitool, file, gawk, iproute2, qemu-utils, python3-dnspython, lsb-release, init-system-helpers (>= 1.14~), python3-setuptools
 Conflicts: cloud-server, cloud-client, cloud-client-ui
 Description: CloudStack server library
  The CloudStack management server
 
 Package: cloudstack-agent
 Architecture: all
-Depends: ${python:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), uuid-runtime, iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, lsb-release, aria2
+Depends: ${python:Depends}, ${python3:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), uuid-runtime, iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, lsb-release, aria2
 Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent

debian/rules

@@ -6,7 +6,7 @@ SYSCONFDIR = "/etc"
 DESTDIR = "debian/tmp"
 
 %:
-	dh $@ --with python2,systemd
+	dh $@ --with systemd
 
 override_dh_auto_configure:
 	cp packaging/debian/replace.properties replace.properties.tmp

engine/schema/src/main/java/com/cloud/upgrade/DatabaseCreator.java

@@ -104,8 +104,10 @@ public class DatabaseCreator {
             List<String> queries = new ArrayList<String>();
             queries.add(String.format("drop database if exists `%s`", dbName));
             queries.add(String.format("create database `%s`", dbName));
-            queries.add(String.format("GRANT ALL ON %s.* to '%s'@`localhost` identified by '%s'", dbName, username, password));
+            queries.add(String.format("CREATE USER IF NOT EXISTS %s@`localhost` identified by '%s'", username, password));
-            queries.add(String.format("GRANT ALL ON %s.* to '%s'@`%%` identified by '%s'", dbName, username, password));
+            queries.add(String.format("CREATE USER IF NOT EXISTS %s@`%%` identified by '%s'", username, password));
+            queries.add(String.format("GRANT ALL ON %s.* to '%s'@`localhost`", dbName, username));
+            queries.add(String.format("GRANT ALL ON %s.* to '%s'@`%%`", dbName, username));
 
             for (String query : queries) {
                 runQuery(host, port, rootPassword, query, dryRun);

engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql

@@ -181,4 +181,16 @@ UPDATE `cloud`.`roles` SET `is_default` = 1 WHERE id IN (1, 2, 3, 4);
 INSERT INTO `cloud`.`roles` (`uuid`, `name`, `role_type`, `description`, `is_default`) VALUES (UUID(), 'Read-Only Admin - Default', 'Admin', 'Default read-only admin role', 1);
 INSERT INTO `cloud`.`roles` (`uuid`, `name`, `role_type`, `description`, `is_default`) VALUES (UUID(), 'Read-Only User - Default', 'User', 'Default read-only user role', 1);
 INSERT INTO `cloud`.`roles` (`uuid`, `name`, `role_type`, `description`, `is_default`) VALUES (UUID(), 'Support Admin - Default', 'Admin', 'Default support admin role', 1);
-INSERT INTO `cloud`.`roles` (`uuid`, `name`, `role_type`, `description`, `is_default`) VALUES (UUID(), 'Support User - Default', 'User', 'Default support user role', 1);
+INSERT INTO `cloud`.`roles` (`uuid`, `name`, `role_type`, `description`, `is_default`) VALUES (UUID(), 'Support User - Default', 'User', 'Default support user role', 1);
+
+-- mysql8 nics table fix for newer distributions
+ALTER TABLE `cloud`.`nics` MODIFY COLUMN update_time timestamp DEFAULT CURRENT_TIMESTAMP;
+
+-- Change guest OS name to support default CentOS 5 template in XenServer8.0
+UPDATE `cloud`.`guest_os_hypervisor` SET guest_os_name='CentOS 7' where guest_os_id=(SELECT guest_os_id from `cloud`.`vm_template` WHERE unique_name='centos56-x86_64-xen') AND hypervisor_type='Xenserver' AND hypervisor_version='8.0.0';
+
+-- Add XenServer 8.1 hypervisor capabilities
+INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported) values (UUID(), 'XenServer', '8.1.0', 1000, 253, 64, 1);
+
+-- Copy XenServer 8.0 hypervisor guest OS mappings to XenServer8.1
+INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '8.1.0', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='8.0.0';

packaging/centos7/cloud.spec

@@ -27,6 +27,7 @@ Summary:   CloudStack IaaS Platform
 %define _maventag %{_fullver}
 Release:   %{_rel}%{dist}
 
+%define __python python3
 %{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")}
 
 Version:   %{_ver}
@@ -47,6 +48,7 @@ BuildRequires: /usr/bin/mkisofs
 BuildRequires: mysql-connector-python
 BuildRequires: maven => 3.0.0
 BuildRequires: python-setuptools
+BuildRequires: wget
 
 %description
 CloudStack is a highly-scalable elastic, open source,
@@ -73,12 +75,12 @@ Requires: /sbin/service
 Requires: /sbin/chkconfig
 Requires: /usr/bin/ssh-keygen
 Requires: genisoimage
-Requires: mysql-connector-python
 Requires: ipmitool
 Requires: %{name}-common = %{_ver}
 Requires: iptables-services
 Requires: qemu-img
-Requires: python-dns
+Requires: python3-pip
+Requires: python3-setuptools
 Group:     System Environment/Libraries
 %description management
 The CloudStack management server is the central point of coordination,
@@ -88,8 +90,7 @@ management, and intelligence in CloudStack.
 Summary: Apache CloudStack common files and scripts
 Requires: python
 Requires: python3
-Requires: python-argparse
+Requires: python3-pip
-Requires: python-netaddr
 Group:   System Environment/Libraries
 %description common
 The Apache CloudStack files shared between agent and management server
@@ -109,7 +110,6 @@ Requires: net-tools
 Requires: iproute
 Requires: ipset
 Requires: perl
-Requires: libvirt-python
 Requires: python36-libvirt
 Requires: qemu-img
 Requires: qemu-kvm
@@ -220,8 +220,8 @@ cp -r scripts/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts
 install -D systemvm/dist/systemvm.iso ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/systemvm.iso
 install python/lib/cloud_utils.py ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py
 cp -r python/lib/cloudutils ${RPM_BUILD_ROOT}%{python_sitearch}/
-python -m py_compile ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py
+python3 -m py_compile ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py
-python -m compileall ${RPM_BUILD_ROOT}%{python_sitearch}/cloudutils
+python3 -m compileall ${RPM_BUILD_ROOT}%{python_sitearch}/cloudutils
 cp build/gitrev.txt ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts
 cp packaging/centos7/cloudstack-sccs ${RPM_BUILD_ROOT}/usr/bin
 
@@ -235,6 +235,7 @@ mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup
 mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management
 mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/run
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel
 
 # Setup Jetty
 ln -sf /etc/%{name}/management ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/conf
@@ -279,6 +280,12 @@ install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}
 touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid
 #install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina
 
+# Package mysql-connector-python
+wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/ee/ff/48bde5c0f013094d729fe4b0316ba2a24774b3ff1c52d924a8a4cb04078a/six-1.15.0-py2.py3-none-any.whl
+wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/e9/93/4860cebd5ad3ff2664ad3c966490ccb46e3b88458b2095145bca11727ca4/setuptools-47.3.1-py3-none-any.whl
+wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/28/05/9867ef8eafd12265267bee138fa2c46ebf34a276ea4cbe184cba4c606e8b/protobuf-3.12.2-cp36-cp36m-manylinux1_x86_64.whl
+wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/d1/53/4cf90d2fe81b9cdb55dc180951bcec44ea8685665f1bdb1412501dc362dd/mysql_connector_python-8.0.20-cp36-cp36m-manylinux1_x86_64.whl
+
 chmod 440 ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management
 chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt
 chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/management
@@ -386,6 +393,9 @@ then
 fi
 
 %post management
+# Install mysql-connector-python
+pip3 install %{_datadir}/%{name}-management/setup/wheel/six-1.15.0-py2.py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/setuptools-47.3.1-py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/protobuf-3.12.2-cp36-cp36m-manylinux1_x86_64.whl %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.20-cp36-cp36m-manylinux1_x86_64.whl
+
 /usr/bin/systemctl on cloudstack-management > /dev/null 2>&1 || true
 
 grep -s -q "db.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties"
@@ -472,7 +482,7 @@ if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then
 fi
 
 %post marvin
-pip install --upgrade http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-2.0.4.zip#md5=3df394d89300db95163f17c843ef49df
+pip install --upgrade https://files.pythonhosted.org/packages/ca/ea/1e2553b088bad2f9fa8120c2624f797b2d7450d3b61bb492d29c72e3d3c2/mysql_connector_python-8.0.20-cp27-cp27mu-manylinux1_x86_64.whl
 pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 
 #No default permission as the permission setup is complex
@@ -515,6 +525,7 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 %{_defaultdocdir}/%{name}-management-%{version}/LICENSE
 %{_defaultdocdir}/%{name}-management-%{version}/NOTICE
 #%attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-catalina
+%{_datadir}/%{name}-management/setup/wheel/*.whl
 
 %files agent
 %attr(0755,root,root) %{_bindir}/%{name}-setup-agent
@@ -543,7 +554,7 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 %attr(0755,root,root) /usr/bin/cloudstack-sccs
 %attr(0644, root, root) %{_datadir}/%{name}-common/vms/systemvm.iso
 %attr(0644,root,root) %{python_sitearch}/cloud_utils.py
-%attr(0644,root,root) %{python_sitearch}/cloud_utils.pyc
+%attr(0644,root,root) %{python_sitearch}/__pycache__/*
 %attr(0644,root,root) %{python_sitearch}/cloudutils/*
 %attr(0644, root, root) %{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar
 %{_defaultdocdir}/%{name}-common-%{version}/LICENSE
@@ -598,3 +609,4 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 
 * Fri Oct 5 2012 Hugo Trippaers <hugo@apache.org> 4.1.0
 - new style spec file
+

packaging/centos7/replace.properties

@@ -48,7 +48,7 @@ MSUSER=cloud
 PIDDIR=/var/run
 PLUGINJAVADIR=/usr/share/cloudstack-management/plugin
 PREMIUMJAVADIR=/usr/share/cloudstack-management/premium
-PYTHONDIR=/usr/lib/python2.7/site-packages/
+PYTHONDIR=/usr/lib/python3.6/site-packages/
 SERVERSYSCONFDIR=/etc/sysconfig
 SETUPDATADIR=/usr/share/cloudstack-management/setup
 SYSCONFDIR=/etc/sysconfig

packaging/centos8/cloud-ipallocator.rc

@@ -0,0 +1,96 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# chkconfig: 35 99 10
+# description: Cloud Agent
+
+# WARNING: if this script is changed, then all other initscripts MUST BE changed to match it as well
+
+. /etc/rc.d/init.d/functions
+
+# set environment variables
+
+SHORTNAME="$(basename $(readlink -f $0))"
+PIDFILE=/var/run/"$SHORTNAME".pid
+LOCKFILE=/var/lock/subsys/"$SHORTNAME"
+LOGFILE=/var/log/cloudstack/ipallocator/ipallocator.log
+PROGNAME="External IPAllocator"
+
+unset OPTIONS
+[ -r /etc/sysconfig/"$SHORTNAME" ] && source /etc/sysconfig/"$SHORTNAME"
+DAEMONIZE=/usr/bin/cloud-daemonize
+PROG=/usr/bin/cloud-external-ipallocator.py
+OPTIONS=8083
+
+start() {
+        echo -n $"Starting $PROGNAME: "
+	if hostname --fqdn >/dev/null 2>&1 ; then
+		daemon --check=$SHORTNAME --pidfile=${PIDFILE} "$DAEMONIZE" \
+			-n "$SHORTNAME" -p "$PIDFILE" -l "$LOGFILE" "$PROG" $OPTIONS
+		RETVAL=$?
+		echo
+	else
+		failure
+		echo
+		echo The host name does not resolve properly to an IP address.  Cannot start "$PROGNAME". > /dev/stderr
+		RETVAL=9
+	fi
+	[ $RETVAL = 0 ] && touch ${LOCKFILE}
+	return $RETVAL
+}
+
+stop() {
+	echo -n $"Stopping $PROGNAME: "
+	killproc -p ${PIDFILE} $SHORTNAME # -d 10 $SHORTNAME
+	RETVAL=$?
+	echo
+	[ $RETVAL = 0 ] && rm -f ${LOCKFILE} ${PIDFILE}
+}
+
+
+# See how we were called.
+case "$1" in
+  start)
+	start
+	;;
+  stop)
+	stop
+	;;
+  status)
+        status -p ${PIDFILE} $SHORTNAME
+	RETVAL=$?
+	;;
+  restart)
+	stop
+	sleep 3
+	start
+	;;
+  condrestart)
+	if status -p ${PIDFILE} $SHORTNAME >&/dev/null; then
+		stop
+		sleep 3
+		start
+	fi
+	;;
+  *)
+	echo $"Usage: $SHORTNAME {start|stop|restart|condrestart|status|help}"
+	RETVAL=3
+esac
+
+exit $RETVAL
+

packaging/centos8/cloud.limits

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cloud hard nofile 4096
+cloud soft nofile 4096

packaging/centos8/cloud.spec

@@ -0,0 +1,609 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+%define __os_install_post %{nil}
+%global debug_package %{nil}
+
+# DISABLE the post-percentinstall java repacking and line number stripping
+# we need to find a way to just disable the java repacking and line number stripping, but not the autodeps
+
+Name:      cloudstack
+Summary:   CloudStack IaaS Platform
+#http://fedoraproject.org/wiki/PackageNamingGuidelines#Pre-Release_packages
+%define _maventag %{_fullver}
+Release:   %{_rel}%{dist}
+
+%define __python python3
+%{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")}
+
+Version:   %{_ver}
+License:   ASL 2.0
+Vendor:    Apache CloudStack <dev@cloudstack.apache.org>
+Packager:  Apache CloudStack <dev@cloudstack.apache.org>
+Group:     System Environment/Libraries
+# FIXME do groups for every single one of the subpackages
+Source0:   %{name}-%{_maventag}.tgz
+BuildRoot: %{_tmppath}/%{name}-%{_maventag}-%{release}-build
+
+BuildRequires: java-11-openjdk-devel
+#BuildRequires: ws-commons-util
+BuildRequires: jpackage-utils
+BuildRequires: gcc
+BuildRequires: glibc-devel
+BuildRequires: /usr/bin/mkisofs
+BuildRequires: maven => 3.0.0
+BuildRequires: python3-setuptools
+BuildRequires: wget
+
+%description
+CloudStack is a highly-scalable elastic, open source,
+intelligent IaaS cloud implementation.
+
+%package management
+Summary:   CloudStack management server UI
+Requires: java-11-openjdk
+Requires: python3
+Requires: bash
+Requires: gawk
+Requires: bzip2
+Requires: gzip
+Requires: unzip
+Requires: /sbin/mount.nfs
+Requires: openssh-clients
+Requires: nfs-utils
+Requires: iproute
+Requires: wget
+Requires: mysql
+Requires: sudo
+Requires: /sbin/service
+Requires: /sbin/chkconfig
+Requires: /usr/bin/ssh-keygen
+Requires: genisoimage
+Requires: ipmitool
+Requires: %{name}-common = %{_ver}
+Requires: iptables-services
+Requires: qemu-img
+Requires: python3-pip
+Requires: python3-setuptools
+Group:     System Environment/Libraries
+%description management
+The CloudStack management server is the central point of coordination,
+management, and intelligence in CloudStack.
+
+%package common
+Summary: Apache CloudStack common files and scripts
+Requires: python3
+Group:   System Environment/Libraries
+%description common
+The Apache CloudStack files shared between agent and management server
+%global __requires_exclude ^(libuuid\\.so\\.1|/usr/bin/python)$
+
+%package agent
+Summary: CloudStack Agent for KVM hypervisors
+Requires: openssh-clients
+Requires: java-11-openjdk
+Requires: %{name}-common = %{_ver}
+Requires: libvirt
+Requires: ebtables
+Requires: iptables
+Requires: ethtool
+Requires: net-tools
+Requires: iproute
+Requires: ipset
+Requires: perl
+Requires: python3-libvirt
+Requires: qemu-img
+Requires: qemu-kvm
+Provides: cloud-agent
+Group: System Environment/Libraries
+%description agent
+The CloudStack agent for KVM hypervisors
+
+%package baremetal-agent
+Summary: CloudStack baremetal agent
+Requires: tftp-server
+Requires: xinetd
+Requires: syslinux
+Requires: chkconfig
+Requires: dhcp
+Requires: httpd
+Group:     System Environment/Libraries
+%description baremetal-agent
+The CloudStack baremetal agent
+
+%package usage
+Summary: CloudStack Usage calculation server
+Requires: java-11-openjdk
+Group: System Environment/Libraries
+%description usage
+The CloudStack usage calculation service
+
+%package cli
+Summary: Apache CloudStack CLI
+Provides: python-marvin
+Group: System Environment/Libraries
+%description cli
+Apache CloudStack command line interface
+
+%package marvin
+Summary: Apache CloudStack Marvin library
+Requires: python3-pip
+Requires: python2-pip
+Requires: gcc
+Requires: python3-devel
+Requires: python2-devel
+Requires: libffi-devel
+Requires: openssl-devel
+Group: System Environment/Libraries
+%description marvin
+Apache CloudStack Marvin library
+
+%package integration-tests
+Summary: Apache CloudStack Marvin integration tests
+Requires: %{name}-marvin = %{_ver}
+Group: System Environment/Libraries
+%description integration-tests
+Apache CloudStack Marvin integration tests
+
+%if "%{_ossnoss}" == "noredist"
+%package mysql-ha
+Summary: Apache CloudStack Balancing Strategy for MySQL
+Group: System Environmnet/Libraries
+%description mysql-ha
+Apache CloudStack Balancing Strategy for MySQL
+
+%endif
+
+%prep
+echo Doing CloudStack build
+
+%setup -q -n %{name}-%{_maventag}
+
+%build
+
+cp packaging/centos8/replace.properties build/replace.properties
+echo VERSION=%{_maventag} >> build/replace.properties
+echo PACKAGE=%{name} >> build/replace.properties
+touch build/gitrev.txt
+echo $(git rev-parse HEAD) > build/gitrev.txt
+
+if [ "%{_ossnoss}" == "NOREDIST" -o "%{_ossnoss}" == "noredist" ] ; then
+   echo "Adding noredist flag to the maven build"
+   FLAGS="$FLAGS -Dnoredist"
+fi
+
+if [ "%{_sim}" == "SIMULATOR" -o "%{_sim}" == "simulator" ] ; then
+   echo "Adding simulator flag to the maven build"
+   FLAGS="$FLAGS -Dsimulator"
+fi
+
+mvn -Psystemvm,developer $FLAGS clean package
+
+%install
+[ ${RPM_BUILD_ROOT} != "/" ] && rm -rf ${RPM_BUILD_ROOT}
+# Common directories
+mkdir -p ${RPM_BUILD_ROOT}%{_bindir}
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/ipallocator
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/work
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/temp
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/management
+mkdir -p ${RPM_BUILD_ROOT}%{_initrddir}
+mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/default
+mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/profile.d
+mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d
+
+# Common
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms
+mkdir -p ${RPM_BUILD_ROOT}%{python_sitearch}/
+mkdir -p ${RPM_BUILD_ROOT}/usr/bin
+cp -r scripts/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts
+install -D systemvm/dist/systemvm.iso ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/systemvm.iso
+install python/lib/cloud_utils.py ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py
+cp -r python/lib/cloudutils ${RPM_BUILD_ROOT}%{python_sitearch}/
+python3 -m py_compile ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py
+python3 -m compileall ${RPM_BUILD_ROOT}%{python_sitearch}/cloudutils
+cp build/gitrev.txt ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts
+cp packaging/centos8/cloudstack-sccs ${RPM_BUILD_ROOT}/usr/bin
+
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts/network/cisco
+cp -r plugins/network-elements/cisco-vnmc/src/main/scripts/network/cisco/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts/network/cisco
+
+# Management
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management
+mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/run
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel
+
+# Setup Jetty
+ln -sf /etc/%{name}/management ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/conf
+ln -sf /var/log/%{name}/management ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/logs
+
+install -D client/target/utilities/bin/cloud-migrate-databases ${RPM_BUILD_ROOT}%{_bindir}/%{name}-migrate-databases
+install -D client/target/utilities/bin/cloud-set-guest-password ${RPM_BUILD_ROOT}%{_bindir}/%{name}-set-guest-password
+install -D client/target/utilities/bin/cloud-set-guest-sshkey ${RPM_BUILD_ROOT}%{_bindir}/%{name}-set-guest-sshkey
+install -D client/target/utilities/bin/cloud-setup-databases ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-databases
+install -D client/target/utilities/bin/cloud-setup-encryption ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-encryption
+install -D client/target/utilities/bin/cloud-setup-management ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-management
+install -D client/target/utilities/bin/cloud-setup-baremetal ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-baremetal
+install -D client/target/utilities/bin/cloud-sysvmadm ${RPM_BUILD_ROOT}%{_bindir}/%{name}-sysvmadm
+install -D client/target/utilities/bin/cloud-update-xenserver-licenses ${RPM_BUILD_ROOT}%{_bindir}/%{name}-update-xenserver-licenses
+
+cp -r client/target/utilities/scripts/db/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup
+
+cp -r client/target/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/
+cp -r client/target/classes/META-INF/webapp ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp
+mv ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib/cloudstack-%{_maventag}.jar
+cp client/target/lib/*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib/
+
+# Don't package the scripts in the management webapp
+rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/scripts
+rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/vms
+
+for name in db.properties server.properties log4j-cloud.xml environment.properties java.security.ciphers
+do
+  cp client/target/conf/$name ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/$name
+done
+
+ln -sf log4j-cloud.xml  ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/log4j.xml
+
+install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/%{name}-external-ipallocator.py
+install -D client/target/pythonlibs/jasypt-1.9.3.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar
+
+install -D packaging/centos8/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator
+install -D packaging/centos8/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud
+install -D packaging/systemd/cloudstack-management.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-management.service
+install -D packaging/systemd/cloudstack-management.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-management
+install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management
+touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid
+#install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina
+
+# Package mysql-connector-python
+wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/ee/ff/48bde5c0f013094d729fe4b0316ba2a24774b3ff1c52d924a8a4cb04078a/six-1.15.0-py2.py3-none-any.whl
+wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/e9/93/4860cebd5ad3ff2664ad3c966490ccb46e3b88458b2095145bca11727ca4/setuptools-47.3.1-py3-none-any.whl
+wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/28/05/9867ef8eafd12265267bee138fa2c46ebf34a276ea4cbe184cba4c606e8b/protobuf-3.12.2-cp36-cp36m-manylinux1_x86_64.whl
+wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/d1/53/4cf90d2fe81b9cdb55dc180951bcec44ea8685665f1bdb1412501dc362dd/mysql_connector_python-8.0.20-cp36-cp36m-manylinux1_x86_64.whl
+
+chmod 440 ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management
+chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt
+chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/management
+chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/work
+chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/temp
+chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management
+chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent
+
+# KVM Agent
+mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/plugins
+install -D packaging/systemd/cloudstack-agent.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-agent.service
+install -D packaging/systemd/cloudstack-rolling-maintenance@.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-rolling-maintenance@.service
+install -D packaging/systemd/cloudstack-agent.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-agent
+install -D agent/target/transformed/agent.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/agent.properties
+install -D agent/target/transformed/environment.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/environment.properties
+install -D agent/target/transformed/log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/log4j-cloud.xml
+install -D agent/target/transformed/cloud-setup-agent ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-agent
+install -D agent/target/transformed/cloudstack-agent-upgrade ${RPM_BUILD_ROOT}%{_bindir}/%{name}-agent-upgrade
+install -D agent/target/transformed/cloud-guest-tool ${RPM_BUILD_ROOT}%{_bindir}/%{name}-guest-tool
+install -D agent/target/transformed/libvirtqemuhook ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/libvirtqemuhook
+install -D agent/target/transformed/rolling-maintenance ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/rolling-maintenance
+install -D agent/target/transformed/cloud-ssh ${RPM_BUILD_ROOT}%{_bindir}/%{name}-ssh
+install -D agent/target/transformed/cloudstack-agent-profile.sh ${RPM_BUILD_ROOT}%{_sysconfdir}/profile.d/%{name}-agent-profile.sh
+install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-agent
+install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar
+cp plugins/hypervisors/kvm/target/dependencies/*  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
+
+# Usage server
+mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib
+install -D usage/target/cloud-usage-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/cloud-usage-%{_maventag}.jar
+install -D usage/target/transformed/db.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage/db.properties
+install -D usage/target/transformed/log4j-cloud_usage.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage/log4j-cloud.xml
+cp usage/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/
+cp client/target/lib/mysql*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/
+install -D packaging/systemd/cloudstack-usage.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-usage.service
+install -D packaging/systemd/cloudstack-usage.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-usage
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/usage/
+
+# CLI
+cp -r cloud-cli/cloudtool ${RPM_BUILD_ROOT}%{python_sitearch}/
+install cloud-cli/cloudapis/cloud.py ${RPM_BUILD_ROOT}%{python_sitearch}/cloudapis.py
+
+# Marvin
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-marvin
+cp tools/marvin/dist/Marvin-*.tar.gz ${RPM_BUILD_ROOT}%{_datadir}/%{name}-marvin/
+
+# integration-tests
+mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-integration-tests
+cp -r test/integration/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-integration-tests/
+
+# MYSQL HA
+if [ "x%{_ossnoss}" == "xnoredist" ] ; then
+  mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-mysql-ha/lib
+  cp -r plugins/database/mysql-ha/target/cloud-plugin-database-mysqlha-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib
+fi
+
+#License files from whisker
+install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-management-%{version}/NOTICE
+install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-management-%{version}/LICENSE
+install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-common-%{version}/NOTICE
+install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-common-%{version}/LICENSE
+install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-agent-%{version}/NOTICE
+install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-agent-%{version}/LICENSE
+install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-usage-%{version}/NOTICE
+install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-usage-%{version}/LICENSE
+install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-cli-%{version}/NOTICE
+install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-cli-%{version}/LICENSE
+install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-marvin-%{version}/NOTICE
+install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-marvin-%{version}/LICENSE
+install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-integration-tests-%{version}/NOTICE
+install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-integration-tests-%{version}/LICENSE
+
+%clean
+[ ${RPM_BUILD_ROOT} != "/" ] && rm -rf ${RPM_BUILD_ROOT}
+
+%preun management
+/usr/bin/systemctl stop cloudstack-management || true
+/usr/bin/systemctl off cloudstack-management || true
+
+%pre management
+id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -c "CloudStack unprivileged user" \
+     -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud|| true
+
+rm -rf %{_localstatedir}/cache/cloudstack
+
+# in case of upgrade to 4.9+ copy commands.properties if not exists in /etc/cloudstack/management/
+if [ "$1" == "2" ] ; then
+    if [ -f "%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/commands.properties" ] && [ ! -f "%{_sysconfdir}/%{name}/management/commands.properties" ] ; then
+        cp -p %{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/commands.properties %{_sysconfdir}/%{name}/management/commands.properties
+    fi
+fi
+
+# Remove old tomcat symlinks and env config file
+if [ -L "%{_datadir}/%{name}-management/lib" ]
+then
+    rm -f %{_datadir}/%{name}-management/bin
+    rm -f %{_datadir}/%{name}-management/lib
+    rm -f %{_datadir}/%{name}-management/temp
+    rm -f %{_datadir}/%{name}-management/work
+    rm -f %{_sysconfdir}/default/%{name}-management
+fi
+
+%post management
+# Install mysql-connector-python
+pip3 install %{_datadir}/%{name}-management/setup/wheel/six-1.15.0-py2.py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/setuptools-47.3.1-py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/protobuf-3.12.2-cp36-cp36m-manylinux1_x86_64.whl %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.20-cp36-cp36m-manylinux1_x86_64.whl
+
+/usr/bin/systemctl on cloudstack-management > /dev/null 2>&1 || true
+
+grep -s -q "db.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties"
+grep -s -q "db.usage.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.usage.driver=jdbc:mysql"  "%{_sysconfdir}/%{name}/management/db.properties"
+grep -s -q "db.simulator.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.simulator.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties"
+
+if [ ! -f %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/vhd-util ] ; then
+    echo Please download vhd-util from http://download.cloudstack.org/tools/vhd-util and put it in
+    echo %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/
+fi
+
+if [ -f %{_sysconfdir}/sysconfig/%{name}-management ] ; then
+    rm -f %{_sysconfdir}/sysconfig/%{name}-management
+fi
+
+chown -R cloud:cloud /var/log/cloudstack/management
+
+systemctl daemon-reload
+
+%preun agent
+/sbin/service cloudstack-agent stop || true
+if [ "$1" == "0" ] ; then
+    /sbin/chkconfig --del cloudstack-agent > /dev/null 2>&1 || true
+fi
+
+%pre agent
+
+# save old configs if they exist (for upgrade). Otherwise we may lose them
+# when the old packages are erased. There are a lot of properties files here.
+if [ -d "%{_sysconfdir}/cloud" ] ; then
+    mv %{_sysconfdir}/cloud %{_sysconfdir}/cloud.rpmsave
+fi
+
+%post agent
+if [ "$1" == "2" ] ; then
+    echo "Running %{_bindir}/%{name}-agent-upgrade to update bridge name for upgrade from CloudStack 4.0.x (and before) to CloudStack 4.1 (and later)"
+    %{_bindir}/%{name}-agent-upgrade
+fi
+if [ ! -d %{_sysconfdir}/libvirt/hooks ] ; then
+    mkdir %{_sysconfdir}/libvirt/hooks
+fi
+cp -a ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/libvirtqemuhook %{_sysconfdir}/libvirt/hooks/qemu
+mkdir -m 0755 -p /usr/share/cloudstack-agent/tmp
+/sbin/service libvirtd restart
+/sbin/systemctl enable cloudstack-agent > /dev/null 2>&1 || true
+/sbin/systemctl enable cloudstack-rolling-maintenance@p > /dev/null 2>&1 || true
+
+# if saved configs from upgrade exist, copy them over
+if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then
+    mv %{_sysconfdir}/%{name}/agent/agent.properties  %{_sysconfdir}/%{name}/agent/agent.properties.rpmnew
+    cp -p %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/%{name}/agent
+    # make sure we only do this on the first install of this RPM, don't want to overwrite on a reinstall
+    mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave
+fi
+
+systemctl daemon-reload
+
+%pre usage
+id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -c "CloudStack unprivileged user" \
+     -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud|| true
+
+%preun usage
+/sbin/service cloudstack-usage stop || true
+if [ "$1" == "0" ] ; then
+    /sbin/chkconfig --del cloudstack-usage > /dev/null 2>&1 || true
+fi
+
+%post usage
+if [ -f "%{_sysconfdir}/%{name}/management/db.properties" ]; then
+    echo Replacing db.properties with management server db.properties
+    rm -f %{_sysconfdir}/%{name}/usage/db.properties
+    ln -s %{_sysconfdir}/%{name}/management/db.properties %{_sysconfdir}/%{name}/usage/db.properties
+    /usr/bin/systemctl enable cloudstack-usage > /dev/null 2>&1 || true
+fi
+
+if [ -f "%{_sysconfdir}/%{name}/management/key" ]; then
+    echo Replacing key with management server key
+    rm -f %{_sysconfdir}/%{name}/usage/key
+    ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key
+fi
+
+if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then
+    ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key
+fi
+
+%post marvin
+pip install --upgrade https://files.pythonhosted.org/packages/ca/ea/1e2553b088bad2f9fa8120c2624f797b2d7450d3b61bb492d29c72e3d3c2/mysql_connector_python-8.0.20-cp27-cp27mu-manylinux1_x86_64.whl
+pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
+
+#No default permission as the permission setup is complex
+%files management
+%defattr(-,root,root,-)
+%dir %{_datadir}/%{name}-management
+%dir %attr(0770,root,cloud) %{_localstatedir}/%{name}/mnt
+%dir %attr(0770,cloud,cloud) %{_localstatedir}/%{name}/management
+%dir %attr(0770,root,cloud) %{_localstatedir}/cache/%{name}/management
+%dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/management
+%config(noreplace) %{_sysconfdir}/default/%{name}-management
+%config(noreplace) %{_sysconfdir}/sudoers.d/%{name}-management
+%config(noreplace) %{_sysconfdir}/security/limits.d/cloud
+%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/db.properties
+%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/server.properties
+%config(noreplace) %{_sysconfdir}/%{name}/management/log4j-cloud.xml
+%config(noreplace) %{_sysconfdir}/%{name}/management/log4j.xml
+%config(noreplace) %{_sysconfdir}/%{name}/management/environment.properties
+%config(noreplace) %{_sysconfdir}/%{name}/management/java.security.ciphers
+%attr(0644,root,root) %{_unitdir}/%{name}-management.service
+%attr(0755,cloud,cloud) %{_localstatedir}/run/%{name}-management.pid
+%attr(0755,root,root) %{_bindir}/%{name}-setup-management
+%attr(0755,root,root) %{_bindir}/%{name}-update-xenserver-licenses
+%{_datadir}/%{name}-management/conf
+%{_datadir}/%{name}-management/lib/*.jar
+%{_datadir}/%{name}-management/logs
+%attr(0755,root,root) %{_bindir}/%{name}-setup-databases
+%attr(0755,root,root) %{_bindir}/%{name}-migrate-databases
+%attr(0755,root,root) %{_bindir}/%{name}-set-guest-password
+%attr(0755,root,root) %{_bindir}/%{name}-set-guest-sshkey
+%attr(0755,root,root) %{_bindir}/%{name}-sysvmadm
+%attr(0755,root,root) %{_bindir}/%{name}-setup-encryption
+%{_datadir}/%{name}-management/setup/*.sql
+%{_datadir}/%{name}-management/setup/*.sh
+%{_datadir}/%{name}-management/setup/server-setup.xml
+%{_datadir}/%{name}-management/webapp/*
+%attr(0755,root,root) %{_bindir}/%{name}-external-ipallocator.py
+%attr(0755,root,root) %{_initrddir}/%{name}-ipallocator
+%dir %attr(0770,root,root) %{_localstatedir}/log/%{name}/ipallocator
+%{_defaultdocdir}/%{name}-management-%{version}/LICENSE
+%{_defaultdocdir}/%{name}-management-%{version}/NOTICE
+#%attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-catalina
+%{_datadir}/%{name}-management/setup/wheel/*.whl
+
+%files agent
+%attr(0755,root,root) %{_bindir}/%{name}-setup-agent
+%attr(0755,root,root) %{_bindir}/%{name}-agent-upgrade
+%attr(0755,root,root) %{_bindir}/%{name}-guest-tool
+%attr(0755,root,root) %{_bindir}/%{name}-ssh
+%attr(0644,root,root) %{_unitdir}/%{name}-agent.service
+%attr(0644,root,root) %{_unitdir}/%{name}-rolling-maintenance@.service
+%config(noreplace) %{_sysconfdir}/default/%{name}-agent
+%attr(0644,root,root) %{_sysconfdir}/profile.d/%{name}-agent-profile.sh
+%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-agent
+%attr(0755,root,root) %{_datadir}/%{name}-common/scripts/network/cisco
+%config(noreplace) %{_sysconfdir}/%{name}/agent
+%dir %{_localstatedir}/log/%{name}/agent
+%attr(0644,root,root) %{_datadir}/%{name}-agent/lib/*.jar
+%attr(0755,root,root) %{_datadir}/%{name}-agent/lib/libvirtqemuhook
+%attr(0755,root,root) %{_datadir}/%{name}-agent/lib/rolling-maintenance
+%dir %{_datadir}/%{name}-agent/plugins
+%{_defaultdocdir}/%{name}-agent-%{version}/LICENSE
+%{_defaultdocdir}/%{name}-agent-%{version}/NOTICE
+
+%files common
+%dir %attr(0755,root,root) %{python_sitearch}/cloudutils
+%dir %attr(0755,root,root) %{_datadir}/%{name}-common/vms
+%attr(0755,root,root) %{_datadir}/%{name}-common/scripts
+%attr(0755,root,root) /usr/bin/cloudstack-sccs
+%attr(0644, root, root) %{_datadir}/%{name}-common/vms/systemvm.iso
+%attr(0644,root,root) %{python_sitearch}/cloud_utils.py
+%attr(0644,root,root) %{python_sitearch}/__pycache__/*
+%attr(0644,root,root) %{python_sitearch}/cloudutils/*
+%attr(0644, root, root) %{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar
+%{_defaultdocdir}/%{name}-common-%{version}/LICENSE
+%{_defaultdocdir}/%{name}-common-%{version}/NOTICE
+
+%files usage
+%attr(0644,root,root) %{_unitdir}/%{name}-usage.service
+%config(noreplace) %{_sysconfdir}/default/%{name}-usage
+%attr(0644,root,root) %{_datadir}/%{name}-usage/*.jar
+%attr(0644,root,root) %{_datadir}/%{name}-usage/lib/*.jar
+%dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/usage
+%attr(0644,root,root) %{_sysconfdir}/%{name}/usage/db.properties
+%attr(0644,root,root) %{_sysconfdir}/%{name}/usage/log4j-cloud.xml
+%{_defaultdocdir}/%{name}-usage-%{version}/LICENSE
+%{_defaultdocdir}/%{name}-usage-%{version}/NOTICE
+
+%files cli
+%attr(0644,root,root) %{python_sitearch}/cloudapis.py
+%attr(0644,root,root) %{python_sitearch}/cloudtool/__init__.py
+%attr(0644,root,root) %{python_sitearch}/cloudtool/utils.py
+%{_defaultdocdir}/%{name}-cli-%{version}/LICENSE
+%{_defaultdocdir}/%{name}-cli-%{version}/NOTICE
+
+%files marvin
+%attr(0644,root,root) %{_datadir}/%{name}-marvin/Marvin*.tar.gz
+%{_defaultdocdir}/%{name}-marvin-%{version}/LICENSE
+%{_defaultdocdir}/%{name}-marvin-%{version}/NOTICE
+
+%files integration-tests
+%attr(0755,root,root) %{_datadir}/%{name}-integration-tests/*
+%{_defaultdocdir}/%{name}-integration-tests-%{version}/LICENSE
+%{_defaultdocdir}/%{name}-integration-tests-%{version}/NOTICE
+
+%if "%{_ossnoss}" == "noredist"
+%files mysql-ha
+%defattr(0644,cloud,cloud,0755)
+%attr(0644,root,root) %{_datadir}/%{name}-management/lib/*mysqlha*jar
+%endif
+
+%files baremetal-agent
+%attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal
+
+%changelog
+* Thu Apr 30 2015 Rohit Yadav <bhaisaab@apache.org> 4.6.0
+- Remove awsapi package
+
+* Wed Nov 19 2014 Hugo Trippaers <hugo@apache.org> 4.6.0
+- Create a specific spec for CentOS 7
+
+* Fri Jul 4 2014 Hugo Trippaers <hugo@apache.org> 4.5.0
+- Add a package for the mysql ha module
+
+* Fri Oct 5 2012 Hugo Trippaers <hugo@apache.org> 4.1.0
+- new style spec file
+

packaging/centos8/cloudstack-agent.te

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module cloudstack-agent 1.0;
+
+require {
+	type nfs_t;
+	type system_conf_t;
+	type mount_t;
+	type qemu_t;
+	class file unlink;
+	class filesystem getattr;
+}
+
+#============= mount_t ==============
+allow mount_t system_conf_t:file unlink;
+
+#============= qemu_t ==============
+allow qemu_t nfs_t:filesystem getattr;

packaging/centos8/cloudstack-sccs

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cat /usr/share/cloudstack-common/scripts/gitrev.txt

packaging/centos8/replace.properties

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+DBUSER=cloud
+DBPW=cloud
+DBROOTPW=
+MSLOG=vmops.log
+APISERVERLOG=api.log
+DBHOST=localhost
+DBDRIVER=jdbc:mysql
+COMPONENTS-SPEC=components-premium.xml
+REMOTEHOST=localhost
+AGENTCLASSPATH=
+AGENTLOG=/var/log/cloudstack/agent/agent.log
+AGENTLOGDIR=/var/log/cloudstack/agent/
+AGENTSYSCONFDIR=/etc/cloudstack/agent
+APISERVERLOG=/var/log/cloudstack/management/apilog.log
+BINDIR=/usr/bin
+COMMONLIBDIR=/usr/share/cloudstack-common
+CONFIGUREVARS=
+DEPSCLASSPATH=
+DOCDIR=
+IPALOCATORLOG=/var/log/cloudstack/management/ipallocator.log
+JAVADIR=/usr/share/java
+LIBEXECDIR=/usr/libexec
+LOCKDIR=/var/lock
+MSCLASSPATH=
+MSCONF=/etc/cloudstack/management
+MSENVIRON=/usr/share/cloudstack-management
+MSLOG=/var/log/cloudstack/management/management-server.log
+MSLOGDIR=/var/log/cloudstack/management/
+MSMNTDIR=/var/cloudstack/mnt
+MSUSER=cloud
+PIDDIR=/var/run
+PLUGINJAVADIR=/usr/share/cloudstack-management/plugin
+PREMIUMJAVADIR=/usr/share/cloudstack-management/premium
+PYTHONDIR=/usr/lib/python3.6/site-packages/
+SERVERSYSCONFDIR=/etc/sysconfig
+SETUPDATADIR=/usr/share/cloudstack-management/setup
+SYSCONFDIR=/etc/sysconfig
+SYSTEMCLASSPATH=
+SYSTEMJARS=
+USAGECLASSPATH=
+USAGELOG=/var/log/cloudstack/usage/usage.log
+USAGESYSCONFDIR=/etc/sysconfig

packaging/debian/replace.properties

@@ -48,7 +48,7 @@ MSUSER=cloud
 PIDDIR=/var/run
 PLUGINJAVADIR=
 PREMIUMJAVADIR=
-PYTHONDIR=/usr/lib/python2.6/site-packages/
+PYTHONDIR=/usr/lib/python3/site-packages/
 SERVERSYSCONFDIR=/etc/cloudstack/server
 SETUPDATADIR=/usr/share/cloudstack-management/setup
 SYSCONFDIR=/etc

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java

@@ -149,6 +149,8 @@ public class QemuImg {
          */
         s.add("-f");
         if (backingFile != null) {
+            s.add(backingFile.getFormat().toString());
+            s.add("-F");
             s.add(backingFile.getFormat().toString());
             s.add("-b");
             s.add(backingFile.getFileName());

plugins/hypervisors/ovm/src/main/scripts/vm/hypervisor/ovm/OvmNetworkModule.py

@@ -184,8 +184,8 @@ class OvmNetwork(OvmObject):
         @return : success
         ex. {bridge:xapi100, attach:eth0.100}
         create bridge interface, and attached it 
-        cmd 1: brctl addbr bridge
+        cmd 1: ip link add bridge
-        cmd 2: brctl addif brdige attach
+        cmd 2: ip link set dev
         """
         
         if "xenbr" not in bridge.name and "vlan" not in bridge.name:
@@ -206,8 +206,8 @@ class OvmNetwork(OvmObject):
             logger.error(self._createBridge, msg)
             raise Exception(msg)
 
-        doCmd(['brctl', 'addbr', bridge.name])
+        doCmd(['ip', 'link', 'add', 'name', bridge.name, 'type', 'bridge'])
-        doCmd(['brctl', 'addif', bridge.name, bridge.attach])
+        doCmd(['ip', 'link', 'set', 'dev', bridge.attach, 'master', bridge.name])
         self.bringUP(bridge.name)
         logger.debug(self._createBridge, "Create bridge %s on %s successfully"%(bridge.name, bridge.attach))
         return self.bridges[bridge.name]
@@ -228,7 +228,7 @@ class OvmNetwork(OvmObject):
             logger.debug(self._deleteBridge, "There are still some interfaces(%s) on bridge %s"%(bridge.interfaces, bridge.name))
             return False
         self.bringDown(bridge.name)
-        doCmd(['brctl', 'delbr', bridge.name])
+        doCmd(['ip', 'link', 'del', bridge.name])
         logger.debug(self._deleteBridge, "Delete bridge %s successfully"%bridge.name)
         return True
         

plugins/hypervisors/ovm/src/main/scripts/vm/hypervisor/ovm/OvmSecurityGroupModule.py

@@ -107,7 +107,7 @@ class OvmSecurityGroup(OvmObject):
                 execute("iptables -I FORWARD -o " + bridge_name + " -j DROP")
                 execute("iptables -I FORWARD -i " + bridge_name + " -m physdev --physdev-is-bridged -j " + brfw)
                 execute("iptables -I FORWARD -o " + bridge_name + " -m physdev --physdev-is-bridged -j " + brfw)
-                phydev = execute("brctl show |grep " + bridge_name + " | awk '{print $4}'").strip()
+                phydev = execute("ip link show type bridge |grep " + bridge_name + " | awk '{print $4}'").strip()
                 execute("iptables -A " + brfw + " -m physdev --physdev-is-bridged --physdev-out " + phydev + " -j ACCEPT")
                 execute("iptables -A " + brfw + " -m state --state RELATED,ESTABLISHED -j ACCEPT")
                 execute("iptables -A " + brfw + " -m physdev --physdev-is-bridged --physdev-is-out -j " + brfwout)

python/bindir/cloud-external-ipallocator.py

@@ -1,4 +1,4 @@
-#! /usr/bin/python
+#! /usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -37,25 +37,25 @@ class dhcp:
 		self.netmask=None
 		self.initialized=False
 
-		options = augtool.match("/files/etc/dnsmasq.conf/dhcp-option").stdout.strip()
+		options = augtool.match("/files/etc/dnsmasq.conf/dhcp-option").stdout.decode('utf-8').strip()
 		for option in options.splitlines():
 			if option.find("option:router") != -1:
 				self.router = option.split("=")[1].strip().split(",")[1]
-				print self.router
+				print(self.router)
 
-		dhcp_range = augtool.get("/files/etc/dnsmasq.conf/dhcp-range").stdout.strip()
+		dhcp_range = augtool.get("/files/etc/dnsmasq.conf/dhcp-range").stdout.decode('utf-8').strip()
 		dhcp_start = dhcp_range.split("=")[1].strip().split(",")[0]
 		dhcp_end = dhcp_range.split("=")[1].strip().split(",")[1]
 		self.netmask = dhcp_range.split("=")[1].strip().split(",")[2]
-		print dhcp_start, dhcp_end, self.netmask
+		print(dhcp_start, dhcp_end, self.netmask)
 
  		start_ip_num = self.ipToNum(dhcp_start);
 		end_ip_num =  self.ipToNum(dhcp_end)
-		print start_ip_num, end_ip_num
+		print(start_ip_num, end_ip_num)
 	
 		for ip in range(start_ip_num, end_ip_num + 1):
 			self.availIP.append(ip)	
-		print self.availIP[0], self.availIP[len(self.availIP) - 1]	
+		print(self.availIP[0], self.availIP[len(self.availIP) - 1])	
 		
 		#load the ip already allocated
 		self.reloadAllocatedIP()
@@ -87,7 +87,7 @@ class dhcp:
 	getInstance = staticmethod(getInstance)
 
 	def reloadAllocatedIP(self):
-		dhcp_hosts = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.strip().splitlines()
+		dhcp_hosts = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.decode('utf-8').strip().splitlines()
 		
 		for host in dhcp_hosts:
 			if host.find("dhcp-host") != -1:
@@ -97,7 +97,7 @@ class dhcp:
 		
 	def allocateIP(self, mac):
 		newIP = self.getFreeIP()
-		dhcp_host = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.strip()
+		dhcp_host = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.decode('utf-8').strip()
 		cnt = len(dhcp_host.splitlines()) + 1
 		script = """set %s %s
 			    save"""%("/files/etc/dnsmasq.conf/dhcp-host[" + str(cnt) + "]", str(mac) + "," + newIP)
@@ -107,17 +107,17 @@ class dhcp:
 		return newIP
 
 	def releaseIP(self, ip):
-		dhcp_host = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.strip()
+		dhcp_host = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.decode('utf-8').strip()
 		path = None
 		for host in dhcp_host.splitlines():
 			if host.find(ip) != -1:
 				path = host.split("=")[0].strip()
 				
 		if path == None:
-			print "Can't find " + str(ip) + " in conf file"
+			print("Can't find " + str(ip) + " in conf file")
 			return None
 
-		print path
+		print(path)
 		script = """rm %s
 			    save"""%(path)
 		augtool < script
@@ -132,7 +132,7 @@ class ipallocator:
 		try:
 			user_data = web.input()
 			command = user_data.command
-			print "Processing: " + command
+			print("Processing: " + command)
 
 			dhcpInit = dhcp.getInstance()
 
@@ -140,11 +140,11 @@ class ipallocator:
 				mac = user_data.mac
 				zone_id = user_data.dc
 				pod_id = user_data.pod
-				print mac, zone_id, pod_id
+				print(mac, zone_id, pod_id)
 				freeIP = dhcpInit.allocateIP(mac)
 				if not freeIP:
 					return "0,0,0"
-				print "Find an available IP: " + freeIP
+				print("Find an available IP: " + freeIP)
 		
 				return freeIP + "," + dhcpInit.getNetmask() + "," + dhcpInit.getRouter()
 			elif command == "releaseIpAddr":

python/bindir/cloud-grab-dependent-library-versions

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -18,16 +18,16 @@
 
 import subprocess
 
-depLibraries = ['python', 'bzip2', 'gzip', 'unzip', 'openssh-clients', 'nfs-utils', 'wget', 'ws-commons-util', 'commons-dbcp',
+depLibraries = ['bzip2', 'gzip', 'unzip', 'openssh-clients', 'nfs-utils', 'wget', 'ws-commons-util', 'commons-dbcp',
-                'commons-collections', 'commons-httpclient', 'jpackage-utils', 'MySQL-python', 'python-paramiko', 'ipmitool', 'commons-httpclient', 'commons-collections',
+                'commons-collections', 'commons-httpclient', 'jpackage-utils', 'mysql-connector-python3', 'python-paramiko', 'ipmitool', 'commons-httpclient', 'commons-collections',
                 'commons-pool', 'commons-dbcp', 'jakarta-commons-logging', 'java-*-openjdk']
 
 def runCmd(cmds):
     process = subprocess.Popen(' '.join(cmds), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, stderr = process.communicate()
     if process.returncode != 0:
-        raise Exception(stderr)
+        raise Exception(stderr.decode('utf-8'))
-    return stdout
+    return stdout.decode('utf-8')
 
 
 def getDependentLibraryInfo():
@@ -43,12 +43,12 @@ def getDependentLibraryInfo():
             key = key.strip()
             value = value.strip()
             if key == 'Name' and "*" not in pkgname and pkgname not in value:
-                print "Required package name %s doesn't equal to package %s installed"%(pkgname, value)
+                print("Required package name %s doesn't equal to package %s installed"%(pkgname, value))
                 return 'UNKNOWN'
             if 'Version' in key: return value
             if 'Description' in key: return 'UNKNOWN' # we hit the end
         return 'UNKNOWN'
-    
+
     libraryMap = {}
     for l in depLibraries:
         cmd = ['yum', 'info', '"%s"'%l]
@@ -56,20 +56,26 @@ def getDependentLibraryInfo():
             result = runCmd(cmd)
             version = getVersion(result, l)
             libraryMap[l] = version
-        except Exception, e:
+        except Exception as e:
-            print "When finding %s, encounters %s"%(l, e)
+            print("When finding %s, encounters %s"%(l, e))
             continue
     return libraryMap
 
 def arrangeOutPut(libraryMap):
     msg = ['\n\n\nBelow is the checking list of library version that CloudStack depends on:']
     for l in depLibraries:
-        if libraryMap.has_key(l):
+        if l in libraryMap:
             entry = "%-40s:    %s"%(l, libraryMap[l])
         else:
             entry = "%-40s:    %s"%(l, 'UNKNOWN')
         msg.append(entry)
-    print '\n'.join(msg)
+    print('\n'.join(msg))
         
 if __name__ == '__main__':
+    pythonDepLibraries = ['python', 'python3']
+    relver = runCmd(['rpm', '-q', 'centos-release'])
+    if relver.startswith('centos-release-') and int(relver[len('centos-release-')]) >= 8:
+        pythonDepLibraries = ['python2', 'python36']
+    depLibraries = pythonDepLibraries + depLibraries
+
     arrangeOutPut(getDependentLibraryInfo())

python/bindir/cloud-setup-baremetal

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

python/incubation/cloud-web-ipallocator.py

@@ -1,4 +1,4 @@
-#! /usr/bin/python
+#! /usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -37,25 +37,25 @@ class dhcp:
 		self.netmask=None
 		self.initialized=False
 
-		options = augtool.match("/files/etc/dnsmasq.conf/dhcp-option").stdout.strip()
+		options = augtool.match("/files/etc/dnsmasq.conf/dhcp-option").stdout.decode('utf-8').strip()
 		for option in options.splitlines():
 			if option.find("option:router") != -1:
 				self.router = option.split("=")[1].strip().split(",")[1]
-				print self.router
+				print(self.router)
 
-		dhcp_range = augtool.get("/files/etc/dnsmasq.conf/dhcp-range").stdout.strip()
+		dhcp_range = augtool.get("/files/etc/dnsmasq.conf/dhcp-range").stdout.decode('utf-8').strip()
 		dhcp_start = dhcp_range.split("=")[1].strip().split(",")[0]
 		dhcp_end = dhcp_range.split("=")[1].strip().split(",")[1]
 		self.netmask = dhcp_range.split("=")[1].strip().split(",")[2]
-		print dhcp_start, dhcp_end, self.netmask
+		print(dhcp_start, dhcp_end, self.netmask)
 
  		start_ip_num = self.ipToNum(dhcp_start);
 		end_ip_num =  self.ipToNum(dhcp_end)
-		print start_ip_num, end_ip_num
+		print(start_ip_num, end_ip_num)
 	
 		for ip in range(start_ip_num, end_ip_num + 1):
 			self.availIP.append(ip)	
-		print self.availIP[0], self.availIP[len(self.availIP) - 1]	
+		print(self.availIP[0], self.availIP[len(self.availIP) - 1])	
 		
 		#load the ip already allocated
 		self.reloadAllocatedIP()
@@ -87,7 +87,7 @@ class dhcp:
 	getInstance = staticmethod(getInstance)
 
 	def reloadAllocatedIP(self):
-		dhcp_hosts = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.strip().splitlines()
+		dhcp_hosts = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.decode('utf-8').strip().splitlines()
 		
 		for host in dhcp_hosts:
 			if host.find("dhcp-host") != -1:
@@ -97,7 +97,7 @@ class dhcp:
 		
 	def allocateIP(self, mac):
 		newIP = self.getFreeIP()
-		dhcp_host = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.strip()
+		dhcp_host = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.decode('utf-8').strip()
 		cnt = len(dhcp_host.splitlines()) + 1
 		script = """set %s %s
 			    save"""%("/files/etc/dnsmasq.conf/dhcp-host[" + str(cnt) + "]", str(mac) + "," + newIP)
@@ -107,17 +107,17 @@ class dhcp:
 		return newIP
 
 	def releaseIP(self, ip):
-		dhcp_host = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.strip()
+		dhcp_host = augtool.match("/files/etc/dnsmasq.conf/dhcp-host").stdout.decode('utf-8').strip()
 		path = None
 		for host in dhcp_host.splitlines():
 			if host.find(ip) != -1:
 				path = host.split("=")[0].strip()
 				
 		if path == None:
-			print "Can't find " + str(ip) + " in conf file"
+			print("Can't find " + str(ip) + " in conf file")
 			return None
 
-		print path
+		print(path)
 		script = """rm %s
 			    save"""%(path)
 		augtool < script
@@ -132,7 +132,7 @@ class ipallocator:
 		try:
 			user_data = web.input()
 			command = user_data.command
-			print "Processing: " + command
+			print("Processing: " + command)
 
 			dhcpInit = dhcp.getInstance()
 
@@ -140,11 +140,11 @@ class ipallocator:
 				mac = user_data.mac
 				zone_id = user_data.dc
 				pod_id = user_data.pod
-				print mac, zone_id, pod_id
+				print(mac, zone_id, pod_id)
 				freeIP = dhcpInit.allocateIP(mac)
 				if not freeIP:
 					return "0,0,0"
-				print "Find an available IP: " + freeIP
+				print("Find an available IP: " + freeIP)
 		
 				return freeIP + "," + dhcpInit.getNetmask() + "," + dhcpInit.getRouter()
 			elif command == "releaseIpAddr":

python/lib/cloud_utils.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -24,7 +24,7 @@
 """CloudStack Python utility library"""
 
 import sys, os, subprocess, errno, re, time, glob
-import urllib2
+import urllib.request, urllib.error, urllib.parse
 import xml.dom.minidom
 import logging
 import socket
@@ -65,14 +65,14 @@ IPV6 = 6
 if os.path.exists("/etc/fedora-release"): distro = Fedora
 elif os.path.exists("/etc/centos-release"): distro = CentOS
 elif os.path.exists("/etc/redhat-release"):
-    version = file("/etc/redhat-release").readline()
+    version = open("/etc/redhat-release").readline()
     if version.find("Red Hat Enterprise Linux Server release 6") != -1:
         distro = RHEL6
     elif version.find("CentOS") != -1:
         distro = CentOS
     else:
         distro = CentOS
-elif os.path.exists("/etc/legal") and "Ubuntu" in file("/etc/legal").read(-1): distro = Ubuntu
+elif os.path.exists("/etc/legal") and "Ubuntu" in open("/etc/legal").read(-1): distro = Ubuntu
 else: distro = Unknown
 logFileName=None
 # ==================  LIBRARY UTILITY CODE=============
@@ -80,7 +80,7 @@ def setLogFile(logFile):
 	global logFileName
 	logFileName=logFile
 def read_properties(propfile):
-	if not hasattr(propfile,"read"): propfile = file(propfile)
+	if not hasattr(propfile,"read"): propfile = open(propfile)
 	properties = propfile.read().splitlines()
 	properties = [ s.strip() for s in properties ]
 	properties = [ s for s in properties if
@@ -211,7 +211,6 @@ augtool = Command("augtool")
 ifconfig = Command("ifconfig")
 ifdown = Command("ifdown")
 ifup = Command("ifup")
-brctl = Command("brctl")
 uuidgen = Command("uuidgen")
 
 
@@ -225,7 +224,7 @@ def is_service_running(servicename):
 		else:
 			# retcode 0, service running
 			return True
-	except CalledProcessError,e:
+	except CalledProcessError as e:
 		# retcode nonzero, service not running
 		return False
 
@@ -261,7 +260,7 @@ def enable_service(servicename,forcestart=False):
 
 
 def replace_line(f,startswith,stanza,always_add=False):
-	lines = [ s.strip() for s in file(f).readlines() ]
+	lines = [ s.strip() for s in open(f).readlines() ]
 	newlines = []
 	replaced = False
 	for line in lines:
@@ -271,7 +270,7 @@ def replace_line(f,startswith,stanza,always_add=False):
 		else: newlines.append(line)
 	if not replaced and always_add: newlines.append(stanza)
 	newlines = [ s + '\n' for s in newlines ]
-	file(f,"w").writelines(newlines)
+	open(f,"w").writelines(newlines)
 
 def replace_or_add_line(f,startswith,stanza):
 	return replace_line(f,startswith,stanza,always_add=True)
@@ -301,11 +300,11 @@ def check_kvm():
 			return True
 		except CalledProcessError:
 			raise CheckFailed("KVM is not correctly installed on this system, or support for it is not enabled in the BIOS")
-		except OSError,e:
+		except OSError as e:
 			if e.errno is errno.ENOENT: raise CheckFailed("KVM is not correctly installed on this system, or support for it is not enabled in the BIOS")
 			raise
 		return True
-	raise AssertionError, "check_kvm() should have never reached this part"
+	raise AssertionError("check_kvm() should have never reached this part")
 
 def check_cgroups():
 	return glob.glob("/*/cpu.shares")
@@ -316,14 +315,14 @@ def check_selinux():
 	enforcing = False
 	config_enforcing = False
 	try:
-		output = getenforce().stdout.strip()
+		output = getenforce().stdout.decode('utf-8').strip()
 		if "nforcing" in output:
 			enforcing = True
-		if any ( [ s.startswith("SELINUX=enforcing") for s in file("/etc/selinux/config").readlines() ] ):
+		if any ( [ s.startswith("SELINUX=enforcing") for s in open("/etc/selinux/config").readlines() ] ):
 			config_enforcing = True
 		else:
 			config_enforcing = False
-	except (IOError,OSError),e:
+	except (IOError,OSError) as e:
 		if e.errno == 2: pass
 		else: raise CheckFailed("An unknown error (%s) took place while checking for SELinux"%str(e))
 	if enforcing:
@@ -334,8 +333,8 @@ def check_selinux():
 We strongly suggest you doing the option 1 that makes sure SELinux goes into permissive after system reboot.\n''')
 
 	if config_enforcing:
-		print "WARNING: We detected that your SELinux is not configured in permissive. to make sure cloudstack won't block by \
+		print("WARNING: We detected that your SELinux is not configured in permissive. to make sure cloudstack won't block by \
-SELinux after system reboot, we strongly suggest you setting it in permissive in /etc/selinux/config, then reboot the machine."
+SELinux after system reboot, we strongly suggest you setting it in permissive in /etc/selinux/config, then reboot the machine.")
 
 
 def preflight_checks(do_check_kvm=True):
@@ -399,7 +398,7 @@ class SetupNetworking(ConfigTask):
 	def __init__(self,brname, pubNic, prvNic):
 		ConfigTask.__init__(self)
 		self.brname = brname
-  	        self.pubNic = pubNic
+		self.pubNic = pubNic
 		self.prvNic = prvNic
 		self.runtime_state_changed = False
 		self.was_nm_service_running = None
@@ -431,7 +430,7 @@ class SetupNetworking(ConfigTask):
 				if not alreadysetup:
 					alreadysetup = augtool.match("/files/etc/network/interfaces/iface",self.brname).stdout.strip()
 			return alreadysetup
-		except OSError,e:
+		except OSError as e:
 			if e.errno is 2: raise TaskFailed("augtool has not been properly installed on this system")
 			raise
 
@@ -441,8 +440,8 @@ class SetupNetworking(ConfigTask):
 		try:
 			o = ifconfig(self.brname)
 			bridge_exists = True
-		except CalledProcessError,e:
+		except CalledProcessError as e:
-			print e.stdout + e.stderr
+			print(e.stdout + e.stderr)
 			bridge_exists = False
 			
 		if bridge_exists:
@@ -454,7 +453,7 @@ class SetupNetworking(ConfigTask):
 			except CalledProcessError: pass
 			try: ifconfig(self.brname,"down")
 			except CalledProcessError: pass
-			try: brctl("delbr",self.brname)
+			try: ip("link del",self.brname)
 			except CalledProcessError: pass
 			try: ifdown("--force",self.brname)
 			except CalledProcessError: pass
@@ -471,7 +470,7 @@ class SetupNetworking(ConfigTask):
 			stop_service(self.netservice,force=True)
 			time.sleep(1)
 			try: start_service(self.netservice,force=True)
-			except CalledProcessError,e:
+			except CalledProcessError as e:
 				if e.returncode == 1: pass
 				else: raise
 			time.sleep(1)
@@ -543,17 +542,17 @@ class SetupNetworking(ConfigTask):
 		yield "Creating Cloud bridging device and making device %s member of this bridge"%dev
 
 		if distro in (Fedora, CentOS, RHEL6):
-			ifcfgtext = file(pathtoconfigfile).read()
+			ifcfgtext = open(pathtoconfigfile).read()
 			newf = "/etc/sysconfig/network-scripts/ifcfg-%s"%self.brname
 			#def restore():
 				#try: os.unlink(newf)
 				#except OSError,e:
 					#if errno == 2: pass
 					#raise
-				#try: file(pathtoconfigfile,"w").write(ifcfgtext)
+				#try: open(pathtoconfigfile,"w").write(ifcfgtext)
 				#except OSError,e: raise
 
-			f = file(newf,"w") ; f.write(ifcfgtext) ; f.flush() ; f.close()
+			f = open(newf,"w") ; f.write(ifcfgtext) ; f.flush() ; f.close()
 			innewconfigfile = "/files" + newf
 
 			script = """set %s/DEVICE %s
@@ -580,18 +579,18 @@ save"""%(innewconfigfile,self.brname,innewconfigfile,self.brname,innewconfigfile
 			try:
 				returned = augtool < script
 				if "Saved 2 file" not in returned.stdout:
-					print returned.stdout + returned.stderr
+					print(returned.stdout + returned.stderr)
 					#restore()
 					raise TaskFailed("Network reconfiguration failed.")
 				else:
 					yield "Network reconfiguration complete"
-			except CalledProcessError,e:
+			except CalledProcessError as e:
 				#restore()
-				print e.stdout + e.stderr
+				print(e.stdout + e.stderr)
 				raise TaskFailed("Network reconfiguration failed")
 		else: # Not fedora
-			backup = file("/etc/network/interfaces").read(-1)
+			backup = open("/etc/network/interfaces").read(-1)
-			#restore = lambda: file("/etc/network/interfaces","w").write(backup)
+			#restore = lambda: open("/etc/network/interfaces","w").write(backup)
 
 			script = """set %s %s
 set %s %s
@@ -607,15 +606,15 @@ save"""%(automatic,self.brname,inconfigfile,self.brname,inconfigfile,dev)
 					raise TaskFailed("Network reconfiguration failed.")
 				else:
 					yield "Network reconfiguration complete"
-			except CalledProcessError,e:
+			except CalledProcessError as e:
 				#restore()
-				print e.stdout + e.stderr
+				print(e.stdout + e.stderr)
 				raise TaskFailed("Network reconfiguration failed")
 		
 		yield "We are going to restart network services now, to make the network changes take effect.  Hit ENTER when you are ready."
 		if self.isAutoMode(): pass
-        	else:
+		else:
-		    raw_input()
+		    input()
 		
 		# if we reach here, then if something goes wrong we should attempt to revert the runinng state
 		# if not, then no point
@@ -625,15 +624,15 @@ save"""%(automatic,self.brname,inconfigfile,self.brname,inconfigfile,dev)
 		if distro is Ubuntu: ifup(self.brname,stdout=None,stderr=None)
 		stop_service(self.netservice)
 		try: enable_service(self.netservice,forcestart=True)
-		except CalledProcessError,e:
+		except CalledProcessError as e:
 			if e.returncode == 1: pass
 			else: raise
 		
 		yield "Verifying that the bridge is up"
 		try:
 			o = ifconfig(self.brname)
-		except CalledProcessError,e:
+		except CalledProcessError as e:
-			print e.stdout + e.stderr
+			print(e.stdout + e.stderr)
 			raise TaskFailed("The bridge could not be set up properly")
 		
 		yield "Networking restart done"
@@ -645,13 +644,13 @@ class SetupCgConfig(ConfigTask):
 	def done(self):
 		
 		try:
-			return "group virt" in file("/etc/cgconfig.conf","r").read(-1)
+			return "group virt" in open("/etc/cgconfig.conf","r").read(-1)
-		except IOError,e:
+		except IOError as e:
 			if e.errno is 2: raise TaskFailed("cgconfig has not been properly installed on this system")
 			raise
 		
 	def execute(self):
-		cgconfig = file("/etc/cgconfig.conf","r").read(-1)
+		cgconfig = open("/etc/cgconfig.conf","r").read(-1)
 		cgconfig = cgconfig + """
 group virt {
 	cpu {
@@ -659,7 +658,7 @@ group virt {
 	}
 }
 """
-		file("/etc/cgconfig.conf","w").write(cgconfig)
+		open("/etc/cgconfig.conf","w").write(cgconfig)
 		
 		stop_service("cgconfig")
 		enable_service("cgconfig",forcestart=True)
@@ -671,15 +670,15 @@ class SetupCgRules(ConfigTask):
 	
 	def done(self):
 		try:
-			return self.cfgline in file("/etc/cgrules.conf","r").read(-1)
+			return self.cfgline in open("/etc/cgrules.conf","r").read(-1)
-		except IOError,e:
+		except IOError as e:
 			if e.errno is 2: raise TaskFailed("cgrulesd has not been properly installed on this system")
 			raise
 	
 	def execute(self):
-		cgrules = file("/etc/cgrules.conf","r").read(-1)
+		cgrules = open("/etc/cgrules.conf","r").read(-1)
 		cgrules = cgrules + "\n" + self.cfgline + "\n"
-		file("/etc/cgrules.conf","w").write(cgrules)
+		open("/etc/cgrules.conf","w").write(cgrules)
 		
 		stop_service("cgred")
 		enable_service("cgred")
@@ -692,15 +691,15 @@ class SetupSecurityDriver(ConfigTask):
 	
 	def done(self):
 		try:
-			return self.cfgline in file(self.filename,"r").read(-1)
+			return self.cfgline in open(self.filename,"r").read(-1)
-		except IOError,e:
+		except IOError as e:
 			if e.errno is 2: raise TaskFailed("qemu has not been properly installed on this system")
 			raise
 	
 	def execute(self):
-		libvirtqemu = file(self.filename,"r").read(-1)
+		libvirtqemu = open(self.filename,"r").read(-1)
 		libvirtqemu = libvirtqemu + "\n" + self.cfgline + "\n"
-		file("/etc/libvirt/qemu.conf","w").write(libvirtqemu)
+		open("/etc/libvirt/qemu.conf","w").write(libvirtqemu)
 
 
 class SetupLibvirt(ConfigTask):
@@ -710,19 +709,19 @@ class SetupLibvirt(ConfigTask):
 		try:
 			if distro in (Fedora,CentOS, RHEL6): 	 libvirtfile = "/etc/sysconfig/libvirtd"
 			elif distro is Ubuntu:	 libvirtfile = "/etc/default/libvirt-bin"
-			else: raise AssertionError, "We should not reach this"
+			else: raise AssertionError("We should not reach this")
-			return self.cfgline in file(libvirtfile,"r").read(-1)
+			return self.cfgline in open(libvirtfile,"r").read(-1)
-		except IOError,e:
+		except IOError as e:
 			if e.errno is 2: raise TaskFailed("libvirt has not been properly installed on this system")
 			raise
 	
 	def execute(self):
 		if distro in (Fedora,CentOS, RHEL6): 	 libvirtfile = "/etc/sysconfig/libvirtd"
 		elif distro is Ubuntu:	 libvirtfile = "/etc/default/libvirt-bin"
-		else: raise AssertionError, "We should not reach this"
+		else: raise AssertionError("We should not reach this")
-		libvirtbin = file(libvirtfile,"r").read(-1)
+		libvirtbin = open(libvirtfile,"r").read(-1)
 		libvirtbin = libvirtbin + "\n" + self.cfgline + "\n"
-		file(libvirtfile,"w").write(libvirtbin)
+		open(libvirtfile,"w").write(libvirtbin)
 		
 		if distro in (CentOS, Fedora, RHEL6):	svc = "libvirtd"
 		else:					svc = "libvirt-bin"
@@ -740,9 +739,9 @@ class SetupLiveMigration(ConfigTask):
 	
 	def done(self):
 		try:
-			lines = [ s.strip() for s in file("/etc/libvirt/libvirtd.conf").readlines() ]
+			lines = [ s.strip() for s in open("/etc/libvirt/libvirtd.conf").readlines() ]
 			if all( [ stanza in lines for stanza in self.stanzas ] ): return True
-		except IOError,e:
+		except IOError as e:
 			if e.errno is 2: raise TaskFailed("libvirt has not been properly installed on this system")
 			raise
 	
@@ -807,7 +806,7 @@ class SetupFirewall(ConfigTask):
 		ports = "22 1798 16509 16514".split()
 		if distro in (Fedora , CentOS, RHEL6):
 			for p in ports: iptables("-I","INPUT","1","-p","tcp","--dport",p,'-j','ACCEPT')
-			o = service.iptables.save() ; print o.stdout + o.stderr
+			o = service.iptables.save() ; print(o.stdout + o.stderr)
 		else:
 			for p in ports: ufw.allow(p)
 
@@ -894,7 +893,7 @@ def remove_backup(targetdir):
 	check_call( ["rm","-rf",targetdir] )
 
 def list_zonespods(host):
-	text = urllib2.urlopen('http://%s:8096/client/api?command=listPods'%host).read(-1)
+	text = urllib.request.urlopen('http://%s:8096/client/api?command=listPods'%host).read(-1)
 	dom = xml.dom.minidom.parseString(text) 
 	x = [ (zonename,podname)
 		for pod in dom.childNodes[0].childNodes  
@@ -908,13 +907,13 @@ def prompt_for_hostpods(zonespods):
 	Returns (zone,pod) or None if the user made the default selection."""
 	while True:
 		stderr("Type the number of the zone and pod combination this host belongs to (hit ENTER to skip this step)")
-		print "  N) ZONE, POD" 
+		print("  N) ZONE, POD") 
-		print "================"
+		print("================")
 		for n,(z,p) in enumerate(zonespods):
-			print "%3d) %s, %s"%(n,z,p)
+			print("%3d) %s, %s"%(n,z,p))
-		print "================"
+		print("================")
-		print "> ",
+		print("> ", end=' ')
-		zoneandpod = raw_input().strip()
+		zoneandpod = input().strip()
 		
 		if not zoneandpod:
 			# we go with default, do not touch anything, just break
@@ -923,8 +922,8 @@ def prompt_for_hostpods(zonespods):
 		try:
 			# if parsing fails as an int, just vomit and retry
 			zoneandpod = int(zoneandpod)
-			if zoneandpod >= len(zonespods) or zoneandpod < 0: raise ValueError, "%s out of bounds"%zoneandpod
+			if zoneandpod >= len(zonespods) or zoneandpod < 0: raise ValueError("%s out of bounds"%zoneandpod)
-		except ValueError,e:
+		except ValueError as e:
 			stderr(str(e))
 			continue # re-ask
 		
@@ -941,13 +940,13 @@ def device_exist(devName):
 		else:
 			alreadysetup = augtool.match("/files/etc/network/interfaces/iface",devName).stdout.strip()
 		return alreadysetup
-	except OSError,e:
+	except OSError as e:
 		return False		
 	
 def setup_agent_config(configfile, host, zone, pod, cluster, guid, pubNic, prvNic):
 	stderr("Examining Agent configuration")
 	fn = configfile
-	text = file(fn).read(-1)
+	text = open(fn).read(-1)
 	lines = [ s.strip() for s in text.splitlines() ]
 	confopts = dict([ m.split("=",1) for m in lines if "=" in m and not m.startswith("#") ])
 	confposes = dict([ (m.split("=",1)[0],n) for n,m in enumerate(lines) if "=" in m and not m.startswith("#") ])
@@ -963,8 +962,8 @@ def setup_agent_config(configfile, host, zone, pod, cluster, guid, pubNic, prvNi
 		try: host = confopts["host"]
 		except KeyError: host = "localhost"
 		stderr("Please enter the host name of the management server that this agent will connect to: (just hit ENTER to go with %s)",host)
-		print "> ",
+		print("> ", end=' ')
-		newhost = raw_input().strip()
+		newhost = input().strip()
 		if newhost: host = newhost
 
 	confopts["host"] = host
@@ -982,54 +981,7 @@ def setup_agent_config(configfile, host, zone, pod, cluster, guid, pubNic, prvNi
 	stderr("Querying %s for zones and pods",host)
 	
 	try:
-	    if zone == None or pod == None:
+		if zone == None or pod == None:
-			x = list_zonespods(confopts['host'])
-			zoneandpod = prompt_for_hostpods(x)
-			if zoneandpod:
-				confopts["zone"],confopts["pod"] = zoneandpod
-				stderr("You selected zone %s pod %s",confopts["zone"],confopts["pod"])
-			else:
-				stderr("Skipped -- using the previous zone %s pod %s",confopts["zone"],confopts["pod"])
-	    else:
-			confopts["zone"] = zone
-			confopts["pod"] = pod
-			confopts["cluster"] = cluster
-	except (urllib2.URLError,urllib2.HTTPError),e:
-		stderr("Query failed: %s.  Defaulting to zone %s pod %s",str(e),confopts["zone"],confopts["pod"])
-
-	for opt,val in confopts.items():
-		line = "=".join([opt,val])
-		if opt not in confposes: lines.append(line)
-		else: lines[confposes[opt]] = line
-	
-	text = "\n".join(lines)
-	file(fn,"w").write(text)
-
-def setup_consoleproxy_config(configfile, host, zone, pod):
-	stderr("Examining Console Proxy configuration")
-	fn = configfile
-	text = file(fn).read(-1)
-	lines = [ s.strip() for s in text.splitlines() ]
-	confopts = dict([ m.split("=",1) for m in lines if "=" in m and not m.startswith("#") ])
-	confposes = dict([ (m.split("=",1)[0],n) for n,m in enumerate(lines) if "=" in m and not m.startswith("#") ])
-
-	if not "guid" in confopts:
-		stderr("Generating GUID for this Console Proxy")
-		confopts['guid'] = uuidgen().stdout.strip()
-
-        if host == None:
-		try: host = confopts["host"]
-		except KeyError: host = "localhost"
-		stderr("Please enter the host name of the management server that this console-proxy will connect to: (just hit ENTER to go with %s)",host)
-		print "> ",
-		newhost = raw_input().strip()
-		if newhost: host = newhost
-	confopts["host"] = host
-
-	stderr("Querying %s for zones and pods",host)
-	
-	try:
-                if zone == None or pod == None:
 			x = list_zonespods(confopts['host'])
 			zoneandpod = prompt_for_hostpods(x)
 			if zoneandpod:
@@ -1040,16 +992,63 @@ def setup_consoleproxy_config(configfile, host, zone, pod):
 		else:
 			confopts["zone"] = zone
 			confopts["pod"] = pod
-	except (urllib2.URLError,urllib2.HTTPError),e:
+			confopts["cluster"] = cluster
+	except (urllib.error.URLError,urllib.error.HTTPError) as e:
 		stderr("Query failed: %s.  Defaulting to zone %s pod %s",str(e),confopts["zone"],confopts["pod"])
 
-	for opt,val in confopts.items():
+	for opt,val in list(confopts.items()):
 		line = "=".join([opt,val])
 		if opt not in confposes: lines.append(line)
 		else: lines[confposes[opt]] = line
 	
 	text = "\n".join(lines)
-	file(fn,"w").write(text)
+	open(fn,"w").write(text)
+
+def setup_consoleproxy_config(configfile, host, zone, pod):
+	stderr("Examining Console Proxy configuration")
+	fn = configfile
+	text = open(fn).read(-1)
+	lines = [ s.strip() for s in text.splitlines() ]
+	confopts = dict([ m.split("=",1) for m in lines if "=" in m and not m.startswith("#") ])
+	confposes = dict([ (m.split("=",1)[0],n) for n,m in enumerate(lines) if "=" in m and not m.startswith("#") ])
+
+	if not "guid" in confopts:
+		stderr("Generating GUID for this Console Proxy")
+		confopts['guid'] = uuidgen().stdout.strip()
+
+	if host == None:
+		try: host = confopts["host"]
+		except KeyError: host = "localhost"
+		stderr("Please enter the host name of the management server that this console-proxy will connect to: (just hit ENTER to go with %s)",host)
+		print("> ", end=' ')
+		newhost = input().strip()
+		if newhost: host = newhost
+	confopts["host"] = host
+
+	stderr("Querying %s for zones and pods",host)
+	
+	try:
+		if zone == None or pod == None:
+			x = list_zonespods(confopts['host'])
+			zoneandpod = prompt_for_hostpods(x)
+			if zoneandpod:
+				confopts["zone"],confopts["pod"] = zoneandpod
+				stderr("You selected zone %s pod %s",confopts["zone"],confopts["pod"])
+			else:
+				stderr("Skipped -- using the previous zone %s pod %s",confopts["zone"],confopts["pod"])
+		else:
+			confopts["zone"] = zone
+			confopts["pod"] = pod
+	except (urllib.error.URLError,urllib.error.HTTPError) as e:
+		stderr("Query failed: %s.  Defaulting to zone %s pod %s",str(e),confopts["zone"],confopts["pod"])
+
+	for opt,val in list(confopts.items()):
+		line = "=".join([opt,val])
+		if opt not in confposes: lines.append(line)
+		else: lines[confposes[opt]] = line
+	
+	text = "\n".join(lines)
+	open(fn,"w").write(text)
 
 # =========================== DATABASE MIGRATION SUPPORT CODE ===================
 
@@ -1086,13 +1085,13 @@ class Migrator:
 			if not new:
 				try: idx= [ i for i,s in enumerate(self.evolvers)
 					if s.from_level == INITIAL_LEVEL ][0] # initial evolver
-				except IndexError,e:
+				except IndexError as e:
-					raise IndexError, "no initial evolver (from_level is None) could be found"
+					raise IndexError("no initial evolver (from_level is None) could be found")
 			else:
 				try: idx= [ i for i,s in enumerate(self.evolvers)
 					if new[-1].to_level == s.from_level ][0]
-				except IndexError,e:
+				except IndexError as e:
-					raise IndexError, "no evolver could be found to evolve from level %s"%new[-1].to_level
+					raise IndexError("no evolver could be found to evolve from level %s"%new[-1].to_level)
 			new.append(self.evolvers.pop(idx))
 		self.evolvers = new
 	
@@ -1101,11 +1100,11 @@ class Migrator:
 		
 	def get_evolver_by_starting_level(self,level):
 		try: return [ s for s in self.evolvers if s.from_level == level][0]
-		except IndexError: raise NoMigrator, "No evolver knows how to evolve the database from schema level %r"%level
+		except IndexError: raise NoMigrator("No evolver knows how to evolve the database from schema level %r"%level)
 	
 	def get_evolver_by_ending_level(self,level):
 		try: return [ s for s in self.evolvers if s.to_level == level][0]
-		except IndexError: raise NoMigrator, "No evolver knows how to evolve the database to schema level %r"%level
+		except IndexError: raise NoMigrator("No evolver knows how to evolve the database to schema level %r"%level)
 	
 	def run(self, context, dryrun = False, starting_level = None, ending_level = None):
 		"""Runs each one of the steps in sequence, passing the migration context to each. At the end of the process, context.commit() is called to save the changes, or context.rollback() is called if dryrun = True.
@@ -1124,8 +1123,8 @@ class Migrator:
 		evolution_path = evolution_path[idx:]
 		try: idx = evolution_path.index(self.get_evolver_by_ending_level(ending_level))
 		except ValueError:
-			raise NoEvolutionPath, "No evolution path from schema level %r to schema level %r" % \
+			raise NoEvolutionPath("No evolution path from schema level %r to schema level %r" % \
-				(starting_level,ending_level)
+				(starting_level,ending_level))
 		evolution_path = evolution_path[:idx+1]
 		
 		logging.info("Starting migration on %s"%context)
@@ -1202,4 +1201,3 @@ class MigrationContext:
 	def get_schema_level(self):raise NotImplementedError
 	def set_schema_level(self,l):raise NotImplementedError
 
-

python/lib/cloudutils/configFileOps.py

@@ -17,7 +17,7 @@
 import re
 import tempfile
 import shutil
-from utilities import bash
+from .utilities import bash
 class configFileOps:
     class entry:
         def __init__(self, name, value, op, separator):
@@ -49,7 +49,7 @@ class configFileOps:
     
     def getEntry(self, name, separator="="):
         try:
-            ctx = file(self.fileName).read(-1)
+            ctx = open(self.fileName).read(-1)
             match = re.search("^" + name + ".*", ctx, re.MULTILINE)
             if match is None:
                 return ""
@@ -103,10 +103,10 @@ class configFileOps:
 
         fp.close()
         
-        file(self.fileName, "w").writelines(newLines)
+        open(self.fileName, "w").writelines(newLines)
 
     def replace_line(self, startswith,stanza,always_add=False):
-        lines = [ s.strip() for s in file(self.fileName).readlines() ]
+        lines = [ s.strip() for s in open(self.fileName).readlines() ]
         newlines = []
         replaced = False
         for line in lines:
@@ -120,36 +120,36 @@ class configFileOps:
             newlines.append(stanza)
             self.backups.append([None, stanza])
         newlines = [ s + '\n' for s in newlines ]
-        file(self.fileName,"w").writelines(newlines)
+        open(self.fileName,"w").writelines(newlines)
 
     def replace_or_add_line(self, startswith,stanza):
         return self.replace_line(startswith,stanza,always_add=True)
 
     def add_lines(self, lines, addToBackup=True):
-        fp = file(self.fileName).read(-1) 
+        fp = open(self.fileName).read(-1) 
         sh = re.escape(lines)
         match = re.search(sh, fp, re.MULTILINE) 
         if match is not None:
             return
     
         fp += lines
-        file(self.fileName, "w").write(fp)
+        open(self.fileName, "w").write(fp)
         self.backups.append([None, lines])
         
     def replace_lines(self, src, dst, addToBackup=True):
-        fp = file(self.fileName).read(-1) 
+        fp = open(self.fileName).read(-1) 
         sh = re.escape(src)
         if dst is None:
             dst = ""
         repl,nums = re.subn(sh, dst, fp)
         if nums <=0:
             return
-        file(self.fileName, "w").write(repl)
+        open(self.fileName, "w").write(repl)
         if addToBackup:
             self.backups.append([src, dst])
 
     def append_lines(self, match_lines, append_lines):
-        fp = file(self.fileName).read(-1)
+        fp = open(self.fileName).read(-1)
         sh = re.escape(match_lines)
         match = re.search(sh, fp, re.MULTILINE)
         if match is None:
@@ -160,14 +160,14 @@ class configFileOps:
             return
 
         newlines = []
-        for line in file(self.fileName).readlines():
+        for line in open(self.fileName).readlines():
             if re.search(match_lines, line) is not None:
                 newlines.append(line + append_lines)
                 self.backups.append([line, line + append_lines])
             else:
                 newlines.append(line)
 
-        file(self.fileName, "w").writelines(newlines)
+        open(self.fileName, "w").writelines(newlines)
             
     def backup(self):
         for oldLine, newLine in self.backups:

python/lib/cloudutils/db.py

@@ -15,8 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 import os
-from utilities import bash
+from .utilities import bash
-from cloudException import CloudRuntimeException
+from .cloudException import CloudRuntimeException
 import mysql.connector
 
 class Database(object):

python/lib/cloudutils/networkConfig.py

@@ -14,8 +14,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from utilities import bash
+from .utilities import bash
-from cloudException import CloudRuntimeException, CloudInternalException
+from .cloudException import CloudRuntimeException, CloudInternalException
 import logging
 import os
 import re
@@ -37,7 +37,7 @@ class networkConfig:
     @staticmethod
     def listNetworks():
         devs = os.listdir("/sys/class/net/") 
-        devs = filter(networkConfig.isBridge, devs) 
+        devs = list(filter(networkConfig.isBridge, devs)) 
         return devs
     @staticmethod
     def getDefaultNetwork():
@@ -66,10 +66,10 @@ class networkConfig:
 
         cmds = ""
         if not networkConfig.isBridge(brName):
-            cmds = "brctl addbr %s ;"%brName
+            cmds = "ip link add name %s type bridge ;"%brName
     
         cmds += "ifconfig %s up;"%brName
-        cmds += "brctl addif %s %s"%(brName, dev)
+        cmds += "ip link set dev %s master %s"%(dev, brName)
         return bash(cmds).isSuccess()
 
     @staticmethod
@@ -87,7 +87,7 @@ class networkConfig:
         if os.path.exists("/proc/sys/net/bridge"):
             return True
 
-        return bash("modprobe -b bridge").isSucess()
+        return bash("modprobe -b bridge").isSuccess()
 
     @staticmethod
     def isNetworkDev(devName):
@@ -125,7 +125,7 @@ class networkConfig:
             return None
 
         for dev in os.listdir("/sys/class/net/%s/brif"%br):
-            br_port = int(file("/sys/class/net/%s/brif/%s/port_no"%(br,dev)).readline().strip("\n"), 16)
+            br_port = int(open("/sys/class/net/%s/brif/%s/port_no"%(br,dev)).readline().strip("\n"), 16)
             if br_port == brPort:
                 return dev
 
@@ -164,4 +164,3 @@ class networkConfig:
 
         return networkConfig.devInfo(macAddr, ipAddr, netmask, None, type, dev)
 
-

python/lib/cloudutils/serviceConfig.py

@@ -14,12 +14,12 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from utilities import writeProgressBar, bash
+from .utilities import writeProgressBar, bash
-from cloudException import CloudRuntimeException, CloudInternalException, formatExceptionInfo
+from .cloudException import CloudRuntimeException, CloudInternalException, formatExceptionInfo
 import logging
-from networkConfig import networkConfig
+from .networkConfig import networkConfig
 import re
-from configFileOps import configFileOps
+from .configFileOps import configFileOps
 import os
 import shutil
 
@@ -27,25 +27,31 @@ import shutil
 Unknown = 0
 CentOS6 = 1
 CentOS7 = 2
-Ubuntu = 3
+CentOS8 = 3
-RHEL6 = 4
+Ubuntu = 4
-RHEL7 = 5
+RHEL6 = 5
+RHEL7 = 6
+RHEL8 = 7
 distro = None
 
 #=================== DISTRIBUTION DETECTION =================
 if os.path.exists("/etc/centos-release"):
-    version = file("/etc/centos-release").readline()
+    version = open("/etc/centos-release").readline()
     if version.find("CentOS release 6") != -1:
       distro = CentOS6
     elif version.find("CentOS Linux release 7") != -1:
       distro = CentOS7
+    elif version.find("CentOS Linux release 8") != -1:
+      distro = CentOS8
 elif os.path.exists("/etc/redhat-release"):
-    version = file("/etc/redhat-release").readline()
+    version = open("/etc/redhat-release").readline()
     if version.find("Red Hat Enterprise Linux Server release 6") != -1:
       distro = RHEL6
     elif version.find("Red Hat Enterprise Linux Server 7") != -1:
       distro = RHEL7
-elif os.path.exists("/etc/lsb-release") and "Ubuntu" in file("/etc/lsb-release").read(-1): distro = Ubuntu
+    elif version.find("Red Hat Enterprise Linux Server 8") != -1:
+      distro = RHEL8
+elif os.path.exists("/etc/lsb-release") and "Ubuntu" in open("/etc/lsb-release").read(-1): distro = Ubuntu
 else: distro = Unknown
 #=================== DISTRIBUTION DETECTION =================
 
@@ -68,12 +74,12 @@ class serviceCfgBase(object):
             self.status = result
             writeProgressBar(None, result)
             return result
-        except CloudRuntimeException, e:
+        except CloudRuntimeException as e:
             self.status = result
             writeProgressBar(None, result)
             logging.debug(e.getDetails())
             raise e
-        except CloudInternalException, e:
+        except CloudInternalException as e:
             self.status = result
             writeProgressBar(None, result)
             raise e
@@ -95,7 +101,7 @@ class serviceCfgBase(object):
                 cfo.backup()
 
             result = self.restore()
-        except (CloudRuntimeException, CloudInternalException), e:
+        except (CloudRuntimeException, CloudInternalException) as e:
             logging.debug(e)
 
         writeProgressBar(None, result)
@@ -178,7 +184,7 @@ class networkConfigUbuntu(serviceCfgBase, networkConfigBase):
 
     def addBridge(self, br, dev):
         bash("ifdown %s"%dev.name)
-        for line in file(self.netCfgFile).readlines():
+        for line in open(self.netCfgFile).readlines():
             match = re.match("^ *iface %s.*"%dev.name, line)
             if match is not None:
                 dev.method = self.getNetworkMethod(match.group(0))
@@ -213,7 +219,7 @@ class networkConfigUbuntu(serviceCfgBase, networkConfigBase):
         logging.debug("Haven't implement yet")
 
     def writeToCfgFile(self, br, dev):
-        cfg = file(self.netCfgFile).read()
+        cfg = open(self.netCfgFile).read()
         ifaceDev = re.search("^ *iface %s.*"%dev.name, cfg, re.MULTILINE)
         ifaceBr = re.search("^ *iface %s.*"%br, cfg, re.MULTILINE)
         if ifaceDev is not None and ifaceBr is not None:
@@ -344,7 +350,8 @@ class networkConfigRedhat(serviceCfgBase, networkConfigBase):
             cfo.save()
 
             if not bash("service network restart").isSuccess():
-                raise CloudInternalException("Can't restart network")
+                if not bash("systemctl restart NetworkManager.service").isSuccess():
+                    raise CloudInternalException("Can't restart network")
 
             self.syscfg.env.nics.append(self.brName)
             self.syscfg.env.nics.append(self.brName)
@@ -566,6 +573,12 @@ class libvirtConfigUbuntu(serviceCfgBase):
         elif os.path.exists("/etc/default/libvirtd"):
             cfo = configFileOps("/etc/default/libvirtd", self)
             cfo.replace_or_add_line("libvirtd_opts=","libvirtd_opts='-l'")
+            if os.path.exists("/lib/systemd/system/libvirtd.socket"):
+                bash("/bin/systemctl mask libvirtd.socket");
+                bash("/bin/systemctl mask libvirtd-ro.socket");
+                bash("/bin/systemctl mask libvirtd-admin.socket");
+                bash("/bin/systemctl mask libvirtd-tls.socket");
+                bash("/bin/systemctl mask libvirtd-tcp.socket");
 
     def config(self):
         try:

python/lib/cloudutils/serviceConfigServer.py

@@ -14,11 +14,11 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from db import Database
+from .db import Database
-from configFileOps import configFileOps
+from .configFileOps import configFileOps
-from serviceConfig import serviceCfgBase
+from .serviceConfig import serviceCfgBase
-from cloudException import CloudRuntimeException, CloudInternalException
+from .cloudException import CloudRuntimeException, CloudInternalException
-from utilities import bash
+from .utilities import bash
 import os
 
 class cloudManagementConfig(serviceCfgBase):
@@ -49,7 +49,7 @@ class cloudManagementConfig(serviceCfgBase):
 
             try:
                 db.testConnection()
-            except CloudRuntimeException, e:
+            except CloudRuntimeException as e:
                 raise e
             except:
                 raise CloudInternalException("Failed to connect to Mysql server")
@@ -93,5 +93,5 @@ class cloudManagementConfig(serviceCfgBase):
             else:
                 raise CloudRuntimeException("Failed to configure %s, please see the /var/log/cloudstack/management/setupManagement.log for detail"%self.serviceName)
         else:
-            print "Configured successfully, but not starting management server."
+            print("Configured successfully, but not starting management server.")
             return True

python/lib/cloudutils/syscfg.py

@@ -14,8 +14,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from utilities import Distribution, serviceOpsRedhat,serviceOpsUbuntu,serviceOpsRedhat7
+from .utilities import Distribution, serviceOpsRedhat,serviceOpsUbuntu,serviceOpsRedhat7Later
-from serviceConfig import *
+from .serviceConfig import *
 class sysConfigFactory:
     @staticmethod
     def getSysConfigFactory(glbEnv):
@@ -43,8 +43,10 @@ class sysConfigAgentFactory:
             return sysConfigRedhat5(glbEnv)
         elif distribution == "RHEL7":
             return sysConfigRedhat7(glbEnv)
+        elif distribution == "RHEL8":
+            return sysConfigRedhat8(glbEnv)
         else:
-            print "Can't find the distribution version"
+            print("Can't find the distribution version")
             return sysConfig()
 
 class sysConfigServerFactory:
@@ -57,7 +59,7 @@ class sysConfigServerFactory:
         elif distribution != "Unknown":
             return sysConfigServerRedhat(glbEnv)
         else:
-            print "Can't find the distribution version"
+            print("Can't find the distribution version")
             return sysConfig()
     
 class sysConfigDbFactory:
@@ -144,9 +146,14 @@ class sysConfigAgentRedhatBase(sysConfigAgent):
 
 class sysConfigAgentRedhat7Base(sysConfigAgent):
     def __init__(self, env):
-        self.svo = serviceOpsRedhat7()
+        self.svo = serviceOpsRedhat7Later()
         super(sysConfigAgentRedhat7Base, self).__init__(env)
 
+class sysConfigAgentRedhat8Base(sysConfigAgent):
+    def __init__(self, env):
+        self.svo = serviceOpsRedhat7Later()
+        super(sysConfigAgentRedhat8Base, self).__init__(env)
+
 class sysConfigAgentUbuntu(sysConfigAgent):
     def __init__(self, glbEnv):
         super(sysConfigAgentUbuntu, self).__init__(glbEnv)
@@ -193,6 +200,17 @@ class sysConfigRedhat7(sysConfigAgentRedhat7Base):
                          nfsConfig(self),
                          cloudAgentConfig(self)]
 
+#it covers RHEL8
+class sysConfigRedhat8(sysConfigAgentRedhat8Base):
+    def __init__(self, glbEnv):
+        super(sysConfigRedhat8, self).__init__(glbEnv)
+        self.services = [securityPolicyConfigRedhat(self),
+                         networkConfigRedhat(self),
+                         libvirtConfigRedhat(self),
+                         firewallConfigAgent(self),
+                         nfsConfig(self),
+                         cloudAgentConfig(self)]
+
 class sysConfigServer(sysConfig):
     def check(self):
         if os.geteuid() != 0:

python/lib/cloudutils/utilities.py

@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from cloudException import CloudRuntimeException, formatExceptionInfo
+from .cloudException import CloudRuntimeException, formatExceptionInfo
 import logging
 from subprocess import PIPE, Popen
 from signal import alarm, signal, SIGALRM, SIGKILL
@@ -60,13 +60,13 @@ class bash:
         return self.success
     
     def getStdout(self):
-        return self.stdout.strip("\n")
+        return self.stdout.decode('utf-8').strip('\n')
     
     def getLines(self):
-        return self.stdout.split("\n")
+        return self.stdout.decode('utf-8').strip('\n')
 
     def getStderr(self):
-        return self.stderr.strip("\n")
+        return self.stderr.decode('utf-8').strip('\n')
     
     def getErrMsg(self):
         if self.isSuccess():
@@ -109,16 +109,18 @@ class Distribution:
         if os.path.exists("/etc/fedora-release"):
             self.distro = "Fedora"
         elif os.path.exists("/etc/redhat-release"):
-            version = file("/etc/redhat-release").readline()
+            version = open("/etc/redhat-release").readline()
             if version.find("Red Hat Enterprise Linux Server release 6") != -1 or version.find("Scientific Linux release 6") != -1 or version.find("CentOS Linux release 6") != -1 or version.find("CentOS release 6.") != -1:
                 self.distro = "RHEL6"
             elif version.find("Red Hat Enterprise Linux Server release 7") != -1 or version.find("Scientific Linux release 7") != -1 or version.find("CentOS Linux release 7") != -1 or version.find("CentOS release 7.") != -1:
                 self.distro = "RHEL7"
+            elif version.find("Red Hat Enterprise Linux Server release 8") != -1 or version.find("Scientific Linux release 8") != -1 or version.find("CentOS Linux release 8") != -1 or version.find("CentOS release 8.") != -1:
+                self.distro = "RHEL8"
             elif version.find("CentOS") != -1:
                 self.distro = "CentOS"
             else:
                 self.distro = "RHEL5"
-        elif os.path.exists("/etc/legal") and "Ubuntu" in file("/etc/legal").read(-1):
+        elif os.path.exists("/etc/legal") and "Ubuntu" in open("/etc/legal").read(-1):
             self.distro = "Ubuntu"
             kernel = bash("uname -r").getStdout()
             if kernel.find("2.6.32") != -1:
@@ -211,9 +213,9 @@ class serviceOpsUbuntu(serviceOps):
         return self.startService(servicename,force=forcestart)
 
     def isKVMEnabled(self):
-        return bash("kvm-ok").isSuccess() 
+        return bash("kvm-ok").isSuccess()
 
-class serviceOpsRedhat7(serviceOps):
+class serviceOpsRedhat7Later(serviceOps):
     def isServiceRunning(self, servicename):
         try:
             o = bash("systemctl is-active " + servicename)

scripts/storage/qcow2/managesnapshot.sh

@@ -187,6 +187,7 @@ backup_snapshot() {
   local snapshotname="$2"
   local destPath=$3
   local destName=$4
+  local forceShareFlag="-U"
 
   if [ ! -d $destPath ]
   then
@@ -211,20 +212,49 @@ backup_snapshot() {
       return 1
     fi
 
-    ${qemu_img} convert -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}" || \
+    qemuimg_ret=$($qemu_img $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}")
-     ( printf "${qemu_img} failed to create backup of snapshot ${snapshotname} for disk ${disk} to ${destPath}.\n" >&2; return 2 )
+    ret_code=$?
-
+    if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"snapshot: invalid option -- 'U'"* ]]
+    then
+      forceShareFlag=""
+      $qemu_img $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}"
+      ret_code=$?
+    fi
+    if [ $ret_code -gt 0 ]
+    then
+      printf "${qemu_img} failed to create backup of snapshot ${snapshotname} for disk ${disk} to ${destPath}.\n" >&2
+      return 2
+    fi
   elif [ -f ${disk} ]; then
     # Does the snapshot exist?
-    $qemu_img snapshot -l $disk|grep -w "$snapshotname" >& /dev/null
+    qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk 2>&1)
-    if [ $? -gt 0 ]
+    ret_code=$?
+    if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"snapshot: invalid option -- 'U'"* ]]
+    then
+      forceShareFlag=""
+      qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk)
+      ret_code=$?
+    fi
+    if [ $ret_code -gt 0 ] || [[ ! $qemuimg_ret == *"$snapshotname"* ]]
     then
       printf "there is no $snapshotname on disk $disk\n" >&2
       return 1
     fi
 
-    $qemu_img convert -f qcow2 -O qcow2 -s $snapshotname $disk $destPath/$destName >& /dev/null
+    qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1 > /dev/null)
-    if [ $? -gt 0 ]
+    ret_code=$?
+    if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"convert: invalid option -- 'U'"* ]]
+    then
+      forceShareFlag=""
+      qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1 > /dev/null)
+      ret_code=$?
+    fi
+    if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"convert: invalid option -- 'l'"* ]]
+    then
+      $qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -s $snapshotname $disk $destPath/$destName >& /dev/null
+      ret_code=$?
+    fi
+    if [ $ret_code -gt 0 ]
     then
       printf "Failed to backup $snapshotname for disk $disk to $destPath\n" >&2
       return 2

scripts/util/migrate-dynamicroles.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -110,7 +110,7 @@ def main():
         sys.exit(1)
 
     while True:
-        choice = raw_input("Running this migration tool will remove any " +
+        choice = input("Running this migration tool will remove any " +
                            "default-role permissions from cloud.role_permissions. " +
                            "Do you want to continue? [y/N]").lower()
         if choice == 'y':

scripts/util/qemu-ifup

@@ -17,5 +17,5 @@
 # under the License.
 
  switch=$(/sbin/ip route list | awk '/^default / { print $NF }') 
-/sbin/ifconfig $1 0.0.0.0 up 
+/sbin/ifconfig $1 0.0.0.0 up
-/usr/sbin/brctl addif ${switch} $1 
+/sbin/ip link set dev $1 master ${switch}

scripts/vm/hypervisor/vmware/discover_networks.py

@@ -16,7 +16,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import print_function
 from pyVim.connect import SmartConnect, SmartConnectNoSSL, Disconnect
 from pyVmomi import vim
 import atexit

scripts/vm/network/security_group.py

@@ -148,7 +148,7 @@ def split_ips_by_family(ips):
 def destroy_network_rules_for_nic(vm_name, vm_ip, vm_mac, vif, sec_ips):
     try:
         rules = execute("""iptables-save -t filter | awk '/ %s / { sub(/-A/, "-D", $1) ; print }'""" % vif ).split("\n")
-        for rule in filter(None, rules):
+        for rule in [_f for _f in rules if _f]:
             try:
                 execute("iptables " + rule)
             except:
@@ -158,7 +158,7 @@ def destroy_network_rules_for_nic(vm_name, vm_ip, vm_mac, vif, sec_ips):
 
     try:
         dnats = execute("""iptables-save -t nat | awk '/ %s / { sub(/-A/, "-D", $1) ; print }'""" % vif ).split("\n")
-        for dnat in filter(None, dnats):
+        for dnat in [_f for _f in dnats if _f]:
             try:
                 execute("iptables -t nat " + dnat)
             except:
@@ -205,14 +205,14 @@ def destroy_network_rules_for_vm(vm_name, vif=None):
     destroy_ebtables_rules(vm_name, vif)
 
     chains = [vmchain_default, vmchain, vmchain_egress]
-    for chain in filter(None, chains):
+    for chain in [_f for _f in chains if _f]:
         try:
             execute("iptables -F " + chain)
             execute('ip6tables -F ' + chain)
         except:
             logging.debug("Ignoring failure to flush chain: " + chain)
 
-    for chain in filter(None, chains):
+    for chain in [_f for _f in chains if _f]:
         try:
             execute("iptables -X " + chain)
             execute('ip6tables -X ' + chain)
@@ -229,7 +229,7 @@ def destroy_network_rules_for_vm(vm_name, vif=None):
     if vif:
         try:
             dnats = execute("""iptables -t nat -S | awk '/%s/ { sub(/-A/, "-D", $1) ; print }'""" % vif ).split("\n")
-            for dnat in filter(None, dnats):
+            for dnat in [_f for _f in dnats if _f]:
                 try:
                     execute("iptables -t nat " + dnat)
                 except:
@@ -247,14 +247,14 @@ def destroy_ebtables_rules(vm_name, vif):
     delcmd = "ebtables -t nat -L PREROUTING | grep " + eb_vm_chain
     delcmds = []
     try:
-        delcmds = filter(None, execute(delcmd).split('\n'))
+        delcmds = [_f for _f in execute(delcmd).split('\n') if _f]
         delcmds = ["-D PREROUTING " + x for x in delcmds ]
     except:
         pass
     postcmds = []
     try:
         postcmd = "ebtables -t nat -L POSTROUTING | grep " + eb_vm_chain
-        postcmds = filter(None, execute(postcmd).split('\n'))
+        postcmds = [_f for _f in execute(postcmd).split('\n') if _f]
         postcmds = ["-D POSTROUTING " + x for x in postcmds]
     except:
         pass
@@ -493,7 +493,7 @@ def ebtables_rules_vmip (vmname, vmmac, ips, action):
     if action and action.strip() == "-A":
         action = "-I"
 
-    for ip in filter(None, ips):
+    for ip in [_f for _f in ips if _f]:
         logging.debug("ip = " + ip)
         if ip == 0 or ip == "0":
             continue
@@ -736,7 +736,7 @@ def delete_rules_for_vm_in_bridge_firewall_chain(vmName):
     vmchain = iptables_chain_name(vm_name)
 
     delcmd = """iptables-save | awk '/BF(.*)physdev-is-bridged(.*)%s/ { sub(/-A/, "-D", $1) ; print }'""" % vmchain
-    delcmds = filter(None, execute(delcmd).split('\n'))
+    delcmds = [_f for _f in execute(delcmd).split('\n') if _f]
     for cmd in delcmds:
         try:
             execute("iptables " + cmd)
@@ -744,7 +744,7 @@ def delete_rules_for_vm_in_bridge_firewall_chain(vmName):
               logging.exception("Ignoring failure to delete rules for vm " + vmName)
 
     delcmd = """ip6tables-save | awk '/BF(.*)physdev-is-bridged(.*)%s/ { sub(/-A/, "-D", $1) ; print }'""" % vmchain
-    delcmds = filter(None, execute(delcmd).split('\n'))
+    delcmds = [_f for _f in execute(delcmd).split('\n') if _f]
     for cmd in delcmds:
         try:
             execute('ip6tables ' + cmd)
@@ -847,7 +847,7 @@ def network_rules_for_rebooted_vm(vmName):
         ipts = []
         for cmd in [delcmd, inscmd]:
             logging.debug(cmd)
-            cmds = filter(None, execute(cmd).split('\n'))
+            cmds = [_f for _f in execute(cmd).split('\n') if _f]
             for c in cmds:
                     ipt = "iptables " + c
                     ipts.append(ipt)
@@ -897,7 +897,7 @@ def cleanup_bridge(bridge):
 
     # Delete iptables/bridge rules
     rules = execute("""iptables-save | grep %s | grep '^-A' | sed 's/-A/-D/' """ % bridge_name).split("\n")
-    for rule in filter(None, rules):
+    for rule in [_f for _f in rules if _f]:
         try:
             command = "iptables " + rule
             execute(command)
@@ -945,13 +945,13 @@ def cleanup_rules():
                     cleanup.append(vm_name)
 
         bridge_tables = execute("""grep -E '^ebtable_' /proc/modules | cut -f1 -d' ' | sed s/ebtable_//""").split('\n')
-        for table in filter(None, bridge_tables):
+        for table in [_f for _f in bridge_tables if _f]:
             chainscmd = """ebtables -t %s -L | awk '/chain:/ { gsub(/(^.*chain: |-(in|out|ips).*)/, ""); print $1}' | sort | uniq""" % table
             chains = execute(chainscmd).split('\n')
 
         logging.debug(" ebtables chains in the host: %s ", chains)
 
-        for chain in filter(None, chains):
+        for chain in [_f for _f in chains if _f]:
             if 1 in [chain.startswith(c) for c in ['r-', 'i-', 's-', 'v-']]:
                 vm_name = chain
                 vmpresent = False
@@ -1451,7 +1451,7 @@ def verify_iptables_rules_for_bridge(brname):
     expected_rules.append("-A %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % (brfw))
     expected_rules.append("-A %s -m physdev --physdev-is-in --physdev-is-bridged -j %s" % (brfw, brfwin))
     expected_rules.append("-A %s -m physdev --physdev-is-out --physdev-is-bridged -j %s" % (brfw, brfwout))
-    phydev = execute("brctl show | awk '/^%s[ \t]/ {print $4}'" % brname ).strip()
+    phydev = execute("ip link show type bridge | awk '/^%s[ \t]/ {print $4}'" % brname ).strip()
     expected_rules.append("-A %s -m physdev --physdev-out %s --physdev-is-bridged -j ACCEPT" % (brfw, phydev))
 
     rules = execute("iptables-save |grep -w %s |grep -v \"^:\"" % brfw).split('\n')

scripts/vm/network/vnet/cloudstack_pluginlib.py

@@ -17,7 +17,7 @@
 
 # cloudstack_pluginlib for openvswitch on KVM hypervisor
 
-import ConfigParser
+import configparser
 import logging
 import os
 import subprocess
@@ -49,7 +49,7 @@ def setup_logging(log_file=None):
     log_date_format = DEFAULT_LOG_DATE_FORMAT
     # try to read plugin configuration file
     if os.path.exists(PLUGIN_CONFIG_PATH):
-        config = ConfigParser.ConfigParser()
+        config = configparser.ConfigParser()
         config.read(PLUGIN_CONFIG_PATH)
         try:
             options = config.options('LOGGING')
@@ -67,7 +67,7 @@ def setup_logging(log_file=None):
             # configuration file contained invalid attributes
             # ignore them
             pass
-        except ConfigParser.NoSectionError:
+        except configparser.NoSectionError:
             # Missing 'Logging' section in configuration file
             pass
 
@@ -114,7 +114,7 @@ def _is_process_run(pidFile, name):
         fpid = open(pidFile, "r")
         pid = fpid.readline()
         fpid.close()
-    except IOError, e:
+    except IOError as e:
         return -1
 
     pid = pid[:-1]
@@ -304,11 +304,11 @@ class jsonLoader(object):
 
   def __repr__(self):
         return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v)
-                                      in self.__dict__.iteritems()))
+                                      in self.__dict__.items()))
 
   def __str__(self):
         return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v)
-                                      in self.__dict__.iteritems()))
+                                      in self.__dict__.items()))
 
 def configure_bridge_for_network_topology(bridge, this_host_id, json_config):
     vpconfig = jsonLoader(json.loads(json_config)).vpc

scripts/vm/network/vnet/ovstunnel.py

@@ -22,7 +22,7 @@
 
 import cloudstack_pluginlib as lib
 import logging
-import commands
+import subprocess
 import os
 import sys
 import subprocess

setup/bindir/cloud-migrate-databases.in

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -244,7 +244,7 @@ def main(*args):
 	
 	try:
 		props = cloud_utils.read_properties(os.path.join(configdir,'db.properties'))
-	except (IOError,OSError),e:
+	except (IOError,OSError) as e:
 		logging.error("Cannot read from config file: %s",e)
 		logging.error("You may want to point to a specific config directory with the --config= option")
 		return 2
@@ -261,13 +261,13 @@ def main(*args):
 	database = props["db.cloud.name"]
 	
 	# tell the migrator to load its steps from the globals list
-	migrator = cloud_utils.Migrator(globals().values())
+	migrator = cloud_utils.Migrator(list(globals().values()))
 	
 	if opts.dumpevolvers:
-		print "Evolution steps:"
+		print("Evolution steps:")
-		print "	%s	%s	%s"%("From","To","Evolver in charge")
+		print("	%s	%s	%s"%("From","To","Evolver in charge"))
 		for f,t,e in migrator.get_evolver_chain():
-			print "	%s	%s	%s"%(f,t,e)
+			print("	%s	%s	%s"%(f,t,e))
 		return
 	
 	#initialize a context with the read configuration
@@ -277,7 +277,7 @@ def main(*args):
 		migrator.run(context,dryrun=opts.dryrun,starting_level=opts.fromlevel,ending_level=opts.tolevel)
 	    finally:
 		context.close()
-	except (cloud_utils.NoMigrationPath,cloud_utils.NoMigrator),e:
+	except (cloud_utils.NoMigrationPath,cloud_utils.NoMigrator) as e:
 		logging.error("%s",e)
 		return 4
 

setup/bindir/cloud-set-guest-password-configdrive.in

@@ -1,4 +1,4 @@
-                                  #!/bin/bash
+#!/bin/bash
 #
 # Init file for Password Download Client
 #

setup/bindir/cloud-setup-databases.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -24,7 +24,7 @@ import glob
 from random import choice
 import string
 from optparse import OptionParser
-import commands
+import subprocess
 import shutil
 import socket
 
@@ -49,7 +49,7 @@ def runCmd(cmds):
     stdout, stderr = process.communicate()
     if process.returncode != 0:
         raise Exception(stderr)
-    return stdout
+    return stdout.decode('utf-8')
 
 class DBDeployer(object):
     parser = None
@@ -115,7 +115,7 @@ class DBDeployer(object):
         sys.stdout.flush()
 
     def putDbProperty(self, key, value):
-        if self.dbDotProperties.has_key(key):
+        if key in self.dbDotProperties:
             (oldValue, index) = self.dbDotProperties[key]
             self.dbDotProperties[key] = (value, index)
         else:
@@ -123,7 +123,7 @@ class DBDeployer(object):
             self.dbDotPropertiesIndex += 1
 
     def getDbProperty(self, key):
-        if not self.dbDotProperties.has_key(key):
+        if key not in self.dbDotProperties:
             return None
         (value, index) = self.dbDotProperties[key]
         return value
@@ -145,14 +145,14 @@ class DBDeployer(object):
           mysqlPath = "mysql"
         try:
             mysqlCmds = [mysqlPath, '--user=%s'%kwargs['user'], '--host=%s'%kwargs['host'], '--port=%s'%kwargs['port']]
-            if kwargs.has_key('passwd'):
+            if 'passwd' in kwargs:
                 mysqlCmds.append('--password=%s'%kwargs['passwd'])
-            file(self.tmpMysqlFile, 'w').write(text)
+            open(self.tmpMysqlFile, 'w').write(text)
             mysqlCmds.append('<')
             mysqlCmds.append(self.tmpMysqlFile)
             runCmd(mysqlCmds)
 
-        except Exception, e:
+        except Exception as e:
             err = '''Encountering an error when executing mysql script
 ----------------------------------------------------------------------
 table:
@@ -222,7 +222,7 @@ for full help
         for f in ["create-database","create-schema", "create-database-premium","create-schema-premium"]:
             p = os.path.join(self.dbFilesPath,"%s.sql"%f)
             if not os.path.exists(p): continue
-            text = file(p).read()
+            text = open(p).read()
             for t, r in replacements: text = text.replace(t,r)
             self.info("Applying %s"%p)
             self.runMysql(text, p, True)
@@ -235,32 +235,32 @@ for full help
             try:
                 output = runCmd(['build-classpath'] + systemjars)
                 systemcp = output.strip()
-            except Exception, e:
+            except Exception as e:
                 systemcp = r"@SYSTEMCLASSPATH@"
             mscp = r"@MSCLASSPATH@"
             depscp = r"@DEPSCLASSPATH@"
             classpath = os.path.pathsep.join([pcp,systemcp,depscp,mscp,conf])
             try:
                 runCmd(["java","-cp",classpath,"com.cloud.test.DatabaseConfig", self.serversetup])
-            except Exception, e:
+            except Exception as e:
                 self.errorAndExit("Apply %s failed"%self.serversetup)
         else:
             p = os.path.join(self.dbFilesPath, 'server-setup.sql')
-            text = file(p).read()
+            text = open(p).read()
             self.info("Applying %s"%p)
             self.runMysql(text, p, True)
             self.info(None, True)
 
         for f in ["templates"]:
             p = os.path.join(self.dbFilesPath,"%s.sql"%f)
-            text = file(p).read()
+            text = open(p).read()
             self.info("Applying %s"%p)
             self.runMysql(text, p, True)
             self.info(None, True)
 
         p = os.path.join(self.dbFilesPath,"schema-level.sql")
         if os.path.isfile(p):
-            text = file(p).read()
+            text = open(p).read()
             self.info("Applying %s"%p)
             self.runMysql(text, p, True)
             self.info(None, True)
@@ -268,7 +268,7 @@ for full help
     def prepareDBFiles(self):
         def prepareDBDotProperties():
             dbpPath = os.path.join(self.dbConfPath, 'db.properties')
-            dbproperties = file(dbpPath).read().splitlines()
+            dbproperties = open(dbpPath).read().splitlines()
             newdbp = []
             emptyLine = 0
             for line in dbproperties:
@@ -289,7 +289,7 @@ for full help
                         if key == "db.usage.password": value = self.password
                         if key == "db.usage.host": value = self.host
                         if key == "db.usage.port": value = self.port
-                except Exception, e:
+                except Exception as e:
                     err = '''Wrong format in %s (%s):
 Besides comments beginning "#" and empty line, all key-value pairs must be in formula of
                     key=value
@@ -306,7 +306,7 @@ for example:
     def finalize(self):
         def finalizeDbProperties():
             entries = []
-            for key in self.dbDotProperties.keys():
+            for key in list(self.dbDotProperties.keys()):
                 (value, index) = self.dbDotProperties[key]
                 if key.startswith("#"):
                     entries.insert(index, key)
@@ -314,7 +314,7 @@ for example:
                     entries.insert(index, '')
                 else:
                     entries.insert(index, "%s=%s"%(key, value))
-            file(os.path.join(self.dbConfPath, 'db.properties'), 'w').write('\n'.join(entries))
+            open(os.path.join(self.dbConfPath, 'db.properties'), 'w').write('\n'.join(entries))
 
         self.info("Finalizing setup ...", None)
         finalizeDbProperties()
@@ -326,7 +326,7 @@ for example:
             try:
                 ip = socket.gethostbyname(socket.gethostname())
                 return ip
-            except Exception, e:
+            except Exception as e:
                 return "127.0.0.1"
 
         if not self.ip:
@@ -363,7 +363,7 @@ for example:
             self.info("Checking local machine hostname ...", None)
             try:
                 output= socket.getfqdn()
-            except Exception, e:
+            except Exception as e:
                 err = "The host name of this computer does not resolve to an IP address.\nPlease use your operating system's network setup tools to fix this ('hostname --fqdn' %s).\n"%e.__str__()
                 self.errorAndExit(err)
             self.info(None, True)
@@ -372,10 +372,10 @@ for example:
             self.info("Checking SELinux setup ...", None)
             try:
                 check_selinux()
-            except CheckFailed, e:
+            except CheckFailed as e:
                 self.info("checkSelinux failed with error continuing...", None)
                 self.errorAndContinue(e.__str__())
-            except OSError, e:
+            except OSError as e:
                 if e.errno == 2: pass
                 else:
                     self.info("checkSelinux failed with error continuing...", None)
@@ -389,11 +389,11 @@ for example:
     def processEncryptionStuff(self):
         def encrypt(input):
             cmd = ['java','-Djava.security.egd=file:/dev/urandom','-classpath','"' + self.encryptionJarPath + '"','org.jasypt.intf.cli.JasyptPBEStringEncryptionCLI', 'encrypt.sh', 'input=%s'%input, 'password=%s'%self.mgmtsecretkey,'verbose=false']
-            return runCmd(cmd).strip('\r\n')
+            return str(runCmd(cmd)).strip('\r\n')
 
         def saveMgmtServerSecretKey():
             if self.encryptiontype == 'file':
-                file(self.encryptionKeyFile, 'w').write(self.mgmtsecretkey)
+                open(self.encryptionKeyFile, 'w').write(self.mgmtsecretkey)
 
         def formatEncryptResult(value):
             return 'ENC(%s)'%value
@@ -595,9 +595,9 @@ for example:
         finally:
             self.postRun()
 
-        print ''
+        print('')
-        print "CloudStack has successfully initialized database, you can check your database configuration in %s"%os.path.join(self.dbConfPath, 'db.properties')
+        print("CloudStack has successfully initialized database, you can check your database configuration in %s"%os.path.join(self.dbConfPath, 'db.properties'))
-        print ''
+        print('')
 
 if __name__ == "__main__":
    o = DBDeployer()

setup/bindir/cloud-setup-encryption.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 
 # Licensed to the Apache Software Foundation (ASF) under one
@@ -26,7 +26,7 @@ import glob
 from random import choice
 import string
 from optparse import OptionParser
-import commands
+import subprocess
 import shutil
 
 # squelch mysqldb spurious warnings
@@ -49,7 +49,7 @@ def runCmd(cmds):
     stdout, stderr = process.communicate()
     if process.returncode != 0:
         raise Exception(stderr)
-    return stdout
+    return stdout.decode('utf-8')
 
 class DBDeployer(object):
     parser = None
@@ -107,7 +107,7 @@ class DBDeployer(object):
         sys.stdout.flush()
 
     def putDbProperty(self, key, value):
-        if self.dbDotProperties.has_key(key):
+        if key in self.dbDotProperties:
             (oldValue, index) = self.dbDotProperties[key]
             self.dbDotProperties[key] = (value, index)
         else:
@@ -115,7 +115,7 @@ class DBDeployer(object):
             self.dbDotPropertiesIndex += 1
     
     def getDbProperty(self, key):
-        if not self.dbDotProperties.has_key(key):
+        if key not in self.dbDotProperties:
             return None
         (value, index) = self.dbDotProperties[key]
         return value
@@ -151,7 +151,7 @@ for full help
                 try:
                     if not passed:
                         (key, value) = line.split('=', 1)
-                except Exception, e:
+                except Exception as e:
                     err = '''Wrong format in %s (%s):
 Besides comments beginning "#" and empty line, all key-value pairs must be in formula of
                     key=value
@@ -167,7 +167,7 @@ for example:
     def finalize(self):
         def finalizeDbProperties():
             entries = []
-            for key in self.dbDotProperties.keys():
+            for key in list(self.dbDotProperties.keys()):
                 (value, index) = self.dbDotProperties[key]
                 if key.startswith("#"):
                     entries.insert(index, key)
@@ -257,9 +257,9 @@ for example:
         finally:
             self.postRun()
         
-        print ''
+        print('')
-        print "CloudStack has successfully setup Encryption"
+        print("CloudStack has successfully setup Encryption")
-        print ''
+        print('')
 
 if __name__ == "__main__":
    o = DBDeployer()

setup/db/create-database-simulator.sql

@@ -20,8 +20,8 @@ DROP DATABASE IF EXISTS `simulator`;
 
 CREATE DATABASE `simulator`;
 
-GRANT ALL ON simulator.* to cloud@`localhost` identified by 'cloud';
+GRANT ALL ON simulator.* to cloud@`localhost`;
-GRANT ALL ON simulator.* to cloud@`%` identified by 'cloud';
+GRANT ALL ON simulator.* to cloud@`%`;
 
 GRANT process ON *.* TO cloud@`localhost`;
 GRANT process ON *.* TO cloud@`%`;

setup/db/create-database.sql

@@ -55,10 +55,11 @@ DROP DATABASE IF EXISTS `cloud`;
 
 CREATE DATABASE `cloud`;
 
-CREATE USER cloud identified by 'cloud';
+CREATE USER cloud@`localhost` identified by 'cloud';
+CREATE USER cloud@`%` identified by 'cloud';
 
-GRANT ALL ON cloud.* to cloud@`localhost` identified by 'cloud';
+GRANT ALL ON cloud.* to cloud@`localhost`;
-GRANT ALL ON cloud.* to cloud@`%` identified by 'cloud';
+GRANT ALL ON cloud.* to cloud@`%`;
 
 GRANT process ON *.* TO cloud@`localhost`;
 GRANT process ON *.* TO cloud@`%`;

test/integration/smoke/test_deploy_vm_extra_config_data.py

@@ -32,7 +32,7 @@ from marvin.cloudstackAPI import (updateVirtualMachine,
 from marvin.cloudstackTestCase import cloudstackTestCase
 from marvin.lib.base import (Account,
                              ServiceOffering,
-                             )
+                             Host)
 from marvin.lib.common import (get_domain,
                                get_zone,
                                get_template,
@@ -79,16 +79,42 @@ class TestAddConfigtoDeployVM(cloudstackTestCase):
             cls.service_offering
         ]
 
+        cls.hosts_hugepages = cls.set_hosts_hugepages()
+
     @classmethod
     def tearDownClass(cls):
         try:
             cls.apiclient = super(TestAddConfigtoDeployVM, cls).getClsTestClient().getApiClient()
+            cls.reset_hosts_hugepages()
             # Clean up, terminate the created templates
             cleanup_resources(cls.apiclient, cls.cleanup)
 
         except Exception as e:
             raise Exception("Warning: Exception during cleanup : %s" % e)
 
+    @classmethod
+    def set_hosts_hugepages(cls):
+        hosts_hugepages = []
+        listHost = Host.list(
+               cls.apiclient,
+               type='Routing',
+               zoneid=cls.zone.id
+           )
+        for host in listHost:
+            if host.hypervisor.lower() == 'kvm':
+                sshClient = SshClient(host.ipaddress, port=22, user=cls.hostConfig["username"], passwd=cls.hostConfig["password"])
+                result = sshClient.execute("sysctl -n vm.nr_hugepages")
+                sshClient.execute("sysctl -w vm.nr_hugepages=1024")
+                if result and len(result) > 0:
+                    hosts_hugepages.append({ "ipaddress": host.ipaddress, "vm.nr_hugepages": result[0].strip()})
+        return hosts_hugepages
+
+    @classmethod
+    def reset_hosts_hugepages(cls):
+        for host in cls.hosts_hugepages:
+            sshClient = SshClient(host["ipaddress"], port=22, user=cls.hostConfig["username"], passwd=cls.hostConfig["password"])
+            sshClient.execute("sysctl -w vm.nr_hugepages=%s" % host["vm.nr_hugepages"])
+
     def setUp(self):
         self.apiclient = self.testClient.getApiClient()
         self.hypervisor = self.testClient.getHypervisorInfo()

test/integration/smoke/test_host_maintenance.py

@@ -21,7 +21,7 @@
 from marvin.cloudstackTestCase import *
 from marvin.lib.utils import *
 from marvin.lib.base import *
-from marvin.lib.common import (get_zone, get_pod, get_template, list_ssvms)
+from marvin.lib.common import (get_zone, get_pod, get_suitable_test_template, list_ssvms)
 from nose.plugins.attrib import attr
 from marvin.lib.decoratorGenerators import skipTestIf
 from distutils.util import strtobool
@@ -115,20 +115,19 @@ class TestHostMaintenance(TestHostMaintenanceBase):
             raise Exception("Warning: Exception during cleanup : %s" % e)
 
         return
-    
+
     def createVMs(self, hostId, number, offering_key="tiny"):
-        
+        self.template = get_suitable_test_template(
-        self.template = get_template(
             self.apiclient,
             self.zone.id,
+            None,
             self.hypervisor
         )
-            
         if self.template == FAILED:
-            assert False, "get_template() failed to return template"
+            assert False, "get_suitable_test_template() failed to return template"
-            
+
         self.logger.debug("Using template %s " % self.template.id)
-                
+
         self.service_offering = ServiceOffering.create(
             self.apiclient,
             self.services["service_offerings"][offering_key]
@@ -196,7 +195,7 @@ class TestHostMaintenance(TestHostMaintenanceBase):
     def checkVmMigratingOnHost(self, hostId):
         vm_migrating=False
         listVms1 = VirtualMachine.list(
-                                   self.apiclient, 
+                                   self.apiclient,
                                    hostid=hostId
                                    )
 
@@ -209,11 +208,11 @@ class TestHostMaintenance(TestHostMaintenanceBase):
                     break
 
         return (vm_migrating, None)
-    
+
     def migrationsFinished(self, hostId):
         migrations_finished=True
         listVms1 = VirtualMachine.list(
-                                   self.apiclient, 
+                                   self.apiclient,
                                    hostid=hostId
                                    )
 
@@ -225,7 +224,7 @@ class TestHostMaintenance(TestHostMaintenanceBase):
 
     def noOfVMsOnHost(self, hostId):
         listVms = VirtualMachine.list(
-                                       self.apiclient, 
+                                       self.apiclient,
                                        hostid=hostId
                                        )
         no_of_vms=0
@@ -459,11 +458,16 @@ class TestHostMaintenanceAgents(TestHostMaintenanceBase):
 
         if not cls.hypervisorNotSupported:
             cls.initialsshvalue = cls.is_ssh_enabled()
-            cls.template = get_template(
+
+            cls.template = get_suitable_test_template(
                 cls.apiclient,
                 cls.zone.id,
+                None,
                 cls.hypervisor
             )
+            if cls.template == FAILED:
+                assert False, "get_suitable_test_template() failed to return template"
+
             cls.services["virtual_machine"]["zoneid"] = cls.zone.id
             cls.services["virtual_machine"]["template"] = cls.template.id
             cls.services["virtual_machine"]["hypervisor"] = cls.hypervisor

test/integration/smoke/test_nic.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 """ NIC tests for VM """
+from marvin.codes import FAILED
 from marvin.cloudstackTestCase import cloudstackTestCase
 from marvin.lib.base import (Account,
                              ServiceOffering,
@@ -22,7 +23,7 @@ from marvin.lib.base import (Account,
                              VirtualMachine,
                              NetworkOffering)
 from marvin.lib.common import (get_zone,
-                               get_template,
+                               get_suitable_test_template,
                                get_domain)
 from marvin.lib.utils import validateList
 from marvin.codes import PASS
@@ -75,12 +76,14 @@ class TestNic(cloudstackTestCase):
             if self.zone.localstorageenabled:
                 self.services["service_offerings"][
                     "tiny"]["storagetype"] = 'local'
-
+            template = get_suitable_test_template(
-            template = get_template(
                 self.apiclient,
                 self.zone.id,
-                self.services["ostype"]
+                self.services["ostype"],
+                self.hypervisor
             )
+            if template == FAILED:
+                assert False, "get_suitable_test_template() failed to return template with description %s" % self.services["ostype"]
             # Set Zones and disk offerings
             self.services["small"]["zoneid"] = self.zone.id
             self.services["small"]["template"] = template.id

test/integration/smoke/test_primary_storage.py

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,6 +19,7 @@
 
 # Import System modules
 # Import Local Modules
+from marvin.codes import FAILED
 from marvin.cloudstackTestCase import *
 from marvin.lib.base import *
 from marvin.lib.common import *
@@ -41,7 +42,14 @@ class TestPrimaryStorageServices(cloudstackTestCase):
         self.pod = get_pod(self.apiclient, self.zone.id)
         self.hypervisor = self.testClient.getHypervisorInfo()
         self.domain = get_domain(self.apiclient)
-        self.template = get_template(self.apiclient, self.zone.id, self.services["ostype"])
+        self.template = get_suitable_test_template(
+            self.apiclient,
+            self.zone.id,
+            self.services["ostype"],
+            self.hypervisor
+        )
+        if self.template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % self.services["ostype"]
 
         return
 
@@ -371,7 +379,14 @@ class TestStorageTags(cloudstackTestCase):
         cls.pod = get_pod(cls.apiclient, cls.zone.id)
         cls.hypervisor = testClient.getHypervisorInfo()
         cls.domain = get_domain(cls.apiclient)
-        cls.template = get_template(cls.apiclient, cls.zone.id, cls.services["ostype"])
+        cls.template = get_suitable_test_template(
+            cls.apiclient,
+            cls.zone.id,
+            cls.services["ostype"],
+            cls.hypervisor
+        )
+        if cls.template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
         cls.services["virtual_machine"]["zoneid"] = cls.zone.id
         cls.services["virtual_machine"]["template"] = cls.template.id
         cls.services["storage_tags"] = StorageTagsServices().storage_tags

test/integration/smoke/test_usage.py

@@ -17,6 +17,7 @@
 """ P1 tests for Snapshots
 """
 # Import Local Modules
+from marvin.codes import FAILED
 from nose.plugins.attrib import attr
 from marvin.cloudstackTestCase import cloudstackTestCase, unittest
 from marvin.cloudstackAPI import deleteVolume
@@ -37,7 +38,7 @@ from marvin.lib.base import (Account,
                              Volume)
 from marvin.lib.common import (get_zone,
                                get_domain,
-                               get_template,
+                               get_suitable_test_template,
                                find_storage_pool_type)
 
 
@@ -136,12 +137,15 @@ class TestVmUsage(cloudstackTestCase):
         cls.domain = get_domain(cls.api_client)
         cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests())
         cls.services['mode'] = cls.zone.networktype
-
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
-        template = get_template(
+        template = get_suitable_test_template(
             cls.api_client,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
+        if template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
         cls.services["server"]["zoneid"] = cls.zone.id
 
         cls.services["template"] = template.id
@@ -335,12 +339,15 @@ class TestPublicIPUsage(cloudstackTestCase):
         cls.domain = get_domain(cls.api_client)
         cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests())
         cls.services['mode'] = cls.zone.networktype
-
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
-        template = get_template(
+        template = get_suitable_test_template(
             cls.api_client,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
+        if template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
         cls.services["server"]["zoneid"] = cls.zone.id
 
         cls.services["template"] = template.id
@@ -508,11 +515,15 @@ class TestVolumeUsage(cloudstackTestCase):
             cls.services["disk_offering"]
         )
         cls._cleanup.append(cls.disk_offering)
-        template = get_template(
+
+        template = get_suitable_test_template(
             cls.api_client,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
+        if template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
         cls.services["server"]["zoneid"] = cls.zone.id
         cls.services["server"]["diskoffering"] = cls.disk_offering.id
         cls.services["template"] = template.id
@@ -803,11 +814,15 @@ class TestTemplateUsage(cloudstackTestCase):
         cls.services['mode'] = cls.zone.networktype
         cls.hypervisor = cls.testClient.getHypervisorInfo()
         cls.services["server"]["zoneid"] = cls.zone.id
-        template = get_template(
+
+        template = get_suitable_test_template(
             cls.api_client,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
+        if template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
         cls.services["server"]["zoneid"] = cls.zone.id
         cls._cleanup = []
         try:
@@ -1131,11 +1146,16 @@ class TestLBRuleUsage(cloudstackTestCase):
         cls.domain = get_domain(cls.api_client)
         cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests())
         cls.services['mode'] = cls.zone.networktype
-        template = get_template(
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+
+        template = get_suitable_test_template(
             cls.api_client,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
+        if template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
         cls.services["server"]["zoneid"] = cls.zone.id
 
         cls.services["template"] = template.id
@@ -1307,11 +1327,14 @@ class TestSnapshotUsage(cloudstackTestCase):
             cls.unsupportedHypervisor = True
             return
 
-        template = get_template(
+        template = get_suitable_test_template(
             cls.api_client,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
+        if template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
         cls.services["server"]["zoneid"] = cls.zone.id
 
         cls.services["template"] = template.id
@@ -1488,11 +1511,16 @@ class TestNatRuleUsage(cloudstackTestCase):
         cls.domain = get_domain(cls.api_client)
         cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests())
         cls.services['mode'] = cls.zone.networktype
-        template = get_template(
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+
+        template = get_suitable_test_template(
             cls.api_client,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
+        if template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
         cls.services["server"]["zoneid"] = cls.zone.id
 
         cls.services["template"] = template.id
@@ -1657,11 +1685,16 @@ class TestVpnUsage(cloudstackTestCase):
         cls.domain = get_domain(cls.api_client)
         cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests())
         cls.services['mode'] = cls.zone.networktype
-        template = get_template(
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+
+        template = get_suitable_test_template(
             cls.api_client,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
+        if template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
         cls.services["server"]["zoneid"] = cls.zone.id
 
         cls.services["template"] = template.id

test/integration/smoke/test_vm_life_cycle.py

@@ -44,7 +44,7 @@ from marvin.lib.base import (Account,
                              Network)
 from marvin.lib.common import (get_domain,
                                get_zone,
-                               get_template,
+                               get_suitable_test_template,
                                list_hosts,
                                list_virtual_machines)
 from marvin.codes import FAILED, PASS
@@ -68,6 +68,7 @@ class TestDeployVM(cloudstackTestCase):
         cls.domain = get_domain(cls.apiclient)
         cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
         cls.services['mode'] = cls.zone.networktype
+        cls.hypervisor = testClient.getHypervisorInfo()
 
         # If local storage is enabled, alter the offerings to use localstorage
         # this step is needed for devcloud
@@ -76,13 +77,14 @@ class TestDeployVM(cloudstackTestCase):
             cls.services["service_offerings"]["small"]["storagetype"] = 'local'
             cls.services["service_offerings"]["medium"]["storagetype"] = 'local'
 
-        template = get_template(
+        template = get_suitable_test_template(
             cls.apiclient,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
         if template == FAILED:
-            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
 
         # Set Zones and disk offerings
         cls.services["small"]["zoneid"] = cls.zone.id
@@ -286,13 +288,14 @@ class TestVMLifeCycle(cloudstackTestCase):
             cls.services["service_offerings"]["small"]["storagetype"] = 'local'
             cls.services["service_offerings"]["medium"]["storagetype"] = 'local'
 
-        template = get_template(
+        template = get_suitable_test_template(
             cls.apiclient,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
         if template == FAILED:
-            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
 
         # Set Zones and disk offerings
         cls.services["small"]["zoneid"] = cls.zone.id
@@ -734,62 +737,69 @@ class TestVMLifeCycle(cloudstackTestCase):
         except Exception as e:
             self.fail("SSH failed for virtual machine: %s - %s" %
                       (self.virtual_machine.ipaddress, e))
-
         mount_dir = "/mnt/tmp"
         cmds = "mkdir -p %s" % mount_dir
         self.assert_(ssh_client.execute(cmds) == [], "mkdir failed within guest")
 
+        iso_unsupported = False
         for diskdevice in self.services["diskdevice"]:
             res = ssh_client.execute("mount -rt iso9660 {} {}".format(diskdevice, mount_dir))
             if res == []:
                 self.services["mount"] = diskdevice
                 break
+            if str(res).find("mount: unknown filesystem type 'iso9660'") != -1:
+                iso_unsupported = True
+                self.debug("Test template does not supports iso9660 filesystem. Proceeding with test without mounting.")
+                print "Test template does not supports iso9660 filesystem. Proceeding with test without mounting."
+                break
         else:
             self.fail("No mount points matched. Mount was unsuccessful")
 
-        c = "mount |grep %s|head -1" % self.services["mount"]
+        if iso_unsupported == False:
-        res = ssh_client.execute(c)
+            c = "mount |grep %s|head -1" % self.services["mount"]
-        size = ssh_client.execute("du %s | tail -1" % self.services["mount"])
+            res = ssh_client.execute(c)
-        self.debug("Found a mount point at %s with size %s" % (res, size))
+            size = ssh_client.execute("du %s | tail -1" % self.services["mount"])
+            self.debug("Found a mount point at %s with size %s" % (res, size))
 
-        # Get ISO size
+            # Get ISO size
-        iso_response = Iso.list(
+            iso_response = Iso.list(
-            self.apiclient,
+                self.apiclient,
-            id=iso.id
+                id=iso.id
-        )
+            )
-        self.assertEqual(
+            self.assertEqual(
-            isinstance(iso_response, list),
+                isinstance(iso_response, list),
-            True,
+                True,
-            "Check list response returns a valid list"
+                "Check list response returns a valid list"
-        )
+            )
 
-        try:
+            try:
-            # Unmount ISO
+                # Unmount ISO
-            command = "umount %s" % mount_dir
+                command = "umount %s" % mount_dir
-            ssh_client.execute(command)
+                ssh_client.execute(command)
-        except Exception as e:
+            except Exception as e:
-            self.fail("SSH failed for virtual machine: %s - %s" %
+                self.fail("SSH failed for virtual machine: %s - %s" %
-                      (self.virtual_machine.ipaddress, e))
+                          (self.virtual_machine.ipaddress, e))
 
         # Detach from VM
         cmd = detachIso.detachIsoCmd()
         cmd.virtualmachineid = self.virtual_machine.id
         self.apiclient.detachIso(cmd)
 
-        try:
+        if iso_unsupported == False:
-            res = ssh_client.execute(c)
+            try:
-        except Exception as e:
+                res = ssh_client.execute(c)
-            self.fail("SSH failed for virtual machine: %s - %s" %
+            except Exception as e:
-                      (self.virtual_machine.ipaddress, e))
+                self.fail("SSH failed for virtual machine: %s - %s" %
+                          (self.virtual_machine.ipaddress, e))
 
-        # Check if ISO is properly detached from VM (using fdisk)
+            # Check if ISO is properly detached from VM (using fdisk)
-        result = self.services["mount"] in str(res)
+            result = self.services["mount"] in str(res)
 
-        self.assertEqual(
+            self.assertEqual(
-            result,
+                result,
-            False,
+                False,
-            "Check if ISO is detached from virtual machine"
+                "Check if ISO is detached from virtual machine"
-        )
+            )
         return
 
     @attr(tags = ["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
@@ -850,13 +860,14 @@ class TestSecuredVmMigration(cloudstackTestCase):
             0].__dict__
         cls.management_ip = cls.config.__dict__["mgtSvr"][0].__dict__["mgtSvrIp"]
 
-        template = get_template(
+        template = get_suitable_test_template(
             cls.apiclient,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
         if template == FAILED:
-            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
 
         # Set Zones and disk offerings
         cls.services["small"]["zoneid"] = cls.zone.id
@@ -972,7 +983,7 @@ class TestSecuredVmMigration(cloudstackTestCase):
                       sleep 30 && \
                       service cloudstack-agent restart")
         print("Unsecuring Host: %s" % (host.name))
-        self.waitUntilHostInState(hostId=host.id, state="Up") 
+        self.waitUntilHostInState(hostId=host.id, state="Up")
         self.check_connection(host=host, secured='false')
         return host
 
@@ -1122,13 +1133,14 @@ class TestMigrateVMwithVolume(cloudstackTestCase):
             0].__dict__
         cls.management_ip = cls.config.__dict__["mgtSvr"][0].__dict__["mgtSvrIp"]
 
-        template = get_template(
+        template = get_suitable_test_template(
             cls.apiclient,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
         if template == FAILED:
-            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
 
         # Set Zones and disk offerings
         cls.services["small"]["zoneid"] = cls.zone.id
@@ -1334,13 +1346,14 @@ class TestKVMLiveMigration(cloudstackTestCase):
             0].__dict__
         cls.management_ip = cls.config.__dict__["mgtSvr"][0].__dict__["mgtSvrIp"]
 
-        template = get_template(
+        template = get_suitable_test_template(
             cls.apiclient,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
         if template == FAILED:
-            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
 
         # Set Zones and disk offerings
         cls.services["small"]["zoneid"] = cls.zone.id
@@ -1533,15 +1546,14 @@ class TestUnmanageVM(cloudstackTestCase):
         cls.domain = get_domain(cls.apiclient)
         cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests())
         cls.services['mode'] = cls.zone.networktype
-
+        cls.template = get_suitable_test_template(
-        cls.template = get_template(
             cls.apiclient,
             cls.zone.id,
             cls.services["ostype"],
-            hypervisor=cls.hypervisor.lower()
+            cls.hypervisor
         )
         if cls.template == FAILED:
-            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
 
         cls.hypervisorNotSupported = cls.hypervisor.lower() != "vmware"
 
@@ -1666,4 +1678,4 @@ class TestUnmanageVM(cloudstackTestCase):
             unmanaged_vm.powerstate,
             "PowerOn",
             "Unmanaged VM is still running"
-        )
+        )

test/integration/smoke/test_vm_snapshots.py

@@ -26,7 +26,7 @@ from marvin.lib.base import (Account,
                              VmSnapshot)
 from marvin.lib.common import (get_zone,
                                get_domain,
-                               get_template,
+                               get_suitable_test_template,
                                list_snapshots,
                                list_virtual_machines)
 import time
@@ -50,13 +50,14 @@ class TestVmSnapshot(cloudstackTestCase):
         cls.domain = get_domain(cls.apiclient)
         cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
 
-        template = get_template(
+        template = get_suitable_test_template(
             cls.apiclient,
             cls.zone.id,
-            cls.services["ostype"]
+            cls.services["ostype"],
+            cls.hypervisor
         )
         if template == FAILED:
-            assert False, "get_template() failed to return template\
+            assert False, "get_suitable_test_template() failed to return template\
                     with description %s" % cls.services["ostype"]
 
         cls.services["domainid"] = cls.domain.id
@@ -87,7 +88,7 @@ class TestVmSnapshot(cloudstackTestCase):
             mode=cls.zone.networktype
         )
         cls.random_data_0 = random_gen(size=100)
-        cls.test_dir = "/tmp"
+        cls.test_dir = "$HOME"
         cls.random_data = "random.data"
         return
 
@@ -288,15 +289,15 @@ class TestVmSnapshot(cloudstackTestCase):
         )
 
 class Utils:
-    
+
     def __init__(self):
         self.added_service_offerings = {
-            'testOffering1' : {'displaytext': 'Test Offering 1', 'cpuspeed': 600, 'cpunumber': 1, 'name': 'Test Offering 1', 'memory': 256}, 
+            'testOffering1' : {'displaytext': 'Test Offering 1', 'cpuspeed': 600, 'cpunumber': 1, 'name': 'Test Offering 1', 'memory': 256},
-            'testOffering2' : {'displaytext': 'Test Offering 2', 'cpuspeed': 600, 'cpunumber': 2, 'name': 'Test Offering 2', 'memory': 512}                            
+            'testOffering2' : {'displaytext': 'Test Offering 2', 'cpuspeed': 600, 'cpunumber': 2, 'name': 'Test Offering 2', 'memory': 512}
         }
-    
+
 class TestChangeServiceOfferingForVmWithSnapshots(cloudstackTestCase):
-        
+
     @classmethod
     def setUpClass(cls):
         try:
@@ -309,46 +310,48 @@ class TestChangeServiceOfferingForVmWithSnapshots(cloudstackTestCase):
             if cls.hypervisor.lower() in (KVM.lower(), "hyperv", "lxc"):
                 cls.unsupportedHypervisor = True
                 return
-            
+
             cls.domain = get_domain(cls.api_client)
             cls.zone = get_zone(
                 cls.api_client,
                 cls.testClient.getZoneForTests()
             )
             cls.services["small"]["zoneid"] = cls.zone.id
-            cls.template = get_template(
+
+            cls.template = get_suitable_test_template(
                 cls.api_client,
                 cls.zone.id,
-                cls.services["ostype"]
+                cls.services["ostype"],
+                cls.hypervisor
             )
             if cls.template == FAILED:
-                assert False, "get_template() failed to return template\
+                assert False, "get_suitable_test_template() failed to return template\
                     with description %s" % cls.services["ostype"]
-            
+
             test_offerings = Utils().added_service_offerings
             for offering in test_offerings:
                 cls.services["service_offerings"][offering] = test_offerings[offering]
-                
+
             # Create 2 different service offerings
             cls.service_offering_1 = ServiceOffering.create(
                 cls.api_client,
                 cls.services["service_offerings"]["testOffering1"]
             )
             cls._cleanup.append(cls.service_offering_1)
-            
+
             cls.service_offering_2 = ServiceOffering.create(
                 cls.api_client,
                 cls.services["service_offerings"]["testOffering2"]
             )
             cls._cleanup.append(cls.service_offering_2)
-            
+
             cls.account = Account.create(
                 cls.api_client,
                 cls.services["account"],
                 domainid=cls.domain.id
             )
             cls._cleanup.append(cls.account)
-            
+
         except Exception as e:
             cls.tearDownClass()
             raise Exception("Warning: Exception in setup : %s" % e)
@@ -358,11 +361,11 @@ class TestChangeServiceOfferingForVmWithSnapshots(cloudstackTestCase):
         self.apiclient = self.testClient.getApiClient()
         self.dbclient = self.testClient.getDbConnection()
         self.cleanup = []
-        
+
         if self.unsupportedHypervisor:
             self.skipTest("Skipping test because unsupported hypervisor\
                     %s" % self.hypervisor)
-        
+
     def tearDown(self):
         # Clean up, terminate the created resources
         cleanup_resources(self.apiclient, self.cleanup)
@@ -376,7 +379,7 @@ class TestChangeServiceOfferingForVmWithSnapshots(cloudstackTestCase):
             raise Exception("Warning: Exception during cleanup : %s" % e)
 
         return
-    
+
     def wait_vm_start(self, apiclient, vmid, timeout, sleep):
         while timeout:
             vms = VirtualMachine.list(apiclient, id=vmid)
@@ -387,7 +390,7 @@ class TestChangeServiceOfferingForVmWithSnapshots(cloudstackTestCase):
             timeout = timeout - 1
 
         return timeout
-    
+
     def checkCPUAndMemory(self, ssh, service_offering):
         cpuinfo = ssh.execute("cat /proc/cpuinfo")
         cpu_cnt = len([i for i in cpuinfo if "processor" in i])
@@ -421,7 +424,7 @@ class TestChangeServiceOfferingForVmWithSnapshots(cloudstackTestCase):
     def test_change_service_offering_for_vm_with_snapshots(self):
         """Test to change service offering for instances with vm snapshots
         """
-        
+
         # 1) Create Virtual Machine using service offering 1
         self.debug("Creating VM using Service Offering 1")
         virtual_machine = VirtualMachine.create(
@@ -435,46 +438,46 @@ class TestChangeServiceOfferingForVmWithSnapshots(cloudstackTestCase):
             mode=self.zone.networktype,
             serviceofferingid=self.service_offering_1.id
         )
-        
+
         # Verify Service OFfering 1 CPU cores and memory
         try:
             ssh_client = virtual_machine.get_ssh_client(reconnect=True)
             self.checkCPUAndMemory(ssh_client, self.service_offering_1)
         except Exception as e:
             self.fail("SSH failed for virtual machine: %s - %s" % (virtual_machine.ipaddress, e))
-        
+
         # 2) Take VM Snapshot
         self.debug("Taking VM Snapshot for VM - ID: %s" % virtual_machine.id)
         vm_snapshot = VmSnapshot.create(
             self.apiclient,
             virtual_machine.id,
         )
-        
+
         # 3) Stop Virtual Machine
         self.debug("Stopping VM - ID: %s" % virtual_machine.id)
         try:
             virtual_machine.stop(self.apiclient)
         except Exception as e:
             self.fail("Failed to stop VM: %s" % e)
-        
+
         # 4) Change service offering for VM with snapshots from Service Offering 1 to Service Offering 2
         self.debug("Changing service offering from Service Offering 1 to Service Offering 2 for VM - ID: %s" % virtual_machine.id)
         virtual_machine.change_service_offering(self.apiclient, self.service_offering_2.id)
-        
+
         # 5) Start VM
         self.debug("Starting VM - ID: %s" % virtual_machine.id)
         try:
             virtual_machine.start(self.apiclient)
         except Exception as e:
             self.fail("Failed to start virtual machine: %s, %s" % (virtual_machine.name, e))
-        
+
         # Wait for vm to start
         timeout = self.wait_vm_start(self.apiclient, virtual_machine.id, self.services["timeout"],
                             self.services["sleep"])
         if timeout == 0:
             self.fail("The virtual machine %s failed to start even after %s minutes"
                    % (virtual_machine.name, self.services["timeout"]))
-        
+
         list_vm_response = list_virtual_machines(
             self.apiclient,
             id=virtual_machine.id
@@ -499,14 +502,14 @@ class TestChangeServiceOfferingForVmWithSnapshots(cloudstackTestCase):
             virtual_machine.id,
             "Check virtual machine id"
         )
-        
+
         # 6) Verify service offering has changed
         try:
             ssh_client_2 = virtual_machine.get_ssh_client(reconnect=True)
             self.checkCPUAndMemory(ssh_client_2, self.service_offering_2)
         except Exception as e:
             self.fail("SSH failed for virtual machine: %s - %s" % (virtual_machine.ipaddress, e))
-        
+
         # 7) Stop Virtual Machine
         self.debug("Stopping VM - ID: %s" % virtual_machine.id)
         try:
@@ -524,19 +527,19 @@ class TestChangeServiceOfferingForVmWithSnapshots(cloudstackTestCase):
             )
         except Exception as e:
             self.fail("Failed to revert to VM Snapshot: %s - %s" % (vm_snapshot.id, e))
-        
+
         # 9) Start VM
         self.debug("Starting VM - ID: %s" % virtual_machine.id)
         try:
             virtual_machine.start(self.apiclient)
         except Exception as e:
             self.fail("Failed to start virtual machine: %s, %s" % (virtual_machine.name, e))
-            
+
         # 10) Verify service offering has changed to Service Offering 1 (from VM Snapshot)
         try:
             ssh_client_3 = virtual_machine.get_ssh_client(reconnect=True)
             self.checkCPUAndMemory(ssh_client_3, self.service_offering_1)
         except Exception as e:
             self.fail("SSH failed for virtual machine: %s - %s" % (virtual_machine.ipaddress, e))
-        
+
-        return
+        return

test/integration/smoke/test_volumes.py

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -34,8 +34,8 @@ from marvin.lib.base import (ServiceOffering,
                              DiskOffering,
                              StoragePool,)
 from marvin.lib.common import (get_domain,
+                                get_suitable_test_template,
                                 get_zone,
-                                get_template,
                                 find_storage_pool_type,
                                 get_pod,
                                 list_disk_offering)
@@ -83,13 +83,15 @@ class TestCreateVolume(cloudstackTestCase):
                                     cls.services["disk_offering"],
                                     custom=True
                                     )
-        template = get_template(
+
-                            cls.apiclient,
+        template = get_suitable_test_template(
-                            cls.zone.id,
+            cls.apiclient,
-                            cls.services["ostype"]
+            cls.zone.id,
-                            )
+            cls.services["ostype"],
+            cls.hypervisor
+        )
         if template == FAILED:
-            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
 
         cls.services["domainid"] = cls.domain.id
         cls.services["zoneid"] = cls.zone.id
@@ -302,17 +304,18 @@ class TestVolumes(cloudstackTestCase):
                                     custom=True
                                     )
 
-        template = get_template(
+        cls.template = get_suitable_test_template(
-                            cls.apiclient,
+            cls.apiclient,
-                            cls.zone.id,
+            cls.zone.id,
-                            cls.services["ostype"]
+            cls.services["ostype"],
-                            )
+            cls.hypervisor
-        if template == FAILED:
+        )
-            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+        if cls.template == FAILED:
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
 
         cls.services["domainid"] = cls.domain.id
         cls.services["zoneid"] = cls.zone.id
-        cls.services["template"] = template.id
+        cls.services["template"] = cls.template.id
         cls.services["diskofferingid"] = cls.disk_offering.id
         cls.services['resizeddiskofferingid'] = cls.resized_disk_offering.id
         cls.services['customresizeddiskofferingid'] = cls.custom_resized_disk_offering.id
@@ -530,8 +533,12 @@ class TestVolumes(cloudstackTestCase):
         self.debug("Extract detached Volume ID: %s" % self.volume.id)
 
         self.virtual_machine.attach_volume(self.apiClient, self.volume)
+        #Sleep to ensure the current state will reflected in other calls
+        time.sleep(self.services["sleep"])
         self.virtual_machine.detach_volume(self.apiClient, self.volume)
         self.attached = False
+        #Sleep to ensure the current state will reflected in other calls
+        time.sleep(self.services["sleep"])
 
         cmd = extractVolume.extractVolumeCmd()
         cmd.id = self.volume.id
@@ -871,6 +878,7 @@ class TestVolumes(cloudstackTestCase):
         test_vm = VirtualMachine.create(
             self.apiclient,
             self.services,
+            templateid=self.template.id,
             accountid=self.account.name,
             domainid=self.account.domainid,
             serviceofferingid=self.service_offering.id,
@@ -906,6 +914,8 @@ class TestVolumes(cloudstackTestCase):
             None,
             "Check if volume state (attached) is reflected"
         )
+        #Sleep to ensure the current state will reflected in other calls
+        time.sleep(self.services["sleep"])
 
         test_vm.detach_volume(self.apiClient, self.volume)
         self.cleanup.append(test_vm)
@@ -978,7 +988,7 @@ class TestVolumes(cloudstackTestCase):
             pool = pools[0]
         else:
             raise self.skipTest("Not enough storage pools found, skipping test")
-        
+
         if hasattr(pool, 'tags'):
             StoragePool.update(self.apiclient, id=pool.id, tags="")
 

tools/marvin/marvin/lib/common.py

@@ -445,7 +445,29 @@ def get_windows_template(
 
     return FAILED
 
-
+def get_suitable_test_template(apiclient, zoneid, ostypeid, hypervisor):
+    '''
+    @Name : get_suitable_test_template
+    @Desc : Retrieves the test template information based upon inputs provided
+            For Xenserver, get_test_template is used for retrieving the template
+            while get_template is used for other hypervisors or when
+            get_test_template fails
+    @Input : returns a template"
+    @Output : FAILED in case of any failure
+              template Information matching the inputs
+    '''
+    template = FAILED
+    if hypervisor.lower() in ["xenserver"]:
+        template = get_test_template(
+            apiclient,
+            zoneid,
+            hypervisor)
+    if template == FAILED:
+        template = get_template(
+            apiclient,
+            zoneid,
+            ostypeid)
+    return template
 
 def download_systemplates_sec_storage(server, services):
     """Download System templates on sec storage"""

utils/conf/db.properties

@@ -44,7 +44,7 @@ db.cloud.testWhileIdle=true
 db.cloud.timeBetweenEvictionRunsMillis=40000
 db.cloud.minEvictableIdleTimeMillis=240000
 db.cloud.poolPreparedStatements=false
-db.cloud.url.params=prepStmtCacheSize=517&cachePrepStmts=true&prepStmtCacheSqlLimit=4096&sessionVariables=sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'&serverTimezone=UTC
+db.cloud.url.params=prepStmtCacheSize=517&cachePrepStmts=true&prepStmtCacheSqlLimit=4096&sessionVariables=sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'&serverTimezone=UTC
 
 # usage database settings
 db.usage.username=cloud