cloudstack/wscript_build

#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""builds the entire stack"""
#For every matching build change here, that produces new installable
#files, the cloud.spec file and the Debian control files must be
#revised and tested.

import shutil,os
import Utils,Node,Options,Logs,Scripting,Environment,Build,Configure
from os import unlink as _unlink, makedirs as _makedirs, getcwd as _getcwd, chdir as _chdir
from os.path import abspath as _abspath, basename as _basename, dirname as _dirname, exists as _exists, isdir as _isdir, split as _split, join as _join, sep, pathsep, pardir, curdir
from glob import glob as _glob
from subprocess import Popen as _Popen,PIPE
try: set([1,2,3])
except Exception: from Sets import set
import re
import zipfile,tarfile
import time
try:
  from os import chmod as _chmod,chown as _chown
  import pwd,stat,grp
except ImportError:
  _chmod,_chown,pwd,stat,grp = (None,None,None,None,None)


# Global variables setup
sourcedir = bld.srcnode.abspath()
builddir = bld.path.abspath(bld.env)
#buildpremium = _exists(_join(sourcedir,"cloudstack-proprietary"))
buildpremium = False
filelist = bld.path.ant_glob
distdir = Utils.relpath(_join(sourcedir,"dist"))
targetdir = Utils.relpath(_join(sourcedir,"target"))

def gitinfo(dir=None):
	if dir and not _isdir(dir): return ''
	try: p = _Popen(['git','remote','show','-n','origin'],stdin=PIPE,stdout=PIPE,stderr=PIPE,cwd=dir)
	except OSError,e:
		if e.errno == 2: return '' # svn command is not installed
		raise
	stdout,stderr = p.communicate('')
	retcode = p.wait()
	# If the guess fails, just return nothing.
	if retcode: return
	stdout = [ s.strip() for s in stdout.splitlines() ]
	try: url = [ s[11:] for s in stdout if s.startswith("Fetch URL") ][0]
	except IndexError: url = [ s[5:] for s in stdout if s.startswith("URL") ][0]
	assert url
	
	p = _Popen(['git','log','-1'],stdin=PIPE,stdout=PIPE,stderr=PIPE,cwd=dir)
	stdout,stderr = p.communicate('')
	retcode = p.wait()
	if retcode: return
	# If the guess fails, just return nothing.
	stdout = [ s.strip() for s in stdout.splitlines() ]
	commitid = [ s.split()[1] for s in stdout if s.startswith("commit") ][0]
	assert commitid
	
	return "Git Revision: %s"%commitid + "\n" + "Git URL: %s"%url + "\n"

def build_utils_docs ():
	stdout = gitinfo()
	if stdout:
		f = file("sccs-info","w")
		f.write(stdout)
		f.flush()
		f.close()
	else:
		if _exists("sccs-info"):
			# If the file already existed, we preserve it
			return
		else:
			f = file("sccs-info","w")
			f.write("No revision control information could be detected when the source distribution was built.")
			f.flush()
			f.close()

	sccsinfo = _join(sourcedir,"sccs-info")
	if _exists(sccsinfo): bld.install_files("${DOCDIR}","sccs-info")

	tgen = bld(features='subst', name='configure-info', source="configure-info.in", target="configure-info")
	tgen.dict = {"CONFIGUREVARS":bld.getconfig()}
	bld.install_files("${DOCDIR}","configure-info")

# compile jar files using ant
# ant only needs to be reinvoked if the version with build number changes
# we here trim all the depended targets from the target list:
def build_jars ():
	Implementation_Version = bld.env.VERSION

	# this is to trigger recompilation / cache avoidance if the relevant environment for ant changes
	ant_args = [
		"build-all",
		"-Dimpl.version=%s" % Implementation_Version,
		"-Dtarget.dir=%s" % targetdir,
		"-Ddist.dir=%s" % distdir,
		"-Dbase.dir=%s" % sourcedir,
		"-f %s" % Utils.relpath (_join(sourcedir, "build.xml")),
	]

	if buildpremium:
		ant_args.append("-Dbuild.premium=true")

	tgen = bld(features='subst', name='version-info', source="version-info.in", target="version-info")
	tgen.dict = { "Implementation_Version":Implementation_Version,"ant_args":ant_args }
	bld.install_files("${DOCDIR}","version-info")

	bld.srcnode.ensure_dir_node_from_path("target/jar")
	bld.srcnode.ensure_dir_node_from_path("dist")

	tgen = bld.new_task_gen (rule = Utils.runant,
							 name = "runant",
							 antargs = ant_args)

	jarnode = bld.srcnode.find_dir ('target/jar')
	jars_str = jarnode.ant_glob ('*.jar').split ()
	ant_jars = []
	excludes = ["cloud-xstream-1.3.1.jar", "cloud-commons-dbcp-1.2.2.jar", 
			"cloud-commons-httpclient-3.1.jar", "cloud-commons-pool-1.4.jar",
			"cloud-servlet-api.jar", "cloud-commons-logging-1.1.1.jar",
			"cloud-ws-commons-util-1.0.2.jar",
			"cloud-commons-collections-3.2.1.jar", "vmware*.jar", "cloud-secstorage-extras.jar",
			"cloud-agent-simulator.jar", "cloud-awsapi.jar", "cloud-test.jar", "cloud-wsdl4j.jar", "cloud-console-proxy.jar"]

	for a in jars_str:
		if _basename (a).startswith ("cloud-") \
			and not _basename (a) in excludes:
			a = jarnode.abspath () + os.sep + a
			ant_jars.append (a)
	
	bld.install_files ('${JAVADIR}', ant_jars)


def build_premium ():
	if buildpremium: bld.recurse(["cloudstack-proprietary/"],'build')

def build_thirdparty_dir ():
	start_path = bld.path.find_dir ("thirdparty")
	bld.install_files('${PREMIUMJAVADIR}','*.jar', cwd = start_path)
	Utils.pprint ("GREEN", "Installed files of thirdparty/")

def build_dependences ():
	excludes = ["cloud-xstream-1.3.1.jar", "cloud-servlet-api.jar", "cloud-commons-logging-1.1.1.jar",
				"cloud-ws-commons-util-1.0.2.jar",
				"cloud-commons-collections-3.2.1.jar", "cloud-wsdl4j.jar"]

	start_path = bld.path.find_dir ("deps")

        bld.install_files('${JAVADIR}',start_path.ant_glob(["CAStorSDK-*.jar", "javax.persistence-2.0.0.jar", "apache-log4j-extras-1.1.jar", "libvirt-0.4.9.jar", "axis2-1.5.1.jar", "jstl-1.2.jar", "commons-discovery-0.5.jar", "commons-codec-1.6.jar", "ejb-api-3.0.jar", "xmlrpc-client-3.1.3.jar", "commons-dbcp-1.4.jar", "commons-pool-1.6.jar", "gson-1.7.1.jar", 
         "netscaler-1.0.jar", "netscaler-sdx-1.0.jar", "backport-util-concurrent-3.1.jar", "ehcache-1.5.0.jar", "httpcore-4.0.jar", "log4j-1.2.16.jar", "trilead-ssh2-build213-svnkit-1.3-patch.jar", "cglib-nodep-2.2.2.jar", "xmlrpc-common-3.*.jar",
         "axiom*.jar", "axis2*.jar", "antlr*.jar", "XmlSchema*.jar", "json-simple*.jar", "neethi*.jar", "woden*.jar", "xercesImpl*.jar", "xml-apis*.jar", "dom4j*.jar", "javassist*.jar", "commons-fileupload*.jar",
         "xmlrpc-client-3.*.jar", "wsdl4j-1.6.2.jar", "bcprov-jdk16-1.45.jar", "jsch-0.1.42.jar", "jasypt-1.9.0.jar", "commons-configuration-1.8.jar", "mail-1.4.jar", "activation-1.1.jar",  "xapi-5.6.100-1-SNAPSHOT.jar"], excl = excludes), cwd=start_path)

#def build_console_proxy ():
	# binary unsubstitutable files:
#	start_path = bld.path.find_dir ("console-proxy")
#	bld.install_files("${CPLIBDIR}",start_path.ant_glob("images/**",src=True,bld=False,dir=False,flat=True),cwd=start_path,relative_trick=True)

	# text substitutable files (substitute with tokens from the environment bld.env):
#	bld.substitute('css/** js/** ui/** scripts/**',install_to="${CPLIBDIR}", cwd=start_path)
	
	# config files (do not replace them if preserve config option is true)
#	if not Options.options.PRESERVECONFIG: bld.install_files_filtered("${CPSYSCONFDIR}","conf.dom0/*", cwd=start_path)

def build_patches ():
	# done here because the patches require substituted files
	start_path = bld.path.find_dir ("patches")
	bld.substitute("*/**",name="patchsubst", cwd = start_path)

	for virttech in Utils.to_list(start_path.ant_glob("*",dir=True)):
		if virttech in ["shared"]: continue
		patchfiles = start_path.ant_glob('shared/** %s/debian/config/**'%virttech,src=False,bld=True,dir=False,flat=True)
		tmp = bld.path
		bld.path = start_path
		tgen = bld(
			features  = 'tar',#Utils.tar_up,
			source = patchfiles,
			target = 'cloud-scripts.tgz',
			name   = 'cloud-scripts_tgz',
			root = os.path.join("patches", virttech + "/debian/config"),
			rename = lambda x: re.sub(".subst$","",x),
		)
		bld.path = tmp


def build_systemvm_patch ():
	if bld.env.DISTRO not in ["Windows","Mac"]:
		# patch creation
		bld.install_files ("${COMMONLIBDIR}/vms", "%s/systemvm.zip" % distdir)
		# ISO creation
		bld.install_as("${COMMONLIBDIR}/vms/systemvm.iso", "%s/systemvm.iso" % distdir)

def build_systemvm_iso ():
	if buildpremium:
		bld.install_as("${AGENTLIBDIR}/vms/systemvm-premium.iso", "%s/systemvm-premium.iso" % distdir)

# =================== Empty directory / symlink creation on install target ====================

def build_dirs_symlinks ():
	bld.createuser(bld.env.MSUSER,bld.env.MSENVIRON,'/bin/sh')
	
	# 7. make log and cache dirs (this actually runs first)
	if bld.env.DISTRO in 'Windows Mac': pass
	else:
		x = ("root",bld.env.MSUSER)
		directories = [
			("${MSLOGDIR}",0770,x),
			("${AGENTLOGDIR}",0770,x),
			("${USAGELOGDIR}",0770,x),
#			("${CPLOGDIR}",0770,x),
			("${IPALLOCATORLOGDIR}",0770,x),
			("${LOCALSTATEDIR}/cache/${MSPATH}",0770,x),
			("${LOCALSTATEDIR}/cache/${MSPATH}/temp",0770,x),
			("${LOCALSTATEDIR}/cache/${MSPATH}/work",0770,x),
			("${SHAREDSTATEDIR}/${MSPATH}",0770,x),
			("${MSMNTDIR}",0770,x),
			("${MSCONF}/Catalina",0770,x),
			("${MSCONF}/Catalina/localhost",0770,x),
			("${MSCONF}/Catalina/localhost/client",0770,x),
			("${PIDDIR}",0755,("root","root")),
			("${LOCKDIR}",0755,("root","root")),
		]
		
		for a,mode,owner in directories:
			s = bld.subst_add_destdir(a,bld)
			if Options.is_install:
				bld.install_dir(a)
				bld.setownership(a,owner[0],owner[1],mode)
		
		# 8. create environment symlinks
		symlinks = [
			('${MSENVIRON}/bin', '${TOMCATHOME}/bin'),
			('${MSENVIRON}/lib',  '${TOMCATHOME}/lib'),
			('${MSENVIRON}/logs', "${MSLOGDIR}"),
			('${MSENVIRON}/temp', '${LOCALSTATEDIR}/cache/${MSPATH}/temp'),
			('${MSENVIRON}/work','${LOCALSTATEDIR}/cache/${MSPATH}/work'),
			('${MSENVIRON}/conf', '${SYSCONFDIR}/${MSPATH}'),
#			("${AGENTLIBDIR}/css", '${CPLIBDIR}/css'),
#			("${AGENTLIBDIR}/images", '${CPLIBDIR}/images'),
#			("${AGENTLIBDIR}/js", '${CPLIBDIR}/js'),
#			("${AGENTLIBDIR}/ui", '${CPLIBDIR}/ui'),
            ("${MSCONF}/server.xml", '${MSCONF}/server-nonssl.xml'),
            ("${MSCONF}/tomcat6.conf", '${MSCONF}/tomcat6-nonssl.conf'),
		]
		
		for lnk,dst in symlinks: bld.symlink_as(lnk,Utils.subst_vars(dst,bld.env))


def build_scripts ():
	start_path = bld.path.find_dir ("scripts")
	bld.substitute('**',"${COMMONLIBDIR}/scripts",chmod=0755, cwd=start_path)

def build_bin_exec_dirs ():
	#bld.install_files_filtered("${LIBEXECDIR}","*/libexec/* cloudstack-proprietary/*/libexec/*",chmod=0755)
	#bld.install_files_filtered("${BINDIR}","*/bindir/* cloudstack-proprietary/*/bindir/*",chmod=0755)
	#bld.install_files_filtered("${SBINDIR}","*/sbindir/* cloudstack-proprietary/*/sbindir/*",chmod=0755)

	bld.install_files_filtered("${LIBEXECDIR}","*/libexec/*",chmod=0755)
	bld.install_files_filtered("${BINDIR}","*/bindir/*",chmod=0755)
	bld.install_files_filtered("${SBINDIR}","*/sbindir/*",chmod=0755)

def build_server_client ():
	start_path = bld.path.find_dir("client/WEB-INF")
	bld.install_files('${MSENVIRON}/webapps/client/WEB-INF',
		start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True),
		cwd=start_path,relative_trick=True)
	
	start_path = bld.path.find_dir("client")
	bld.install_files("${MSCONF}/resources",'WEB-INF/classes/resources/*.properties',chmod=0640, cwd=start_path)
	
	if not Options.options.PRESERVECONFIG:
		bld.install_files_filtered("${MSCONF}","tomcatconf/*", cwd=start_path)
		bld.install_files("${MSCONF}",'tomcatconf/db.properties',chmod=0640, cwd=start_path)
		bld.setownership("${MSCONF}/db.properties","root",bld.env.MSUSER)
	
	start_path = bld.path.find_dir("server")
	if not Options.options.PRESERVECONFIG:
		bld.install_files_filtered("${SERVERSYSCONFDIR}","conf/*", cwd=start_path)

def build_agent ():
	start_path = bld.path.find_dir ("agent")
	bld.install_files("${AGENTLIBDIR}",
		start_path.ant_glob("storagepatch/**",src=True,bld=False,dir=False,flat=True),
		cwd=start_path,relative_trick=True)
	if not Options.options.PRESERVECONFIG:
		bld.install_files_filtered("${AGENTSYSCONFDIR}","conf/*", cwd = start_path)

def build_ui ():
	# binary unsubstitutable files:
	start_path = bld.path.find_dir ("ui")
	bld.install_files ("${MSENVIRON}/webapps/client",start_path.ant_glob("*.ico **/*png **/*jpg **/*gif",src=True,bld=False,dir=False,flat=True),cwd=start_path,relative_trick=True)
	
	# text substitutable files (substitute with tokens from the environment bld.env):
	bld.substitute ("*html **/*html **/*js **/*css **/*properties **/*jsp *jsp",install_to="${MSENVIRON}/webapps/client", cwd=start_path)

def build_conf_files ():
	# apply distro-specific config on top of the 'all' generic cloud-management config
	globspec = _join("*","distro",bld.env.DISTRO.lower(),"*") # matches premium/distro/centos/SYSCONFDIR
	#distrospecificdirs=_glob(globspec) + _glob(_join("cloudstack-proprietary",globspec))
	distrospecificdirs=_glob(globspec)
	for dsdir in distrospecificdirs:
		start_path = bld.srcnode.find_dir(dsdir)
		subpath,varname = _split(dsdir)
		dsdirwithvar = _join("${%s}"%varname)
		files = filelist('%s/**'%dsdir,src=True,bld=False,dir=False,flat=True)
		mode = 0644
		if "SYSCONFDIR" in dsdir:
			mode = 0755
			if Options.options.PRESERVECONFIG: continue
		bld.install_files_filtered(dsdirwithvar, files,  cwd=start_path, relative_trick=True,chmod=mode)

# cloudstack-proprietary still has db files for usage
def build_db_files ():
	#bld.install_files_filtered("${SETUPDATADIR}",filelist("*/db/* cloudstack-proprietary/*/db/*",excl=Node.exclude_regs + "\ncloud-gate\ncloud-bridge"))
	start_path = bld.path.find_dir ("setup/db")
	bld.substitute('**',"${SETUPDATADIR}", cwd=start_path)
	if buildpremium: 
		start_path = bld.path.find_dir ("cloudstack-proprietary/premium/db")
		bld.substitute('**',"${SETUPDATADIR}", cwd=start_path)


def build_plugins ():
	# ====================== Feature-specific plugins ========================
	
	for plugin in _glob(_join("plugins","*")) + _glob(_join("cloudstack-proprietary","plugins","*")):
		if not _exists(_join(plugin,"build.xml")): continue
		pluginname = _basename(plugin)
		target = 'target/jar/cloud-%s.jar' % pluginname
		sources = filelist( '%s/**/*.java' % plugin.replace(sep,"/") , src=True, bld=False, dir=False )
		tgen = bld(rule=lambda x: runant("compile-%s"%pluginname), name='compile_%s'%pluginname, source=sources, target=target, after='runant')
		bld.install_files('${PLUGINJAVADIR}',target)
	
	# ====================== End feature-specific plugins ====================
	
	
	# ====================== Vendor-specific plugins ========================
	
	for vendor in _glob(_join("vendor","*")) + _glob(_join("cloudstack-proprietary","vendor","*")):
		if not Options.options.PRESERVECONFIG:
			bld.install_files_filtered("${MSCONF}/%s"%vendor,filelist("%s/tomcatconf/*"%vendor))
	
	# ====================== End vendor-specific plugins ====================

def build_xml_api_description ():
	def generate_xml_api_description(task):
		relationship = Utils.relpath(sourcedir,os.getcwd())
		cp = [ _join(relationship,x) for x in task.generator.env.CLASSPATH.split(pathsep) ]

		jarnames = ['utils','server','core', 'api', 'server-extras']
		props = ["client/tomcatconf/commands.properties.in"]

		sources = []
		for i in jarnames:
			str = 'target/jar/cloud-%s.jar' % i
			sources.append (str)
		sources.append ("client/tomcatconf/commands.properties.in")
		if buildpremium:
			sources.append("client/tomcatconf/commands-ext.properties.in")

		buildproducts = []
		for i in sources:
			path = bld.path.abspath() + os.sep + i
			buildproducts.append (path)

		jars = [ x for x in buildproducts if x.endswith("jar") ]
		properties = [ x for x in buildproducts if x.endswith("properties.in") ]
		cp += jars
		cp = pathsep.join(cp)
		arguments = ["-f",",".join(properties),"-d",builddir]
		ret = Utils.exec_command(["java","-cp",cp,"com.cloud.api.doc.ApiXmlDocWriter"]+arguments,log=True)
		return ret
		
	#TODO: We can't use 'source' token here because task_gen check if sources
	# exist before all task_gen. This bring a problem that 'runant' task doesn't
	# run when the check happen, which results in no source found at target/jar.
	# Ask waf community to fix that
	tgen = bld.new_task_gen (
			rule = generate_xml_api_description,
			target = 'commands.xml',
			name   = 'xmlapi',
			after  = 'runant',
			install_path="${CLIDIR}"
		)
	
	bld.install_as("${PYTHONDIR}/cloud_utils.py", 'python/lib/cloud_utils.py')
	bld.install_files("${PYTHONDIR}/cloudtool", 'cloud-cli/cloudtool/*')
	bld.install_files("${PYTHONDIR}/cloudutils", 'python/lib/cloudutils/*')
	bld.install_as("${PYTHONDIR}/cloudapis.py", 'cloud-cli/cloudapis/cloud.py')

def build_ovm():
	start_path = bld.path.find_dir("plugins/hypervisors/ovm/scripts")
	bld.substitute('**',"${COMMONLIBDIR}/scripts",chmod=0755, cwd=start_path)

def build_test():
	start_path = bld.path.find_dir("test/scripts")
	bld.install_files('${LIBDIR}/${PACKAGE}/test', \
		start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), \
		cwd=start_path,relative_trick=True)

	start_path = bld.path.find_dir("test/metadata")
	bld.install_files('${SHAREDSTATEDIR}/${PACKAGE}/test', \
		start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), \
		cwd=start_path,relative_trick=True)

	if not Options.options.PRESERVECONFIG:
		start_path = bld.path.find_dir("test")
		bld.install_files('${SYSCONFDIR}/${PACKAGE}/test','conf/*', cwd = start_path)
	Utils.pprint ("GREEN", "Installed files of test/")

def build_usage_dir ():
	start_path = bld.path.find_dir ("usage")
	bld.install_files_filtered("${USAGESYSCONFDIR}","conf/*", cwd = start_path)
	bld.symlink_as("${USAGESYSCONFDIR}/db.properties", \
			Utils.subst_vars("${MSCONF}/db.properties",bld.env))
	Utils.pprint ("GREEN", "Installed files of usage/")


# Get started to execute here
build_utils_docs ()
build_jars ()
build_premium ()
#build_thirdparty_dir()
build_dependences ()
#build_patches ()
build_systemvm_patch ()
build_dirs_symlinks ()
build_scripts ()
build_bin_exec_dirs ()
build_server_client ()
build_agent ()
build_ui ()
build_conf_files ()
build_db_files ()
build_plugins ()
build_xml_api_description ()
build_ovm ()
#build_test()
build_usage_dir()

# ====================== Magic! =========================================
bld.use_the_magic()