mirror of
https://github.com/apache/cloudstack.git
synced 2025-11-04 00:02:37 +01:00
450 lines
17 KiB
Python
450 lines
17 KiB
Python
#!/usr/bin/env python
|
|
|
|
"""builds the entire stack"""
|
|
#For every matching build change here, that produces new installable
|
|
#files, the cloud.spec file and the Debian control files must be
|
|
#revised and tested.
|
|
|
|
import shutil,os
|
|
import Utils,Node,Options,Logs,Scripting,Environment,Build,Configure
|
|
from os import unlink as _unlink, makedirs as _makedirs, getcwd as _getcwd, chdir as _chdir
|
|
from os.path import abspath as _abspath, basename as _basename, dirname as _dirname, exists as _exists, isdir as _isdir, split as _split, join as _join, sep, pathsep, pardir, curdir
|
|
from glob import glob as _glob
|
|
try: set([1,2,3])
|
|
except Exception: from Sets import set
|
|
import re
|
|
import zipfile,tarfile
|
|
try:
|
|
from os import chmod as _chmod,chown as _chown
|
|
import pwd,stat,grp
|
|
except ImportError:
|
|
_chmod,_chown,pwd,stat,grp = (None,None,None,None,None)
|
|
|
|
required_env = [
|
|
"APISERVERLOG",
|
|
"MSLOG",
|
|
"PIDDIR",
|
|
"CPPATH",
|
|
"AGENTSIMULATORCLASSPATH",
|
|
"SYSTEMJAVADIR",
|
|
"USAGELOG",
|
|
"PLUGINJAVADIR",
|
|
]
|
|
for e in required_env:
|
|
if e not in bld.env: raise Utils.WafError("configure required: new variable %s added"%e)
|
|
|
|
sourcedir = bld.srcnode.abspath()
|
|
builddir = bld.path.abspath(bld.env)
|
|
buildpremium = _exists(_join(sourcedir,"cloudstack-proprietary"))
|
|
filelist = bld.path.ant_glob
|
|
|
|
sccsinfo = _join(sourcedir,"sccs-info")
|
|
if _exists(sccsinfo): bld.install_files("${DOCDIR}","sccs-info")
|
|
|
|
tgen = bld(features='subst', name='configure-info', source="configure-info.in", target="configure-info")
|
|
tgen.dict = {"CONFIGUREVARS":bld.getconfig()}
|
|
bld.install_files("${DOCDIR}","configure-info")
|
|
|
|
# ==================== Java compilation ===========================
|
|
|
|
# discover ant targets and properties
|
|
antxmlfiles = _glob(_join("build","*.xml")) + _glob(_join("build","*.properties"))
|
|
if buildpremium: antxmlfiles += _glob(_join("build","premium","*.xml")) + _glob(_join("cloudstack-proprietary","build","*.xml"))
|
|
targets,antprops = Utils.discover_ant_targets_and_properties(antxmlfiles)
|
|
|
|
Implementation_Version = bld.env.VERSION
|
|
buildnumber = Utils.getbuildnumber()
|
|
if buildnumber: Implementation_Version += "." + buildnumber
|
|
|
|
# this is to trigger recompilation / cache avoidance if the relevant environment for ant changes
|
|
ant_args = [
|
|
"-Dimpl.version=%s"%Implementation_Version,
|
|
"-Dtarget.dir=%s"%Utils.relpath(_join(builddir,"target")),
|
|
"-Ddist.dir=%s"%Utils.relpath(_join(builddir,"ant-dist")),
|
|
]
|
|
if not buildpremium: ant_args += ["-DOSS=true"]
|
|
|
|
tgen = bld(features='subst', name='version-info', source="version-info.in", target="version-info")
|
|
tgen.dict = { "Implementation_Version":Implementation_Version,"ant_args":ant_args }
|
|
bld.install_files("${DOCDIR}","version-info")
|
|
|
|
# compile jar files using ant
|
|
# ant only needs to be reinvoked if the version with build number changes
|
|
# we here trim all the depended targets from the target list:
|
|
build_targets = [ x[0] for x in targets ]
|
|
nondep_build_targets = list ( set([ x[0] for x in targets ]) - set([ x for dependencies in [ y[4] for y in targets ] for x in dependencies ]) )
|
|
ant_sources = " ".join(antxmlfiles + [ filelist(x[1] + "/**/*.java",src=True,bld=False,dir=False,flat=True) for x in targets ]) + ' version-info'
|
|
ant_jars = [ x[3] for x in targets ]
|
|
|
|
bld.path.ensure_dir_node_from_path("target/jar")
|
|
tgen = bld(rule=Utils.runant, name='runant', source=ant_sources, target=ant_jars, anttgts = nondep_build_targets, antargs=ant_args)
|
|
|
|
# install jar files
|
|
[ bld.install_files('${JAVADIR}',jar) for jar in ant_jars if _basename(jar).startswith("cloud-") ]
|
|
|
|
# ======================= End Java compilation ======================
|
|
|
|
|
|
# =================== C / Python compilation =========================
|
|
|
|
bld.recurse(["vnet","daemonize","python"],'build')
|
|
|
|
# ===================== End C / Python compilation ==========================
|
|
|
|
|
|
# ================ Third-party / dependency installation ===============
|
|
|
|
bld.recurse(["deps"],'build')
|
|
if buildpremium: bld.recurse(['cloudstack-proprietary/thirdparty'],'build')
|
|
|
|
# =================== End 3rdparty/dep install ========================
|
|
|
|
|
|
# =================== Build install declaratoin of console proxy project ========
|
|
|
|
bld.recurse(["console-proxy"],'build')
|
|
|
|
# ================== End console proxy ===================
|
|
|
|
|
|
# ================ Creation of patch.tgz's ============================
|
|
|
|
# done here because the patches require substituted files
|
|
bld.recurse(["patches"],'build')
|
|
|
|
# ================== End creation of patch.tgz's ====================
|
|
|
|
|
|
# ================== systemvm patch creation ====================
|
|
|
|
deps = " ".join( Utils.to_list(
|
|
"""
|
|
deps/cloud-xmlrpc-client-3.1.3.jar
|
|
deps/cloud-xmlrpc-common-3.1.3.jar
|
|
deps/cloud-log4j.jar
|
|
deps/cloud-gson-1.3.jar
|
|
deps/cloud-apache-log4j-extras-1.0.jar
|
|
deps/cloud-libvirt-0.4.5.jar
|
|
deps/cloud-jna.jar
|
|
deps/cloud-cglib.jar
|
|
deps/cloud-commons-codec-1.4.jar
|
|
deps/cloud-commons-httpclient-3.1.jar
|
|
deps/cloud-commons-logging-1.1.1.jar
|
|
deps/cloud-commons-pool-1.4.jar
|
|
deps/cloud-ws-commons-util-1.0.2.jar
|
|
deps/cloud-commons-collections-3.2.1.jar
|
|
"""
|
|
) )
|
|
thirdparties = " ".join( Utils.to_list(
|
|
"""
|
|
cloudstack-proprietary/thirdparty/xmlrpc-client-3.1.3.jar
|
|
cloudstack-proprietary/thirdparty/xmlrpc-common-3.1.3.jar
|
|
cloudstack-proprietary/thirdparty/ws-commons-util-1.0.2.jar
|
|
cloudstack-proprietary/thirdparty/log4j-1.2.15.jar
|
|
cloudstack-proprietary/thirdparty/gson-1.3.jar
|
|
cloudstack-proprietary/thirdparty/apache-log4j-extras-1.0.jar
|
|
cloudstack-proprietary/thirdparty/commons-httpclient-3.1.jar
|
|
cloudstack-proprietary/thirdparty/commons-logging-1.1.1.jar
|
|
cloudstack-proprietary/thirdparty/commons-collections-3.2.1.jar
|
|
cloudstack-proprietary/thirdparty/commons-codec-1.4.jar
|
|
cloudstack-proprietary/thirdparty/commons-pool-1.4.jar
|
|
cloudstack-proprietary/thirdparty/libvirt-0.4.5.jar
|
|
cloudstack-proprietary/thirdparty/jna.jar
|
|
cloudstack-proprietary/thirdparty/cglib-nodep-2.2.jar
|
|
"""
|
|
) )
|
|
|
|
patterns = Utils.to_list(
|
|
"""
|
|
console-proxy/css/**
|
|
console-proxy/images/**
|
|
console-proxy/js/**
|
|
console-proxy/ui/**
|
|
console-proxy/conf/**
|
|
console-proxy/scripts/*.sh
|
|
console-proxy/vm-script/*.sh
|
|
scripts/storage/secondary/*sh
|
|
"""
|
|
)
|
|
premiumpatterns = patterns + ["cloudstack-proprietary/console-proxy-premium/certs/**"]
|
|
|
|
artifacts = "target/jar/VMOpsConsoleApplet.jar " + " ".join( "target/jar/cloud-%s.jar"%j for j in "console-proxy console-common agent utils api core".split() )
|
|
premiumartifacts = artifacts + " target/jar/cloud-console-proxy-premium.jar"
|
|
|
|
sources = " ".join( [ filelist(x,src=True,bld=False,dir=False,flat=True) for x in patterns ] )
|
|
premiumsources = " ".join( [ filelist(x,src=True,bld=False,dir=False,flat=True) for x in premiumpatterns ] )
|
|
|
|
systemfiles = [ (f,_join(".",_basename(f))) for f in bld.env.SYSTEMCLASSPATH.split(pathsep) ]
|
|
systems = " ".join( [ n[1] for n in systemfiles ] )
|
|
systemtargetdir = _join(builddir,".")
|
|
def copydeps(task):
|
|
Utils.check_dir(systemtargetdir)
|
|
for src,tgt in systemfiles: shutil.copy(src,systemtargetdir)
|
|
return 0
|
|
def zip_up(task):
|
|
tgt = task.outputs[0].bldpath(task.env)
|
|
if _exists(tgt): _unlink(tgt)
|
|
z = zipfile.ZipFile(tgt,"w")
|
|
for inp in task.inputs:
|
|
if inp.id&3==Node.BUILD:
|
|
src = inp.bldpath(task.env)
|
|
srcname = src
|
|
srcname = "/".join(srcname.split("/")[1:]) # chop off default/
|
|
if srcname.startswith("target/jar"): srcname = "/".join(srcname.split("/")[2:]) # chop off target/jar
|
|
else:
|
|
src = inp.srcpath(task.env)
|
|
srcname = src
|
|
srcname = "/".join(srcname.split("/")[1:]) # chop off ../
|
|
if srcname.startswith("cloudstack-proprietary"): srcname = "/".join(srcname.split("/")[1:]) # chop off cloudstack proprietary
|
|
srcname = "/".join(srcname.split("/")[1:]) # chop off project name
|
|
# post-process the paths
|
|
if True in [ srcname.startswith(strt) for strt in [ 'scripts/run','scripts/_run','scripts/config_ssl','scripts/config_auth','scripts/ssvm-check' ] ]:
|
|
srcname = "/".join(srcname.split("/")[1:])
|
|
elif srcname.startswith('storage'):
|
|
srcname = "scripts/" + srcname
|
|
elif srcname.startswith('VMOpsConsoleApplet'):
|
|
srcname = "applet/" + srcname
|
|
elif srcname.startswith('certs'):
|
|
srcname = srcname
|
|
z.write(src,srcname)
|
|
z.close()
|
|
return 0
|
|
|
|
bld.path.ensure_dir_node_from_path("target/oss")
|
|
bld.path.ensure_dir_node_from_path("target/premium")
|
|
if bld.env.DISTRO not in ["Windows","Mac"]:
|
|
tgen = bld(
|
|
rule = copydeps,
|
|
source = 'wscript_configure',
|
|
target = systems,
|
|
name = 'getsystemjars',
|
|
)
|
|
tgen = bld(
|
|
rule = zip_up,
|
|
source = " ".join( [sources,artifacts,deps,systems] ),
|
|
target = 'target/oss/systemvm.zip',
|
|
name = 'systemvm_zip',
|
|
)
|
|
bld.install_files("${AGENTLIBDIR}/vms", "target/oss/systemvm.zip")
|
|
|
|
if buildpremium:
|
|
tgen = bld(
|
|
rule = zip_up,
|
|
source = " ".join( [premiumsources,premiumartifacts,thirdparties] ),
|
|
target = 'target/premium/systemvm.zip',
|
|
name = 'systemvm-premium_zip',
|
|
)
|
|
#no need to install the premium one, we have ISOs below
|
|
#bld.install_files("${AGENTLIBDIR}/vms", "systemvm-premium.zip")
|
|
|
|
# ================== End systemvm patch creation ====================
|
|
|
|
|
|
# ================== systemvm ISO creation ====================
|
|
|
|
def iso_up(task):
|
|
tgt = task.outputs[0].bldpath(task.env)
|
|
if os.path.exists(tgt): os.unlink(tgt)
|
|
inps = []
|
|
for inp in task.inputs:
|
|
if inp.id&3==Node.BUILD:
|
|
src = inp.bldpath(task.env)
|
|
srcname = src
|
|
srcname = sep.join(srcname.split(sep)[1:]) # chop off default/
|
|
else:
|
|
src = inp.srcpath(task.env)
|
|
srcname = src
|
|
srcname = sep.join(srcname.split(sep)[1:]) # chop off ../
|
|
if task.generator.rename: srcname = task.generator.rename(srcname)
|
|
inps.append(srcname+'='+src)
|
|
ret = Utils.exec_command(
|
|
[
|
|
task.generator.env.MKISOFS,
|
|
"-quiet",
|
|
"-r",
|
|
"-graft-points",
|
|
"-o",tgt,
|
|
] + inps, shell=False)
|
|
if ret != 0: return ret
|
|
|
|
if bld.env.DISTRO not in ["Windows","Mac"]:
|
|
# systemvm.zip cannot be built on Windows or Mac because system deps do not exist there
|
|
tgen = bld(
|
|
rule = iso_up,
|
|
source = "patches/cloud-scripts.tgz target/oss/systemvm.zip patches/systemvm/debian/config/root/.ssh/authorized_keys",
|
|
target = 'target/oss/systemvm.iso',
|
|
name = 'systemvm_iso',
|
|
rename = lambda x: x.split(sep)[-1].replace('systemvm-',''),
|
|
)
|
|
bld.install_as("${AGENTLIBDIR}/vms/systemvm.iso", "target/oss/systemvm.iso")
|
|
|
|
if buildpremium:
|
|
tgen = bld(
|
|
rule = iso_up,
|
|
source = "patches/cloud-scripts.tgz target/premium/systemvm.zip patches/systemvm/debian/config/root/.ssh/authorized_keys",
|
|
target = 'target/premium/systemvm.iso',
|
|
name = 'systemvm-premium_iso',
|
|
rename = lambda x: x.split(sep)[-1].replace('systemvm-',''),
|
|
)
|
|
bld.install_as("${AGENTLIBDIR}/vms/systemvm-premium.iso", "target/premium/systemvm.iso")
|
|
|
|
# ================== End systemvm ISO creation ====================
|
|
|
|
|
|
# =================== Empty directory / symlink creation on install target ====================
|
|
|
|
bld.createuser(bld.env.MSUSER,bld.env.MSENVIRON,'/bin/sh')
|
|
|
|
# 7. make log and cache dirs (this actually runs first)
|
|
if bld.env.DISTRO in 'Windows Mac': pass
|
|
else:
|
|
x = ("root",bld.env.MSUSER)
|
|
directories = [
|
|
("${MSLOGDIR}",0770,x),
|
|
("${AGENTLOGDIR}",0770,x),
|
|
("${USAGELOGDIR}",0770,x),
|
|
("${CPLOGDIR}",0770,x),
|
|
("${IPALLOCATORLOGDIR}",0770,x),
|
|
("${LOCALSTATEDIR}/cache/${MSPATH}",0770,x),
|
|
("${LOCALSTATEDIR}/cache/${MSPATH}/temp",0770,x),
|
|
("${LOCALSTATEDIR}/cache/${MSPATH}/work",0770,x),
|
|
("${SHAREDSTATEDIR}/${MSPATH}",0770,x),
|
|
("${MSMNTDIR}",0770,x),
|
|
("${MSCONF}/Catalina",0770,x),
|
|
("${MSCONF}/Catalina/localhost",0770,x),
|
|
("${MSCONF}/Catalina/localhost/client",0770,x),
|
|
("${PIDDIR}",0755,("root","root")),
|
|
("${LOCKDIR}",0755,("root","root")),
|
|
]
|
|
|
|
for a,mode,owner in directories:
|
|
s = bld.subst_add_destdir(a,bld)
|
|
if Options.is_install:
|
|
bld.install_dir(a)
|
|
bld.setownership(a,owner[0],owner[1],mode)
|
|
|
|
# 8. create environment symlinks
|
|
symlinks = [
|
|
('${MSENVIRON}/bin', '${TOMCATHOME}/bin'),
|
|
('${MSENVIRON}/lib', '${TOMCATHOME}/lib'),
|
|
('${MSENVIRON}/logs', "${MSLOGDIR}"),
|
|
('${MSENVIRON}/temp', '${LOCALSTATEDIR}/cache/${MSPATH}/temp'),
|
|
('${MSENVIRON}/work','${LOCALSTATEDIR}/cache/${MSPATH}/work'),
|
|
('${MSENVIRON}/conf', '${SYSCONFDIR}/${MSPATH}'),
|
|
("${AGENTLIBDIR}/css", '${CPLIBDIR}/css'),
|
|
("${AGENTLIBDIR}/images", '${CPLIBDIR}/images'),
|
|
("${AGENTLIBDIR}/js", '${CPLIBDIR}/js'),
|
|
("${AGENTLIBDIR}/ui", '${CPLIBDIR}/ui'),
|
|
]
|
|
|
|
for lnk,dst in symlinks: bld.symlink_as(lnk,Utils.subst_vars(dst,bld.env))
|
|
|
|
# ================== End empty directory / symlink creation on install target =====================
|
|
|
|
|
|
# =================== Subst / installation of agent scripts project ========
|
|
|
|
bld.recurse(["scripts"],'build')
|
|
if buildpremium: bld.recurse(["cloudstack-proprietary/scripts"],'build')
|
|
|
|
# ================== End agent scripts ===================
|
|
|
|
|
|
# ================== Subst / installation of scripts in bin directories ========================
|
|
|
|
bld.install_files_filtered("${LIBEXECDIR}","*/libexec/* cloudstack-proprietary/*/libexec/*",chmod=0755)
|
|
bld.install_files_filtered("${BINDIR}","*/bindir/* cloudstack-proprietary/*/bindir/*",chmod=0755)
|
|
bld.install_files_filtered("${SBINDIR}","*/sbindir/* cloudstack-proprietary/*/sbindir/*",chmod=0755)
|
|
|
|
# ================== End subst / installation of scripts in bin directories ========================
|
|
|
|
|
|
# ================== Installation of scripts / bindirs / configuration files ===========================
|
|
|
|
# build / install declarations of test project
|
|
if buildpremium: bld.recurse(["cloudstack-proprietary/test"],'build')
|
|
|
|
# build / install declarations of server project <- this is actually now in client project
|
|
bld.recurse(["client","server"],'build')
|
|
if buildpremium: bld.recurse(["cloudstack-proprietary/premium"],'build')
|
|
|
|
# build / install declarations of agent project
|
|
bld.recurse(["agent"],'build')
|
|
|
|
# build / install declarations of client UI project
|
|
bld.recurse(["ui"],'build')
|
|
|
|
# apply distro-specific config on top of the 'all' generic cloud-management config
|
|
globspec = _join("*","distro",bld.env.DISTRO.lower(),"*") # matches premium/distro/centos/SYSCONFDIR
|
|
distrospecificdirs=_glob(globspec) + _glob(_join("cloudstack-proprietary",globspec))
|
|
for dsdir in distrospecificdirs:
|
|
start_path = bld.srcnode.find_dir(dsdir)
|
|
subpath,varname = _split(dsdir)
|
|
dsdirwithvar = _join("${%s}"%varname)
|
|
files = filelist('%s/**'%dsdir,src=True,bld=False,dir=False,flat=True)
|
|
mode = 0644
|
|
if "SYSCONFDIR" in dsdir:
|
|
mode = 0755
|
|
if Options.options.PRESERVECONFIG: continue
|
|
bld.install_files_filtered(dsdirwithvar, files, cwd=start_path, relative_trick=True,chmod=mode)
|
|
|
|
# build / install declarations of usage
|
|
if buildpremium: bld.recurse(["cloudstack-proprietary/usage"],'build')
|
|
|
|
# install db data files
|
|
bld.install_files_filtered("${SETUPDATADIR}",filelist("*/db/* cloudstack-proprietary/*/db/*",excl=Node.exclude_regs + "\ncloud-gate\ncloud-bridge"))
|
|
|
|
# ================== End installation of scripts / bindirs / configuration files ===========================
|
|
|
|
|
|
# ====================== Feature-specific plugins ========================
|
|
|
|
for plugin in _glob(_join("plugins","*")) + _glob(_join("cloudstack-proprietary","plugins","*")):
|
|
if not _exists(_join(plugin,"build.xml")): continue
|
|
pluginname = _basename(plugin)
|
|
target = 'target/jar/cloud-%s.jar' % pluginname
|
|
sources = filelist( '%s/**/*.java' % plugin.replace(sep,"/") , src=True, bld=False, dir=False )
|
|
tgen = bld(rule=lambda x: runant("compile-%s"%pluginname), name='compile_%s'%pluginname, source=sources, target=target, after='runant')
|
|
bld.install_files('${PLUGINJAVADIR}',target)
|
|
|
|
# ====================== End feature-specific plugins ====================
|
|
|
|
|
|
# ====================== Vendor-specific plugins ========================
|
|
|
|
for vendor in _glob(_join("vendor","*")) + _glob(_join("cloudstack-proprietary","vendor","*")):
|
|
if not Options.options.PRESERVECONFIG:
|
|
bld.install_files_filtered("${MSCONF}/%s"%vendor,filelist("%s/tomcatconf/*"%vendor))
|
|
|
|
# ====================== End vendor-specific plugins ====================
|
|
|
|
def generate_xml_api_description(task):
|
|
relationship = Utils.relpath(sourcedir,os.getcwd())
|
|
cp = [ _join(relationship,x) for x in task.generator.env.CLASSPATH.split(pathsep) ]
|
|
buildproducts = [ x.bldpath(task.env) for x in task.inputs ]
|
|
jars = [ x for x in buildproducts if x.endswith("jar") ]
|
|
properties = [ x for x in buildproducts if x.endswith("properties") ]
|
|
cp += jars
|
|
cp = pathsep.join(cp)
|
|
arguments = ["-f",",".join(properties),"-d",builddir]
|
|
ret = Utils.exec_command(["java","-cp",cp,"com.cloud.api.doc.ApiXmlDocWriter"]+arguments,log=True)
|
|
return ret
|
|
props = " client/tomcatconf/commands.properties"
|
|
jarnames = ['utils','server','core', 'api']
|
|
tgen = bld(
|
|
rule = generate_xml_api_description,
|
|
source = " ".join( [ 'target/jar/cloud-%s.jar'%x for x in jarnames ] ) + props,
|
|
target = 'commands.xml',
|
|
name = 'xmlapi',
|
|
after = 'runant',
|
|
install_path="${CLIDIR}"
|
|
)
|
|
#bld.process_after(tgen)
|
|
|
|
bld.install_files("${PYTHONDIR}/cloudtool", 'cloud-cli/cloudtool/*')
|
|
bld.install_as("${PYTHONDIR}/cloudapis.py", 'cloud-cli/cloudapis/cloud.py')
|
|
|
|
# ====================== Magic! =========================================
|
|
|
|
bld.use_the_magic()
|