diff --git a/build.xml b/build.xml index f03c94b4734..a676328389a 100755 --- a/build.xml +++ b/build.xml @@ -10,7 +10,15 @@ - + + + + + + + + + @@ -21,57 +29,57 @@ - + - + - + - + - + - + - + - + - + - + - - - - - - + + + + + + diff --git a/build/build-cloud.xml b/build/build-cloud.xml index 515fdb34999..416ab36ee1b 100755 --- a/build/build-cloud.xml +++ b/build/build-cloud.xml @@ -60,7 +60,9 @@ - + + + diff --git a/build/package.xml b/build/package.xml index 1551d044a15..4d15dd2021a 100755 --- a/build/package.xml +++ b/build/package.xml @@ -232,7 +232,8 @@ - + + diff --git a/cloud.spec b/cloud.spec index f167ba65dd1..00065cc3583 100644 --- a/cloud.spec +++ b/cloud.spec @@ -35,7 +35,7 @@ BuildRequires: jpackage-utils BuildRequires: gcc BuildRequires: glibc-devel -%global _premium %(tar jtvmf %{SOURCE0} '*/premium/' --occurrence=1 2>/dev/null | wc -l) +%global _premium %(tar jtvmf %{SOURCE0} '*/cloudstack-proprietary/' --occurrence=1 2>/dev/null | wc -l) %description This is the Cloud.com Stack, a highly-scalable elastic, open source, diff --git a/wscript b/wscript index 482ae1b1a30..84dd09e1602 100644 --- a/wscript +++ b/wscript @@ -135,9 +135,41 @@ def svninfo(*args): retcode = p.wait() # If the guess fails, just return nothing. if retcode: return - return stdout + # SVN available + rev = [ x for x in stdout.splitlines() if x.startswith('Revision') ] + if not rev: rev = '' + else: rev = "SVN " + rev[0].strip() + url = [ x for x in stdout.splitlines() if x.startswith('URL') ] + if not url: url = '' + else: url = "SVN " + url[0].strip() + return rev + "\n" + url -def _getbuildnumber(): +def gitinfo(*args): + try: p = _Popen(['git','remote','show','-n','origin']+list(args),stdin=PIPE,stdout=PIPE,stderr=PIPE) + except OSError,e: + if e.errno == 2: return '' # svn command is not installed + raise + stdout,stderr = p.communicate('') + retcode = p.wait() + # If the guess fails, just return nothing. + if retcode: return + stdout = [ s.strip() for s in stdout.splitlines() ] + try: url = [ s[11:] for s in stdout if s.startswith("Fetch URL") ][0] + except IndexError: url = [ s[5:] for s in stdout if s.startswith("URL") ][0] + assert url + + p = _Popen(['git','log','-1']+list(args),stdin=PIPE,stdout=PIPE,stderr=PIPE) + stdout,stderr = p.communicate('') + retcode = p.wait() + if retcode: return + # If the guess fails, just return nothing. + stdout = [ s.strip() for s in stdout.splitlines() ] + commitid = [ s.split()[1] for s in stdout if s.startswith("commit") ][0] + assert commitid + + return "Git Revision: %s"%commitid + "\n" + "Git URL: %s"%url + +def _getbuildnumber(): # FIXME implement for git n = Options.options.BUILDNUMBER if n: # luntbuild prepends "build-" to the build number. we work around this here: @@ -150,10 +182,10 @@ def _getbuildnumber(): stdout = svninfo() if not stdout: return '' # Filter lines. - rev = [ x for x in stdout.splitlines() if x.startswith('Revision') ] + rev = [ x for x in stdout.splitlines() if x.startswith('SVN Revision') ] if not rev: return '' # Parse revision number. - rev = rev[0][10:].strip() + rev = rev[0][14:].strip() return rev Utils.getbuildnumber = _getbuildnumber @@ -289,7 +321,7 @@ def _install_files_filtered(self,destdir,listoffiles,**kwargs): if _isdir(f): continue if f.endswith(".in"): source = f ; target = f[:-3] - tgen = self(features='subst', source=source[len(self.path.abspath())+1:], target=target[len(self.path.abspath())+1:]) + tgen = self(features='subst', source=source[len(self.path.abspath())+1:], target=target[len(self.path.abspath())+1:], name="filtered_%s"%source) tgen.dict = self.env.get_merged_dict() else: source = f ; target = f @@ -466,19 +498,23 @@ def list_targets(ctx): for name in lst: print(name) -def dist(context): - '''makes a tarball for redistributing the sources -- if --skip-dist is specified, does nothing''' - if Options.options.DONTDIST: - appname=Utils.g_module.APPNAME - version=Utils.g_module.VERSION - tmp_folder=appname+'-'+version - if Scripting.g_gz in['gz','bz2']: - arch_name=tmp_folder+'.tar.'+Scripting.g_gz +def decorate_dist(f): + def dist(appname='',version=''): + '''makes a tarball for redistributing the sources -- if --skip-dist is specified, does nothing''' + if Options.options.DONTDIST: + if not appname: appname=Utils.g_module.APPNAME + if not version: version=Utils.g_module.VERSION + tmp_folder=appname+'-'+version + if Scripting.g_gz in['gz','bz2']: + arch_name=tmp_folder+'.tar.'+Scripting.g_gz + else: + arch_name=tmp_folder+'.'+'zip' + Logs.info('New archive skipped: %s'%(arch_name)) + return arch_name else: - arch_name=tmp_folder+'.'+'zip' - return arch_name - else: - return Scripting.dist() + return f(appname,version) + return dist +Scripting.dist = decorate_dist(Scripting.dist) def dist_hook(): # Clean the GARBAGE that clogs our repo to the tune of 300 MB @@ -486,33 +522,16 @@ def dist_hook(): # package over 90 MB in size [ shutil.rmtree(f) for f in _glob(_join("*","bin")) if _isdir(f) ] [ shutil.rmtree(f) for f in [ _join("build","deploy") ] if _isdir(f) ] - [ shutil.rmtree(f) for f in _glob(_join("thirdparty","*")) if _isdir(f) ] - [ shutil.rmtree(_join("tools",f)) for f in [ - "bin", - "meld", - "misc", - "tomcat", - "pdfdoclet", - "taglets", - ] if _exists(_join("tools",f)) ] - + [ shutil.rmtree(f) for f in _glob(_join("cloudstack-proprietary","thirdparty","*")) if _isdir(f) ] + [ shutil.rmtree(f) for f in [ _join("cloudstack-proprietary","tools") ] if _isdir(f) ] if Options.options.OSS: - [ shutil.rmtree(f) for f in "premium usage thirdparty console-proxy-premium test agent-simulator core-premium plugins vendor".split() if _exists(f) ] - [ shutil.rmtree(f) for f in [ _join("build","premium") ] if _exists(f) ] + [ shutil.rmtree(f) for f in "cloudstack-proprietary".split() if _exists(f) ] - stdout = svninfo("..") + stdout = svninfo("..") or gitinfo() if stdout: - # SVN available - rev = [ x for x in stdout.splitlines() if x.startswith('Revision') ] - if not rev: rev = '' - else: rev = "SVN " + rev[0].strip() - url = [ x for x in stdout.splitlines() if x.startswith('URL') ] - if not url: url = '' - else: url = "SVN " + url[0].strip() f = file("sccs-info","w") - if rev: f.write("%s\n"%rev) - if url: f.write("%s\n"%url) + f.write(stdout) f.flush() f.close() else: @@ -560,12 +579,11 @@ def rpm(context): if not Options.options.blddir: outputdir = _join(context.curdir,blddir,"rpmbuild") else: outputdir = _join(_abspath(Options.options.blddir),"rpmbuild") Utils.pprint("GREEN","Building RPMs") - if not Options.options.DONTDIST: Scripting.dist() + tarball = Scripting.dist() #if _isdir(outputdir): shutil.rmtree(outputdir) for a in ["RPMS/noarch","SRPMS","BUILD","SPECS","SOURCES"]: mkdir_p(_join(outputdir,a)) specfile = "%s.spec"%APPNAME - tarball = "%s-%s.tar.%s"%(APPNAME,VERSION,Scripting.g_gz) shutil.copy(tarball,_join(outputdir,"SOURCES")) checkdeps = lambda: c(["rpmbuild","--define","_topdir %s"%outputdir,"--nobuild",specfile]) dorpm = lambda: c(["rpmbuild","--define","_topdir %s"%outputdir,"-ba",specfile]+buildnumber+prerelease) @@ -604,11 +622,10 @@ def deb(context): if not Options.options.blddir: outputdir = _join(context.curdir,blddir,"debbuild") else: outputdir = _join(_abspath(Options.options.blddir),"debbuild") Utils.pprint("GREEN","Building DEBs") - if not Options.options.DONTDIST: Scripting.dist() + tarball = Scripting.dist() #if _isdir(outputdir): shutil.rmtree(outputdir) mkdir_p(outputdir) - tarball = "%s-%s.tar.%s"%(APPNAME,VERSION,Scripting.g_gz) f = tarfile.open(tarball,'r:bz2') f.extractall(path=outputdir) srcdir = "%s/%s-%s"%(outputdir,APPNAME,VERSION) diff --git a/wscript_build b/wscript_build index c8bfdf3d52b..c349af18440 100644 --- a/wscript_build +++ b/wscript_build @@ -35,7 +35,7 @@ for e in required_env: sourcedir = bld.srcnode.abspath() builddir = bld.path.abspath(bld.env) -buildpremium = _exists(_join(sourcedir,"premium")) +buildpremium = _exists(_join(sourcedir,"cloudstack-proprietary")) filelist = bld.path.ant_glob sccsinfo = _join(sourcedir,"sccs-info") @@ -153,7 +153,7 @@ if bld.env.DISTRO not in ['Windows','Mac']: # ================ Third-party / dependency installation =============== bld.install_files('${JAVADIR}','deps/*.jar') -if buildpremium: bld.install_files('${PREMIUMJAVADIR}','thirdparty/*.jar') +if buildpremium: bld.install_files('${PREMIUMJAVADIR}','cloudstack-proprietary/thirdparty/*.jar') # =================== End 3rdparty/dep install ======================== @@ -265,20 +265,20 @@ deps = " ".join( Utils.to_list( ) ) thirdparties = " ".join( Utils.to_list( """ - thirdparty/xmlrpc-client-3.1.3.jar - thirdparty/xmlrpc-common-3.1.3.jar - thirdparty/ws-commons-util-1.0.2.jar - thirdparty/log4j-1.2.15.jar - thirdparty/gson-1.3.jar - thirdparty/apache-log4j-extras-1.0.jar - thirdparty/commons-httpclient-3.1.jar - thirdparty/commons-logging-1.1.1.jar - thirdparty/commons-collections-3.2.1.jar - thirdparty/commons-codec-1.4.jar - thirdparty/commons-pool-1.4.jar - thirdparty/libvirt-0.4.5.jar - thirdparty/jna.jar - thirdparty/cglib-nodep-2.2.jar + cloudstack-proprietary/thirdparty/xmlrpc-client-3.1.3.jar + cloudstack-proprietary/thirdparty/xmlrpc-common-3.1.3.jar + cloudstack-proprietary/thirdparty/ws-commons-util-1.0.2.jar + cloudstack-proprietary/thirdparty/log4j-1.2.15.jar + cloudstack-proprietary/thirdparty/gson-1.3.jar + cloudstack-proprietary/thirdparty/apache-log4j-extras-1.0.jar + cloudstack-proprietary/thirdparty/commons-httpclient-3.1.jar + cloudstack-proprietary/thirdparty/commons-logging-1.1.1.jar + cloudstack-proprietary/thirdparty/commons-collections-3.2.1.jar + cloudstack-proprietary/thirdparty/commons-codec-1.4.jar + cloudstack-proprietary/thirdparty/commons-pool-1.4.jar + cloudstack-proprietary/thirdparty/libvirt-0.4.5.jar + cloudstack-proprietary/thirdparty/jna.jar + cloudstack-proprietary/thirdparty/cglib-nodep-2.2.jar """ ) ) @@ -294,7 +294,7 @@ patterns = Utils.to_list( scripts/storage/secondary/*sh """ ) -premiumpatterns = patterns + ["console-proxy-premium/certs/**"] +premiumpatterns = patterns + ["cloudstack-proprietary/console-proxy-premium/certs/**"] artifacts = "target/jar/VMOpsConsoleApplet.jar " + " ".join( "target/jar/cloud-%s.jar"%j for j in "console-proxy console-common agent utils api core".split() ) premiumartifacts = artifacts + " target/jar/cloud-console-proxy-premium.jar" @@ -316,16 +316,24 @@ def zip_up(task): for inp in task.inputs: if inp.id&3==Node.BUILD: src = inp.bldpath(task.env) - srcname = "/".join(src.split("/")[1:]) + srcname = src + srcname = "/".join(srcname.split("/")[1:]) # chop off default/ if srcname.startswith("target/jar"): srcname = "/".join(srcname.split("/")[2:]) # chop off target/jar else: src = inp.srcpath(task.env) - srcname = "/".join(src.split("/")[2:]) - if srcname.startswith('scripts/run') or srcname.startswith('scripts/config_ssl') \ - or srcname.startswith('scripts/config_auth') or srcname.startswith('scripts/ssvm-check'): srcname = "/".join(srcname.split("/")[1:]) - elif srcname.startswith('storage'): srcname = "scripts/" + srcname - elif srcname.startswith('VMOpsConsoleApplet'): srcname = "applet/" + srcname - elif srcname.startswith('certs'): srcname = srcname + srcname = src + srcname = "/".join(srcname.split("/")[1:]) # chop off ../ + if srcname.startswith("cloudstack-proprietary"): srcname = "/".join(srcname.split("/")[1:]) # chop off cloudstack proprietary + srcname = "/".join(srcname.split("/")[1:]) # chop off project name + # post-process the paths + if True in [ srcname.startswith(strt) for strt in [ 'scripts/run','scripts/config_ssl','scripts/config_auth','scripts/ssvm-check' ] ]: + srcname = "/".join(srcname.split("/")[1:]) + elif srcname.startswith('storage'): + srcname = "scripts/" + srcname + elif srcname.startswith('VMOpsConsoleApplet'): + srcname = "applet/" + srcname + elif srcname.startswith('certs'): + srcname = srcname z.write(src,srcname) z.close() return 0 @@ -428,9 +436,9 @@ for src,tgt,inst in zip(src_files,subst_files,inst_files): # ================== Subst / installation of scripts in bin directories ======================== -bld.install_files_filtered("${LIBEXECDIR}","*/libexec/*",chmod=0755) -bld.install_files_filtered("${BINDIR}","*/bindir/*",chmod=0755) -bld.install_files_filtered("${SBINDIR}","*/sbindir/*",chmod=0755) +bld.install_files_filtered("${LIBEXECDIR}","*/libexec/* cloudstack-proprietary/*/libexec/*",chmod=0755) +bld.install_files_filtered("${BINDIR}","*/bindir/* cloudstack-proprietary/*/bindir/*",chmod=0755) +bld.install_files_filtered("${SBINDIR}","*/sbindir/* cloudstack-proprietary/*/sbindir/*",chmod=0755) # ================== End subst / installation of scripts in bin directories ======================== @@ -440,16 +448,16 @@ bld.install_files_filtered("${SBINDIR}","*/sbindir/*",chmod=0755) # build / install declarations of test project if buildpremium: proj = 'test' - start_path = bld.path.find_dir("test/scripts") + start_path = bld.path.find_dir("cloudstack-proprietary/test/scripts") bld.install_files('${LIBDIR}/${PACKAGE}/test', start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), cwd=start_path,relative_trick=True) - start_path = bld.path.find_dir("test/metadata") + start_path = bld.path.find_dir("cloudstack-proprietary/test/metadata") bld.install_files('${SHAREDSTATEDIR}/${PACKAGE}/test', start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), cwd=start_path,relative_trick=True) if not Options.options.PRESERVECONFIG: - bld.install_files('${SYSCONFDIR}/%s/%s'%(bld.env.PACKAGE,proj),'%s/conf/*'%proj) # install config + bld.install_files('${SYSCONFDIR}/%s/%s'%(bld.env.PACKAGE,proj),'cloudstack-proprietary/test/conf/*') # build / install declarations of server project <- this is actually now in client project bld.install_files("${MSENVIRON}/webapps/client/WEB-INF",'client/WEB-INF/web.xml') # install web.xml file @@ -464,7 +472,6 @@ bld.install_files("${AGENTLIBDIR}", cwd=start_path,relative_trick=True) if not Options.options.PRESERVECONFIG: bld.install_files_filtered("${AGENTSYSCONFDIR}","%s/conf/*"%proj) -bld.install_files_filtered("${BINDIR}","%s/bindir/*"%proj,chmod=0755) # build / install declarations of client UI project @@ -521,12 +528,13 @@ bld.install_files("${MSENVIRON}/webapps/client/scripts", "ui/scripts/cloud.min.j # substitute and install generic tomcat config if not Options.options.PRESERVECONFIG: - bld.install_files_filtered("${MSCONF}",filelist("*/tomcatconf/*")) + bld.install_files_filtered("${MSCONF}","*/tomcatconf/* cloudstack-proprietary/*/tomcatconf/*") bld.install_files("${MSCONF}",'client/tomcatconf/db.properties',chmod=0640) bld.setownership("${MSCONF}/db.properties","root",bld.env.MSUSER) # apply distro-specific config on top of the 'all' generic cloud-management config -distrospecificdirs=_glob(_join("*","distro",bld.env.DISTRO.lower(),"*")) +globspec = _join("*","distro",bld.env.DISTRO.lower(),"*") # matches premium/distro/centos/SYSCONFDIR +distrospecificdirs=_glob(globspec) + _glob(_join("cloudstack-proprietary",globspec)) for dsdir in distrospecificdirs: start_path = bld.srcnode.find_dir(dsdir) subpath,varname = _split(dsdir) @@ -547,7 +555,7 @@ if buildpremium: bld.symlink_as("${USAGESYSCONFDIR}/db.properties",Utils.subst_vars("${MSCONF}/db.properties",bld.env)) # install db data files -bld.install_files_filtered("${SETUPDATADIR}",filelist("*/db/* cloudstack-proprietary/db/*",excl=Node.exclude_regs + "\ncloud-gate\ncloud-bridge")) +bld.install_files_filtered("${SETUPDATADIR}",filelist("*/db/* cloudstack-proprietary/*/db/*",excl=Node.exclude_regs + "\ncloud-gate\ncloud-bridge")) # ================== End installation of scripts / bindirs / configuration files =========================== diff --git a/wscript_configure b/wscript_configure index a5d16f3f816..5185f87fbf2 100644 --- a/wscript_configure +++ b/wscript_configure @@ -234,7 +234,7 @@ conf.env.USAGECLASSPATH = pathsep.join(usageclasspath) compilecp = []# list(srcdirs) # 2.a) the thirdparty/ directory in the source if on Windows / Mac # 2.b) the deps/ directory in the source if on Linux -if conf.env.DISTRO in ["Windows","Mac"]: compilecp+= _glob(_join("thirdparty","*.jar")) +if conf.env.DISTRO in ["Windows","Mac"]: compilecp+= _glob(_join("cloudstack-proprietary","thirdparty","*.jar")) else: compilecp+= _glob(_join("deps","*.jar")) # 3. the system classpath (system-installed JARs) compilecp+= [ conf.env.SYSTEMCLASSPATH ]