CLOUDSTACK-6701, CLOUDSTACK-6702:

1. Integrate System Seed Template into MSI Installer
2. Added progress bar status messages for custom actions at needed places.

Signed-off-by: Abhinandan Prateek <aprateek@apache.org>

client/pom.xml

@@ -693,6 +693,7 @@
                         <include name="**/*" />
                       </fileset>
                     </copy>
+                    <copy file="../scripts/storage/secondary/cloud-install-sys-tmplt.py" tofile="target/scripts/cloud-install-sys-tmplt.py" />
                     <copy todir="./target/scripts">
                       <fileset dir="./target/utilities/bin">
                         <include name="**/*" />

scripts/installer/windows/WixInstallerDialog.wxs

@@ -57,6 +57,19 @@
             <Publish Dialog="MaintenanceTypeDlg" Control="RepairButton" Event="NewDialog" Value="VerifyReadyDlg">1</Publish>
             <Publish Dialog="MaintenanceTypeDlg" Control="RemoveButton" Event="NewDialog" Value="VerifyReadyDlg">1</Publish>
             <Publish Dialog="MaintenanceTypeDlg" Control="Back" Event="NewDialog" Value="MaintenanceWelcomeDlg">1</Publish>
+
+            <!-- Progress Text for Custom Actions STARTs here -->
+            <ProgressText Action="GenerateSSLKey">!(loc.GenerateSSLKey)</ProgressText>
+            <ProgressText Action="DeployDB">!(loc.DeployDB)</ProgressText>
+            <ProgressText Action="SetupDatabases">!(loc.SetupDatabases)</ProgressText>
+            <ProgressText Action="DeleteDirectory">!(loc.DeleteDirectory)</ProgressText>
+            <ProgressText Action="CopySitePackages">!(loc.CopySitePackages)</ProgressText>
+            <ProgressText Action="SetuptoolsInstallation">!(loc.SetuptoolsInstallation)</ProgressText>
+            <ProgressText Action="DeleteFiles">!(loc.DeleteFiles)</ProgressText>
+            <ProgressText Action="UpdateTomcatCatalinaBase">!(loc.UpdateTomcatCatalinaBase)</ProgressText>
+            <ProgressText Action="UpdateTomcatClassPath">!(loc.UpdateTomcatClassPath)</ProgressText>
+
+            <!-- Progress Text for Custom Actions ENDs here -->
         </UI>
         <UIRef Id="WixUI_Common" />
     </Fragment>

scripts/installer/windows/acs.wxs

@@ -81,8 +81,11 @@
                 Name="CATALINA_OPTS" Permanent="no" System="yes"
                 Value="-XX:MaxPermSize=512m -Xmx1024m -Xms256m" />
               <Environment Id="Path" Action="set" Name="Path"
-                Permanent="yes" System="yes" Part="last"
+                Permanent="no" System="yes" Part="last"
-                Value="[JAVA_HOME]\bin;[PYTHON_HOME];[SEVENZ_HOME];[ProgramFilesFolder]\cdrtools;[MYSQL]\bin" />
+                Value="[JAVA_HOME]bin;[PYTHON_HOME];[SEVENZ_HOME];[ProgramFilesFolder]cdrtools;[MYSQL]bin;[INSTALLDIR]scripts" />
+              <Environment Id="PATHEXT" Action="set" Name="PATHEXT"
+                Permanent="no" System="yes" Part="last"
+                Value=".py" />
             </Component>
             <Component Id='setuptools' Guid='019a51dd-7fc4-4d6c-9277-13cc7b600789'>
               <File Id="ez_setup" Source="ez_setup.py" />
@@ -235,31 +238,31 @@
     </CustomAction>
     <CustomAction Id="UpdateTomcatCatalinaBase"
       ExeCommand='//US//Tomcat6 --JvmOptions=-Dcatalina.base=[CSMANAGEMENT];-Djava.io.tmpdir=[CSMANAGEMENT]\temp;-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager;-Dcatalina.home="[TOMCATDIRECTORY]";-Xms512m;-Xmx1024m;-XX:MaxPermSize=512m;-Duser.name=cloud'
-      Property="TOMCATDIRECTORY1" Execute="commit" Return="check" />
+      Property="TOMCATDIRECTORY1" Execute="deferred" Return="check" />
     <CustomAction Id="UpdateTomcatClassPath"
       ExeCommand='//US//Tomcat6 --Classpath="[TOMCATDIRECTORY]\bin\bootstrap.jar";"[TOMCATDIRECTORY]\bin\tomcat-juli.jar";[CSMANAGEMENT]\conf;[CSMANAGEMENT]\lib;[CSMANAGEMENT]\setup'
-      Property="TOMCATDIRECTORY1" Execute="commit" Return="check" />
+      Property="TOMCATDIRECTORY1" Execute="deferred" Return="check" />
     <CustomAction Id="CopySitePackages" Directory='INSTALLDIR'
       ExeCommand='[SystemFolder]cmd.exe /c xcopy /S "[INSTALLDIR]\python-site-packages" [PYTHON_HOME]\Lib\site-packages'
-      Execute="commit" Return="check"/>
+      Execute="deferred" Return="check"/>
     <CustomAction Id="DeleteDirectory" Directory='INSTALLDIR'
       ExeCommand='[SystemFolder]cmd.exe /c RD /S /Q "[INSTALLDIR]\python-site-packages"'
-      Execute="commit" Return="check" />
+      Execute="deferred" Return="check" />
     <CustomAction Id="DeleteFiles" Directory='CSMANAGEMENT'
       ExeCommand='[SystemFolder]cmd.exe /c del "[CSMANAGEMENT]\webapps\client\WEB-INF\classes\db.properties" "[CSMANAGEMENT]\webapps\client\WEB-INF\classes\log4j*.xml"'
-      Execute="commit" Return="check" />
+      Execute="deferred" Return="check" />
     <CustomAction Id="SetuptoolsInstallation" Directory='INSTALLDIR'
       ExeCommand='[PYTHON_HOME]\python "[INSTALLDIR]\ez_setup.py"'
-      Execute="commit" Return="check" />
+      Execute="deferred" Return="check" />
     <CustomAction Id="GenerateSSLKey" Directory='CSMANAGEMENT'
       ExeCommand='[PYTHON_HOME]\python "[CSMANAGEMENT]\webapps\client\WEB-INF\classes\scripts\common\keys\ssl-keys.py" "[CSMANAGEMENT]\lib"'
-      Execute="commit" Return="check" />
+      Execute="deferred" Return="check" />
     <CustomAction Id="DeployDB" Directory='CSMANAGEMENT'
       ExeCommand='[PYTHON_HOME]\python "[INSTALLDIR]\scripts\cloud-setup-databases" [DB_USERNAME]:[DB_PASSWORD]@[DB_HOSTNAME] --deploy-as=root:[DB_ROOT_PASSWORD] -c "[CSMANAGEMENT]\lib" -f "[CSMANAGEMENT]\setup" -j "[CSMANAGEMENT]\webapps\client\WEB-INF\lib\jasypt-1.9.0.jar" -n "[CSMANAGEMENT]\lib\key"'
-      Execute="commit" Return="check" />
+      Execute="deferred" Return="check" Impersonate="no"/>
     <CustomAction Id="SetupDatabases" Directory='CSMANAGEMENT'
       ExeCommand='[PYTHON_HOME]\python "[INSTALLDIR]\scripts\cloud-setup-databases" [DB_USERNAME]:[DB_PASSWORD]@[DB_HOSTNAME] -c "[CSMANAGEMENT]\lib" -f "[CSMANAGEMENT]\setup" -j "[CSMANAGEMENT]\webapps\client\WEB-INF\lib\jasypt-1.9.0.jar" -n "[CSMANAGEMENT]\lib\key"'
-      Execute="commit" Return="check" />
+      Execute="deferred" Return="check" Impersonate="no"/>
 
     <InstallExecuteSequence>
       <InstallServices Sequence="4990"></InstallServices>
@@ -270,9 +273,9 @@
         Installed</Custom>
       <Custom Action="DeleteFiles" Before="SetuptoolsInstallation">NOT Installed</Custom>
       <Custom Action="SetuptoolsInstallation" Before="GenerateSSLKey">NOT Installed</Custom>
-      <Custom Action="GenerateSSLKey" Before="InstallFinalize">NOT Installed
+      <Custom Action="GenerateSSLKey" After="PublishProduct">NOT Installed
       </Custom>
-      <Custom Action="CopySitePackages" Before="InstallFinalize">NOT Installed</Custom>
+      <Custom Action="CopySitePackages" After="GenerateSSLKey">NOT Installed</Custom>
       <Custom Action="DeleteDirectory" After="CopySitePackages">NOT Installed</Custom>
       <Custom Action="DeployDB" After="DeleteDirectory">(NOT Installed) AND (CREATE_DATABASE = "1")
       </Custom>

scripts/installer/windows/en-us.wxl

@@ -28,6 +28,15 @@
   <String Id="DbPasswordLabel">Password</String>
   <String Id="DbRootPasswordLabel">Root Password</String>
   <String Id="CreateDBCheckBoxLabel">Create Cloud Stack Database</String>
+  <String Id="GenerateSSLKey">Generating SSL Keys which is needed to talk to Management Server.</String>
+  <String Id="DeployDB">Creating Database and needed Schemas for management server.</String>
+  <String Id="SetupDatabases">Updating db.properties with the given database configuration.</String>
+  <String Id="DeleteDirectory">Deleting un-necessary directories from installation directory.</String>
+  <String Id="CopySitePackages">Copying Management Server related python libraries.</String>
+  <String Id="SetuptoolsInstallation">Installing Setup Tools for Python.</String>
+  <String Id="DeleteFiles">Deleting un-necessary files from installation directory.</String>
+  <String Id="UpdateTomcatCatalinaBase">Updating Tomcat Catalina Base with Management Server Home.</String>
+  <String Id="UpdateTomcatClassPath">Updating Tomcat Classpath with necessary paths.</String>
 
   <String Id="InstallVersion">Version 4.0.0.0</String>
 </WixLocalization>

scripts/storage/secondary/cloud-install-sys-tmplt.py

@@ -0,0 +1,236 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import sys
+import urllib
+import uuid
+import subprocess
+import os
+import shutil
+import gzip
+import zipfile
+import bz2
+
+class InstallSysTemplate(object):
+  parser = None
+  mountpoint = None
+  args = None
+  hypervisor = None
+  systemvmtemplatepath = None
+  systemvmtemplateurl = None
+  managementsecretkey = None
+  forcecleanup = None
+  databasehostname = None
+  databaseusername = None
+  databaseuserpassword = None
+  templatesuffix = None
+  template = None
+  fileextension = None
+  templateName = None
+  destDir = None
+  fileSize = None
+  dictionary = None
+
+  def __init__(self):
+    self.dictionary = dict(xenserver=('XenServer', 'vhd'), kvm=('KVM', 'qcow2'), vmware=('VMware', 'ova'), lxc=('LXC', 'qcow2'), hyperv=('Hyperv', 'vhd'))
+
+  def parseOptions(self):
+    self.parser = argparse.ArgumentParser(prog="System Template Installer")
+    self.parser.add_argument("-m", "--mount-point", action="store", dest="mountpoint", help="Secondary Storage Mount Point where to install the temlate.", required="true")
+    self.parser.add_argument("-H", "--hypervisor", action="store", dest="hypervisor", help="The Hypervisor name for which template need to be installed", required="true", choices=['kvm','xenserver','vmware','lxc','hyperv'])
+    group = self.parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("-f", "--system-vm-template", action="store", dest="systemvmtemplatepath", help="The local system vm template file path")
+    group.add_argument("-u", "--system-vm-template-url", action="store", dest="systemvmtemplateurl", help="Url to download system vm template")
+    self.parser.add_argument("-s", "--management-secret-key", action="store", dest="managementsecretkey", help="mgmt server secret key, if you specified any when running cloudstack-setup-database, default is password", default="password")
+    self.parser.add_argument("-F", "--force-clean-up", action="store_true", dest="forcecleanup", help="clean up system templates of specified hypervisor", default="false")
+    self.parser.add_argument("-d", "--database-host-name", action="store", dest="databasehostname", help="Database server hostname or ip, e.g localhost", default="localhost", required="true")
+    self.parser.add_argument("-r", "--database-user-name", action="store", dest="databaseusername", help="Database user name, e.g root", default="root", required="true")
+    self.parser.add_argument("-p", "--database-user-password", nargs='?', action="store", dest="databaseuserpassword", help="Database password. Followed by nothing if the password is empty", default="", required="true")
+    self.parser.add_argument("-e", "--template-suffix", action="store", dest="templatesuffix", help="Template suffix, e.g vhd, ova, qcow2",default="vhd")
+    self.parser.add_argument("-t", "--file-extension", action="store", dest="fileextension", help="The template file extension", default="", required="true", choices=['bz2','gz','zip'])
+
+    self.args = self.parser.parse_args()
+
+  def populateOptions(self):
+    self.mountpoint = self.args.mountpoint
+    self.hypervisor = self.args.hypervisor
+    self.fileextension = self.args.fileextension
+    if self.args.systemvmtemplatepath:
+      self.systemvmtemplatepath = self.args.systemvmtemplatepath
+    if self.args.systemvmtemplateurl:
+      self.systemvmtemplateurl = self.args.systemvmtemplateurl
+    if self.args.managementsecretkey:
+      self.managementsecretkey = self.args.managementsecretkey
+    if self.args.forcecleanup:
+      self.forcecleanup = self.args.forcecleanup
+    if self.args.databasehostname:
+      self.databasehostname = self.args.databasehostname
+    if self.args.databaseusername:
+      self.databaseusername = self.args.databaseusername
+    if self.args.databaseuserpassword:
+      self.databaseuserpassword = self.args.databaseuserpassword
+    else:
+      self.databaseuserpassword = ""
+    if self.args.templatesuffix:
+      self.templatesuffix = self.args.templatesuffix
+    print 'Password for DB: %s'%self.databaseuserpassword
+
+  def errorAndExit(self, msg):
+    err = '''\n\nWe apologize for below error:
+***************************************************************
+%s
+***************************************************************
+Please run:
+
+    cloud-install-sys-tmplt -h
+
+for full help
+''' % msg
+    sys.stderr.write(err)
+    sys.stderr.flush()
+    sys.exit(1)
+
+  def runCmd(self, cmds):
+    process = subprocess.Popen(' '.join(cmds), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+    print(stdout)
+    if process.returncode != 0:
+        raise Exception(stderr)
+    return stdout
+
+  def runMysql(self, query):
+    try:
+      print 'Running Query: %s' % query
+      mysqlCmds = ['mysql', '--user=%s'%self.databaseusername, '--host=%s'%self.databasehostname, '--password=%s'%self.databaseuserpassword, '--skip-column-names', '-U', 'cloud', '-e "%s"'%query]
+      templateId = self.runCmd(mysqlCmds)
+      print 'TemplateId is : %s' % templateId
+    except Exception, e:
+      err = '''Encountering an error when executing mysql script\n%s''' % str(e)
+      self.errorAndExit(err)
+    return templateId
+
+  def fetchTemplateDetails(self):
+    mysqlQuery = "select max(id) from cloud.vm_template where type = 'SYSTEM' and hypervisor_type = '%s' and removed is null"
+    ht = None
+    hypervisorInfo = self.dictionary[self.hypervisor]
+    ht = hypervisorInfo[0]
+    self.templatesuffix = hypervisorInfo[1]
+
+    self.template = int(self.runMysql(mysqlQuery%ht))
+
+  def downloadTemplate(self):
+    self.systemvmtemplatepath = self.templateName + "." + self.fileextension
+    print 'Downloading template from %s To %s' % (self.systemvmtemplateurl, self.systemvmtemplatepath)
+    try:
+      templateFileDownloadUrl = urllib.urlretrieve(self.systemvmtemplateurl, self.systemvmtemplatepath, reporthook=self.report)
+    except Exception:
+      self.errorAndExit("Failed to download template file from %s" % self.systemvmtemplateurl)
+
+  def report(tmp, blocknr, blocksize, size):
+    current = blocknr*blocksize
+    sys.stdout.write("\rDownloading completed: {0:.2f}%".format(100.0*current/size))
+
+  def installTemplate(self):
+    destDir = self.mountpoint + os.sep + "template" + os.sep + "tmpl" + os.sep + "1" + os.sep + str(self.template)
+    self.destDir = destDir
+    print 'The desination Directory is : %s' % destDir
+    try:
+      if self.forcecleanup:
+        if os.path.exists(destDir):
+          shutil.rmtree(destDir)
+      if not os.path.exists(destDir):
+        os.makedirs(destDir)
+    except Exception, e:
+      self.errorAndExit('Failed to create directories on the mounted path.. %s' % str (e))
+    print 'Installing Template to : %s' % destDir
+    tmpFile = self.templateName + "." + "tmp"
+    self.uncompressFile(tmpFile)
+    print 'Moving the decompressed file to destination directory %s... which could take a long time, please wait' % destDir
+    shutil.move(tmpFile, destDir + os.sep + self.templateName)
+
+  def uncompressFile(self, fileName):
+    print 'Uncompressing the file %s... which could take a long time, please wait' % self.systemvmtemplatepath
+    if self.fileextension == 'gz':
+      compressedFile = gzip.GzipFile(self.systemvmtemplatepath, 'rb')
+      decompressedData = compressedFile.read()
+      compressedFile.close()
+      decompressedFile = file(fileName, 'wb')
+      decompressedFile.write(decompressedData)
+      decompressedFile.close()
+    elif self.fileextension == 'bz2':
+      compressedFile = bz2.BZ2File(self.systemvmtemplatepath)
+      decompressedData = compressedFile.read()
+      compressedFile.close()
+      decompressedFile = file(fileName, 'wb')
+      decompressedFile.write(decompressedData)
+      decompressedFile.close()
+      print ''
+    elif self.fileextension == 'zip':
+      zippedFile = zipfile.ZipFile(self.systemvmtemplatepath, 'r')
+      zippedFiles = zippedFile.namelist()
+      compressedFile = zippedFiles[0]
+      decompressedData = zippedFile.read(compressedFile)
+      decompressedFile = file(fileName, 'wb')
+      decompressedFile.write(decompressedData)
+      decompressedFile.close()
+      zippedFile.close()
+      print ''
+    else:
+      self.errorAndExit('Not supported file type %s to decompress' % self.fileextension)
+    self.fileSize = os.path.getsize(fileName)
+
+  def writeProperties(self):
+    propertiesFile = file(self.destDir + os.sep + 'template.properties', 'wb')
+    propertiesFile.write('filename=%s\n'%self.templateName)
+    propertiesFile.write('description=SystemVM Template\n')
+    propertiesFile.write('checksum=\n')
+    propertiesFile.write('hvm=false\n')
+    propertiesFile.write('size=%s\n'%str(self.fileSize))
+    propertiesFile.write('%s=true\n'%self.templatesuffix)
+    propertiesFile.write('id=%s\n'%str(self.template))
+    propertiesFile.write('public=true\n')
+    propertiesFile.write('%s.filename=%s\n'%(self.templatesuffix, self.templateName))
+    propertiesFile.write('uniquename=routing-%s\n'%str(self.template))
+    propertiesFile.write('%s.virtualsize=%s\n'%(self.templatesuffix, str(self.fileSize)))
+    propertiesFile.write('virtualsize=%s\n'%str(self.fileSize))
+    propertiesFile.write('%s.size=%s'%(self.templatesuffix, str(self.fileSize)))
+
+    propertiesFile.close()
+
+  def run(self):
+    try:
+      self.parseOptions()
+      self.populateOptions()
+      self.fetchTemplateDetails()
+      randomUUID = uuid.uuid1()
+      self.templateName = str(randomUUID) + "." + self.templatesuffix
+      if self.args.systemvmtemplateurl:
+        self.downloadTemplate()
+      self.installTemplate()
+      self.writeProperties()
+    finally:
+      print ''
+    print ''
+    print "CloudStack has successfully installed system template"
+    print ''
+
+if __name__ == "__main__":
+   o = InstallSysTemplate()
+   o.run()