CLOUDSTACK-9462: Build packages on Ubuntu 14.04/16.04 and support tomcat6/tomcat7

Changes
(1) add systemd support in debian/control and debian/rules for 16.04
(2) add python-setuptools in Build-Depends in debian/control
(3) seperate cloudstack-management.service and cloudstack-management.default for CentOS7 and Ubuntu 16.04
(4) add server7-ssl.xml and server7-nonssl.xml in management installation
(5) link /usr/share/cloudstack-management/lib and /usr/share/cloudstack-management/bin to correct path (tomcat6 or tomcat7)
(6) link /etc/cloudstack/management/server.xml to correct file path (server-nonssl.xml or server7-nonssl.xml)
(7) remove *.zip from .gitignore to avoid build error caused by missing /vhds/test.vhd.zip

Instruction
(1) build packages on Ubuntu 16.04: dpkg-buildpackage -uc -us
    Output on Ubuntu 16.04:
-rw-r--r-- 1 root root      4090 Feb 17 10:12 cloudstack_4.10.0.0-SNAPSHOT_amd64.changes
-rw-r--r-- 1 root root      1235 Feb 17 09:53 cloudstack_4.10.0.0-SNAPSHOT.dsc
-rw-r--r-- 1 root root   8018248 Feb 17 09:53 cloudstack_4.10.0.0-SNAPSHOT.tar.xz
-rw-r--r-- 1 root root  91868746 Feb 17 10:11 cloudstack-agent_4.10.0.0-SNAPSHOT_all.deb
-rw-r--r-- 1 root root     52882 Feb 17 10:12 cloudstack-cli_4.10.0.0-SNAPSHOT_all.deb
-rw-r--r-- 1 root root  98556216 Feb 17 10:08 cloudstack-common_4.10.0.0-SNAPSHOT_all.deb
-rw-r--r-- 1 root root     52864 Feb 17 10:12 cloudstack-docs_4.10.0.0-SNAPSHOT_all.deb
-rw-r--r-- 1 root root    585434 Feb 17 10:12 cloudstack-integration-tests_4.10.0.0-SNAPSHOT_all.deb
-rw-r--r-- 1 root root 323459934 Feb 17 10:10 cloudstack-management_4.10.0.0-SNAPSHOT_all.deb
-rw-r--r-- 1 root root    442656 Feb 17 10:12 cloudstack-marvin_4.10.0.0-SNAPSHOT_all.deb
-rw-r--r-- 1 root root  87037576 Feb 17 10:12 cloudstack-usage_4.10.0.0-SNAPSHOT_all.deb

(2) setup tomcat6/tomcat7 on management server:
    tomcat6: cloudstack-setup-management --tomcat6
    tomcat7: cloudstack-setup-management --tomcat7

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

.gitignore

@@ -51,7 +51,6 @@ tools/cli/build/
 *.jar
 *.war
 *.mar
-*.zip
 *.iso
 *.tar.gz
 *.tgz

debian/cloudstack-management.install

@@ -21,7 +21,8 @@
 /etc/cloudstack/management/ehcache.xml
 /etc/cloudstack/management/server-ssl.xml
 /etc/cloudstack/management/server-nonssl.xml
-/etc/cloudstack/management/server.xml
+/etc/cloudstack/management/server7-ssl.xml
+/etc/cloudstack/management/server7-nonssl.xml
 /etc/cloudstack/management/classpath.conf
 /etc/cloudstack/management/db.properties
 /etc/cloudstack/management/tomcat6-ssl.conf
@@ -33,6 +34,7 @@
 /etc/cloudstack/management/log4j-cloud.xml
 /etc/cloudstack/management/tomcat-users.xml
 /etc/cloudstack/management/context.xml
+/etc/default/cloudstack-management
 /etc/init.d/cloudstack-management
 /etc/security/limits.d/cloudstack-limits.conf
 /etc/sudoers.d/cloudstack

debian/cloudstack-management.postinst

@@ -60,7 +60,26 @@ if [ "$1" = configure ]; then
 
     chmod 0640 ${CONFDIR}/${DBPROPS}
     chgrp cloud ${CONFDIR}/${DBPROPS}
-    invoke-rc.d tomcat6 stop || true
+
+    if [ -d "/usr/share/tomcat7" ]; then
+        rm -f /usr/share/cloudstack-management/bin
+        rm -f /usr/share/cloudstack-management/lib
+        ln -s /usr/share/tomcat7/bin /usr/share/cloudstack-management/bin
+        ln -s /usr/share/tomcat7/lib /usr/share/cloudstack-management/lib
+        invoke-rc.d tomcat7 stop || true
+        if [ ! -f "/etc/cloudstack/management/server.xml" ]; then
+            ln -s /etc/cloudstack/management/server7-nonssl.xml /etc/cloudstack/management/server.xml
+        fi
+    elif [ -d "/usr/share/tomcat6" ]; then
+        rm -f /usr/share/cloudstack-management/bin
+        rm -f /usr/share/cloudstack-management/lib
+        ln -s /usr/share/tomcat6/bin /usr/share/cloudstack-management/bin
+        ln -s /usr/share/tomcat6/lib /usr/share/cloudstack-management/lib
+        invoke-rc.d tomcat6 stop || true
+        if [ ! -f "/etc/cloudstack/management/server.xml" ]; then
+            ln -s /etc/cloudstack/management/server-nonssl.xml /etc/cloudstack/management/server.xml
+        fi
+    fi
 
     # Add jdbc MySQL driver settings to db.properties if not present
     grep -s -q "db.cloud.driver=jdbc:mysql" ${CONFDIR}/${DBPROPS} || sed -i -e "\$adb.cloud.driver=jdbc:mysql" ${CONFDIR}/${DBPROPS}

debian/control

@@ -3,7 +3,7 @@ Section: libs
 Priority: extra
 Maintainer: Wido den Hollander <wido@widodh.nl>
 Build-Depends: debhelper (>= 9), openjdk-8-jdk | java8-sdk | java8-jdk, genisoimage,
- python-mysql.connector, maven (>= 3) | maven3, python (>= 2.7), lsb-release, dh-systemd
+ python-mysql.connector, maven (>= 3) | maven3, python (>= 2.7), lsb-release, dh-systemd, python-setuptools
 Standards-Version: 3.8.1
 Homepage: http://www.cloudstack.org/
 
@@ -15,15 +15,14 @@ Description: A common package which contains files which are shared by several C
 
 Package: cloudstack-management
 Architecture: all
-Depends: ${misc:Depends}, ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), tomcat6 | tomcat7, sudo, jsvc, python-mysql.connector, libmysql-java, augeas-tools, mysql-client, adduser, bzip2, ipmitool, lsb-release
+Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime | openjdk-7-jre-headless, cloudstack-common (= ${source:Version}), tomcat6 | tomcat7, sudo, jsvc, python-mysql.connector, libmysql-java, augeas-tools, mysql-client, adduser, bzip2, ipmitool, lsb-release, init-system-helpers (>= 1.14~)
 Conflicts: cloud-server, cloud-client, cloud-client-ui
 Description: CloudStack server library
  The CloudStack management server
 
 Package: cloudstack-agent
 Architecture: all
-Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 0.9.8), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, lsb-release
+Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 0.9.8), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, lsb-release, init-system-helpers (>= 1.14~)
-Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent
  The CloudStack agent is in charge of managing shared computing resources in
@@ -32,8 +31,7 @@ Description: CloudStack agent
 
 Package: cloudstack-usage
 Architecture: all
-Depends: openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), jsvc, libmysql-java
+Depends: openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), jsvc, libmysql-java, init-system-helpers (>= 1.14~)
-Recommends: init-system-helpers
 Description: CloudStack usage monitor
  The CloudStack usage monitor provides usage accounting across the entire cloud for
  cloud operators to charge based on usage parameters.

debian/rules

@@ -85,19 +85,20 @@ override_dh_auto_install:
 	chmod 0440 $(DESTDIR)/$(SYSCONFDIR)/sudoers.d/$(PACKAGE)
 
 	ln -s tomcat6-nonssl.conf $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/management/tomcat6.conf
-	ln -s server-nonssl.xml $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/management/server.xml
 	install -D packaging/debian/init/cloud-management $(DESTDIR)/$(SYSCONFDIR)/init.d/$(PACKAGE)-management
 	install -D client/target/utilities/bin/cloud-update-xenserver-licenses $(DESTDIR)/usr/bin/cloudstack-update-xenserver-licenses
-	ln -s /usr/share/tomcat6/bin $(DESTDIR)/usr/share/$(PACKAGE)-management/bin
 	# Remove configuration in /ur/share/cloudstack-management/webapps/client/WEB-INF
 	# This should all be in /etc/cloudstack/management
 	rm $(DESTDIR)/usr/share/$(PACKAGE)-management/webapps/client/WEB-INF/classes/*.*
 	ln -s ../../..$(SYSCONFDIR)/$(PACKAGE)/management $(DESTDIR)/usr/share/$(PACKAGE)-management/conf
-	ln -s ../../../usr/share/tomcat6/lib $(DESTDIR)/usr/share/$(PACKAGE)-management/lib
 	ln -s ../../../var/log/$(PACKAGE)/management $(DESTDIR)/usr/share/$(PACKAGE)-management/logs
 	ln -s ../../../var/cache/$(PACKAGE)/management/temp $(DESTDIR)/usr/share/$(PACKAGE)-management/temp
 	ln -s ../../../var/cache/$(PACKAGE)/management/work $(DESTDIR)/usr/share/$(PACKAGE)-management/work
 
+	install -d -m0755 debian/$(PACKAGE)-management/lib/systemd/system
+	install -m0644 packaging/systemd/$(PACKAGE)-management.service.ubuntu debian/$(PACKAGE)-management/lib/systemd/system/$(PACKAGE)-management.service
+	install -m0644 packaging/systemd/$(PACKAGE)-management.default.ubuntu $(DESTDIR)/$(SYSCONFDIR)/default/$(PACKAGE)-management
+
 	# cloudstack-common
 	mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-common
 	mkdir $(DESTDIR)/usr/share/$(PACKAGE)-common/scripts
@@ -149,7 +150,7 @@ override_dh_installinit:
 	dh_installinit -pcloudstack-management -pcloudstack-agent -pcloudstack-usage --onlyscripts --no-start
 
 override_dh_systemd_enable:
-	dh_systemd_enable -pcloudstack-agent -pcloudstack-usage
+	dh_systemd_enable -pcloudstack-management -pcloudstack-agent -pcloudstack-usage
 
 override_dh_installdocs:
 	dh_installdocs -A tools/whisker/LICENSE tools/whisker/NOTICE INSTALL.md

packaging/build-deb.sh

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
+#set -e
 
 #
 # This script builds Debian packages for CloudStack and does
@@ -43,11 +43,21 @@ set -e
 cd `dirname $0`
 cd ..
 
-dpkg-checkbuilddeps
+DCH=$(which dch)
+if [ -z "$DCH" ] ; then
+    echo -e "dch not found, please install devscripts at first. \nDEB Build Failed"
+    exit
+fi
 
-VERSION=$(grep '^  <version>' pom.xml| cut -d'>' -f2 |cut -d'<' -f1)
+VERSION=$(head -n1 debian/changelog  |awk -F [\(\)] '{print $2}')
 DISTCODE=$(lsb_release -sc)
 
-dch -b -v "${VERSION}~${DISTCODE}" -u low -m "Apache CloudStack Release ${VERSION}"
+/bin/cp debian/changelog /tmp/changelog.orig
 
-dpkg-buildpackage -j2 -b -uc -us
+dch -b -v "${VERSION}~${DISTCODE}" -u low -m "Apache CloudStack Release ${VERSION}"
+sed -i '0,/ UNRELEASED;/s// unstable;/g' debian/changelog
+
+dpkg-checkbuilddeps
+dpkg-buildpackage -uc -us
+
+/bin/mv /tmp/changelog.orig debian/changelog

packaging/systemd/cloudstack-management.default.ubuntu

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Required for a tocmat cloned service
+SERVICE_NAME=cloudstack-management
+
+# Where your java installation lives
+#JAVA_HOME="/usr/lib/jvm/java"
+
+# Where your cloudstack-management installation lives
+CATALINA_BASE="/usr/share/cloudstack-management"
+CATALINA_HOME="/usr/share/cloudstack-management"
+JASPER_HOME="/usr/share/cloudstack-management"
+CATALINA_TMPDIR="/usr/share/cloudstack-management/temp"
+
+
+if [ -r "/etc/cloudstack/management/cloudmanagementserver.keystore" ] ; then
+  JAVA_OPTS="-Djava.awt.headless=true -Dcom.sun.management.jmxremote=false -Xmx2g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:PermSize=512M -XX:MaxPermSize=800m -Djavax.net.ssl.trustStore=/etc/cloudstack/management/cloudmanagementserver.keystore -Djavax.net.ssl.trustStorePassword=vmops.com "
+else
+  JAVA_OPTS="-Djava.awt.headless=true -Dcom.sun.management.jmxremote=false -Xmx2g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:PermSize=512M -XX:MaxPermSize=800m"
+fi
+
+TOMCAT_USER="cloud"
+
+SECURITY_MANAGER="false"
+
+SHUTDOWN_WAIT="30"
+
+SHUTDOWN_VERBOSE="false"
+
+# Set the TOMCAT_PID location
+CATALINA_PID="/var/run/cloudstack-management.pid"
+
+CLASSPATH=/usr/share/java/commons-daemon.jar:/usr/share/cloudstack-management/bin/bootstrap.jar:/usr/share/tomcat7/bin/tomcat-juli.jar:/etc/cloudstack/management:/usr/share/cloudstack-common:/usr/share/cloudstack-management/setup
+
+BOOTSTRAP_CLASS=org.apache.catalina.startup.Bootstrap

packaging/systemd/cloudstack-management.service.ubuntu

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Systemd unit file for CloudStack Management server
+#
+# clone tomcat service, see /usr/lib/systemd/system/tomcat.service
+
+[Unit]
+Description=CloudStack Management Server
+After=syslog.target network.target
+
+[Service]
+UMask=0022
+Type=forking
+Environment="NAME=cloudstack-management"
+EnvironmentFile=-/etc/default/cloudstack-management
+ExecStartPre=/bin/bash -c "/bin/systemctl set-environment JAVA_HOME=$( readlink -f $( which java ) | sed s:bin/.*$:: )"
+ExecStart=/usr/bin/jsvc -user "${TOMCAT_USER}" -cp "$CLASSPATH" \
+    -outfile SYSLOG -errfile SYSLOG \
+    -pidfile "${CATALINA_PID}" ${JAVA_OPTS} \
+    -Dcatalina.base="${CATALINA_BASE}" -Dcatalina.home="${CATALINA_HOME}" -Djava.io.tmpdir="${CATALINA_TMPDIR}" "${BOOTSTRAP_CLASS}"
+ExecStop=/usr/bin/jsvc -cp "$CLASSPATH" -pidfile "$CATALINA_PID" \
+    -stop "$BOOTSTRAP_CLASS"
+
+[Install]
+WantedBy=multi-user.target

python/lib/cloudutils/serviceConfigServer.py

@@ -89,6 +89,10 @@ class cloudManagementConfig(serviceCfgBase):
                 if os.path.exists("/etc/cloudstack/management/server.xml"):
                     bash("rm -f /etc/cloudstack/management/server.xml")
                 bash("ln -s /etc/cloudstack/management/server7-ssl.xml /etc/cloudstack/management/server.xml")
+                bash("rm -f /usr/share/cloudstack-management/bin")
+                bash("rm -f /usr/share/cloudstack-management/lib")
+                bash("ln -s /usr/share/tomcat7/bin /usr/share/cloudstack-management/bin")
+                bash("ln -s /usr/share/tomcat7/lib /usr/share/cloudstack-management/lib")
             else:
                 if not os.path.exists("/etc/cloudstack/management/server-ssl.xml") or not os.path.exists("/etc/cloudstack/management/tomcat6-ssl.conf"):
                     raise CloudRuntimeException("Cannot find /etc/cloudstack/management/server-ssl.xml or /etc/cloudstack/management/tomcat6-ssl.conf, https enable failed")
@@ -98,6 +102,10 @@ class cloudManagementConfig(serviceCfgBase):
                     bash("rm -f /etc/cloudstack/management/tomcat6.conf")
                 bash("ln -s /etc/cloudstack/management/server-ssl.xml /etc/cloudstack/management/server.xml")
                 bash("ln -s /etc/cloudstack/management/tomcat6-ssl.conf /etc/cloudstack/management/tomcat6.conf")
+                bash("rm -f /usr/share/cloudstack-management/bin")
+                bash("rm -f /usr/share/cloudstack-management/lib")
+                bash("ln -s /usr/share/tomcat6/bin /usr/share/cloudstack-management/bin")
+                bash("ln -s /usr/share/tomcat6/lib /usr/share/cloudstack-management/lib")
             if not bash("iptables-save |grep PREROUTING | grep 6443").isSuccess():
                 bash("iptables -A PREROUTING -t nat -p tcp --dport 443 -j REDIRECT --to-port 6443")
         else:
@@ -107,6 +115,10 @@ class cloudManagementConfig(serviceCfgBase):
                 if os.path.exists("/etc/cloudstack/management/server.xml"):
                     bash("rm -f /etc/cloudstack/management/server.xml")
                 bash("ln -s /etc/cloudstack/management/server7-nonssl.xml /etc/cloudstack/management/server.xml")
+                bash("rm -f /usr/share/cloudstack-management/bin")
+                bash("rm -f /usr/share/cloudstack-management/lib")
+                bash("ln -s /usr/share/tomcat7/bin /usr/share/cloudstack-management/bin")
+                bash("ln -s /usr/share/tomcat7/lib /usr/share/cloudstack-management/lib")
             else:
                 if not os.path.exists("/etc/cloudstack/management/server-nonssl.xml") or not os.path.exists("/etc/cloudstack/management/tomcat6-nonssl.conf"):
                     raise CloudRuntimeException("Cannot find /etc/cloudstack/management/server-nonssl.xml or /etc/cloudstack/management/tomcat6-nonssl.conf, https enable failed")
@@ -116,6 +128,10 @@ class cloudManagementConfig(serviceCfgBase):
                     bash("rm -f /etc/cloudstack/management/tomcat6.conf")
                 bash("ln -s /etc/cloudstack/management/server-nonssl.xml /etc/cloudstack/management/server.xml")
                 bash("ln -s /etc/cloudstack/management/tomcat6-nonssl.conf /etc/cloudstack/management/tomcat6.conf")
+                bash("rm -f /usr/share/cloudstack-management/bin")
+                bash("rm -f /usr/share/cloudstack-management/lib")
+                bash("ln -s /usr/share/tomcat6/bin /usr/share/cloudstack-management/bin")
+                bash("ln -s /usr/share/tomcat6/lib /usr/share/cloudstack-management/lib")
         bash("touch /var/run/cloudstack-management.pid")
         bash("chown cloud.cloud /var/run/cloudstack-management.pid")
         #distro like sl 6.1 needs this folder, or tomcat6 failed to start