diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 84d9648a883..90a1ad95164 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -107,12 +107,7 @@ BUILD_NUMBER="${4:-${BUILD_NUMBER:-}}" # (debian) os architecture to build arch="${5:-${arch:-i386}}" -if [ "${appliance}" == "systemvm64template" ]; then - arch="amd64" - export VM_ARCH="${arch}" - rm -rf definitions/systemvm64template - cp -r definitions/systemvmtemplate definitions/systemvm64template -fi +export VM_ARCH="${arch}" # optional root SSH public key to write to /root/.ssh/authorized_keys # note the cs management server overwrites this, so the only reason to @@ -231,6 +226,14 @@ function retry() { ### function create_definition() { + if [ "${appliance}" == "systemvm64template" ]; then + arch="amd64" + export VM_ARCH="${arch}" + rm -rf definitions/systemvm64template # in case of left-over cruft from failed build + cp -r definitions/systemvmtemplate definitions/systemvm64template + add_on_exit rm -rf definitions/systemvm64template + fi + if [ "${appliance}" != "${appliance_build_name}" ]; then cp -r "definitions/${appliance}" "definitions/${appliance_build_name}" set +e diff --git a/tools/appliance/definitions/debianbase/cleanup.sh b/tools/appliance/definitions/debianbase/cleanup.sh new file mode 100644 index 00000000000..cd32d5b42aa --- /dev/null +++ b/tools/appliance/definitions/debianbase/cleanup.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function cleanup_apt() { + #apt-get -y remove linux-headers-$(uname -r) build-essential + apt-get -y remove dictionaries-common busybox + apt-get -y autoremove + apt-get autoclean + apt-get clean +} + +# Removing leftover leases and persistent rules +function cleanup_dhcp() { + rm -f /var/lib/dhcp/* +} + +# Make sure Udev doesn't block our network +function cleanup_dev() { + echo "cleaning up udev rules" + rm -f /etc/udev/rules.d/70-persistent-net.rules + rm -rf /dev/.udev/ + rm -f /lib/udev/rules.d/75-persistent-net-generator.rules +} + +function cleanup() { + cleanup_apt + cleanup_dhcp + cleanup_dev +} + +return 2>/dev/null || cleanup diff --git a/tools/appliance/definitions/debianbase/configure_login.sh b/tools/appliance/definitions/debianbase/configure_login.sh new file mode 100644 index 00000000000..36fccabc5fa --- /dev/null +++ b/tools/appliance/definitions/debianbase/configure_login.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +function add_admin_group() { + groupadd -f -r admin +} + +function configure_cloud_user() { + usermod -a -G admin cloud + mkdir -p /home/cloud/.ssh + chmod 700 /home/cloud/.ssh + echo "cloud:`openssl rand -base64 32`" | chpasswd +} + +function configure_sudoers() { + cat >/etc/sudoers < /etc/sudoers.d/cloud +} + +# sshd_config is overwritten from cloud_scripts +#function configure_sshd() { +# grep "UseDNS no" /etc/ssh/sshd_config && \ +# grep "PasswordAuthentication no" /etc/ssh/sshd_config && \ +# return +# # Tweak sshd to prevent DNS resolution (speed up logins) +# echo 'UseDNS no' >> /etc/ssh/sshd_config +# +# # Require ssh keys for login +# sed -i -e 's/^.*PasswordAuthentication .*$/PasswordAuthentication no/g' /etc/ssh/sshd_config +#} + +function configure_inittab() { + grep "vc:2345:respawn:/sbin/getty" /etc/inittab && return + + # Fix inittab + cat >> /etc/inittab << EOF + +vc:2345:respawn:/sbin/getty 38400 hvc0 +EOF +} + +function configure_login() { + add_admin_group + configure_cloud_user + configure_sudoers + # configure_sshd + configure_inittab +} + +return 2>/dev/null || configure_login diff --git a/tools/appliance/definitions/debianbase/definition.rb b/tools/appliance/definitions/debianbase/definition.rb new file mode 100644 index 00000000000..f430f33548f --- /dev/null +++ b/tools/appliance/definitions/debianbase/definition.rb @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arch = ENV['VM_ARCH'] || 'i386' + +architectures = { + :i386 => { + :os_type_id => 'Debian', + :iso_file => 'debian-7.6.0-i386-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/i386/iso-cd/debian-7.6.0-i386-netinst.iso', + :iso_md5 => '528e1a7315da1bbf50bd4d187880a519', + }, + :amd64 => { + :os_type_id => 'Debian_64', + :iso_file => 'debian-7.6.0-amd64-netinst.iso', + :iso_src => 'http://cdimage.debian.org/debian-cd/7.6.0/amd64/iso-cd/debian-7.6.0-amd64-netinst.iso', + :iso_md5 => '8a3c2ad7fd7a9c4c7e9bcb5cae38c135' + } +} + +config = { + :cpu_count => '1', + :memory_size => '256', + :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', + :iso_download_timeout => '1200', + :boot_wait => '10', + :boot_cmd_sequence => [ + '', + 'install ', + 'preseed/url=http://%IP%:%PORT%/preseed.cfg ', + 'debian-installer=en_US ', + 'auto ', + 'locale=en_US ', + 'kbd-chooser/method=us ', + 'netcfg/get_hostname=systemvm ', + 'netcfg/get_domain=apache.org ', + 'fb=false ', + 'debconf/frontend=noninteractive ', + 'console-setup/ask_detect=false ', + 'console-keymaps-at/keymap=us ', + 'keyboard-configuration/xkb-keymap=us ', + '' + ], + :kickstart_port => '7122', + :kickstart_timeout => '1200', + :kickstart_file => 'preseed.cfg', + :ssh_login_timeout => '1200', + :ssh_user => 'root', + :ssh_password => 'password', + :ssh_key => '', + :ssh_host_port => '7222', + :ssh_guest_port => '22', + :sudo_cmd => "echo '%p'|sudo -S sh '%f'", + :shutdown_cmd => 'halt -p', + :postinstall_files => [ + # basic minimal vm creation + 'build_time.sh', + 'apt_upgrade.sh', + 'configure_grub.sh', + 'configure_locale.sh', + 'configure_login.sh', + 'configure_networking.sh', + 'configure_acpid.sh', + # turning it into a systemvm + 'install_systemvm_packages.sh', + 'configure_conntrack.sh', + '../../cloud_scripts_shar_archive.sh', + 'configure_systemvm_services.sh', + 'authorized_keys.sh', + # cleanup & space-saving + 'cleanup.sh', + 'zerodisk.sh' + ], + :postinstall_timeout => '1200' +} + +config.merge! architectures[arch.to_sym] + +Veewee::Definition.declare(config) diff --git a/tools/appliance/definitions/debianbase/preseed.cfg b/tools/appliance/definitions/debianbase/preseed.cfg new file mode 100644 index 00000000000..e6e5adbdd32 --- /dev/null +++ b/tools/appliance/definitions/debianbase/preseed.cfg @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +### Localization +# Locale sets language and country. +d-i debian-installer/locale string en_US + +# Keyboard selection. +d-i console-keymaps-at/keymap select us + +### Network configuration +d-i netcfg/choose_interface select auto +d-i netcfg/get_hostname string debianbase +d-i netcfg/get_domain string cloudstack.org + +### Mirror settings +d-i mirror/country string manual +d-i mirror/http/hostname string http.us.debian.org +d-i mirror/http/directory string /debian +d-i mirror/http/proxy string + +### Clock and time zone setup +d-i clock-setup/utc boolean true +d-i time/zone string UTC +d-i clock-setup/ntp boolean true + +### Partitioning +d-i partman-auto/disk string /dev/sda +d-i partman-auto/method string regular +d-i partman-auto/choose_recipe select atomic +d-i partman-auto/expert_recipe string \ + boot-root :: \ + 30 50 100 ext4 \ + $primary{ } $bootable{ } \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /boot } \ + . \ + 300 40 400 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ / } \ + . \ + 50 100 200 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /home } \ + . \ + 650 20 1100 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /usr } \ + . \ + 400 40 500 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /opt } \ + . \ + 450 60 1000 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /var } \ + . \ + 50 70 400 ext4 \ + method{ format } format{ } \ + use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ /tmp } \ + . \ + 70 512 300% linux-swap \ + method{ swap } format{ } \ + . +d-i partman/confirm_write_new_label boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true + +### Base system installation +# ... + +### Account setup +d-i passwd/root-login boolean true +d-i passwd/root-password password password +d-i passwd/root-password-again password password +d-i passwd/user-fullname string Cloud Stack +d-i passwd/username string cloud +d-i passwd/user-password password cloud +d-i passwd/user-password-again password cloud +d-i user-setup/encrypt-home boolean false +d-i user-setup/allow-password-weak boolean true +d-i passwd/user-default-groups string audio cdrom video admin + +### Apt setup +# ... + +### Package selection +tasksel tasksel/first multiselect ssh-server +d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 +# Allowed values: none, safe-upgrade, full-upgrade +d-i pkgsel/upgrade select none + +popularity-contest popularity-contest/participate boolean false + +### Boot loader installation +d-i grub-installer/only_debian boolean true +d-i finish-install/reboot_in_progress note + +### Preseeding other packages +libssl1.0.0 libssl1.0.0/restart-services string +libssl1.0.0 libssl1.0.0/restart-failed error + +#### Advanced options +# Prevent packaged version of VirtualBox Guest Additions being installed: +d-i preseed/early_command string sed -i \ + '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ + /usr/lib/pre-pkgsel.d/20install-hwpackages diff --git a/tools/appliance/definitions/debianbase/zerodisk.sh b/tools/appliance/definitions/debianbase/zerodisk.sh new file mode 100644 index 00000000000..581e868edc9 --- /dev/null +++ b/tools/appliance/definitions/debianbase/zerodisk.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +# clean up stuff copied in by veewee +function cleanup_veewee() { + # this has to be here since it is the last file to run (and we remove ourselves) + rm -fv /root/*.iso + rm -fv /root/{apt_upgrade,authorized_keys,build_time,cleanup,install_systemvm_packages,zerodisk}.sh + rm -fv /root/configure_{acpid,conntrack,grub,locale,login,networking,systemvm_services}.sh + rm -fv .veewee_version .veewee_params .vbox_version +} + +# Zero out the free space to save space in the final image: +function zero_disk() { + cleanup_veewee + + for path in / /boot /usr /var /opt /tmp /home + do + dd if=/dev/zero of=${path}/zero bs=1M || true + sync + rm -f ${path}/zero + done +} + +return 2>/dev/null || zero_disk diff --git a/tools/appliance/test.sh b/tools/appliance/test.sh new file mode 100644 index 00000000000..5cd7316eac3 --- /dev/null +++ b/tools/appliance/test.sh @@ -0,0 +1,180 @@ +#!/bin/bash -xl +# note: the -l is needed here for bash to always make a login shell and load rvm if it hasn't been loaded +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# test script for build.sh which tries a variety of options/configs to make different vms + +set -e + +DEBUG="${DEBUG:-}" +TRACE="${TRACE:-0}" + +### +### Configuration +### + +if [[ "${DEBUG}" == "1" ]]; then + set -x +fi + +# which test to run +test_to_run=${1:-} +# build.sh settings for running the tests +appliance=debianbase +version=`date "+%Y%m%d%H%M%S"` +branch=`git status | grep '# On branch' | awk '{print $4}'` +BUILD_NUMBER="${BUILD_NUMBER:-}" +ssh_key= + +# where we are running the tests from +CURR_DIR=${PWD} +# where this script is +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# ensure we are running in isolation +if [ ${CURR_DIR} == ${SCRIPT_DIR} ]; then + mkdir -p ../appliance-work + cd ../appliance-work + CURR_DIR=${PWD} +fi + +### +### testing 'framework' +### + +function test_result() { + log INFO "$@" + add_on_exit log INFO "$@" +} + +function run_test() { + set +e + cleanup + fixture + log INFO running test: "$@" + eval $@ + result=$? + if ${result}; then + test_result "$@" FAIL + else + test_result "$@" OK + fi + cleanup + set -e +} + +function cleanup() { + ( + cd ${CURR_DIR}; + rm -rf iso definitions Gemfile shar_cloud_scripts convert_ovf_vbox_to_esx.xslt .rvmrc; + ) +} + +function fixture() { + ( + cd ${CURR_DIR}; + mkdir -p ${SCRIPT_DIR}/iso; + ln -s ${SCRIPT_DIR}/iso; + mkdir definitions; + ln -s ${SCRIPT_DIR}/definitions/${appliance} definitions/${appliance}; + + ln -s ${SCRIPT_DIR}/Gemfile; + ln -s ${SCRIPT_DIR}/shar_cloud_scripts.sh; + ln -s ${SCRIPT_DIR}/convert_ovf_vbox_to_esx.xslt; + ln -s ${SCRIPT_DIR}/.rvmrc; + ) +} + +### +### Test definitions +### + +function do_test_vm() { + prepare + create_definition + veewee_build + retry 10 check_appliance_shutdown + retry 10 remove_shares + veewee_destroy +} + +function do_test_export() { + prepare + create_definition + veewee_build + retry 10 check_appliance_shutdown + retry 10 remove_shares + + # Get appliance uuids + local vm_info=`vboxmanage showvminfo "${appliance_build_name}"` + local machine_uuid=`echo "${vm_info}" | grep UUID | head -1 | awk '{print $2}'` + local hdd_uuid=`echo "${vm_info}" | grep vdi | head -1 | awk '{print $8}' | cut -d ')' -f 1` + local hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | \ + cut -c 14- | sed ${sed_regex_option} 's/^ *//'` + + compact_hdd "${hdd_uuid}" + xen_server_export "${hdd_path}" + kvm_export "${hdd_path}" + vmware_export "${machine_uuid}" "${hdd_uuid}" + hyperv_export "${hdd_uuid}" + + veewee_destroy +} + +function test_basic_veewee_invocation() { + appliance=debianbase + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_export() { + appliance=debianbase + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_export +} + +function test_systemvm() { + appliance=systemvmtemplate + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_systemvm64() { + appliance=systemvm64template + appliance_build_name=${appliance}${branch_tag}${version_tag} + do_test_vm +} + +function test_suite() { + if [ "${test_to_run}" == "" ]; then + # list of all tests goes here + run_test test_basic_veewee_invocation + run_test test_systemvm + run_test test_systemvm64 + run_test test_export + else + run_test "${test_to_run}" + fi +} + +### +### Main invocation +### + +source ${SCRIPT_DIR}/build.sh +return 2>/dev/null || test_suite