systemvm: improve SystemVM startup and memory usage (#3126)

In order to reduce memory footprint and improve boot speed/predictability.
The following changes have been made:

- add vm.min_free_kbytes to sysctl
- periodically clear disk cache (depending on memory size)
- only start guest services specific to hypervisor
- use systemvm code to determine hypervisor type (not systemd)
- start cloud service at end of post init rather than through systemd
- reduce initial threads started for httpd
- fix vmtools config file

Fixes #3039

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf

@@ -0,0 +1,18 @@
+# event MPM
+# StartServers: initial number of server processes to start
+# MinSpareThreads: minimum number of worker threads which are kept spare
+# MaxSpareThreads: maximum number of worker threads which are kept spare
+# ThreadsPerChild: constant number of worker threads in each server process
+# MaxRequestWorkers: maximum number of worker threads
+# MaxConnectionsPerChild: maximum number of requests a server process serves
+<IfModule mpm_event_module>
+	StartServers			1
+	MinSpareThreads			25
+	MaxSpareThreads		 	75
+	ThreadLimit			64
+	ThreadsPerChild			25
+	MaxRequestWorkers		30
+	MaxConnectionsPerChild   	1000
+</IfModule>
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

systemvm/debian/etc/cron.daily/clear_cache

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# clear memory cache to ultimately reduce swapping
+
+sync && echo 1 > /proc/sys/vm/drop_caches

systemvm/debian/etc/cron.hourly/clear_cache

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# clear memory cache to ultimately reduce swapping
+
+phymem=$(free|awk '/^Mem:/{print $2}')
+if [ $phymem -lt 513000 ]; then
+  sync && echo 1 > /proc/sys/vm/drop_caches
+fi

systemvm/debian/etc/sysctl.conf

@@ -63,3 +63,9 @@ net.ipv6.conf.all.autoconf = 0
 
 # Minimum swappiness without disabling it
 vm.swappiness=1
+
+# make the kernel more aggressive in reclaiming RAM from the disk and swap caches
+vm.vfs_cache_pressure = 200
+
+# try to maintain 'free' memory thereby reducing the size of disk cache, hence reducing swapping.
+vm.min_free_kbytes = 20480

systemvm/debian/etc/vmware-tools/tools.conf

@@ -0,0 +1,17 @@
+[logging]
+# Turns on logging globally. It can still be disabled for each domain.
+# log = true
+
+# Disables core dumps on fatal errors; they're enabled by default.
+enableCoreDump = false
+
+# Defines the "vmsvc" domain, logging to file
+# vmsvc.level = message
+vmsvc.handler = file
+# Setup file rotation - keep 3 files
+vmsvc.maxOldLogFiles = 2
+# Max log file size kept: 1 MB
+vmsvc.maxLogSize = 1
+
+# Defines the "vmtoolsd" domain, and disable logging for it.
+# vmtoolsd.level = none

systemvm/debian/lib/systemd/system/cloud.service

@@ -1,6 +1,6 @@
 [Unit]
 Description=CloudStack Agent service
-After=cloud-early-config.service network.target local-fs.target
+After=cloud-early-config.service network.target cloud-postinit.service local-fs.target
 
 [Install]
 WantedBy=multi-user.target

systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Hyper-V file copy service (FCOPY) daemon
+ConditionPathExists=/dev/vmbus/hv_fcopy
+
+[Service]
+ExecStart=/usr/sbin/hv_fcopy_daemon -n
+
+[Install]
+WantedBy=multi-user.target

systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Hyper-V key-value pair (KVP) daemon
+
+[Service]
+ExecStart=/usr/sbin/hv_kvp_daemon -n
+
+[Install]
+WantedBy=multi-user.target

systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Hyper-V volume shadow copy service (VSS) daemon
+
+[Service]
+ExecStart=/usr/sbin/hv_vss_daemon -n
+
+[Install]
+WantedBy=multi-user.target

systemvm/debian/lib/systemd/system/open-vm-tools.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Service for virtual machines hosted on VMware
+Documentation=http://open-vm-tools.sourceforge.net/about.php
+DefaultDependencies=no
+Before=cloud-early-config.service
+
+[Service]
+ExecStart=/usr/bin/vmtoolsd
+TimeoutStopSec=5
+
+[Install]
+WantedBy=multi-user.target

systemvm/debian/lib/systemd/system/xe-daemon.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Xen Guest Monitoring Agent
+DefaultDependencies=no
+After=local-fs.target
+Requires=proc-xen.mount
+Before=network.target cloud-early-config.service
+ConditionPathExists=/proc/xen/capabilities
+
+[Service]
+ExecStartPre=/usr/sbin/xe-linux-distribution /var/cache/xe-linux-distribution
+ExecStart=/usr/sbin/xe-daemon
+StandardOutput=journal+console
+
+[Install]
+WantedBy=multi-user.target

systemvm/debian/opt/cloud/bin/setup/bootstrap.sh

@@ -19,6 +19,9 @@
 PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
 CMDLINE=/var/cache/cloud/cmdline
 
+rm -f /var/cache/cloud/enabled_svcs
+rm -f /var/cache/cloud/disabled_svcs
+
 . /lib/lsb/init-functions
 
 log_it() {
@@ -56,25 +59,25 @@ hypervisor() {
 }
 
 config_guest() {
-  if [ "$HYPERVISOR" == "kvm" ]
-  then
-    # Configure hot-plug
-    modprobe acpiphp || true
-    modprobe pci_hotplug || true
-    sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab
-    sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab
-  fi
   [ ! -d /proc/xen ] && sed -i 's/^vc/#vc/' /etc/inittab && telinit q
-  [  -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q
-}
+  [ -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q
+
+  systemctl daemon-reload
 
-get_boot_params() {
   case $HYPERVISOR in
      xen-pv|xen-domU)
+          systemctl stop ntpd
+          systemctl disable ntpd
+          systemctl start xe-daemon
+
           cat /proc/cmdline > $CMDLINE
           sed -i "s/%/ /g" $CMDLINE
           ;;
      xen-hvm)
+          systemctl stop ntpd
+          systemctl disable ntpd
+          systemctl start xe-daemon
+
           if [ ! -f /usr/bin/xenstore-read ]; then
             log_it "ERROR: xentools not installed, cannot found xenstore-read" && exit 5
           fi
@@ -82,7 +85,13 @@ get_boot_params() {
           sed -i "s/%/ /g" $CMDLINE
           ;;
      kvm)
+          # Configure hot-plug
+          modprobe acpiphp || true
+          modprobe pci_hotplug || true
+          sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab
+          sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab
           systemctl enable --now qemu-guest-agent
+
           # Wait for $CMDLINE file to be written by the qemu-guest-agent
           for i in {1..60}; do
             if [ -s $CMDLINE ]; then
@@ -96,13 +105,16 @@ get_boot_params() {
           fi
           ;;
      vmware)
+          # system time sync'd with host via vmware tools
+          systemctl stop ntpd
+          systemctl disable ntpd
+          systemctl start open-vm-tools
+
           vmtoolsd --cmd 'machine.id.get' > $CMDLINE
           ;;
      virtualpc|hyperv)
           # Hyper-V is recognized as virtualpc hypervisor type. Boot args are passed using KVP Daemon
-          #waiting for the hv_kvp_daemon to start up
-          #sleep  need to fix the race condition of hv_kvp_daemon and cloud-early-config
-          [ -f /usr/sbin/hv_kvp_daemon ] && /usr/sbin/hv_kvp_daemon
+          systemctl start hyperv-daemons.hv-fcopy-daemon.service hyperv-daemons.hv-kvp-daemon.service hyperv-daemons.hv-vss-daemon.service
           sleep 5
           cp -f /var/opt/hyperv/.kvp_pool_0 $CMDLINE
           cat /dev/null > /var/opt/hyperv/.kvp_pool_0
@@ -117,13 +129,11 @@ get_boot_params() {
           fi
           ;;
   esac
-}
 
-get_systemvm_type() {
+  # Find and export guest type
   export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE)
 }
 
-
 patch_systemvm() {
   local patchfile=$1
   local backupfolder="/tmp/.conf.backup"
@@ -172,19 +182,29 @@ patch() {
   return 0
 }
 
+config_sysctl() {
+  # When there is more memory reset the cache back pressure to default 100
+  physmem=$(free|awk '/^Mem:/{print $2}')
+  if [ $((physmem)) -lt 409600 ]; then
+      sed  -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 200" /etc/sysctl.conf
+  else
+      sed  -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 100" /etc/sysctl.conf
+  fi
+
+  sync
+  sysctl -p
+}
+
 bootstrap() {
   log_it "Bootstrapping systemvm appliance"
 
   export HYPERVISOR=$(hypervisor)
-  [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out of early init" && exit 10
-  log_it "Detected that we are running inside $HYPERVISOR"
+  [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out" && exit 10
+  log_it "Starting guest services for $HYPERVISOR"
 
   config_guest
-  get_boot_params
-  get_systemvm_type
   patch
-  sync
-  sysctl -p
+  config_sysctl
 
   log_it "Configuring systemvm type=$TYPE"
   if [ -f "/opt/cloud/bin/setup/$TYPE.sh" ]; then
@@ -192,6 +212,7 @@ bootstrap() {
   else
       /opt/cloud/bin/setup/default.sh
   fi
+
   log_it "Finished setting up systemvm"
   exit 0
 }

systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh

@@ -18,14 +18,13 @@
 
 . /opt/cloud/bin/setup/common.sh
 
-consoleproxy_svcs() {
+setup_console_proxy() {
+  log_it "Setting up console proxy system vm"
+
   echo "cloud" > /var/cache/cloud/enabled_svcs
   echo "haproxy dnsmasq apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs
   mkdir -p /var/log/cloud
-}
 
-setup_console_proxy() {
-  log_it "Setting up console proxy system vm"
   setup_common eth0 eth1 eth2
   setup_system_rfc1918_internal
 
@@ -51,10 +50,4 @@ setup_console_proxy() {
   rm -f /etc/logrotate.d/cloud
 }
 
-consoleproxy_svcs
-if [ $? -gt 0 ]
-then
-  log_it "Failed to execute consoleproxy_svcs"
-  exit 1
-fi
 setup_console_proxy

systemvm/debian/opt/cloud/bin/setup/postinit.sh

@@ -29,36 +29,39 @@ if [ "$TYPE" == "router" ] || [ "$TYPE" == "vpcrouter" ] || [ "$TYPE" == "dhcpsr
 then
   if [ -x /opt/cloud/bin/update_config.py ]
   then
-      /opt/cloud/bin/update_config.py cmd_line.json || true
+    /opt/cloud/bin/update_config.py cmd_line.json || true
   fi
 fi
 
 [ ! -f /var/cache/cloud/enabled_svcs ] && touch /var/cache/cloud/enabled_svcs
 for svc in $(cat /var/cache/cloud/enabled_svcs)
 do
-   systemctl enable --now --no-block $svc
+  systemctl enable --now --no-block $svc
 done
 
 [ ! -f /var/cache/cloud/disabled_svcs ] && touch /var/cache/cloud/disabled_svcs
 for svc in $(cat /var/cache/cloud/disabled_svcs)
 do
-   systemctl disable --now --no-block $svc
+  systemctl disable --now --no-block $svc
 done
 
 # Restore the persistent iptables nat, rules and filters for IPv4 and IPv6 if they exist
 ipv4="/etc/iptables/rules.v4"
 if [ -e $ipv4 ]
 then
-   iptables-restore < $ipv4
+  iptables-restore < $ipv4
 fi
 
 ipv6="/etc/iptables/rules.v6"
 if [ -e $ipv6 ]
 then
-   ip6tables-restore < $ipv6
+  ip6tables-restore < $ipv6
 fi
 
-# Enable SSH
+# Patch known systemd/sshd memory leak - https://github.com/systemd/systemd/issues/8015#issuecomment-476160981
+echo '@include null' >> /etc/pam.d/systemd-user
+
+# Enable and Start SSH
 systemctl enable --now --no-block ssh
 
 date > /var/cache/cloud/boot_up_done

systemvm/debian/opt/cloud/bin/setup/secstorage.sh

@@ -18,15 +18,12 @@
 
 . /opt/cloud/bin/setup/common.sh
 
-secstorage_svcs() {
-  echo "apache2 cloud nfs-common portmap" > /var/cache/cloud/enabled_svcs
-  echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs
-  mkdir -p /var/log/cloud
-}
-
 setup_secstorage() {
   log_it "Setting up secondary storage system vm"
-  sysctl vm.min_free_kbytes=8192
+
+  echo "cloud apache2 nfs-common portmap" > /var/cache/cloud/enabled_svcs
+  echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs
+  mkdir -p /var/log/cloud
 
   setup_common eth0 eth1 eth2
   setup_storage_network
@@ -80,10 +77,4 @@ CORS
   rm -f /etc/logrotate.d/cloud
 }
 
-secstorage_svcs
-if [ $? -gt 0 ]
-then
-  log_it "Failed to execute secstorage_svcs"
-  exit 1
-fi
 setup_secstorage

tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh

@@ -46,7 +46,7 @@ function install_cloud_scripts() {
   rsync -av ./cloud_scripts/ /
   chmod +x /opt/cloud/bin/* /opt/cloud/bin/setup/* \
     /root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \
-    /etc/profile.d/cloud.sh
+    /etc/profile.d/cloud.sh /etc/cron.daily/* /etc/cron.hourly/*
 
   chmod -x /etc/systemd/system/*
 
@@ -64,6 +64,7 @@ function do_signature() {
 
 function configure_issue() {
   cat > /etc/issue <<EOF
+ESC [ 2J
    __?.o/  Apache CloudStack SystemVM $CLOUDSTACK_RELEASE
   (  )#    https://cloudstack.apache.org
  (___(_)   Debian GNU/Linux 9 \n \l
@@ -108,6 +109,18 @@ function configure_services() {
   systemctl disable strongswan
   systemctl disable x11-common
   systemctl disable xl2tpd
+  systemctl disable vgauth
+  systemctl disable sshd
+  systemctl disable nfs-common
+  systemctl disable portmap
+
+  # Disable guest services which will selectively be started based on hypervisor
+  systemctl disable open-vm-tools
+  systemctl disable xe-daemon
+  systemctl disable hyperv-daemons.hv-fcopy-daemon.service
+  systemctl disable hyperv-daemons.hv-kvp-daemon.service
+  systemctl disable hyperv-daemons.hv-vss-daemon.service
+  systemctl disable qemu-guest-agent
 
   configure_apache2
   configure_strongswan