apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

systemvm: improve SystemVM startup and memory usage (#3126)

Browse Source 
In order to reduce memory footprint and improve boot speed/predictability.
The following changes have been made:

- add vm.min_free_kbytes to sysctl
- periodically clear disk cache (depending on memory size)
- only start guest services specific to hypervisor
- use systemvm code to determine hypervisor type (not systemd)
- start cloud service at end of post init rather than through systemd
- reduce initial threads started for httpd
- fix vmtools config file

Fixes #3039

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
This commit is contained in:
Paul Angus 2019-06-26 10:10:59 +01:00 committed by Rohit Yadav
parent 0833cf1dd7
commit 033199972e
17 changed files with 181 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf Normal file
View File

@ -0,0 +1,18 @@
# event MPM
# StartServers: initial number of server processes to start
# MinSpareThreads: minimum number of worker threads which are kept spare
# MaxSpareThreads: maximum number of worker threads which are kept spare
# ThreadsPerChild: constant number of worker threads in each server process
# MaxRequestWorkers: maximum number of worker threads
# MaxConnectionsPerChild: maximum number of requests a server process serves
<IfModule mpm_event_module>
StartServers 1
MinSpareThreads 25
MaxSpareThreads 75
ThreadLimit 64
ThreadsPerChild 25
MaxRequestWorkers 30
MaxConnectionsPerChild 1000
</IfModule>
# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

5
systemvm/debian/etc/cron.daily/clear_cache Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
# clear memory cache to ultimately reduce swapping
sync && echo 1 > /proc/sys/vm/drop_caches

8
systemvm/debian/etc/cron.hourly/clear_cache Executable file
View File

@ -0,0 +1,8 @@
#!/bin/bash
# clear memory cache to ultimately reduce swapping
phymem=$(free|awk '/^Mem:/{print $2}')
if [ $phymem -lt 513000 ]; then
sync && echo 1 > /proc/sys/vm/drop_caches
fi

6
systemvm/debian/etc/sysctl.conf
View File

@ -63,3 +63,9 @@ net.ipv6.conf.all.autoconf = 0
# Minimum swappiness without disabling it
vm.swappiness=1
# make the kernel more aggressive in reclaiming RAM from the disk and swap caches
vm.vfs_cache_pressure = 200
# try to maintain 'free' memory thereby reducing the size of disk cache, hence reducing swapping.
vm.min_free_kbytes = 20480

17
systemvm/debian/etc/vmware-tools/tools.conf Normal file
View File

@ -0,0 +1,17 @@
[logging]
# Turns on logging globally. It can still be disabled for each domain.
# log = true
# Disables core dumps on fatal errors; they're enabled by default.
enableCoreDump = false
# Defines the "vmsvc" domain, logging to file
# vmsvc.level = message
vmsvc.handler = file
# Setup file rotation - keep 3 files
vmsvc.maxOldLogFiles = 2
# Max log file size kept: 1 MB
vmsvc.maxLogSize = 1
# Defines the "vmtoolsd" domain, and disable logging for it.
# vmtoolsd.level = none

0
systemvm/debian/etc/systemd/system/baremetal-vr.service → systemvm/debian/lib/systemd/system/baremetal-vr.service
View File

2
systemvm/debian/etc/systemd/system/cloud.service → systemvm/debian/lib/systemd/system/cloud.service
View File

@ -1,6 +1,6 @@
[Unit]
Description=CloudStack Agent service
After=cloud-early-config.service network.target local-fs.target
After=cloud-early-config.service network.target cloud-postinit.service local-fs.target
[Install]
WantedBy=multi-user.target

9
systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service Normal file
View File

@ -0,0 +1,9 @@
[Unit]
Description=Hyper-V file copy service (FCOPY) daemon
ConditionPathExists=/dev/vmbus/hv_fcopy
[Service]
ExecStart=/usr/sbin/hv_fcopy_daemon -n
[Install]
WantedBy=multi-user.target

8
systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service Normal file
View File

@ -0,0 +1,8 @@
[Unit]
Description=Hyper-V key-value pair (KVP) daemon
[Service]
ExecStart=/usr/sbin/hv_kvp_daemon -n
[Install]
WantedBy=multi-user.target

8
systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service Normal file
View File

@ -0,0 +1,8 @@
[Unit]
Description=Hyper-V volume shadow copy service (VSS) daemon
[Service]
ExecStart=/usr/sbin/hv_vss_daemon -n
[Install]
WantedBy=multi-user.target

12
systemvm/debian/lib/systemd/system/open-vm-tools.service Normal file
View File

@ -0,0 +1,12 @@
[Unit]
Description=Service for virtual machines hosted on VMware
Documentation=http://open-vm-tools.sourceforge.net/about.php
DefaultDependencies=no
Before=cloud-early-config.service
[Service]
ExecStart=/usr/bin/vmtoolsd
TimeoutStopSec=5
[Install]
WantedBy=multi-user.target

15
systemvm/debian/lib/systemd/system/xe-daemon.service Normal file
View File

@ -0,0 +1,15 @@
[Unit]
Description=Xen Guest Monitoring Agent
DefaultDependencies=no
After=local-fs.target
Requires=proc-xen.mount
Before=network.target cloud-early-config.service
ConditionPathExists=/proc/xen/capabilities
[Service]
ExecStartPre=/usr/sbin/xe-linux-distribution /var/cache/xe-linux-distribution
ExecStart=/usr/sbin/xe-daemon
StandardOutput=journal+console
[Install]
WantedBy=multi-user.target

65
systemvm/debian/opt/cloud/bin/setup/bootstrap.sh
View File

@ -19,6 +19,9 @@
PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
CMDLINE=/var/cache/cloud/cmdline
rm -f /var/cache/cloud/enabled_svcs
rm -f /var/cache/cloud/disabled_svcs
. /lib/lsb/init-functions
log_it() {
@ -56,25 +59,25 @@ hypervisor() {
}
config_guest() {
if [ "$HYPERVISOR" == "kvm" ]
then
# Configure hot-plug
modprobe acpiphp || true
modprobe pci_hotplug || true
sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab
sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab
fi
[ ! -d /proc/xen ] && sed -i 's/^vc/#vc/' /etc/inittab && telinit q
[ -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q
}
get_boot_params() {
systemctl daemon-reload
case $HYPERVISOR in
xen-pv|xen-domU)
systemctl stop ntpd
systemctl disable ntpd
systemctl start xe-daemon
cat /proc/cmdline > $CMDLINE
sed -i "s/%/ /g" $CMDLINE
;;
xen-hvm)
systemctl stop ntpd
systemctl disable ntpd
systemctl start xe-daemon
if [ ! -f /usr/bin/xenstore-read ]; then
log_it "ERROR: xentools not installed, cannot found xenstore-read" && exit 5
fi
@ -82,7 +85,13 @@ get_boot_params() {
sed -i "s/%/ /g" $CMDLINE
;;
kvm)
# Configure hot-plug
modprobe acpiphp || true
modprobe pci_hotplug || true
sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab
sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab
systemctl enable --now qemu-guest-agent
# Wait for $CMDLINE file to be written by the qemu-guest-agent
for i in {1..60}; do
if [ -s $CMDLINE ]; then
@ -96,13 +105,16 @@ get_boot_params() {
fi
;;
vmware)
# system time sync'd with host via vmware tools
systemctl stop ntpd
systemctl disable ntpd
systemctl start open-vm-tools
vmtoolsd --cmd 'machine.id.get' > $CMDLINE
;;
virtualpc|hyperv)
# Hyper-V is recognized as virtualpc hypervisor type. Boot args are passed using KVP Daemon
#waiting for the hv_kvp_daemon to start up
#sleep need to fix the race condition of hv_kvp_daemon and cloud-early-config
[ -f /usr/sbin/hv_kvp_daemon ] && /usr/sbin/hv_kvp_daemon
systemctl start hyperv-daemons.hv-fcopy-daemon.service hyperv-daemons.hv-kvp-daemon.service hyperv-daemons.hv-vss-daemon.service
sleep 5
cp -f /var/opt/hyperv/.kvp_pool_0 $CMDLINE
cat /dev/null > /var/opt/hyperv/.kvp_pool_0
@ -117,13 +129,11 @@ get_boot_params() {
fi
;;
esac
}
get_systemvm_type() {
# Find and export guest type
export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE)
}
patch_systemvm() {
local patchfile=$1
local backupfolder="/tmp/.conf.backup"
@ -172,19 +182,29 @@ patch() {
return 0
}
config_sysctl() {
# When there is more memory reset the cache back pressure to default 100
physmem=$(free|awk '/^Mem:/{print $2}')
if [ $((physmem)) -lt 409600 ]; then
sed -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 200" /etc/sysctl.conf
else
sed -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 100" /etc/sysctl.conf
fi
sync
sysctl -p
}
bootstrap() {
log_it "Bootstrapping systemvm appliance"
export HYPERVISOR=$(hypervisor)
[ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out of early init" && exit 10
log_it "Detected that we are running inside $HYPERVISOR"
[ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out" && exit 10
log_it "Starting guest services for $HYPERVISOR"
config_guest
get_boot_params
get_systemvm_type
patch
sync
sysctl -p
config_sysctl
log_it "Configuring systemvm type=$TYPE"
if [ -f "/opt/cloud/bin/setup/$TYPE.sh" ]; then
@ -192,6 +212,7 @@ bootstrap() {
else
/opt/cloud/bin/setup/default.sh
fi
log_it "Finished setting up systemvm"
exit 0
}

13
systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh
View File

@ -18,14 +18,13 @@
. /opt/cloud/bin/setup/common.sh
consoleproxy_svcs() {
setup_console_proxy() {
log_it "Setting up console proxy system vm"
echo "cloud" > /var/cache/cloud/enabled_svcs
echo "haproxy dnsmasq apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs
mkdir -p /var/log/cloud
}
setup_console_proxy() {
log_it "Setting up console proxy system vm"
setup_common eth0 eth1 eth2
setup_system_rfc1918_internal
@ -51,10 +50,4 @@ setup_console_proxy() {
rm -f /etc/logrotate.d/cloud
}
consoleproxy_svcs
if [ $? -gt 0 ]
then
log_it "Failed to execute consoleproxy_svcs"
exit 1
fi
setup_console_proxy

5
systemvm/debian/opt/cloud/bin/setup/postinit.sh
View File

@ -58,7 +58,10 @@ then
ip6tables-restore < $ipv6
fi
# Enable SSH
# Patch known systemd/sshd memory leak - https://github.com/systemd/systemd/issues/8015#issuecomment-476160981
echo '@include null' >> /etc/pam.d/systemd-user
# Enable and Start SSH
systemctl enable --now --no-block ssh
date > /var/cache/cloud/boot_up_done

17
systemvm/debian/opt/cloud/bin/setup/secstorage.sh
View File

@ -18,15 +18,12 @@
. /opt/cloud/bin/setup/common.sh
secstorage_svcs() {
echo "apache2 cloud nfs-common portmap" > /var/cache/cloud/enabled_svcs
echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs
mkdir -p /var/log/cloud
}
setup_secstorage() {
log_it "Setting up secondary storage system vm"
sysctl vm.min_free_kbytes=8192
echo "cloud apache2 nfs-common portmap" > /var/cache/cloud/enabled_svcs
echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs
mkdir -p /var/log/cloud
setup_common eth0 eth1 eth2
setup_storage_network
@ -80,10 +77,4 @@ CORS
rm -f /etc/logrotate.d/cloud
}
secstorage_svcs
if [ $? -gt 0 ]
then
log_it "Failed to execute secstorage_svcs"
exit 1
fi
setup_secstorage

15
tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh
View File

@ -46,7 +46,7 @@ function install_cloud_scripts() {
rsync -av ./cloud_scripts/ /
chmod +x /opt/cloud/bin/* /opt/cloud/bin/setup/* \
/root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \
/etc/profile.d/cloud.sh
/etc/profile.d/cloud.sh /etc/cron.daily/* /etc/cron.hourly/*
chmod -x /etc/systemd/system/*
@ -64,6 +64,7 @@ function do_signature() {
function configure_issue() {
cat > /etc/issue <<EOF
ESC [ 2J
__?.o/ Apache CloudStack SystemVM $CLOUDSTACK_RELEASE
( )# https://cloudstack.apache.org
(___(_) Debian GNU/Linux 9 \n \l
@ -108,6 +109,18 @@ function configure_services() {
systemctl disable strongswan
systemctl disable x11-common
systemctl disable xl2tpd
systemctl disable vgauth
systemctl disable sshd
systemctl disable nfs-common
systemctl disable portmap
# Disable guest services which will selectively be started based on hypervisor
systemctl disable open-vm-tools
systemctl disable xe-daemon
systemctl disable hyperv-daemons.hv-fcopy-daemon.service
systemctl disable hyperv-daemons.hv-kvp-daemon.service
systemctl disable hyperv-daemons.hv-vss-daemon.service
systemctl disable qemu-guest-agent
configure_apache2
configure_strongswan