From 033199972ecf0c83975f2ae43c266e92c62ec1a0 Mon Sep 17 00:00:00 2001 From: Paul Angus Date: Wed, 26 Jun 2019 10:10:59 +0100 Subject: [PATCH 01/13] systemvm: improve SystemVM startup and memory usage (#3126) In order to reduce memory footprint and improve boot speed/predictability. The following changes have been made: - add vm.min_free_kbytes to sysctl - periodically clear disk cache (depending on memory size) - only start guest services specific to hypervisor - use systemvm code to determine hypervisor type (not systemd) - start cloud service at end of post init rather than through systemd - reduce initial threads started for httpd - fix vmtools config file Fixes #3039 Signed-off-by: Rohit Yadav --- .../conf-enabled/mods-enabled/mpm_event.conf | 18 +++++ systemvm/debian/etc/cron.daily/clear_cache | 5 ++ systemvm/debian/etc/cron.hourly/clear_cache | 8 +++ systemvm/debian/etc/sysctl.conf | 6 ++ systemvm/debian/etc/vmware-tools/tools.conf | 17 +++++ .../systemd/system/baremetal-vr.service | 0 .../{etc => lib}/systemd/system/cloud.service | 2 +- .../hyperv-daemons.hv-fcopy-daemon.service | 9 +++ .../hyperv-daemons.hv-kvp-daemon.service | 8 +++ .../hyperv-daemons.hv-vss-daemon.service | 8 +++ .../lib/systemd/system/open-vm-tools.service | 12 ++++ .../lib/systemd/system/xe-daemon.service | 15 +++++ .../debian/opt/cloud/bin/setup/bootstrap.sh | 67 ++++++++++++------- .../opt/cloud/bin/setup/consoleproxy.sh | 13 +--- .../debian/opt/cloud/bin/setup/postinit.sh | 15 +++-- .../debian/opt/cloud/bin/setup/secstorage.sh | 17 ++--- .../scripts/configure_systemvm_services.sh | 15 ++++- 17 files changed, 181 insertions(+), 54 deletions(-) create mode 100644 systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf create mode 100755 systemvm/debian/etc/cron.daily/clear_cache create mode 100755 systemvm/debian/etc/cron.hourly/clear_cache create mode 100644 systemvm/debian/etc/vmware-tools/tools.conf rename systemvm/debian/{etc => lib}/systemd/system/baremetal-vr.service (100%) rename systemvm/debian/{etc => lib}/systemd/system/cloud.service (70%) create mode 100644 systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service create mode 100644 systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service create mode 100644 systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service create mode 100644 systemvm/debian/lib/systemd/system/open-vm-tools.service create mode 100644 systemvm/debian/lib/systemd/system/xe-daemon.service diff --git a/systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf b/systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf new file mode 100644 index 00000000000..ea06042344f --- /dev/null +++ b/systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf @@ -0,0 +1,18 @@ +# event MPM +# StartServers: initial number of server processes to start +# MinSpareThreads: minimum number of worker threads which are kept spare +# MaxSpareThreads: maximum number of worker threads which are kept spare +# ThreadsPerChild: constant number of worker threads in each server process +# MaxRequestWorkers: maximum number of worker threads +# MaxConnectionsPerChild: maximum number of requests a server process serves + + StartServers 1 + MinSpareThreads 25 + MaxSpareThreads 75 + ThreadLimit 64 + ThreadsPerChild 25 + MaxRequestWorkers 30 + MaxConnectionsPerChild 1000 + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/systemvm/debian/etc/cron.daily/clear_cache b/systemvm/debian/etc/cron.daily/clear_cache new file mode 100755 index 00000000000..e2662f9380f --- /dev/null +++ b/systemvm/debian/etc/cron.daily/clear_cache @@ -0,0 +1,5 @@ +#!/bin/bash + +# clear memory cache to ultimately reduce swapping + +sync && echo 1 > /proc/sys/vm/drop_caches diff --git a/systemvm/debian/etc/cron.hourly/clear_cache b/systemvm/debian/etc/cron.hourly/clear_cache new file mode 100755 index 00000000000..5daa16744d0 --- /dev/null +++ b/systemvm/debian/etc/cron.hourly/clear_cache @@ -0,0 +1,8 @@ +#!/bin/bash + +# clear memory cache to ultimately reduce swapping + +phymem=$(free|awk '/^Mem:/{print $2}') +if [ $phymem -lt 513000 ]; then + sync && echo 1 > /proc/sys/vm/drop_caches +fi diff --git a/systemvm/debian/etc/sysctl.conf b/systemvm/debian/etc/sysctl.conf index 5b5f818e76d..5dd1ae44bc1 100644 --- a/systemvm/debian/etc/sysctl.conf +++ b/systemvm/debian/etc/sysctl.conf @@ -63,3 +63,9 @@ net.ipv6.conf.all.autoconf = 0 # Minimum swappiness without disabling it vm.swappiness=1 + +# make the kernel more aggressive in reclaiming RAM from the disk and swap caches +vm.vfs_cache_pressure = 200 + +# try to maintain 'free' memory thereby reducing the size of disk cache, hence reducing swapping. +vm.min_free_kbytes = 20480 diff --git a/systemvm/debian/etc/vmware-tools/tools.conf b/systemvm/debian/etc/vmware-tools/tools.conf new file mode 100644 index 00000000000..26f96ffc277 --- /dev/null +++ b/systemvm/debian/etc/vmware-tools/tools.conf @@ -0,0 +1,17 @@ +[logging] +# Turns on logging globally. It can still be disabled for each domain. +# log = true + +# Disables core dumps on fatal errors; they're enabled by default. +enableCoreDump = false + +# Defines the "vmsvc" domain, logging to file +# vmsvc.level = message +vmsvc.handler = file +# Setup file rotation - keep 3 files +vmsvc.maxOldLogFiles = 2 +# Max log file size kept: 1 MB +vmsvc.maxLogSize = 1 + +# Defines the "vmtoolsd" domain, and disable logging for it. +# vmtoolsd.level = none diff --git a/systemvm/debian/etc/systemd/system/baremetal-vr.service b/systemvm/debian/lib/systemd/system/baremetal-vr.service similarity index 100% rename from systemvm/debian/etc/systemd/system/baremetal-vr.service rename to systemvm/debian/lib/systemd/system/baremetal-vr.service diff --git a/systemvm/debian/etc/systemd/system/cloud.service b/systemvm/debian/lib/systemd/system/cloud.service similarity index 70% rename from systemvm/debian/etc/systemd/system/cloud.service rename to systemvm/debian/lib/systemd/system/cloud.service index 22439239caa..e3824bf53c2 100644 --- a/systemvm/debian/etc/systemd/system/cloud.service +++ b/systemvm/debian/lib/systemd/system/cloud.service @@ -1,6 +1,6 @@ [Unit] Description=CloudStack Agent service -After=cloud-early-config.service network.target local-fs.target +After=cloud-early-config.service network.target cloud-postinit.service local-fs.target [Install] WantedBy=multi-user.target diff --git a/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service new file mode 100644 index 00000000000..12a0b63c970 --- /dev/null +++ b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service @@ -0,0 +1,9 @@ +[Unit] +Description=Hyper-V file copy service (FCOPY) daemon +ConditionPathExists=/dev/vmbus/hv_fcopy + +[Service] +ExecStart=/usr/sbin/hv_fcopy_daemon -n + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service new file mode 100644 index 00000000000..534a25a7f24 --- /dev/null +++ b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service @@ -0,0 +1,8 @@ +[Unit] +Description=Hyper-V key-value pair (KVP) daemon + +[Service] +ExecStart=/usr/sbin/hv_kvp_daemon -n + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service new file mode 100644 index 00000000000..d4fd675bdc8 --- /dev/null +++ b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service @@ -0,0 +1,8 @@ +[Unit] +Description=Hyper-V volume shadow copy service (VSS) daemon + +[Service] +ExecStart=/usr/sbin/hv_vss_daemon -n + +[Install] +WantedBy=multi-user.target diff --git a/systemvm/debian/lib/systemd/system/open-vm-tools.service b/systemvm/debian/lib/systemd/system/open-vm-tools.service new file mode 100644 index 00000000000..e7cdde7e3ff --- /dev/null +++ b/systemvm/debian/lib/systemd/system/open-vm-tools.service @@ -0,0 +1,12 @@ +[Unit] +Description=Service for virtual machines hosted on VMware +Documentation=http://open-vm-tools.sourceforge.net/about.php +DefaultDependencies=no +Before=cloud-early-config.service + +[Service] +ExecStart=/usr/bin/vmtoolsd +TimeoutStopSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/systemvm/debian/lib/systemd/system/xe-daemon.service b/systemvm/debian/lib/systemd/system/xe-daemon.service new file mode 100644 index 00000000000..6dfef7bfdb2 --- /dev/null +++ b/systemvm/debian/lib/systemd/system/xe-daemon.service @@ -0,0 +1,15 @@ +[Unit] +Description=Xen Guest Monitoring Agent +DefaultDependencies=no +After=local-fs.target +Requires=proc-xen.mount +Before=network.target cloud-early-config.service +ConditionPathExists=/proc/xen/capabilities + +[Service] +ExecStartPre=/usr/sbin/xe-linux-distribution /var/cache/xe-linux-distribution +ExecStart=/usr/sbin/xe-daemon +StandardOutput=journal+console + +[Install] +WantedBy=multi-user.target diff --git a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh index 0208b36d435..5df59a4438e 100755 --- a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh +++ b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh @@ -19,6 +19,9 @@ PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" CMDLINE=/var/cache/cloud/cmdline +rm -f /var/cache/cloud/enabled_svcs +rm -f /var/cache/cloud/disabled_svcs + . /lib/lsb/init-functions log_it() { @@ -56,25 +59,25 @@ hypervisor() { } config_guest() { - if [ "$HYPERVISOR" == "kvm" ] - then - # Configure hot-plug - modprobe acpiphp || true - modprobe pci_hotplug || true - sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab - sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab - fi [ ! -d /proc/xen ] && sed -i 's/^vc/#vc/' /etc/inittab && telinit q - [ -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q -} + [ -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q + + systemctl daemon-reload -get_boot_params() { case $HYPERVISOR in xen-pv|xen-domU) + systemctl stop ntpd + systemctl disable ntpd + systemctl start xe-daemon + cat /proc/cmdline > $CMDLINE sed -i "s/%/ /g" $CMDLINE ;; xen-hvm) + systemctl stop ntpd + systemctl disable ntpd + systemctl start xe-daemon + if [ ! -f /usr/bin/xenstore-read ]; then log_it "ERROR: xentools not installed, cannot found xenstore-read" && exit 5 fi @@ -82,7 +85,13 @@ get_boot_params() { sed -i "s/%/ /g" $CMDLINE ;; kvm) + # Configure hot-plug + modprobe acpiphp || true + modprobe pci_hotplug || true + sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab + sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab systemctl enable --now qemu-guest-agent + # Wait for $CMDLINE file to be written by the qemu-guest-agent for i in {1..60}; do if [ -s $CMDLINE ]; then @@ -96,13 +105,16 @@ get_boot_params() { fi ;; vmware) + # system time sync'd with host via vmware tools + systemctl stop ntpd + systemctl disable ntpd + systemctl start open-vm-tools + vmtoolsd --cmd 'machine.id.get' > $CMDLINE ;; virtualpc|hyperv) # Hyper-V is recognized as virtualpc hypervisor type. Boot args are passed using KVP Daemon - #waiting for the hv_kvp_daemon to start up - #sleep need to fix the race condition of hv_kvp_daemon and cloud-early-config - [ -f /usr/sbin/hv_kvp_daemon ] && /usr/sbin/hv_kvp_daemon + systemctl start hyperv-daemons.hv-fcopy-daemon.service hyperv-daemons.hv-kvp-daemon.service hyperv-daemons.hv-vss-daemon.service sleep 5 cp -f /var/opt/hyperv/.kvp_pool_0 $CMDLINE cat /dev/null > /var/opt/hyperv/.kvp_pool_0 @@ -117,13 +129,11 @@ get_boot_params() { fi ;; esac -} -get_systemvm_type() { + # Find and export guest type export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE) } - patch_systemvm() { local patchfile=$1 local backupfolder="/tmp/.conf.backup" @@ -172,19 +182,29 @@ patch() { return 0 } +config_sysctl() { + # When there is more memory reset the cache back pressure to default 100 + physmem=$(free|awk '/^Mem:/{print $2}') + if [ $((physmem)) -lt 409600 ]; then + sed -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 200" /etc/sysctl.conf + else + sed -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 100" /etc/sysctl.conf + fi + + sync + sysctl -p +} + bootstrap() { log_it "Bootstrapping systemvm appliance" export HYPERVISOR=$(hypervisor) - [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out of early init" && exit 10 - log_it "Detected that we are running inside $HYPERVISOR" + [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out" && exit 10 + log_it "Starting guest services for $HYPERVISOR" config_guest - get_boot_params - get_systemvm_type patch - sync - sysctl -p + config_sysctl log_it "Configuring systemvm type=$TYPE" if [ -f "/opt/cloud/bin/setup/$TYPE.sh" ]; then @@ -192,6 +212,7 @@ bootstrap() { else /opt/cloud/bin/setup/default.sh fi + log_it "Finished setting up systemvm" exit 0 } diff --git a/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh index 225dc6f5ad2..3f00f3da43a 100755 --- a/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh +++ b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh @@ -18,14 +18,13 @@ . /opt/cloud/bin/setup/common.sh -consoleproxy_svcs() { +setup_console_proxy() { + log_it "Setting up console proxy system vm" + echo "cloud" > /var/cache/cloud/enabled_svcs echo "haproxy dnsmasq apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs mkdir -p /var/log/cloud -} -setup_console_proxy() { - log_it "Setting up console proxy system vm" setup_common eth0 eth1 eth2 setup_system_rfc1918_internal @@ -51,10 +50,4 @@ setup_console_proxy() { rm -f /etc/logrotate.d/cloud } -consoleproxy_svcs -if [ $? -gt 0 ] -then - log_it "Failed to execute consoleproxy_svcs" - exit 1 -fi setup_console_proxy diff --git a/systemvm/debian/opt/cloud/bin/setup/postinit.sh b/systemvm/debian/opt/cloud/bin/setup/postinit.sh index fc92b7e97cd..5e7e4c01a22 100755 --- a/systemvm/debian/opt/cloud/bin/setup/postinit.sh +++ b/systemvm/debian/opt/cloud/bin/setup/postinit.sh @@ -29,36 +29,39 @@ if [ "$TYPE" == "router" ] || [ "$TYPE" == "vpcrouter" ] || [ "$TYPE" == "dhcpsr then if [ -x /opt/cloud/bin/update_config.py ] then - /opt/cloud/bin/update_config.py cmd_line.json || true + /opt/cloud/bin/update_config.py cmd_line.json || true fi fi [ ! -f /var/cache/cloud/enabled_svcs ] && touch /var/cache/cloud/enabled_svcs for svc in $(cat /var/cache/cloud/enabled_svcs) do - systemctl enable --now --no-block $svc + systemctl enable --now --no-block $svc done [ ! -f /var/cache/cloud/disabled_svcs ] && touch /var/cache/cloud/disabled_svcs for svc in $(cat /var/cache/cloud/disabled_svcs) do - systemctl disable --now --no-block $svc + systemctl disable --now --no-block $svc done # Restore the persistent iptables nat, rules and filters for IPv4 and IPv6 if they exist ipv4="/etc/iptables/rules.v4" if [ -e $ipv4 ] then - iptables-restore < $ipv4 + iptables-restore < $ipv4 fi ipv6="/etc/iptables/rules.v6" if [ -e $ipv6 ] then - ip6tables-restore < $ipv6 + ip6tables-restore < $ipv6 fi -# Enable SSH +# Patch known systemd/sshd memory leak - https://github.com/systemd/systemd/issues/8015#issuecomment-476160981 +echo '@include null' >> /etc/pam.d/systemd-user + +# Enable and Start SSH systemctl enable --now --no-block ssh date > /var/cache/cloud/boot_up_done diff --git a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh index 8b6d4ee5a4b..d3a6d21cf65 100755 --- a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh +++ b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh @@ -18,15 +18,12 @@ . /opt/cloud/bin/setup/common.sh -secstorage_svcs() { - echo "apache2 cloud nfs-common portmap" > /var/cache/cloud/enabled_svcs - echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs - mkdir -p /var/log/cloud -} - setup_secstorage() { log_it "Setting up secondary storage system vm" - sysctl vm.min_free_kbytes=8192 + + echo "cloud apache2 nfs-common portmap" > /var/cache/cloud/enabled_svcs + echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs + mkdir -p /var/log/cloud setup_common eth0 eth1 eth2 setup_storage_network @@ -80,10 +77,4 @@ CORS rm -f /etc/logrotate.d/cloud } -secstorage_svcs -if [ $? -gt 0 ] -then - log_it "Failed to execute secstorage_svcs" - exit 1 -fi setup_secstorage diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index 78d868d8b0f..56406b711f5 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -46,7 +46,7 @@ function install_cloud_scripts() { rsync -av ./cloud_scripts/ / chmod +x /opt/cloud/bin/* /opt/cloud/bin/setup/* \ /root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \ - /etc/profile.d/cloud.sh + /etc/profile.d/cloud.sh /etc/cron.daily/* /etc/cron.hourly/* chmod -x /etc/systemd/system/* @@ -64,6 +64,7 @@ function do_signature() { function configure_issue() { cat > /etc/issue < Date: Wed, 26 Jun 2019 12:56:05 +0200 Subject: [PATCH 02/13] systemvm: Fix hostname is localhost in some VRs (#3422) In some virtual routers, 'hostname -f' returns 'localhost'. The hostname is also 'localhost' in `/var/log/messages`. This change can fix the issue in new VRs. --- systemvm/debian/opt/cloud/bin/cs/CsDhcp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py index e7abb902046..56b44195f6d 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py @@ -139,7 +139,8 @@ class CsDhcp(CsDataBag): logging.error("Caught error while trying to delete entries from dnsmasq.leases file: %s" % e) def preseed(self): - self.add_host("127.0.0.1", "localhost %s" % CsHelper.get_hostname()) + self.add_host("127.0.0.1", "localhost") + self.add_host("127.0.1.1", "%s" % CsHelper.get_hostname()) self.add_host("::1", "localhost ip6-localhost ip6-loopback") self.add_host("ff02::1", "ip6-allnodes") self.add_host("ff02::2", "ip6-allrouters") From 240878335570edb8fed657e92f74500109b4bc0a Mon Sep 17 00:00:00 2001 From: ustcweizhou Date: Wed, 26 Jun 2019 12:57:07 +0200 Subject: [PATCH 03/13] ssvm: use secstorage.ssl.cert.domain as hostname if it does not start with '*' when upload a template or volume from local (#3420) If secstorage.ssl.cert.domain does not start with '*', we should use it as host name in url when upload template/volume from local Fixes #3305 --- .../apache/cloudstack/utils/imagestore/ImageStoreUtil.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java index e754a8e1d52..f87669b4bf5 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java @@ -30,9 +30,11 @@ public class ImageStoreUtil { //if ssvm url domain is present, use it to construct hostname in the format 1-2-3-4.domain // if the domain name is not present, ssl validation fails and has to be ignored - if(StringUtils.isNotBlank(ssvmUrlDomain)) { + if(StringUtils.isNotBlank(ssvmUrlDomain) && ssvmUrlDomain.startsWith("*")) { hostname = ipAddress.replace(".", "-"); hostname = hostname + ssvmUrlDomain.substring(1); + } else if (StringUtils.isNotBlank(ssvmUrlDomain)) { + hostname = ssvmUrlDomain; } //only https works with postupload and url format is fixed From 2c3c88e2092dae44de823cf52cb3e2cc787bb3a1 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 26 Jun 2019 23:18:20 +0530 Subject: [PATCH 04/13] console-proxy: fix potential NPE condition (#3419) When checking if the console proxy URL domain starts with *, the code does not check if the provided string is null. When domain is not configured the IP address should be used. Fixes #3164 Signed-off-by: Rohit Yadav --- .../java/com/cloud/info/ConsoleProxyInfo.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/com/cloud/info/ConsoleProxyInfo.java b/core/src/main/java/com/cloud/info/ConsoleProxyInfo.java index 78cb877e7db..7e1be6a9cad 100644 --- a/core/src/main/java/com/cloud/info/ConsoleProxyInfo.java +++ b/core/src/main/java/com/cloud/info/ConsoleProxyInfo.java @@ -55,18 +55,16 @@ public class ConsoleProxyInfo { private String formatProxyAddress(String consoleProxyUrlDomain, String proxyIpAddress) { StringBuffer sb = new StringBuffer(); - // Domain in format *.example.com, proxy IP is 1.2.3.4 --> 1-2-3-4.example.com - if (consoleProxyUrlDomain.startsWith("*")) { + if (StringUtils.isBlank(consoleProxyUrlDomain)) { + // Blank config, we use the proxy IP + sb.append(proxyIpAddress); + } else if (consoleProxyUrlDomain.startsWith("*")) { + // Domain in format *.example.com, proxy IP is 1.2.3.4 --> 1-2-3-4.example.com sb.append(proxyIpAddress.replaceAll("\\.", "-")); sb.append(consoleProxyUrlDomain.substring(1)); // skip the * - - // Otherwise we assume a valid domain if config not blank - } else if (StringUtils.isNotBlank(consoleProxyUrlDomain)) { - sb.append(consoleProxyUrlDomain); - - // Blank config, we use the proxy IP } else { - sb.append(proxyIpAddress); + // Otherwise we assume a valid domain if config not blank + sb.append(consoleProxyUrlDomain); } return sb.toString(); } From f9998e418c483afb2ce3f8b26def31d66223baf2 Mon Sep 17 00:00:00 2001 From: EK <20902920+kioie@users.noreply.github.com> Date: Thu, 27 Jun 2019 06:36:27 +0300 Subject: [PATCH 05/13] server: warn on migration of volumes within the same storage pool (#3424) Added an if statement that catches the destination poolid as an invalid parameter if it is similer to the current poolid. Fixes #3291 --- .../main/java/com/cloud/storage/VolumeApiServiceImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 099f88bde42..182379acbab 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2033,6 +2033,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new InvalidParameterValueException("Volume must be in ready state"); } + if (vol.getPoolId() == storagePoolId) { + throw new InvalidParameterValueException("Volume " + vol + " is already on the destination storage pool"); + } + boolean liveMigrateVolume = false; Long instanceId = vol.getInstanceId(); Long srcClusterId = null; @@ -3384,4 +3388,4 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic public ConfigKey[] getConfigKeys() { return new ConfigKey[] {ConcurrentMigrationsThresholdPerDatastore}; } -} \ No newline at end of file +} From 9f4f2c5348af93eeb790fbf4908cdde4f0b7806a Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 27 Jun 2019 09:14:47 +0530 Subject: [PATCH 06/13] api: instance and template details are free text (#3240) Problem: Users don't know what keys/values to enter for template and VM details. Root Cause: The feature does not exist that can list possible details and options. Solution: Based on the possible VM and template details handled by the codebase, those details were refactored and a list API is introduced that can return users those details along with possible values. When users add details now, they will be presented with a list of key details and their possible options if any. Signed-off-by: Rohit Yadav --- .../java/com/cloud/vm/VmDetailConstants.java | 47 ++++- .../user/resource/ListDetailOptionsCmd.java | 91 +++++++++ .../api/response/DetailOptionsResponse.java | 44 ++++ .../apache/cloudstack/query/QueryService.java | 6 +- .../cloud/vm/VirtualMachineManagerImpl.java | 36 ++-- .../vmware/resource/VmwareResource.java | 2 +- .../resource/CitrixResourceBase.java | 11 +- .../main/java/com/cloud/api/ApiDBUtils.java | 3 +- .../com/cloud/api/query/QueryManagerImpl.java | 67 ++++++- .../cloud/capacity/CapacityManagerImpl.java | 13 +- .../element/ConfigDriveNetworkElement.java | 5 +- .../network/element/VirtualRouterElement.java | 3 +- .../cloud/resource/ResourceManagerImpl.java | 22 +- .../cloud/server/ManagementServerImpl.java | 2 + .../cloud/servlet/ConsoleProxyServlet.java | 7 +- .../cloud/template/TemplateManagerImpl.java | 3 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 77 ++++--- .../com/cloud/vm/UserVmManagerImplTest.java | 17 ++ ui/css/cloudstack3.css | 23 +++ ui/scripts/globalSettings.js | 1 - ui/scripts/instances.js | 2 +- ui/scripts/roles.js | 41 ++-- ui/scripts/templates.js | 1 + ui/scripts/ui-custom/granularSettings.js | 189 +++++++++++++----- 24 files changed, 548 insertions(+), 165 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListDetailOptionsCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/DetailOptionsResponse.java diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index f24c4f587c4..84de8c9ebac 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -17,14 +17,41 @@ package com.cloud.vm; public interface VmDetailConstants { - public static final String KEYBOARD = "keyboard"; - public static final String NIC_ADAPTER = "nicAdapter"; - public static final String ROOT_DISK_CONTROLLER = "rootDiskController"; - public static final String NESTED_VIRTUALIZATION_FLAG = "nestedVirtualizationFlag"; - public static final String HYPERVISOR_TOOLS_VERSION = "hypervisortoolsversion"; - public static final String DATA_DISK_CONTROLLER = "dataDiskController"; - public static final String SVGA_VRAM_SIZE = "svga.vramSize"; - public static final String CPU_NUMBER = "cpuNumber"; - public static final String CPU_SPEED = "cpuSpeed"; - public static final String MEMORY = "memory"; + String KEYBOARD = "keyboard"; + String CPU_CORE_PER_SOCKET = "cpu.corespersocket"; + String ROOT_DISK_SIZE = "rootdisksize"; + + // VMware specific + String NIC_ADAPTER = "nicAdapter"; + String ROOT_DISK_CONTROLLER = "rootDiskController"; + String DATA_DISK_CONTROLLER = "dataDiskController"; + String SVGA_VRAM_SIZE = "svga.vramSize"; + String NESTED_VIRTUALIZATION_FLAG = "nestedVirtualizationFlag"; + + // XenServer specific (internal) + String HYPERVISOR_TOOLS_VERSION = "hypervisortoolsversion"; + String PLATFORM = "platform"; + String TIME_OFFSET = "timeoffset"; + + // KVM specific (internal) + String KVM_VNC_PORT = "kvm.vnc.port"; + String KVM_VNC_ADDRESS = "kvm.vnc.address"; + + // Mac OSX guest specific (internal) + String SMC_PRESENT = "smc.present"; + String FIRMWARE = "firmware"; + + // VM deployment with custom compute offering params + String CPU_NUMBER = "cpuNumber"; + String CPU_SPEED = "cpuSpeed"; + String MEMORY = "memory"; + + // Misc details for internal usage (not to be set/changed by user or admin) + String CPU_OVER_COMMIT_RATIO = "cpuOvercommitRatio"; + String MEMORY_OVER_COMMIT_RATIO = "memoryOvercommitRatio"; + String MESSAGE_RESERVED_CAPACITY_FREED_FLAG = "Message.ReservedCapacityFreed.Flag"; + String DEPLOY_VM = "deployvm"; + String SSH_PUBLIC_KEY = "SSH.PublicKey"; + String PASSWORD = "password"; + String ENCRYPTED_PASSWORD = "Encrypted.Password"; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListDetailOptionsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListDetailOptionsCmd.java new file mode 100644 index 00000000000..e53754c099c --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListDetailOptionsCmd.java @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.resource; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.DetailOptionsResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.server.ResourceTag; +import com.google.common.base.Strings; + +@APICommand(name = ListDetailOptionsCmd.APINAME, + description = "Lists all possible details and their options for a resource type such as a VM or a template", + responseObject = DetailOptionsResponse.class, + since = "4.13", + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListDetailOptionsCmd extends BaseCmd { + public final static String APINAME = "listDetailOptions"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, required = true, + description = "the resource type such as UserVm, Template etc.", + validations = {ApiArgValidator.NotNullOrEmpty} + ) + private String resourceType; + + @Parameter(name = ApiConstants.RESOURCE_ID, type = CommandType.STRING, + description = "the UUID of the resource (optional)") + private String resourceId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public ResourceTag.ResourceObjectType getResourceType() { + return _taggedResourceService.getResourceType(resourceType); + } + + public String getResourceId() { + if (!Strings.isNullOrEmpty(resourceId)) { + return _taggedResourceService.getUuid(resourceId, getResourceType()); + } + return null; + } + + ///////////////////////////////////////////////////// + /////////////////// Implementation ////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + @Override + public void execute() { + final DetailOptionsResponse response = _queryService.listDetailOptions(this); + response.setResponseName(getCommandName()); + response.setObjectName("detailoptions"); + setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DetailOptionsResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DetailOptionsResponse.java new file mode 100644 index 00000000000..5f6bff39597 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/DetailOptionsResponse.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class DetailOptionsResponse extends BaseResponse { + @SerializedName(ApiConstants.DETAILS) + @Param(description = "Map of all possible details and their possible list of values") + private Map> details; + + public DetailOptionsResponse(Map> details) { + this.details = details; + } + + public void setDetails(Map> details) { + this.details = details; + } + + public Map> getDetails() { + return details; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 1f0f933d7d2..618a8f6f8a5 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -20,8 +20,8 @@ import java.util.List; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.command.admin.domain.ListDomainsCmd; -import org.apache.cloudstack.api.command.admin.host.ListHostsCmd; import org.apache.cloudstack.api.command.admin.host.ListHostTagsCmd; +import org.apache.cloudstack.api.command.admin.host.ListHostsCmd; import org.apache.cloudstack.api.command.admin.internallb.ListInternalLBVMsCmd; import org.apache.cloudstack.api.command.admin.management.ListMgmtsCmd; import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd; @@ -40,6 +40,7 @@ import org.apache.cloudstack.api.command.user.offering.ListDiskOfferingsCmd; import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd; import org.apache.cloudstack.api.command.user.project.ListProjectInvitationsCmd; import org.apache.cloudstack.api.command.user.project.ListProjectsCmd; +import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; @@ -50,6 +51,7 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.DetailOptionsResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.DomainRouterResponse; @@ -147,6 +149,8 @@ public interface QueryService { ListResponse listIsos(ListIsosCmd cmd); + DetailOptionsResponse listDetailOptions(ListDetailOptionsCmd cmd); + ListResponse searchForAffinityGroups(ListAffinityGroupsCmd cmd); List listResourceDetails(ListResourceDetailsCmd cmd); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 79570534596..1cc925b3ccf 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1071,16 +1071,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac long destHostId = dest.getHost().getId(); vm.setPodIdToDeployIn(dest.getPod().getId()); final Long cluster_id = dest.getCluster().getId(); - final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio"); - final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio"); - //storing the value of overcommit in the vm_details table for doing a capacity check in case the cluster overcommit ratio is changed. - if (userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio") == null && + final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.CPU_OVER_COMMIT_RATIO); + final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); + //storing the value of overcommit in the user_vm_details table for doing a capacity check in case the cluster overcommit ratio is changed. + if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) == null && (Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f)) { - userVmDetailsDao.addDetail(vm.getId(), "cpuOvercommitRatio", cluster_detail_cpu.getValue(), true); - userVmDetailsDao.addDetail(vm.getId(), "memoryOvercommitRatio", cluster_detail_ram.getValue(), true); - } else if (userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio") != null) { - userVmDetailsDao.addDetail(vm.getId(), "cpuOvercommitRatio", cluster_detail_cpu.getValue(), true); - userVmDetailsDao.addDetail(vm.getId(), "memoryOvercommitRatio", cluster_detail_ram.getValue(), true); + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true); + } else if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) != null) { + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true); } vmProfile.setCpuOvercommitRatio(Float.parseFloat(cluster_detail_cpu.getValue())); @@ -1160,8 +1160,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac // Remove the information on whether it was a deploy vm request.The deployvm=true information // is set only when the vm is being deployed. When a vm is started from a stop state the // information isn't set, - if (userVmDetailsDao.findDetail(vm.getId(), "deployvm") != null) { - userVmDetailsDao.removeDetail(vm.getId(), "deployvm"); + if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_VM) != null) { + userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.DEPLOY_VM); } startedVm = vm; @@ -1455,7 +1455,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (platform != null) { final UserVmVO userVm = _userVmDao.findById(vm.getId()); _userVmDao.loadDetails(userVm); - userVm.setDetail("platform", platform); + userVm.setDetail(VmDetailConstants.PLATFORM, platform); _userVmDao.saveDetails(userVm); } } @@ -1731,7 +1731,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (platform != null) { final UserVmVO userVm = _userVmDao.findById(vm.getId()); _userVmDao.loadDetails(userVm); - userVm.setDetail("platform", platform); + userVm.setDetail(VmDetailConstants.PLATFORM, platform); _userVmDao.saveDetails(userVm); } } @@ -3174,16 +3174,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private void updateVmMetaData(Long vmId, String platform) { UserVmVO userVm = _userVmDao.findById(vmId); _userVmDao.loadDetails(userVm); - if ( userVm.details.containsKey("timeoffset")) { - userVm.details.remove("timeoffset"); + if ( userVm.details.containsKey(VmDetailConstants.TIME_OFFSET)) { + userVm.details.remove(VmDetailConstants.TIME_OFFSET); } - userVm.setDetail("platform", platform); + userVm.setDetail(VmDetailConstants.PLATFORM, platform); String pvdriver = "xenserver56"; if ( platform.contains("device_id")) { pvdriver = "xenserver61"; } - if (!userVm.details.containsKey("hypervisortoolsversion") || !userVm.details.get("hypervisortoolsversion").equals(pvdriver)) { - userVm.setDetail("hypervisortoolsversion", pvdriver); + if (!userVm.details.containsKey(VmDetailConstants.HYPERVISOR_TOOLS_VERSION) || !userVm.details.get(VmDetailConstants.HYPERVISOR_TOOLS_VERSION).equals(pvdriver)) { + userVm.setDetail(VmDetailConstants.HYPERVISOR_TOOLS_VERSION, pvdriver); } _userVmDao.saveDetails(userVm); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index abbe3243c6c..3d6732c0fbe 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1874,7 +1874,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // Check for multi-cores per socket settings int numCoresPerSocket = 1; - String coresPerSocket = vmSpec.getDetails().get("cpu.corespersocket"); + String coresPerSocket = vmSpec.getDetails().get(VmDetailConstants.CPU_CORE_PER_SOCKET); if (coresPerSocket != null) { String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext()); // Property 'numCoresPerSocket' is supported since vSphere API 5.0 diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index ff7b0a3ab53..79a9fb22972 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -134,6 +134,7 @@ import com.cloud.utils.ssh.SSHCmdHelper; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.PowerState; +import com.cloud.vm.VmDetailConstants; import com.trilead.ssh2.SCPClient; import com.xensource.xenapi.Bond; import com.xensource.xenapi.Connection; @@ -1862,18 +1863,18 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe final Map details = vmSpec.getDetails(); if (details != null) { - final String platformstring = details.get("platform"); + final String platformstring = details.get(VmDetailConstants.PLATFORM); if (platformstring != null && !platformstring.isEmpty()) { final Map platform = StringUtils.stringToMap(platformstring); vm.setPlatform(conn, platform); } else { - final String timeoffset = details.get("timeoffset"); + final String timeoffset = details.get(VmDetailConstants.TIME_OFFSET); if (timeoffset != null) { final Map platform = vm.getPlatform(conn); - platform.put("timeoffset", timeoffset); + platform.put(VmDetailConstants.TIME_OFFSET, timeoffset); vm.setPlatform(conn, platform); } - final String coresPerSocket = details.get("cpu.corespersocket"); + final String coresPerSocket = details.get(VmDetailConstants.CPU_CORE_PER_SOCKET); if (coresPerSocket != null) { final Map platform = vm.getPlatform(conn); platform.put("cores-per-socket", coresPerSocket); @@ -1881,7 +1882,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe } } if (!BootloaderType.CD.equals(vmSpec.getBootloader())) { - final String xenservertoolsversion = details.get("hypervisortoolsversion"); + final String xenservertoolsversion = details.get(VmDetailConstants.HYPERVISOR_TOOLS_VERSION); if ((xenservertoolsversion == null || !xenservertoolsversion.equalsIgnoreCase("xenserver61")) && vmSpec.getGpuDevice() == null) { final Map platform = vm.getPlatform(conn); platform.remove("device_id"); diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 22ed2437d4d..40ff827a9cd 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -302,6 +302,7 @@ import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.VmStats; import com.cloud.vm.dao.ConsoleProxyDao; import com.cloud.vm.dao.DomainRouterDao; @@ -1454,7 +1455,7 @@ public class ApiDBUtils { } public static UserVmDetailVO findPublicKeyByVmId(long vmId) { - return s_userVmDetailsDao.findDetail(vmId, "SSH.PublicKey"); + return s_userVmDetailsDao.findDetail(vmId, VmDetailConstants.SSH_PUBLIC_KEY); } public static void getAutoScaleVmGroupPolicies(long vmGroupId, List scaleUpPolicies, List scaleDownPolicies) { diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 8f003185da2..f0596a40a1a 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -17,11 +17,16 @@ package com.cloud.api.query; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.inject.Inject; @@ -61,6 +66,7 @@ import org.apache.cloudstack.api.command.user.offering.ListDiskOfferingsCmd; import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd; import org.apache.cloudstack.api.command.user.project.ListProjectInvitationsCmd; import org.apache.cloudstack.api.command.user.project.ListProjectsCmd; +import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; @@ -71,6 +77,7 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.DetailOptionsResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.DomainRouterResponse; @@ -212,13 +219,16 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Func; import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import com.google.common.base.Strings; @Component public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements QueryService, Configurable { @@ -3381,6 +3391,61 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q hypervisorType, true, cmd.listInReadyState(), permittedAccounts, caller, listProjectResourcesCriteria, tags, showRemovedISO, null, null); } + @Override + public DetailOptionsResponse listDetailOptions(final ListDetailOptionsCmd cmd) { + final ResourceObjectType type = cmd.getResourceType(); + final String resourceUuid = cmd.getResourceId(); + final Map> options = new HashMap<>(); + switch (type) { + case Template: + case UserVm: + HypervisorType hypervisorType = HypervisorType.None; + if (!Strings.isNullOrEmpty(resourceUuid) && ResourceObjectType.Template.equals(type)) { + hypervisorType = _templateDao.findByUuid(resourceUuid).getHypervisorType(); + } + if (!Strings.isNullOrEmpty(resourceUuid) && ResourceObjectType.UserVm.equals(type)) { + hypervisorType = _vmInstanceDao.findByUuid(resourceUuid).getHypervisorType(); + } + fillVMOrTemplateDetailOptions(options, hypervisorType); + break; + default: + throw new CloudRuntimeException("Resource type not supported."); + } + if (CallContext.current().getCallingAccount().getType() != Account.ACCOUNT_TYPE_ADMIN) { + final List userBlacklistedSettings = Stream.of(QueryService.UserVMBlacklistedDetails.value().split(",")) + .map(item -> (item).trim()) + .collect(Collectors.toList()); + for (final String detail : userBlacklistedSettings) { + if (options.containsKey(detail)) { + options.remove(detail); + } + } + } + return new DetailOptionsResponse(options); + } + + private void fillVMOrTemplateDetailOptions(final Map> options, final HypervisorType hypervisorType) { + if (options == null) { + throw new CloudRuntimeException("Invalid/null detail-options response object passed"); + } + + options.put(VmDetailConstants.KEYBOARD, Arrays.asList("uk", "us", "jp", "fr")); + options.put(VmDetailConstants.CPU_CORE_PER_SOCKET, Collections.emptyList()); + options.put(VmDetailConstants.ROOT_DISK_SIZE, Collections.emptyList()); + + if (HypervisorType.KVM.equals(hypervisorType)) { + options.put(VmDetailConstants.ROOT_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "virtio")); + } + + if (HypervisorType.VMware.equals(hypervisorType)) { + options.put(VmDetailConstants.NIC_ADAPTER, Arrays.asList("E1000", "PCNet32", "Vmxnet2", "Vmxnet3")); + options.put(VmDetailConstants.ROOT_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "lsilogic", "lsisas1068", "buslogic", "pvscsi")); + options.put(VmDetailConstants.DATA_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "lsilogic", "lsisas1068", "buslogic", "pvscsi")); + options.put(VmDetailConstants.NESTED_VIRTUALIZATION_FLAG, Arrays.asList("true", "false")); + options.put(VmDetailConstants.SVGA_VRAM_SIZE, Collections.emptyList()); + } + } + @Override public ListResponse searchForAffinityGroups(ListAffinityGroupsCmd cmd) { Pair, Integer> result = searchForAffinityGroupsInternal(cmd); @@ -3687,7 +3752,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } response.setResponses(result); return response; - } + } @Override public String getConfigComponentName() { diff --git a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java index 6775bfc6cff..a7fee9603a0 100644 --- a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java @@ -89,6 +89,7 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Event; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; @@ -138,8 +139,6 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, @Inject MessageBus _messageBus; - private static final String MESSAGE_RESERVED_CAPACITY_FREED_FLAG = "Message.ReservedCapacityFreed.Flag"; - @Override public boolean configure(String name, Map params) throws ConfigurationException { _vmCapacityReleaseInterval = NumbersUtil.parseInt(_configDao.getValue(Config.CapacitySkipcountingHours.key()), 3600); @@ -638,8 +637,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, Float ramOvercommitRatio = 1.0f; long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - vm.getUpdateTime().getTime()) / 1000; if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) { - UserVmDetailVO vmDetailCpu = _userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio"); - UserVmDetailVO vmDetailRam = _userVmDetailsDao.findDetail(vm.getId(), "memoryOvercommitRatio"); + UserVmDetailVO vmDetailCpu = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO); + UserVmDetailVO vmDetailRam = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); if (vmDetailCpu != null) { //if vmDetail_cpu is not null it means it is running in a overcommited cluster. cpuOvercommitRatio = Float.parseFloat(vmDetailCpu.getValue()); @@ -669,14 +668,14 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, } else { // signal if not done already, that the VM has been stopped for skip.counting.hours, // hence capacity will not be reserved anymore. - UserVmDetailVO messageSentFlag = _userVmDetailsDao.findDetail(vm.getId(), MESSAGE_RESERVED_CAPACITY_FREED_FLAG); + UserVmDetailVO messageSentFlag = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.MESSAGE_RESERVED_CAPACITY_FREED_FLAG); if (messageSentFlag == null || !Boolean.valueOf(messageSentFlag.getValue())) { _messageBus.publish(_name, "VM_ReservedCapacity_Free", PublishScope.LOCAL, vm); if (vm.getType() == VirtualMachine.Type.User) { UserVmVO userVM = _userVMDao.findById(vm.getId()); _userVMDao.loadDetails(userVM); - userVM.setDetail(MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "true"); + userVM.setDetail(VmDetailConstants.MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "true"); _userVMDao.saveDetails(userVM); } } @@ -903,7 +902,7 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, UserVmVO userVM = _userVMDao.findById(vm.getId()); _userVMDao.loadDetails(userVM); // free the message sent flag if it exists - userVM.setDetail(MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "false"); + userVM.setDetail(VmDetailConstants.MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "false"); _userVMDao.saveDetails(userVM); } diff --git a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java index 4d2452d45ce..76e4fc03ce7 100644 --- a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java @@ -77,6 +77,7 @@ import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; @@ -195,7 +196,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle } private String getSshKey(VirtualMachineProfile profile) { - final UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey"); + final UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), VmDetailConstants.SSH_PUBLIC_KEY); return (vmDetailSshKey!=null ? vmDetailSshKey.getValue() : null); } @@ -262,7 +263,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle final String password_encrypted = DBEncryptionUtil.encrypt(password); final UserVmVO userVmVO = _userVmDao.findById(vm.getId()); - _userVmDetailsDao.addDetail(vm.getId(), "password", password_encrypted, false); + _userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.PASSWORD, password_encrypted, false); userVmVO.setUpdateParameters(true); _userVmDao.update(userVmVO.getId(), userVmVO); diff --git a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java index b78dcfdbb5d..aefa528bea7 100644 --- a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java @@ -31,6 +31,7 @@ import org.cloud.network.router.deployment.RouterDeploymentDefinitionBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import com.cloud.vm.VmDetailConstants; import com.google.gson.Gson; import org.apache.cloudstack.api.command.admin.router.ConfigureOvsElementCmd; @@ -718,7 +719,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor, RedundantResource, DnsServ final UserVmVO userVmVO = _userVmDao.findById(vm.getId()); _userVmDao.loadDetails(userVmVO); - userVmVO.setDetail("password", password_encrypted); + userVmVO.setDetail(VmDetailConstants.PASSWORD, password_encrypted); _userVmDao.saveDetails(userVmVO); userVmVO.setUpdateParameters(true); diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 27fa42cc4c9..d07a4383d01 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -31,13 +31,6 @@ import java.util.concurrent.ConcurrentHashMap; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.utils.Pair; -import com.cloud.vm.dao.UserVmDetailsDao; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.commons.lang.ObjectUtils; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.cluster.AddClusterCmd; import org.apache.cloudstack.api.command.admin.cluster.DeleteClusterCmd; @@ -49,21 +42,25 @@ import org.apache.cloudstack.api.command.admin.host.ReconnectHostCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostPasswordCmd; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.ObjectUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; -import com.cloud.agent.api.GetVncPortCommand; -import com.cloud.agent.api.GetVncPortAnswer; import com.cloud.agent.api.GetGPUStatsAnswer; import com.cloud.agent.api.GetGPUStatsCommand; import com.cloud.agent.api.GetHostStatsAnswer; import com.cloud.agent.api.GetHostStatsCommand; +import com.cloud.agent.api.GetVncPortAnswer; +import com.cloud.agent.api.GetVncPortCommand; import com.cloud.agent.api.MaintainAnswer; import com.cloud.agent.api.MaintainCommand; import com.cloud.agent.api.PropagateResourceEventCommand; @@ -148,6 +145,7 @@ import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; +import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; import com.cloud.utils.UriUtils; import com.cloud.utils.component.Manager; @@ -177,6 +175,8 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; import com.google.gson.Gson; @@ -1314,8 +1314,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, for (VMInstanceVO vm : vms) { GetVncPortAnswer vmVncPortAnswer = (GetVncPortAnswer) _agentMgr.easySend(hostId, new GetVncPortCommand(vm.getId(), vm.getInstanceName())); if (vmVncPortAnswer != null) { - userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.address", vmVncPortAnswer.getAddress(), true); - userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.port", String.valueOf(vmVncPortAnswer.getPort()), true); + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.KVM_VNC_ADDRESS, vmVncPortAnswer.getAddress(), true); + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.KVM_VNC_PORT, String.valueOf(vmVncPortAnswer.getPort()), true); } } } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 2c3e71543e3..4beddcc6e02 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -411,6 +411,7 @@ import org.apache.cloudstack.api.command.user.region.ha.gslb.ListGlobalLoadBalan import org.apache.cloudstack.api.command.user.region.ha.gslb.RemoveFromGlobalLoadBalancerRuleCmd; import org.apache.cloudstack.api.command.user.region.ha.gslb.UpdateGlobalLoadBalancerRuleCmd; import org.apache.cloudstack.api.command.user.resource.GetCloudIdentifierCmd; +import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.resource.ListHypervisorsCmd; import org.apache.cloudstack.api.command.user.resource.ListResourceLimitsCmd; import org.apache.cloudstack.api.command.user.resource.UpdateResourceCountCmd; @@ -2900,6 +2901,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(ExtractTemplateCmd.class); cmdList.add(ListTemplatePermissionsCmd.class); cmdList.add(ListTemplatesCmd.class); + cmdList.add(ListDetailOptionsCmd.class); cmdList.add(RegisterTemplateCmd.class); cmdList.add(UpdateTemplateCmd.class); cmdList.add(UpdateTemplatePermissionsCmd.class); diff --git a/server/src/main/java/com/cloud/servlet/ConsoleProxyServlet.java b/server/src/main/java/com/cloud/servlet/ConsoleProxyServlet.java index 8cfaa9fd69b..5a6c84f1479 100644 --- a/server/src/main/java/com/cloud/servlet/ConsoleProxyServlet.java +++ b/server/src/main/java/com/cloud/servlet/ConsoleProxyServlet.java @@ -41,6 +41,7 @@ import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import org.springframework.web.context.support.SpringBeanAutowiringSupport; +import com.cloud.vm.VmDetailConstants; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -421,8 +422,8 @@ public class ConsoleProxyServlet extends HttpServlet { Pair portInfo; if (hostVo.getResourceState().equals(ResourceState.ErrorInMaintenance)) { - UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.address"); - UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.port"); + UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.KVM_VNC_ADDRESS); + UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.KVM_VNC_PORT); portInfo = new Pair<>(detailAddress.getValue(), Integer.valueOf(detailPort.getValue())); } else { portInfo = _ms.getVncPort(vm); @@ -441,7 +442,7 @@ public class ConsoleProxyServlet extends HttpServlet { } String sid = vm.getVncPassword(); - UserVmDetailVO details = _userVmDetailsDao.findDetail(vm.getId(), "keyboard"); + UserVmDetailVO details = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.KEYBOARD); String tag = vm.getUuid(); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 9beeb7bc010..5eb96aac11d 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -39,6 +39,7 @@ import com.cloud.utils.EncryptionUtil; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; import com.cloud.utils.EnumUtils; +import com.cloud.vm.VmDetailConstants; import com.google.common.base.Joiner; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -1918,7 +1919,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } if (cmd.getDetails() != null) { - details.remove("Encrypted.Password"); // new password will be generated during vm deployment from password enabled template + details.remove(VmDetailConstants.ENCRYPTED_PASSWORD); // new password will be generated during vm deployment from password enabled template details.putAll(cmd.getDetails()); } if (!details.isEmpty()) { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 106441517e7..0727a2a8593 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -35,6 +35,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -85,6 +86,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -847,7 +849,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } else { final UserVmVO userVm = _vmDao.findById(vmId); _vmDao.loadDetails(userVm); - userVm.setDetail("SSH.PublicKey", sshPublicKey); + userVm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKey); if (template.isEnablePassword()) { userVm.setPassword(password); //update the encrypted password in vm_details table too @@ -2426,11 +2428,36 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (isDisplayVm != null && isDisplayVm != vmInstance.isDisplay()) { updateDisplayVmFlag(isDisplayVm, id, vmInstance); } + final Account caller = CallContext.current().getCallingAccount(); + final List userBlacklistedSettings = Stream.of(QueryService.UserVMBlacklistedDetails.value().split(",")) + .map(item -> (item).trim()) + .collect(Collectors.toList()); if (cleanupDetails){ - userVmDetailsDao.removeDetails(id); - } - else { + if (caller != null && caller.getType() == Account.ACCOUNT_TYPE_ADMIN) { + userVmDetailsDao.removeDetails(id); + } else { + for (final UserVmDetailVO detail : userVmDetailsDao.listDetails(id)) { + if (detail != null && !userBlacklistedSettings.contains(detail.getName())) { + userVmDetailsDao.removeDetail(id, detail.getName()); + } + } + } + } else { if (MapUtils.isNotEmpty(details)) { + if (caller != null && caller.getType() != Account.ACCOUNT_TYPE_ADMIN) { + // Ensure blacklisted detail is not passed by non-root-admin user + for (final String detailName : details.keySet()) { + if (userBlacklistedSettings.contains(detailName)) { + throw new InvalidParameterValueException("You're not allowed to add or edit the restricted setting: " + detailName); + } + } + // Add any hidden/blacklisted detail + for (final UserVmDetailVO detail : userVmDetailsDao.listDetails(id)) { + if (userBlacklistedSettings.contains(detail.getName())) { + details.put(detail.getName(), detail.getValue()); + } + } + } vmInstance.setDetails(details); _vmDao.saveDetails(vmInstance); } @@ -3379,13 +3406,13 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir boolean isIso = Storage.ImageFormat.ISO == template.getFormat(); long size = 0; // custom root disk size, resizes base template to larger size - if (customParameters.containsKey("rootdisksize")) { + if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { // only KVM, XenServer and VMware supports rootdisksize override if (!(hypervisorType == HypervisorType.KVM || hypervisorType == HypervisorType.XenServer || hypervisorType == HypervisorType.VMware || hypervisorType == HypervisorType.Simulator)) { throw new InvalidParameterValueException("Hypervisor " + hypervisorType + " does not support rootdisksize override"); } - Long rootDiskSize = NumbersUtil.parseLong(customParameters.get("rootdisksize"), -1); + Long rootDiskSize = NumbersUtil.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE), -1); if (rootDiskSize <= 0) { throw new InvalidParameterValueException("Root disk size should be a positive number."); } @@ -3768,7 +3795,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } if (sshPublicKey != null) { - vm.setDetail("SSH.PublicKey", sshPublicKey); + vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKey); } if (keyboard != null && !keyboard.isEmpty()) { @@ -3780,9 +3807,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } Long rootDiskSize = null; // custom root disk size, resizes base template to larger size - if (customParameters.containsKey("rootdisksize")) { + if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { // already verified for positive number - rootDiskSize = Long.parseLong(customParameters.get("rootdisksize")); + rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE)); VMTemplateVO templateVO = _templateDao.findById(template.getId()); if (templateVO == null) { @@ -3806,10 +3833,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir // If hypervisor is vSphere and OS is OS X, set special settings. if (hypervisorType.equals(HypervisorType.VMware)) { if (guestOS.getDisplayName().toLowerCase().contains("apple mac os")) { - vm.setDetail("smc.present", "TRUE"); + vm.setDetail(VmDetailConstants.SMC_PRESENT, "TRUE"); vm.setDetail(VmDetailConstants.ROOT_DISK_CONTROLLER, "scsi"); vm.setDetail(VmDetailConstants.DATA_DISK_CONTROLLER, "scsi"); - vm.setDetail("firmware", "efi"); + vm.setDetail(VmDetailConstants.FIRMWARE, "efi"); s_logger.info("guestOS is OSX : overwrite root disk controller to scsi, use smc and efi"); } else { String controllerSetting = _configDao.getValue("vmware.root.disk.controller"); @@ -3838,7 +3865,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir vm.setDetail(key, customParameters.get(key)); } } - vm.setDetail("deployvm", "true"); + vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); _vmDao.saveDetails(vm); s_logger.debug("Allocating in the DB for vm"); @@ -3888,10 +3915,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir s_logger.error(error); throw new InvalidParameterValueException(error); } else if ((rootDiskSize << 30) > templateVO.getSize()) { - if (hypervisorType == HypervisorType.VMware && (vm.getDetails() == null || vm.getDetails().get("rootDiskController") == null)) { + if (hypervisorType == HypervisorType.VMware && (vm.getDetails() == null || vm.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER) == null)) { s_logger.warn("If Root disk controller parameter is not overridden, then Root disk resize may fail because current Root disk controller value is NULL."); - } else if (hypervisorType == HypervisorType.VMware && !vm.getDetails().get("rootDiskController").toLowerCase().contains("scsi")) { - String error = "Found unsupported root disk controller: " + vm.getDetails().get("rootDiskController"); + } else if (hypervisorType == HypervisorType.VMware && !vm.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER).toLowerCase().contains("scsi")) { + String error = "Found unsupported root disk controller: " + vm.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER); s_logger.error(error); throw new InvalidParameterValueException(error); } else { @@ -3899,7 +3926,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } else { s_logger.debug("Root disk size specified is " + (rootDiskSize << 30) + "B and Template root disk size is " + templateVO.getSize() + "B. Both are equal so no need to override"); - customParameters.remove("rootdisksize"); + customParameters.remove(VmDetailConstants.ROOT_DISK_SIZE); } } @@ -4182,7 +4209,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); List vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(), - vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); + vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail(VmDetailConstants.SSH_PUBLIC_KEY), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); String vmName = vm.getInstanceName(); String configDriveIsoRootFolder = "/tmp"; String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso"; @@ -4574,8 +4601,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir // this value is not being sent to the backend; need only for api // display purposes if (template.isEnablePassword()) { - if (vm.getDetail("password") != null) { - userVmDetailsDao.removeDetail(vm.getId(), "password"); + if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { + userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.PASSWORD); } vm.setUpdateParameters(false); _vmDao.update(vm.getId(), vm); @@ -6334,7 +6361,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (needRestart) { try { - if (vm.getDetail("password") != null) { + if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { params = new HashMap(); params.put(VirtualMachineProfile.Param.VmPassword, password); } @@ -6347,8 +6374,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (vm.isUpdateParameters()) { vm.setUpdateParameters(false); _vmDao.loadDetails(vm); - if (vm.getDetail("password") != null) { - userVmDetailsDao.removeDetail(vm.getId(), "password"); + if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { + userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.PASSWORD); } _vmDao.update(vm.getId(), vm); } @@ -6526,7 +6553,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } private void encryptAndStorePassword(UserVmVO vm, String password) { - String sshPublicKey = vm.getDetail("SSH.PublicKey"); + String sshPublicKey = vm.getDetail(VmDetailConstants.SSH_PUBLIC_KEY); if (sshPublicKey != null && !sshPublicKey.equals("") && password != null && !password.equals("saved_password")) { if (!sshPublicKey.startsWith("ssh-rsa")) { s_logger.warn("Only RSA public keys can be used to encrypt a vm password."); @@ -6537,7 +6564,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir throw new CloudRuntimeException("Error encrypting password"); } - vm.setDetail("Encrypted.Password", encryptedPasswd); + vm.setDetail(VmDetailConstants.ENCRYPTED_PASSWORD, encryptedPasswd); _vmDao.saveDetails(vm); } } @@ -6569,7 +6596,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir public String getVmUserData(long vmId) { UserVmVO vm = _vmDao.findById(vmId); if (vm == null) { - throw new InvalidParameterValueException("Unable to find virual machine with id " + vmId); + throw new InvalidParameterValueException("Unable to find virtual machine with id " + vmId); } _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 2744714c67b..965377b2c7b 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -22,6 +22,7 @@ import java.util.HashMap; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; import org.apache.cloudstack.context.CallContext; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -44,6 +45,8 @@ import com.cloud.storage.GuestOSVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.UserVO; import com.cloud.uservm.UserVm; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; @@ -76,6 +79,12 @@ public class UserVmManagerImplTest { @Mock private NetworkModel networkModel; + @Mock + private AccountVO callerAccount; + + @Mock + private UserVO callerUser; + private long vmId = 1l; @Before @@ -83,6 +92,14 @@ public class UserVmManagerImplTest { Mockito.when(updateVmCommand.getId()).thenReturn(vmId); Mockito.when(userVmDao.findById(Mockito.eq(vmId))).thenReturn(userVmVoMock); + + Mockito.when(callerAccount.getType()).thenReturn(Account.ACCOUNT_TYPE_ADMIN); + CallContext.register(callerUser, callerAccount); + } + + @After + public void afterTest() { + CallContext.unregister(); } @Test diff --git a/ui/css/cloudstack3.css b/ui/css/cloudstack3.css index a85dee1d909..0d2aca67571 100644 --- a/ui/css/cloudstack3.css +++ b/ui/css/cloudstack3.css @@ -13015,6 +13015,29 @@ div.gpugroups div.list-view { background: transparent url("../images/icons.png") no-repeat -626px -209px; } +ul.ui-autocomplete.ui-menu { + width: 250px; + max-height: 100px; + background: #eee; + padding: 5px; + text-align: left; + overflow-y: auto; + overflow-x: hidden; + z-index: 100; +} + +.ui-menu .ui-menu-item { + cursor: pointer; +} + +.ui-menu .ui-menu-item .ui-state-active { + background: #CBDDF3; +} + +.ui-helper-hidden-accessible { + display: none; +} + .copy-template-destination-list div.text-search { right: 5px; } diff --git a/ui/scripts/globalSettings.js b/ui/scripts/globalSettings.js index ab03978c22a..3e926ea678f 100644 --- a/ui/scripts/globalSettings.js +++ b/ui/scripts/globalSettings.js @@ -168,7 +168,6 @@ }], dataProvider: function(args) { var items = []; - console.log(args); $.ajax({ url: createURL("listLdapConfigurations&hostname=" + args.context.ldapConfiguration[0].hostname), dataType: "json", diff --git a/ui/scripts/instances.js b/ui/scripts/instances.js index 4cc1dbb18be..0a89fdcb8f6 100644 --- a/ui/scripts/instances.js +++ b/ui/scripts/instances.js @@ -3329,6 +3329,7 @@ settings: { title: 'label.settings', custom: cloudStack.uiCustom.granularDetails({ + resourceType: 'UserVm', dataProvider: function(args) { $.ajax({ url: createURL('listVirtualMachines&id=' + args.context.instances[0].id), @@ -3388,7 +3389,6 @@ // It could happen that a stale web page has been opened up when VM was stopped but // vm was turned on through another route - UI or API. so we should check again. var existingDetails = virtualMachine.details; - console.log(existingDetails); var newDetails = {}; for (d in existingDetails) { if (d != data.name) { diff --git a/ui/scripts/roles.js b/ui/scripts/roles.js index 59072bd47c0..ff89dc56a19 100644 --- a/ui/scripts/roles.js +++ b/ui/scripts/roles.js @@ -327,31 +327,24 @@ }); } }); - var setupAutocompletion = function() { - var $target = $($.find('input[name="rule"]')); - if ($target.hasClass('ui-autocomplete')) { - $target.autocomplete('destroy'); + $.ajax({ + url: createURL("listApis"), + dataType: "json", + success: function(json) { + var apis = []; + var response = json.listapisresponse.api; + $.each(response, function(idx, api) { + apis.push(api.name); + }); + $($.find('input[name="rule"]')).autocomplete({ + minLength: 0, + delay: 0, + source: apis.sort() + }).focus(function() { + $(this).data("uiAutocomplete").search($(this).val()); + }); } - $($.find('input[name="rule"]')).autocomplete({ - source: apiList, - autoFocus:true - }); - }; - if (apiList.length == 0) { - $.ajax({ - url: createURL("listApis"), - dataType: "json", - success: function(json) { - var response = json.listapisresponse.api; - $.each(response, function(idx, api) { - apiList.push(api.name); - }); - setupAutocompletion(); - } - }); - } else { - setupAutocompletion(); - } + }); } }); } diff --git a/ui/scripts/templates.js b/ui/scripts/templates.js index 28af9d39f2f..df040001edc 100755 --- a/ui/scripts/templates.js +++ b/ui/scripts/templates.js @@ -2170,6 +2170,7 @@ settings: { title: 'label.settings', custom: cloudStack.uiCustom.granularDetails({ + resourceType: 'Template', dataProvider: function(args) { $.ajax({ url: createURL('listTemplates'), diff --git a/ui/scripts/ui-custom/granularSettings.js b/ui/scripts/ui-custom/granularSettings.js index 5312394127e..42177e32156 100644 --- a/ui/scripts/ui-custom/granularSettings.js +++ b/ui/scripts/ui-custom/granularSettings.js @@ -54,9 +54,10 @@ return $listView; } }; - cloudStack.uiCustom.granularDetails = function(args) { + cloudStack.uiCustom.granularDetails = function(args) { var dataProvider = args.dataProvider; var actions = args.actions; + var resourceType = args.resourceType; return function(args) { var context = args.context; @@ -77,57 +78,141 @@ label: 'label.change.value', action: actions.edit }, - remove: { - label: 'Remove Setting', - messages: { - confirm: function(args) { - return 'Delete Setting'; - }, - notification: function(args) { - return 'Setting deleted'; - } - }, - action: actions.remove, - notification: { - poll: function(args) { - args.complete(); - } - } - }, - add : { - label: 'Add Setting', - messages: { - confirm: function(args) { - return 'Add Setting'; - }, - notification: function(args) { - return 'Setting added'; - } - }, - preFilter: function(args) { - return true; - }, - createForm: { - title: 'Add New Setting', - fields: { - name: { - label: 'label.name', - validation: { - required: true - } - }, - value: { - label: 'label.value', - validation: { - required: true - } - } - } - }, - action: actions.add - } + remove: { + label: 'Remove Setting', + messages: { + confirm: function(args) { + return 'Delete Setting'; + }, + notification: function(args) { + return 'Setting deleted'; + } + }, + action: actions.remove, + notification: { + poll: function(args) { + args.complete(); + } + } + }, + add : { + label: 'Add Setting', + messages: { + confirm: function(args) { + return 'Add Setting'; + }, + notification: function(args) { + return 'Setting added'; + } + }, + preFilter: function(args) { + return true; + }, + createForm: { + title: 'Add New Setting', + preFilter: function(args) { + var data = { + resourcetype: resourceType + }; + if (resourceType === 'UserVm') { + data.resourceid = args.context.instances[0].id; + } + if (resourceType === 'Template') { + data.resourceid = args.context.templates[0].id; + } + + $.ajax({ + url: createURL("listDetailOptions"), + data: data, + dataType: "json", + success: function(json) { + var details = json.listdetailoptionsresponse.detailoptions.details; + var keys = []; + Object.keys(details).forEach(function(key,index) { + keys.push(key); + }); + $(args.$form.find('input[name="name"]')).blur(); + $(args.$form.find('input[name="name"]')).autocomplete({ + minLength: 0, + delay: 0, + source: keys.sort(), + select: function(event, ui) { + const key = ui.item.value; + const options = details[key]; + $(args.$form.find('input[name="value"]')).autocomplete({ + minLength: 0, + delay: 0, + autoFocus: true, + source: options.sort() + }).focus(function() { + $(this).data("uiAutocomplete").search($(this).val()); + }); + } + }).focus(function() { + $(this).data("uiAutocomplete").search($(this).val()); + }); + } + }); + + return true; + }, + fields: { + name: { + label: 'label.name', + validation: { + required: true + } + }, + value: { + label: 'label.value', + validation: { + required: true + } + } + } + }, + action: actions.add + } }, - dataProvider: dataProvider + dataProvider: function(args) { + var data = { + resourcetype: resourceType + }; + if (resourceType === 'UserVm') { + data.resourceid = args.context.instances[0].id; + } + if (resourceType === 'Template') { + data.resourceid = args.context.templates[0].id; + } + + $.ajax({ + url: createURL("listDetailOptions"), + data: data, + dataType: "json", + success: function(json) { + var details = json.listdetailoptionsresponse.detailoptions.details; + var tbody = $.find('#details-tab-settings .data-table tbody'); + $(tbody).on('DOMNodeInserted', "tr", function() { + $.each($.find('#details-tab-settings .data-table tbody tr'), function(idx, row) { + const key = $(row).find('td.name').attr('title'); + const options = details[key]; + if (options) { + $($(row).find('input.edit')).autocomplete({ + minLength: 0, + delay: 0, + autoFocus: true, + source: options.sort() + }).focus(function() { + $(this).data("uiAutocomplete").search(""); + }); + $(row).find('input.edit').blur(); + } + }); + }); + dataProvider(args); + } + }); + } }; var $listView = $('
').listView({ @@ -138,4 +223,4 @@ return $listView; } }; -}(jQuery, cloudStack)); \ No newline at end of file +}(jQuery, cloudStack)); From 14bff7bd034a5697c50a6755c420f121d8a58754 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 27 Jun 2019 09:18:10 +0530 Subject: [PATCH 07/13] server: export granular volume bytes and iops metrics (#3259) Problem: The VM metrics has aggregated volume bytes read/write and iops metrics but not on per volume basis. Root Cause: The volume stats sub-system is not used to export the metrics, the support is not available for VMware. Solution: Use the volume stats sub-system and DB table to export the metrics via the listVolumes and listVolumeMetrics API, and implement support for VMware and fix issue with network and disk metrics in the VM metrics view. Signed-off-by: Rohit Yadav --- .../apache/cloudstack/api/ApiConstants.java | 5 + .../api/response/UserVmResponse.java | 8 +- .../api/response/VolumeResponse.java | 65 ++- .../vmware/resource/VmwareResource.java | 386 +++++++++++++----- .../CitrixGetVmDiskStatsCommandWrapper.java | 2 +- .../metrics/MetricsServiceImpl.java | 65 +-- .../response/VmMetricsResponse.java | 9 +- .../response/VolumeMetricsResponse.java | 14 +- .../api/query/dao/VolumeJoinDaoImpl.java | 12 + .../java/com/cloud/server/StatsCollector.java | 4 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 6 +- ui/scripts/metrics.js | 15 + .../vmware/mo/VirtualMachineMO.java | 2 +- .../hypervisor/vmware/util/VmwareHelper.java | 35 ++ 14 files changed, 461 insertions(+), 167 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 7502b18488e..3c1d7bb93c6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -91,6 +91,11 @@ public class ApiConstants { public static final String DIRECT_DOWNLOAD = "directdownload"; public static final String DISK_OFFERING_ID = "diskofferingid"; public static final String NEW_DISK_OFFERING_ID = "newdiskofferingid"; + public static final String DISK_KBS_READ = "diskkbsread"; + public static final String DISK_KBS_WRITE = "diskkbswrite"; + public static final String DISK_IO_READ = "diskioread"; + public static final String DISK_IO_WRITE = "diskiowrite"; + public static final String DISK_IO_PSTOTAL = "diskiopstotal"; public static final String DISK_SIZE = "disksize"; public static final String UTILIZATION = "utilization"; public static final String DRIVER = "driver"; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java index bbf6b6cf614..8a2f1a169d6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java @@ -188,11 +188,11 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co @Param(description = "the outgoing network traffic on the host") private Long networkKbsWrite; - @SerializedName("diskkbsread") + @SerializedName(ApiConstants.DISK_KBS_READ) @Param(description = "the read (bytes) of disk on the vm") private Long diskKbsRead; - @SerializedName("diskkbswrite") + @SerializedName(ApiConstants.DISK_KBS_WRITE) @Param(description = "the write (bytes) of disk on the vm") private Long diskKbsWrite; @@ -208,11 +208,11 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co @Param(description = "the target memory in vm") private Long memoryTargetKBs; - @SerializedName("diskioread") + @SerializedName(ApiConstants.DISK_IO_READ) @Param(description = "the read (io) of disk on the vm") private Long diskIORead; - @SerializedName("diskiowrite") + @SerializedName(ApiConstants.DISK_IO_WRITE) @Param(description = "the write (io) of disk on the vm") private Long diskIOWrite; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java index d845e414808..01d2c9b3d8f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java @@ -16,17 +16,18 @@ // under the License. package org.apache.cloudstack.api.response; -import com.cloud.serializer.Param; -import com.cloud.storage.Volume; -import com.google.gson.annotations.SerializedName; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.Set; + import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponseWithTagInformation; import org.apache.cloudstack.api.EntityReference; -import java.util.Date; -import java.util.LinkedHashSet; -import java.util.Set; +import com.cloud.serializer.Param; +import com.cloud.storage.Volume; +import com.google.gson.annotations.SerializedName; @EntityReference(value = Volume.class) @SuppressWarnings("unused") @@ -152,13 +153,29 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co private Long bytesWriteRate; @SerializedName("diskIopsReadRate") - @Param(description = "io requests read rate of the disk volume") + @Param(description = "io requests read rate of the disk volume per the disk offering") private Long iopsReadRate; @SerializedName("diskIopsWriteRate") - @Param(description = "io requests write rate of the disk volume") + @Param(description = "io requests write rate of the disk volume per the disk offering") private Long iopsWriteRate; + @SerializedName(ApiConstants.DISK_KBS_READ) + @Param(description = "the read (bytes) of disk on the vm") + private Long diskKbsRead; + + @SerializedName(ApiConstants.DISK_KBS_WRITE) + @Param(description = "the write (bytes) of disk on the vm") + private Long diskKbsWrite; + + @SerializedName(ApiConstants.DISK_IO_READ) + @Param(description = "the read (io) of disk on the vm") + private Long diskIORead; + + @SerializedName(ApiConstants.DISK_IO_WRITE) + @Param(description = "the write (io) of disk on the vm") + private Long diskIOWrite; + @SerializedName(ApiConstants.HYPERVISOR) @Param(description = "Hypervisor the volume belongs to") private String hypervisor; @@ -395,10 +412,42 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co this.iopsWriteRate = iopsWriteRate; } + public Long getDiskKbsRead() { + return diskKbsRead; + } + + public void setDiskKbsRead(Long diskKbsRead) { + this.diskKbsRead = diskKbsRead; + } + + public Long getDiskKbsWrite() { + return diskKbsWrite; + } + + public void setDiskKbsWrite(Long diskKbsWrite) { + this.diskKbsWrite = diskKbsWrite; + } + public Long getIopsWriteRate() { return iopsWriteRate; } + public Long getDiskIORead() { + return diskIORead; + } + + public void setDiskIORead(Long diskIORead) { + this.diskIORead = diskIORead; + } + + public Long getDiskIOWrite() { + return diskIOWrite; + } + + public void setDiskIOWrite(Long diskIOWrite) { + this.diskIOWrite = diskIOWrite; + } + public void setHypervisor(String hypervisor) { this.hypervisor = hypervisor; } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 3d6732c0fbe..c195712e62a 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -42,68 +42,7 @@ import java.util.TimeZone; import java.util.UUID; import javax.naming.ConfigurationException; - -import org.apache.commons.lang.math.NumberUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; -import org.joda.time.Duration; - -import com.google.gson.Gson; -import com.vmware.vim25.AboutInfo; -import com.vmware.vim25.BoolPolicy; -import com.vmware.vim25.ComputeResourceSummary; -import com.vmware.vim25.CustomFieldStringValue; -import com.vmware.vim25.DVPortConfigInfo; -import com.vmware.vim25.DVPortConfigSpec; -import com.vmware.vim25.DasVmPriority; -import com.vmware.vim25.DatastoreSummary; -import com.vmware.vim25.DistributedVirtualPort; -import com.vmware.vim25.DistributedVirtualSwitchPortConnection; -import com.vmware.vim25.DistributedVirtualSwitchPortCriteria; -import com.vmware.vim25.DynamicProperty; -import com.vmware.vim25.GuestInfo; -import com.vmware.vim25.HostCapability; -import com.vmware.vim25.HostHostBusAdapter; -import com.vmware.vim25.HostInternetScsiHba; -import com.vmware.vim25.ManagedObjectReference; -import com.vmware.vim25.ObjectContent; -import com.vmware.vim25.OptionValue; -import com.vmware.vim25.PerfCounterInfo; -import com.vmware.vim25.PerfEntityMetric; -import com.vmware.vim25.PerfEntityMetricBase; -import com.vmware.vim25.PerfMetricId; -import com.vmware.vim25.PerfMetricIntSeries; -import com.vmware.vim25.PerfMetricSeries; -import com.vmware.vim25.PerfQuerySpec; -import com.vmware.vim25.PerfSampleInfo; -import com.vmware.vim25.RuntimeFaultFaultMsg; -import com.vmware.vim25.ToolsUnavailableFaultMsg; -import com.vmware.vim25.VMwareDVSPortSetting; -import com.vmware.vim25.VimPortType; -import com.vmware.vim25.VirtualDevice; -import com.vmware.vim25.VirtualDeviceBackingInfo; -import com.vmware.vim25.VirtualDeviceConfigSpec; -import com.vmware.vim25.VirtualDeviceConfigSpecOperation; -import com.vmware.vim25.VirtualDisk; -import com.vmware.vim25.VirtualDiskFlatVer2BackingInfo; -import com.vmware.vim25.VirtualEthernetCard; -import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo; -import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo; -import com.vmware.vim25.VirtualEthernetCardOpaqueNetworkBackingInfo; -import com.vmware.vim25.VirtualMachineConfigSpec; -import com.vmware.vim25.VirtualMachineFileInfo; -import com.vmware.vim25.VirtualMachineFileLayoutEx; -import com.vmware.vim25.VirtualMachineFileLayoutExFileInfo; -import com.vmware.vim25.VirtualMachineGuestOsIdentifier; -import com.vmware.vim25.VirtualMachinePowerState; -import com.vmware.vim25.VirtualMachineRelocateSpec; -import com.vmware.vim25.VirtualMachineRelocateSpecDiskLocator; -import com.vmware.vim25.VirtualMachineRuntimeInfo; -import com.vmware.vim25.VirtualMachineToolsStatus; -import com.vmware.vim25.VirtualMachineVideoCard; -import com.vmware.vim25.VirtualUSBController; -import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; +import javax.xml.datatype.XMLGregorianCalendar; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.command.CopyCommand; @@ -114,6 +53,11 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; +import org.joda.time.Duration; import com.cloud.agent.IAgentControl; import com.cloud.agent.api.Answer; @@ -211,6 +155,7 @@ import com.cloud.agent.api.UnregisterVMCommand; import com.cloud.agent.api.UpgradeSnapshotCommand; import com.cloud.agent.api.ValidateSnapshotAnswer; import com.cloud.agent.api.ValidateSnapshotCommand; +import com.cloud.agent.api.VmDiskStatsEntry; import com.cloud.agent.api.VmStatsEntry; import com.cloud.agent.api.VolumeStatsEntry; import com.cloud.agent.api.check.CheckSshAnswer; @@ -310,6 +255,60 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.PowerState; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VmDetailConstants; +import com.google.gson.Gson; +import com.vmware.vim25.AboutInfo; +import com.vmware.vim25.BoolPolicy; +import com.vmware.vim25.ComputeResourceSummary; +import com.vmware.vim25.CustomFieldStringValue; +import com.vmware.vim25.DVPortConfigInfo; +import com.vmware.vim25.DVPortConfigSpec; +import com.vmware.vim25.DasVmPriority; +import com.vmware.vim25.DatastoreSummary; +import com.vmware.vim25.DistributedVirtualPort; +import com.vmware.vim25.DistributedVirtualSwitchPortConnection; +import com.vmware.vim25.DistributedVirtualSwitchPortCriteria; +import com.vmware.vim25.DynamicProperty; +import com.vmware.vim25.GuestInfo; +import com.vmware.vim25.HostCapability; +import com.vmware.vim25.HostHostBusAdapter; +import com.vmware.vim25.HostInternetScsiHba; +import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.ObjectContent; +import com.vmware.vim25.OptionValue; +import com.vmware.vim25.PerfCounterInfo; +import com.vmware.vim25.PerfEntityMetric; +import com.vmware.vim25.PerfEntityMetricBase; +import com.vmware.vim25.PerfMetricId; +import com.vmware.vim25.PerfMetricIntSeries; +import com.vmware.vim25.PerfMetricSeries; +import com.vmware.vim25.PerfQuerySpec; +import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.ToolsUnavailableFaultMsg; +import com.vmware.vim25.VMwareDVSPortSetting; +import com.vmware.vim25.VimPortType; +import com.vmware.vim25.VirtualDevice; +import com.vmware.vim25.VirtualDeviceBackingInfo; +import com.vmware.vim25.VirtualDeviceConfigSpec; +import com.vmware.vim25.VirtualDeviceConfigSpecOperation; +import com.vmware.vim25.VirtualDisk; +import com.vmware.vim25.VirtualDiskFlatVer2BackingInfo; +import com.vmware.vim25.VirtualEthernetCard; +import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo; +import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo; +import com.vmware.vim25.VirtualEthernetCardOpaqueNetworkBackingInfo; +import com.vmware.vim25.VirtualMachineConfigSpec; +import com.vmware.vim25.VirtualMachineFileInfo; +import com.vmware.vim25.VirtualMachineFileLayoutEx; +import com.vmware.vim25.VirtualMachineFileLayoutExFileInfo; +import com.vmware.vim25.VirtualMachineGuestOsIdentifier; +import com.vmware.vim25.VirtualMachinePowerState; +import com.vmware.vim25.VirtualMachineRelocateSpec; +import com.vmware.vim25.VirtualMachineRelocateSpecDiskLocator; +import com.vmware.vim25.VirtualMachineRuntimeInfo; +import com.vmware.vim25.VirtualMachineToolsStatus; +import com.vmware.vim25.VirtualMachineVideoCard; +import com.vmware.vim25.VirtualUSBController; +import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; public class VmwareResource implements StoragePoolResource, ServerResource, VmwareHostService, VirtualRouterDeployer { private static final Logger s_logger = Logger.getLogger(VmwareResource.class); @@ -3469,6 +3468,120 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } protected Answer execute(GetVmDiskStatsCommand cmd) { + try { + final VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); + final ManagedObjectReference perfMgr = getServiceContext().getServiceContent().getPerfManager(); + VimPortType service = getServiceContext().getService(); + + final int intervalSeconds = 300; + final XMLGregorianCalendar startTime = VmwareHelper.getXMLGregorianCalendar(new Date(), intervalSeconds); + final XMLGregorianCalendar endTime = VmwareHelper.getXMLGregorianCalendar(new Date(), 0); + + PerfCounterInfo diskReadIOPerfCounterInfo = null; + PerfCounterInfo diskWriteIOPerfCounterInfo = null; + PerfCounterInfo diskReadKbsPerfCounterInfo = null; + PerfCounterInfo diskWriteKbsPerfCounterInfo = null; + + // https://pubs.vmware.com/vsphere-5-5/topic/com.vmware.wssdk.apiref.doc/virtual_disk_counters.html + List cInfo = getServiceContext().getVimClient().getDynamicProperty(perfMgr, "perfCounter"); + for (PerfCounterInfo info : cInfo) { + if ("virtualdisk".equalsIgnoreCase(info.getGroupInfo().getKey()) && "average".equalsIgnoreCase(info.getRollupType().value())) { + if ("numberReadAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskReadIOPerfCounterInfo = info; + } + if ("numberWriteAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskWriteIOPerfCounterInfo = info; + } + if ("read".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskReadKbsPerfCounterInfo = info; + } + if ("write".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskWriteKbsPerfCounterInfo = info; + } + } + } + + final ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter(); + final DatacenterMO dcMo = new DatacenterMO(getServiceContext(), dcMor); + + final HashMap> vmStatsMap = new HashMap<>(); + for (final String vmName : cmd.getVmNames()) { + final VirtualMachineMO vmMo = dcMo.findVm(vmName); + final List diskStats = new ArrayList<>(); + for (final VirtualDisk disk : vmMo.getAllDiskDevice()) { + final String diskBusName = vmMo.getDeviceBusName(vmMo.getAllDeviceList(), disk); + long readReq = 0; + long readBytes = 0; + long writeReq = 0; + long writeBytes = 0; + + final ArrayList perfMetricsIds = new ArrayList(); + if (diskReadIOPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadIOPerfCounterInfo, diskBusName)); + } + if (diskWriteIOPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteIOPerfCounterInfo, diskBusName)); + } + if (diskReadKbsPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadKbsPerfCounterInfo, diskBusName)); + } + if (diskWriteKbsPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteKbsPerfCounterInfo, diskBusName)); + } + + if (perfMetricsIds.size() > 0) { + final PerfQuerySpec qSpec = new PerfQuerySpec(); + qSpec.setEntity(vmMo.getMor()); + qSpec.setFormat("normal"); + qSpec.setIntervalId(intervalSeconds); + qSpec.setStartTime(startTime); + qSpec.setEndTime(endTime); + qSpec.getMetricId().addAll(perfMetricsIds); + + for (final PerfEntityMetricBase perfValue : service.queryPerf(perfMgr, Collections.singletonList(qSpec))) { + if (!(perfValue instanceof PerfEntityMetric)) { + continue; + } + final List values = ((PerfEntityMetric)perfValue).getValue(); + if (values == null || values.isEmpty()) { + continue; + } + for (final PerfMetricSeries value : values) { + if (!(value instanceof PerfMetricIntSeries) || !value.getId().getInstance().equals(diskBusName)) { + continue; + } + final List perfStats = ((PerfMetricIntSeries)value).getValue(); + if (perfStats.size() > 0) { + long sum = 0; + for (long val : perfStats) { + sum += val; + } + long avg = sum / perfStats.size(); + if (value.getId().getCounterId() == diskReadIOPerfCounterInfo.getKey()) { + readReq = avg; + } else if (value.getId().getCounterId() == diskWriteIOPerfCounterInfo.getKey()) { + writeReq = avg; + } else if (value.getId().getCounterId() == diskReadKbsPerfCounterInfo.getKey()) { + readBytes = avg * 1024; + } else if (value.getId().getCounterId() == diskWriteKbsPerfCounterInfo.getKey()) { + writeBytes = avg * 1024; + } + } + } + } + } + diskStats.add(new VmDiskStatsEntry(vmName, VmwareHelper.getDiskDeviceFileName(disk), writeReq, readReq, writeBytes, readBytes)); + } + if (diskStats.size() > 0) { + vmStatsMap.put(vmName, diskStats); + } + } + if (vmStatsMap.size() > 0) { + return new GetVmDiskStatsAnswer(cmd, "", cmd.getHostName(), vmStatsMap); + } + } catch (Exception e) { + s_logger.error("Unable to execute GetVmDiskStatsCommand due to " + VmwareHelper.getExceptionMessage(e), e); + } return new GetVmDiskStatsAnswer(cmd, null, null, null); } @@ -5857,12 +5970,21 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa HashMap vmResponseMap = new HashMap(); ManagedObjectReference perfMgr = getServiceContext().getServiceContent().getPerfManager(); VimPortType service = getServiceContext().getService(); + PerfCounterInfo rxPerfCounterInfo = null; PerfCounterInfo txPerfCounterInfo = null; + PerfCounterInfo diskReadIOPerfCounterInfo = null; + PerfCounterInfo diskWriteIOPerfCounterInfo = null; + PerfCounterInfo diskReadKbsPerfCounterInfo = null; + PerfCounterInfo diskWriteKbsPerfCounterInfo = null; + + final int intervalSeconds = 300; + final XMLGregorianCalendar startTime = VmwareHelper.getXMLGregorianCalendar(new Date(), intervalSeconds); + final XMLGregorianCalendar endTime = VmwareHelper.getXMLGregorianCalendar(new Date(), 0); List cInfo = getServiceContext().getVimClient().getDynamicProperty(perfMgr, "perfCounter"); for (PerfCounterInfo info : cInfo) { - if ("net".equalsIgnoreCase(info.getGroupInfo().getKey())) { + if ("net".equalsIgnoreCase(info.getGroupInfo().getKey()) && "average".equalsIgnoreCase(info.getRollupType().value())) { if ("transmitted".equalsIgnoreCase(info.getNameInfo().getKey())) { txPerfCounterInfo = info; } @@ -5870,6 +5992,20 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa rxPerfCounterInfo = info; } } + if ("virtualdisk".equalsIgnoreCase(info.getGroupInfo().getKey())) { + if ("numberReadAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskReadIOPerfCounterInfo = info; + } + if ("numberWriteAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskWriteIOPerfCounterInfo = info; + } + if ("read".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskReadKbsPerfCounterInfo = info; + } + if ("write".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskWriteKbsPerfCounterInfo = info; + } + } } int key = ((HostMO)hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME); @@ -5885,8 +6021,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa final String memMbStr = "config.hardware.memoryMB"; final String allocatedCpuStr = "summary.runtime.maxCpuUsage"; - ObjectContent[] ocs = - hyperHost.getVmPropertiesOnHyperHost(new String[] {"name", numCpuStr, cpuUseStr ,guestMemUseStr ,memLimitStr ,memMbStr,allocatedCpuStr ,instanceNameCustomField}); + ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[] { + "name", numCpuStr, cpuUseStr, guestMemUseStr, memLimitStr, memMbStr,allocatedCpuStr, instanceNameCustomField + }); if (ocs != null && ocs.length > 0) { for (ObjectContent oc : ocs) { @@ -5923,7 +6060,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } maxCpuUsage = (maxCpuUsage/allocatedCpu)*100; - new VirtualMachineMO(hyperHost.getContext(), oc.getObj()); if (vmInternalCSName != null) { name = vmInternalCSName; } else { @@ -5937,60 +6073,86 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa ManagedObjectReference vmMor = hyperHost.findVmOnHyperHost(name).getMor(); assert (vmMor != null); - ArrayList vmNetworkMetrics = new ArrayList(); - // get all the metrics from the available sample period - List perfMetrics = service.queryAvailablePerfMetric(perfMgr, vmMor, null, null, null); - if (perfMetrics != null) { - for (int index = 0; index < perfMetrics.size(); ++index) { - if (((rxPerfCounterInfo != null) && (perfMetrics.get(index).getCounterId() == rxPerfCounterInfo.getKey())) - || ((txPerfCounterInfo != null) && (perfMetrics.get(index).getCounterId() == txPerfCounterInfo.getKey()))) { - vmNetworkMetrics.add(perfMetrics.get(index)); - } - } - } - double networkReadKBs = 0; double networkWriteKBs = 0; - long sampleDuration = 0; + double diskReadIops = 0; + double diskWriteIops = 0; + double diskReadKbs = 0; + double diskWriteKbs = 0; - if (vmNetworkMetrics.size() != 0) { - PerfQuerySpec qSpec = new PerfQuerySpec(); + final ArrayList perfMetricsIds = new ArrayList(); + if (rxPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(rxPerfCounterInfo, "")); + } + if (txPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(txPerfCounterInfo, "")); + } + if (diskReadIOPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadIOPerfCounterInfo, "*")); + } + if (diskWriteIOPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteIOPerfCounterInfo, "*")); + } + if (diskReadKbsPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadKbsPerfCounterInfo, "")); + } + if (diskWriteKbsPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteKbsPerfCounterInfo, "")); + } + + if (perfMetricsIds.size() > 0) { + final PerfQuerySpec qSpec = new PerfQuerySpec(); qSpec.setEntity(vmMor); - PerfMetricId[] availableMetricIds = vmNetworkMetrics.toArray(new PerfMetricId[0]); - qSpec.getMetricId().addAll(Arrays.asList(availableMetricIds)); - List qSpecs = new ArrayList(); - qSpecs.add(qSpec); - List values = service.queryPerf(perfMgr, qSpecs); - - for (int i = 0; i < values.size(); ++i) { - List infos = ((PerfEntityMetric)values.get(i)).getSampleInfo(); - if (infos != null && infos.size() > 0) { - int endMs = infos.get(infos.size() - 1).getTimestamp().getSecond() * 1000 + infos.get(infos.size() - 1).getTimestamp().getMillisecond(); - int beginMs = infos.get(0).getTimestamp().getSecond() * 1000 + infos.get(0).getTimestamp().getMillisecond(); - sampleDuration = (endMs - beginMs) / 1000; - List vals = ((PerfEntityMetric)values.get(i)).getValue(); - for (int vi = 0; ((vals != null) && (vi < vals.size())); ++vi) { - if (vals.get(vi) instanceof PerfMetricIntSeries) { - PerfMetricIntSeries val = (PerfMetricIntSeries)vals.get(vi); - List perfValues = val.getValue(); - Long sumRate = 0L; - for (int j = 0; j < infos.size(); j++) { // Size of the array matches the size as the PerfSampleInfo - sumRate += perfValues.get(j); - } - Long averageRate = sumRate / infos.size(); - if (vals.get(vi).getId().getCounterId() == rxPerfCounterInfo.getKey()) { - networkReadKBs = sampleDuration * averageRate; //get the average RX rate multiplied by sampled duration - } - if (vals.get(vi).getId().getCounterId() == txPerfCounterInfo.getKey()) { - networkWriteKBs = sampleDuration * averageRate;//get the average TX rate multiplied by sampled duration - } - } + qSpec.setFormat("normal"); + qSpec.setIntervalId(intervalSeconds); + qSpec.setStartTime(startTime); + qSpec.setEndTime(endTime); + qSpec.getMetricId().addAll(perfMetricsIds); + final List perfValues = service.queryPerf(perfMgr, Collections.singletonList(qSpec)); + for (final PerfEntityMetricBase perfValue : perfValues) { + if (!(perfValue instanceof PerfEntityMetric)) { + continue; + } + final List seriesList = ((PerfEntityMetric) perfValue).getValue(); + for (final PerfMetricSeries series : seriesList) { + if (!(series instanceof PerfMetricIntSeries)) { + continue; + } + final List values = ((PerfMetricIntSeries) series).getValue(); + double sum = 0; + for (final Long value : values) { + sum += value; + } + double avg = sum / values.size(); + if (series.getId().getCounterId() == rxPerfCounterInfo.getKey()) { + networkReadKBs = avg; + } + if (series.getId().getCounterId() == txPerfCounterInfo.getKey()) { + networkWriteKBs = avg; + } + if (series.getId().getCounterId() == diskReadIOPerfCounterInfo.getKey()) { + diskReadIops += avg; + } + if (series.getId().getCounterId() == diskWriteIOPerfCounterInfo.getKey()) { + diskWriteIops += avg; + } + if (series.getId().getCounterId() == diskReadKbsPerfCounterInfo.getKey()) { + diskReadKbs = avg; + } + if (series.getId().getCounterId() == diskWriteKbsPerfCounterInfo.getKey()) { + diskWriteKbs = avg; } } } } - vmResponseMap.put(name, new VmStatsEntry( NumberUtils.toDouble(memkb)*1024,NumberUtils.toDouble(guestMemusage)*1024,NumberUtils.toDouble(memlimit)*1024, - maxCpuUsage, networkReadKBs, networkWriteKBs, NumberUtils.toInt(numberCPUs), "vm")); + + final VmStatsEntry vmStats = new VmStatsEntry( NumberUtils.toDouble(memkb)*1024,NumberUtils.toDouble(guestMemusage)*1024,NumberUtils.toDouble(memlimit)*1024, + maxCpuUsage, networkReadKBs, networkWriteKBs, NumberUtils.toInt(numberCPUs), "vm"); + vmStats.setDiskReadIOs(diskReadIops); + vmStats.setDiskWriteIOs(diskWriteIops); + vmStats.setDiskReadKBs(diskReadKbs); + vmStats.setDiskWriteKBs(diskWriteKbs); + vmResponseMap.put(name, vmStats); } } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixGetVmDiskStatsCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixGetVmDiskStatsCommandWrapper.java index 04090ce7436..d15e0a71727 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixGetVmDiskStatsCommandWrapper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixGetVmDiskStatsCommandWrapper.java @@ -33,4 +33,4 @@ public final class CitrixGetVmDiskStatsCommandWrapper extends CommandWrapper volSearch; @@ -102,6 +106,14 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation sc = createSearchCriteriaForHostTypeRoutingStateUpAndNotInMaintenance(); - sc.addAnd("hypervisorType", SearchCriteria.Op.EQ, HypervisorType.KVM); // support KVM only util 2013.06.25 + sc.addAnd("hypervisorType", SearchCriteria.Op.IN, HypervisorType.KVM, HypervisorType.VMware); List hosts = _hostDao.search(sc, null); for (HostVO host : hosts) { List vms = _userVmDao.listRunningByHostId(host.getId()); List vmIds = new ArrayList(); - for (UserVmVO vm : vms) { + for (UserVmVO vm : vms) { if (vm.getType() == VirtualMachine.Type.User) // user vm vmIds.add(vm.getId()); } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 0727a2a8593..f3731ab6ffe 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4670,8 +4670,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Override public void collectVmDiskStatistics(final UserVm userVm) { - // support KVM only util 2013.06.25 - if (!userVm.getHypervisorType().equals(HypervisorType.KVM)) { + // Only supported for KVM and VMware + if (!(userVm.getHypervisorType().equals(HypervisorType.KVM) || userVm.getHypervisorType().equals(HypervisorType.VMware))) { return; } s_logger.debug("Collect vm disk statistics from host before stopping Vm"); @@ -6684,4 +6684,4 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } } -} \ No newline at end of file +} diff --git a/ui/scripts/metrics.js b/ui/scripts/metrics.js index c3c68e1e233..2784eab2d8d 100644 --- a/ui/scripts/metrics.js +++ b/ui/scripts/metrics.js @@ -607,6 +607,21 @@ }, storage: { label: 'label.metrics.storagepool' + }, + disk: { + label: 'label.metrics.disk.usage', + collapsible: true, + columns: { + diskioread: { + label: 'label.metrics.disk.read' + }, + diskiowrite: { + label: 'label.metrics.disk.write' + }, + diskiopstotal: { + label: 'label.metrics.disk.iops.total' + } + } } }, dataProvider: function(args) { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index d9d03d1484a..6a42f7f9aea 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -2709,7 +2709,7 @@ public class VirtualMachineMO extends BaseMO { return pathList; } - private String getDeviceBusName(List allDevices, VirtualDevice theDevice) throws Exception { + public String getDeviceBusName(List allDevices, VirtualDevice theDevice) throws Exception { for (VirtualDevice device : allDevices) { if (device.getKey() == theDevice.getControllerKey().intValue()) { if (device instanceof VirtualIDEController) { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index bb7f3c5a140..6477eb1ecee 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -25,11 +25,16 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Date; +import java.util.GregorianCalendar; import java.util.List; import java.util.Random; import java.util.UUID; import javax.annotation.Nonnull; +import javax.xml.datatype.DatatypeConfigurationException; +import javax.xml.datatype.DatatypeFactory; +import javax.xml.datatype.XMLGregorianCalendar; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; @@ -41,6 +46,8 @@ import com.vmware.vim25.ManagedObjectReference; import com.vmware.vim25.MethodFault; import com.vmware.vim25.ObjectContent; import com.vmware.vim25.OptionValue; +import com.vmware.vim25.PerfCounterInfo; +import com.vmware.vim25.PerfMetricId; import com.vmware.vim25.ResourceAllocationInfo; import com.vmware.vim25.VirtualCdrom; import com.vmware.vim25.VirtualCdromIsoBackingInfo; @@ -637,6 +644,25 @@ public class VmwareHelper { return usbController; } + public static PerfMetricId createPerfMetricId(PerfCounterInfo counterInfo, String instance) { + PerfMetricId metricId = new PerfMetricId(); + metricId.setCounterId(counterInfo.getKey()); + metricId.setInstance(instance); + return metricId; + } + + public static String getDiskDeviceFileName(VirtualDisk diskDevice) { + VirtualDeviceBackingInfo backingInfo = diskDevice.getBacking(); + if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { + final String vmdkName = ((VirtualDiskFlatVer2BackingInfo)backingInfo).getFileName().replace(".vmdk", ""); + if (vmdkName.contains("/")) { + return vmdkName.split("/", 2)[1]; + } + return vmdkName; + } + return null; + } + public static ManagedObjectReference getDiskDeviceDatastore(VirtualDisk diskDevice) throws Exception { VirtualDeviceBackingInfo backingInfo = diskDevice.getBacking(); assert (backingInfo instanceof VirtualDiskFlatVer2BackingInfo); @@ -773,4 +799,13 @@ public class VmwareHelper { return DiskControllerType.getType(dataDiskController) == DiskControllerType.osdefault; } + public static XMLGregorianCalendar getXMLGregorianCalendar(final Date date, final int offsetSeconds) throws DatatypeConfigurationException { + if (offsetSeconds > 0) { + date.setTime(date.getTime() - offsetSeconds * 1000); + } + final GregorianCalendar gregorianCalendar = new GregorianCalendar(); + gregorianCalendar.setTime(date); + return DatatypeFactory.newInstance().newXMLGregorianCalendar(gregorianCalendar); + } + } From 58474530f69825f5108d249ac3f5e4a153d0c8d6 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 27 Jun 2019 09:21:09 +0530 Subject: [PATCH 08/13] api: snapshot, snapshotpolicy tag support (#3228) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: Currently tags cannot be applied to snapshot when it is being created but through separate “create tags” API calls. For snapshot policies tags cannot be set either at creation or through “create tags” API. Root Cause: The “create snapshots” API does not support adding tags during creation and it can only be done through “create tags” API. Snapshot policy as a resource does not support tags and no tags can be set for them through any API. Solution: Tag support for snapshot policy has been added. Snapshot policy with tags when executed will produce snapshots containing the same tags from snapshot policy. Following APIs have been updated: Both “create snapshotpolicy” and “create snapshot” now accepts “tags” as a new parameter. The expected format for “tags” parameter is similar to parameter “tags” in “create tags“ API. Deletion support for tags associated with snapshots policy has been added to “delete snapshotpolicies” API. Tags set for snapshot policies are added to the Response of “list snapshotpolicies“ API. UI support for setting tags to snapshots and snapshot policy is provided through the corresponding menus with a new section in each form to set tags. Signed-off-by: Abhishek Kumar --- .../java/com/cloud/server/ResourceTag.java | 2 +- .../com/cloud/storage/VolumeApiService.java | 3 +- .../user/snapshot/CreateSnapshotCmd.java | 22 +++- .../snapshot/CreateSnapshotPolicyCmd.java | 23 +++- .../api/response/SnapshotPolicyResponse.java | 16 ++- .../api/response/SnapshotResponse.java | 26 ++-- .../command/test/CreateSnapshotCmdTest.java | 30 ++++- .../snapshot/CreateSnapshotPolicyCmdTest.java | 46 +++++++ .../java/com/cloud/api/ApiResponseHelper.java | 10 +- .../cloud/storage/VolumeApiServiceImpl.java | 16 ++- .../storage/snapshot/SnapshotManagerImpl.java | 122 ++++++++++-------- .../snapshot/SnapshotSchedulerImpl.java | 13 ++ .../cloud/tags/TaggedResourceManagerImpl.java | 45 ++++--- .../storage/VolumeApiServiceImplTest.java | 10 +- ui/css/cloudstack3.css | 25 +++- ui/index.html | 20 ++- ui/scripts/storage.js | 34 +++-- ui/scripts/ui-custom/recurringSnapshots.js | 6 +- ui/scripts/ui/dialog.js | 24 ++-- ui/scripts/ui/widgets/tagger.js | 65 +++++++++- 20 files changed, 423 insertions(+), 135 deletions(-) create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java diff --git a/api/src/main/java/com/cloud/server/ResourceTag.java b/api/src/main/java/com/cloud/server/ResourceTag.java index 0bd5d734e30..99eb8603d1c 100644 --- a/api/src/main/java/com/cloud/server/ResourceTag.java +++ b/api/src/main/java/com/cloud/server/ResourceTag.java @@ -58,7 +58,7 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit AutoScaleVmGroup(false, true), LBStickinessPolicy(false, true), LBHealthCheckPolicy(false, true), - SnapshotPolicy(false, true), + SnapshotPolicy(true, true), GuestOs(false, true), NetworkOffering(false, true), VpcOffering(true, false); diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 7b38a6b1af1..aa6d8a664c8 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -19,6 +19,7 @@ package com.cloud.storage; import java.net.MalformedURLException; +import java.util.Map; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; @@ -93,7 +94,7 @@ public interface VolumeApiService { Volume detachVolumeFromVM(DetachVolumeCmd cmd); - Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup) + Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) throws ResourceAllocationException; Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index b77b9854590..72437c8d3fe 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -16,6 +16,10 @@ // under the License. package org.apache.cloudstack.api.command.user.snapshot; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandJobType; import org.apache.cloudstack.api.ApiConstants; @@ -28,6 +32,7 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.SnapshotPolicyResponse; import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; import com.cloud.event.EventTypes; @@ -83,6 +88,9 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.ASYNC_BACKUP, type = CommandType.BOOLEAN, required = false, description = "asynchronous backup if true") private Boolean asyncBackup; + @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)") + private Map tags; + private String syncObjectType = BaseAsyncCmd.snapshotHostSyncObject; // /////////////////////////////////////////////////// @@ -121,6 +129,18 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { } } + public Map getTags() { + Map tagsMap = new HashMap<>(); + if (MapUtils.isNotEmpty(tags)) { + for (Map services : (Collection>)tags.values()) { + String key = services.get("key"); + String value = services.get("value"); + tagsMap.put(key, value); + } + } + return tagsMap; + } + private Long getHostId() { Volume volume = _entityMgr.findById(Volume.class, getVolumeId()); if (volume == null) { @@ -196,7 +216,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { Snapshot snapshot; try { snapshot = - _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup()); + _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags()); if (snapshot != null) { SnapshotResponse response = _responseGenerator.createSnapshotResponse(snapshot); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java index 4e2e6bd3bbd..898bae5919f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java @@ -16,9 +16,11 @@ // under the License. package org.apache.cloudstack.api.command.user.snapshot; -import org.apache.cloudstack.acl.RoleType; -import org.apache.log4j.Logger; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; @@ -27,6 +29,8 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.SnapshotPolicyResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; @@ -68,6 +72,9 @@ public class CreateSnapshotPolicyCmd extends BaseCmd { @Parameter(name = ApiConstants.FOR_DISPLAY, type = CommandType.BOOLEAN, description = "an optional field, whether to the display the policy to the end user or not", since = "4.4", authorized = {RoleType.Admin}) private Boolean display; + @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)") + private Map tags; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -133,6 +140,18 @@ public class CreateSnapshotPolicyCmd extends BaseCmd { return volume.getAccountId(); } + public Map getTags() { + Map tagsMap = new HashMap<>(); + if (MapUtils.isNotEmpty(tags)) { + for (Map services : (Collection>)tags.values()) { + String key = services.get("key"); + String value = services.get("value"); + tagsMap.put(key, value); + } + } + return tagsMap; + } + @Override public void execute() { SnapshotPolicy result = _snapshotService.createPolicy(this, _accountService.getAccount(getEntityOwnerId())); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java index 10710c64803..d1e535ee743 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java @@ -16,18 +16,20 @@ // under the License. package org.apache.cloudstack.api.response; -import com.google.gson.annotations.SerializedName; +import java.util.LinkedHashSet; +import java.util.Set; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.BaseResponseWithTagInformation; import org.apache.cloudstack.api.EntityReference; import com.cloud.serializer.Param; import com.cloud.storage.snapshot.SnapshotPolicy; +import com.google.gson.annotations.SerializedName; @EntityReference(value = SnapshotPolicy.class) -public class SnapshotPolicyResponse extends BaseResponse { +public class SnapshotPolicyResponse extends BaseResponseWithTagInformation { @SerializedName("id") @Param(description = "the ID of the snapshot policy") private String id; @@ -56,6 +58,10 @@ public class SnapshotPolicyResponse extends BaseResponse { @Param(description = "is this policy for display to the regular user", since = "4.4", authorized = {RoleType.Admin}) private Boolean forDisplay; + public SnapshotPolicyResponse() { + tags = new LinkedHashSet(); + } + public String getId() { return id; } @@ -111,4 +117,8 @@ public class SnapshotPolicyResponse extends BaseResponse { public void setForDisplay(Boolean forDisplay) { this.forDisplay = forDisplay; } + + public void setTags(Set tags) { + this.tags = tags; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java index bb2ff7f6d0e..94bb4d14444 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java @@ -16,18 +16,20 @@ // under the License. package org.apache.cloudstack.api.response; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.Set; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponseWithTagInformation; +import org.apache.cloudstack.api.EntityReference; + import com.cloud.serializer.Param; import com.cloud.storage.Snapshot; import com.google.gson.annotations.SerializedName; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseResponse; -import org.apache.cloudstack.api.EntityReference; - -import java.util.Date; -import java.util.List; @EntityReference(value = Snapshot.class) -public class SnapshotResponse extends BaseResponse implements ControlledEntityResponse { +public class SnapshotResponse extends BaseResponseWithTagInformation implements ControlledEntityResponse { @SerializedName(ApiConstants.ID) @Param(description = "ID of the snapshot") private String id; @@ -96,10 +98,6 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe @Param(description = "id of the availability zone") private String zoneId; - @SerializedName(ApiConstants.TAGS) - @Param(description = "the list of resource tags associated with snapshot", responseObject = ResourceTagResponse.class) - private List tags; - @SerializedName(ApiConstants.REVERTABLE) @Param(description = "indicates whether the underlying storage supports reverting the volume to this snapshot") private boolean revertable; @@ -116,6 +114,10 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe @Param(description = "virtual size of backedup snapshot on image store") private long virtualSize; + public SnapshotResponse() { + tags = new LinkedHashSet(); + } + @Override public String getObjectId() { return this.getId(); @@ -206,7 +208,7 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe this.zoneId = zoneId; } - public void setTags(List tags) { + public void setTags(Set tags) { this.tags = tags; } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java index ceb63ab6e56..4739082cf64 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java @@ -19,9 +19,13 @@ package org.apache.cloudstack.api.command.test; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.isNull; +import java.util.HashMap; +import java.util.Map; + import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd; @@ -32,6 +36,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; import com.cloud.storage.Snapshot; import com.cloud.storage.VolumeApiService; @@ -87,9 +92,8 @@ public class CreateSnapshotCmdTest extends TestCase { VolumeApiService volumeApiService = Mockito.mock(VolumeApiService.class); Snapshot snapshot = Mockito.mock(Snapshot.class); try { - Mockito.when(volumeApiService.takeSnapshot(anyLong(), anyLong(), anyLong(), - any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean())).thenReturn(snapshot); + any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean(), anyObject())).thenReturn(snapshot); } catch (Exception e) { Assert.fail("Received exception when success expected " + e.getMessage()); @@ -122,7 +126,7 @@ public class CreateSnapshotCmdTest extends TestCase { try { Mockito.when(volumeApiService.takeSnapshot(anyLong(), anyLong(), anyLong(), - any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean())).thenReturn(null); + any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean(), anyObject())).thenReturn(null); } catch (Exception e) { Assert.fail("Received exception when success expected " + e.getMessage()); } @@ -136,4 +140,24 @@ public class CreateSnapshotCmdTest extends TestCase { Assert.assertEquals("Failed to create snapshot due to an internal error creating snapshot for volume 123", exception.getDescription()); } } + + @Test + public void testParsingTags() { + final CreateSnapshotCmd createSnapshotCmd = new CreateSnapshotCmd(); + final Map tag1 = new HashMap<>(); + tag1.put("key", "key1"); + tag1.put("value", "value1"); + final Map tag2 = new HashMap<>(); + tag2.put("key", "key2"); + tag2.put("value", "value2"); + final Map expectedTags = new HashMap<>(); + expectedTags.put("key1", "value1"); + expectedTags.put("key2", "value2"); + + final Map> tagsParams = new HashMap<>(); + tagsParams.put("0", tag1); + tagsParams.put("1", tag2); + ReflectionTestUtils.setField(createSnapshotCmd, "tags", tagsParams); + Assert.assertEquals(createSnapshotCmd.getTags(), expectedTags); + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java new file mode 100644 index 00000000000..860c2c2c3e7 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.snapshot; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +public class CreateSnapshotPolicyCmdTest { + @Test + public void testParsingTags() { + final CreateSnapshotPolicyCmd createSnapshotPolicyCmd = new CreateSnapshotPolicyCmd(); + final Map tag1 = new HashMap<>(); + tag1.put("key", "key1"); + tag1.put("value", "value1"); + final Map tag2 = new HashMap<>(); + tag2.put("key", "key2"); + tag2.put("value", "value2"); + final Map expectedTags = new HashMap<>(); + expectedTags.put("key1", "value1"); + expectedTags.put("key2", "value2"); + + final Map> tagsParams = new HashMap<>(); + tagsParams.put("0", tag1); + tagsParams.put("1", tag2); + ReflectionTestUtils.setField(createSnapshotPolicyCmd, "tags", tagsParams); + Assert.assertEquals(createSnapshotPolicyCmd.getTags(), expectedTags); + } +} \ No newline at end of file diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 9bea30a32b3..4e9a2bc629f 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -552,7 +552,7 @@ public class ApiResponseHelper implements ResponseGenerator { ResourceTagResponse tagResponse = createResourceTagResponse(tag, true); CollectionUtils.addIgnoreNull(tagResponses, tagResponse); } - snapshotResponse.setTags(tagResponses); + snapshotResponse.setTags(new HashSet<>(tagResponses)); snapshotResponse.setObjectName("snapshot"); return snapshotResponse; @@ -654,6 +654,14 @@ public class ApiResponseHelper implements ResponseGenerator { policyResponse.setForDisplay(policy.isDisplay()); policyResponse.setObjectName("snapshotpolicy"); + List tags = _resourceTagDao.listBy(policy.getId(), ResourceObjectType.SnapshotPolicy); + List tagResponses = new ArrayList(); + for (ResourceTag tag : tags) { + ResourceTagResponse tagResponse = createResourceTagResponse(tag, false); + CollectionUtils.addIgnoreNull(tagResponses, tagResponse); + } + policyResponse.setTags(new HashSet<>(tagResponses)); + return policyResponse; } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 182379acbab..2022d5b5be1 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -80,6 +80,7 @@ import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -114,6 +115,8 @@ import com.cloud.hypervisor.HypervisorCapabilitiesVO; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.org.Grouping; import com.cloud.serializer.GsonHelper; +import com.cloud.server.ResourceTag; +import com.cloud.server.TaggedResourceService; import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.dao.DiskOfferingDao; @@ -256,6 +259,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic private StoragePoolTagsDao storagePoolTagsDao; @Inject private StorageUtil storageUtil; + @Inject + public TaggedResourceService taggedResourceService; protected Gson _gson; @@ -2343,7 +2348,16 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Override @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_CREATE, eventDescription = "taking snapshot", async = true) - public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup) + public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) + throws ResourceAllocationException { + final Snapshot snapshot = takeSnapshotInternal(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup); + if (snapshot != null && MapUtils.isNotEmpty(tags)) { + taggedResourceService.createTags(Collections.singletonList(snapshot.getUuid()), ResourceTag.ResourceObjectType.Snapshot, tags, null); + } + return snapshot; + } + + private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup) throws ResourceAllocationException { VolumeInfo volume = volFactory.getVolume(volumeId); if (volume == null) { diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index ee65486a127..5d98c503ab9 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -16,8 +16,53 @@ // under the License. package com.cloud.storage.snapshot; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; +import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; +import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + import com.cloud.agent.api.Answer; -import com.cloud.utils.db.GlobalLock; import com.cloud.agent.api.Command; import com.cloud.agent.api.DeleteSnapshotsDirCommand; import com.cloud.alert.AlertManager; @@ -42,6 +87,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.projects.Project.ListProjectResourcesCriteria; import com.cloud.resource.ResourceManager; import com.cloud.server.ResourceTag.ResourceObjectType; +import com.cloud.server.TaggedResourceService; import com.cloud.storage.CreateSnapshotPayload; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ScopeType; @@ -80,6 +126,7 @@ import com.cloud.utils.Ternary; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -92,48 +139,6 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; -import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; -import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; -import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; -import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; -import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; -import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; @Component public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implements SnapshotManager, SnapshotApiService, Configurable { @@ -192,6 +197,8 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement ResourceManager _resourceMgr; @Inject StorageStrategyFactory _storageStrategyFactory; + @Inject + public TaggedResourceService taggedResourceService; private int _totalRetries; private int _pauseInterval; @@ -902,6 +909,11 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement policy.setDisplay(display); _snapshotPolicyDao.update(policy.getId(), policy); _snapSchedMgr.scheduleOrCancelNextSnapshotJobOnDisplayChange(policy, previousDisplay); + taggedResourceService.deleteTags(Collections.singletonList(policy.getUuid()), ResourceObjectType.SnapshotPolicy, null); + } + final Map tags = cmd.getTags(); + if (MapUtils.isNotEmpty(tags)) { + taggedResourceService.createTags(Collections.singletonList(policy.getUuid()), ResourceObjectType.SnapshotPolicy, tags, null); } } finally { createSnapshotPolicyLock.unlock(); @@ -916,9 +928,10 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement } } - protected boolean deletePolicy(long userId, Long policyId) { + protected boolean deletePolicy(Long policyId) { SnapshotPolicyVO snapshotPolicy = _snapshotPolicyDao.findById(policyId); _snapSchedMgr.removeSchedule(snapshotPolicy.getVolumeId(), snapshotPolicy.getId()); + taggedResourceService.deleteTags(Collections.singletonList(snapshotPolicy.getUuid()), ResourceObjectType.SnapshotPolicy, null); return _snapshotPolicyDao.remove(policyId); } @@ -963,8 +976,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement public void deletePoliciesForVolume(Long volumeId) { List policyInstances = listPoliciesforVolume(volumeId); for (SnapshotPolicyVO policyInstance : policyInstances) { - Long policyId = policyInstance.getId(); - deletePolicy(1L, policyId); + deletePolicy(policyInstance.getId()); } // We also want to delete the manual snapshots scheduled for this volume // We can only delete the schedules in the future, not the ones which are already executing. @@ -1321,7 +1333,6 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement public boolean deleteSnapshotPolicies(DeleteSnapshotPoliciesCmd cmd) { Long policyId = cmd.getId(); List policyIds = cmd.getIds(); - Long userId = getSnapshotUserId(); if ((policyId == null) && (policyIds == null)) { throw new InvalidParameterValueException("No policy id (or list of ids) specified."); @@ -1335,6 +1346,10 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement throw new InvalidParameterValueException("There are no policy ids"); } + if (policyIds.contains(Snapshot.MANUAL_POLICY_ID)) { + throw new InvalidParameterValueException("Invalid Policy id given: " + Snapshot.MANUAL_POLICY_ID); + } + for (Long policy : policyIds) { SnapshotPolicyVO snapshotPolicyVO = _snapshotPolicyDao.findById(policy); if (snapshotPolicyVO == null) { @@ -1348,21 +1363,14 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, volume); } - boolean success = true; - - if (policyIds.contains(Snapshot.MANUAL_POLICY_ID)) { - throw new InvalidParameterValueException("Invalid Policy id given: " + Snapshot.MANUAL_POLICY_ID); - } - for (Long pId : policyIds) { - if (!deletePolicy(userId, pId)) { - success = false; + if (!deletePolicy(pId)) { s_logger.warn("Failed to delete snapshot policy with Id: " + policyId); - return success; + return false; } } - return success; + return true; } @Override diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java index dc71b365353..bccf8c68b23 100644 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java @@ -43,6 +43,8 @@ import com.cloud.api.ApiDispatcher; import com.cloud.api.ApiGsonHelper; import com.cloud.event.ActionEventUtils; import com.cloud.event.EventTypes; +import com.cloud.server.ResourceTag; +import com.cloud.server.TaggedResourceService; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotScheduleVO; @@ -97,6 +99,8 @@ public class SnapshotSchedulerImpl extends ManagerBase implements SnapshotSchedu protected VMSnapshotDao _vmSnapshotDao; @Inject protected VMSnapshotManager _vmSnaphostManager; + @Inject + public TaggedResourceService taggedResourceService; protected AsyncJobDispatcher _asyncDispatcher; @@ -305,6 +309,15 @@ public class SnapshotSchedulerImpl extends ManagerBase implements SnapshotSchedu params.put("ctxUserId", "1"); params.put("ctxAccountId", "" + volume.getAccountId()); params.put("ctxStartEventId", String.valueOf(eventId)); + List resourceTags = taggedResourceService.listByResourceTypeAndId(ResourceTag.ResourceObjectType.SnapshotPolicy, policyId); + if (resourceTags != null && !resourceTags.isEmpty()) { + int tagNumber = 0; + for (ResourceTag resourceTag : resourceTags) { + params.put("tags[" + tagNumber + "].key", resourceTag.getKey()); + params.put("tags[" + tagNumber + "].value", resourceTag.getValue()); + tagNumber++; + } + } final CreateSnapshotCmd cmd = new CreateSnapshotCmd(); ComponentContext.inject(cmd); diff --git a/server/src/main/java/com/cloud/tags/TaggedResourceManagerImpl.java b/server/src/main/java/com/cloud/tags/TaggedResourceManagerImpl.java index ae7949863b4..44876d0ddf5 100644 --- a/server/src/main/java/com/cloud/tags/TaggedResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/tags/TaggedResourceManagerImpl.java @@ -16,13 +16,31 @@ // under the License. package com.cloud.tags; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import javax.persistence.EntityExistsException; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + import com.cloud.dc.DataCenterVO; import com.cloud.domain.PartOf; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; -import com.cloud.offerings.NetworkOfferingVO; import com.cloud.network.LBHealthCheckPolicyVO; import com.cloud.network.as.AutoScaleVmGroupVO; import com.cloud.network.as.AutoScaleVmProfileVO; @@ -43,6 +61,7 @@ import com.cloud.network.vpc.NetworkACLVO; import com.cloud.network.vpc.StaticRouteVO; import com.cloud.network.vpc.VpcOfferingVO; import com.cloud.network.vpc.VpcVO; +import com.cloud.offerings.NetworkOfferingVO; import com.cloud.projects.ProjectVO; import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; @@ -74,24 +93,6 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.NicVO; import com.cloud.vm.UserVmVO; import com.cloud.vm.snapshot.VMSnapshotVO; -import org.apache.cloudstack.api.Identity; -import org.apache.cloudstack.api.InternalIdentity; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; - -import org.apache.commons.collections.MapUtils; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; -import javax.persistence.EntityExistsException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; public class TaggedResourceManagerImpl extends ManagerBase implements TaggedResourceService { public static final Logger s_logger = Logger.getLogger(TaggedResourceManagerImpl.class); @@ -220,6 +221,10 @@ public class TaggedResourceManagerImpl extends ManagerBase implements TaggedReso accountId = ((ProjectVO)entity).getProjectAccountId(); } + if (resourceType == ResourceObjectType.SnapshotPolicy) { + accountId = _entityMgr.findById(VolumeVO.class, ((SnapshotPolicyVO)entity).getVolumeId()).getAccountId(); + } + if (entity instanceof OwnedBy) { accountId = ((OwnedBy)entity).getAccountId(); } @@ -389,7 +394,7 @@ public class TaggedResourceManagerImpl extends ManagerBase implements TaggedReso } if (tagsToDelete.isEmpty()) { - throw new InvalidParameterValueException("Unable to find any tags which conform to specified delete parameters."); + return false; } //Remove the tags diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index bb5599f0a99..9d1426ce9f1 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -18,6 +18,7 @@ package com.cloud.storage; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doNothing; @@ -65,6 +66,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceType; @@ -76,6 +78,7 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.org.Grouping; import com.cloud.serializer.GsonHelper; +import com.cloud.server.TaggedResourceService; import com.cloud.storage.Volume.Type; import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.VolumeDao; @@ -409,7 +412,7 @@ public class VolumeApiServiceImplTest { when(volumeDataFactoryMock.getVolume(anyLong())).thenReturn(volumeInfoMock); when(volumeInfoMock.getState()).thenReturn(Volume.State.Allocated); when(volumeInfoMock.getPoolId()).thenReturn(1L); - volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false); + volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null); } @Test @@ -419,7 +422,10 @@ public class VolumeApiServiceImplTest { when(volumeInfoMock.getInstanceId()).thenReturn(null); when(volumeInfoMock.getPoolId()).thenReturn(1L); when(volumeServiceMock.takeSnapshot(Mockito.any(VolumeInfo.class))).thenReturn(snapshotInfoMock); - volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false); + final TaggedResourceService taggedResourceService = Mockito.mock(TaggedResourceService.class); + Mockito.when(taggedResourceService.createTags(anyObject(), anyObject(), anyObject(), anyObject())).thenReturn(null); + ReflectionTestUtils.setField(volumeApiServiceImpl, "taggedResourceService", taggedResourceService); + volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null); } @Test diff --git a/ui/css/cloudstack3.css b/ui/css/cloudstack3.css index 0d2aca67571..c3d8061857a 100644 --- a/ui/css/cloudstack3.css +++ b/ui/css/cloudstack3.css @@ -4027,7 +4027,7 @@ textarea { font-size: 14px; } -.ui-dialog div.form-container div.value label { +#label_delete_volumes label { display: block; width: 119px; margin-top: 2px; @@ -8447,6 +8447,10 @@ div#details-tab-aclRules td.cidrlist span { display: inline-block; } +.recurring-snapshots .schedule .forms .formContainer { + min-height: 250px; +} + .recurring-snapshots .schedule .add-snapshot-actions { float: left; clear: both; @@ -8510,6 +8514,11 @@ div#details-tab-aclRules td.cidrlist span { left: 5px; } +.recurring-snapshots .schedule .forms .tagger form div.value label { + width: 25px; + top: 10px; +} + .recurring-snapshots .schedule .forms form label.error { float: left; width: 100%; @@ -8522,6 +8531,10 @@ div#details-tab-aclRules td.cidrlist span { margin: 8px 0 0; } +.recurring-snapshots .schedule .forms .tagger form .field { + margin: 0; +} + .recurring-snapshots .schedule .forms form .name { float: left; width: 72px; @@ -8660,6 +8673,11 @@ div#details-tab-aclRules td.cidrlist span { padding: 0; } +.recurring-snapshots .ui-tabs .tagger ul { + margin: 16px auto auto; + padding-bottom: 10px; +} + .recurring-snapshots .ui-tabs ul li a { width: 76px; background: url("../images/sprites.png") no-repeat -521px -533px; @@ -10692,6 +10710,11 @@ div#details-tab-aclRules td.cidrlist span { display: none; } +.ui-dialog .tagger .tag-info.inside-form { + display: block; + text-align: left; +} + .ui-dialog.editTags .ui-button { float: right; } diff --git a/ui/index.html b/ui/index.html index 8645fc252c8..1dddbd3cdc8 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1553,7 +1553,7 @@ -
+
@@ -1583,11 +1583,14 @@
+ + +
-
+
@@ -1617,11 +1620,14 @@
+ + +
-
+
@@ -1659,11 +1665,14 @@
+ + +
-
+
@@ -1701,6 +1710,9 @@
+ + +
diff --git a/ui/scripts/storage.js b/ui/scripts/storage.js index 789af428c01..e6eaacf2f6c 100644 --- a/ui/scripts/storage.js +++ b/ui/scripts/storage.js @@ -921,6 +921,10 @@ asyncBackup: { label: 'label.async.backup', isBoolean: true + }, + tags: { + label: 'label.tags', + tagger: true } } }, @@ -935,6 +939,15 @@ name: args.data.name }); } + if (!$.isEmptyObject(args.data.tags)) { + $(args.data.tags).each(function(idx, tagData) { + var formattedTagData = {}; + formattedTagData["tags[" + _s(idx) + "].key"] = _s(tagData.key); + formattedTagData["tags[" + _s(idx) + "].value"] = _s(tagData.value); + $.extend(data, formattedTagData); + }); + } + $.ajax({ url: createURL("createSnapshot"), data: data, @@ -1000,7 +1013,9 @@ var snap = args.snapshot; var data = { - keep: snap.maxsnaps, + volumeid: args.context.volumes[0].id, + intervaltype: snap['snapshot-type'], + maxsnaps: snap.maxsnaps, timezone: snap.timezone }; @@ -1053,15 +1068,18 @@ break; } + if (!$.isEmptyObject(snap.tags)) { + $(snap.tags).each(function(idx, tagData) { + var formattedTagData = {}; + formattedTagData["tags[" + _s(idx) + "].key"] = _s(tagData.key); + formattedTagData["tags[" + _s(idx) + "].value"] = _s(tagData.value); + $.extend(data, formattedTagData); + }); + } + $.ajax({ url: createURL('createSnapshotPolicy'), - data: { - volumeid: args.context.volumes[0].id, - intervaltype: snap['snapshot-type'], - maxsnaps: snap.maxsnaps, - schedule: data.schedule, - timezone: snap.timezone - }, + data: data, dataType: 'json', async: true, success: function(successData) { diff --git a/ui/scripts/ui-custom/recurringSnapshots.js b/ui/scripts/ui-custom/recurringSnapshots.js index 6d0eefd4032..82b34893047 100644 --- a/ui/scripts/ui-custom/recurringSnapshots.js +++ b/ui/scripts/ui-custom/recurringSnapshots.js @@ -61,6 +61,10 @@ } }); + $($snapshots.find('.taggerContainer')).each(function() { + $('
').taggerInForm().appendTo(this); + }); + // Form validation $snapshots.find('form').validate(); @@ -70,7 +74,7 @@ if (!$form.valid()) return false; - var formData = cloudStack.serializeForm($form); + var formData = $.extend(cloudStack.serializeForm($form), {'tags' : cloudStack.getTagsFromForm($form)}); actions.add({ context: context, diff --git a/ui/scripts/ui/dialog.js b/ui/scripts/ui/dialog.js index 96f2298ff1a..de2709ef57f 100644 --- a/ui/scripts/ui/dialog.js +++ b/ui/scripts/ui/dialog.js @@ -149,7 +149,6 @@ var isAsync = false; var isNoDialog = args.noDialog ? args.noDialog : false; - $(fields).each(function(idx, element) { var key = this; var field = args.form.fields[key]; @@ -190,7 +189,6 @@ closeOnEscape: false }); */ // Label field - var $name = $('
').addClass('name') .appendTo($formItem) .append( @@ -619,9 +617,9 @@ } $input.addClass("disallowSpecialCharacters"); $input.datepicker({ - dateFormat: 'yy-mm-dd', - maxDate: field.maxDate, - minDate: field.minDate + dateFormat: 'yy-mm-dd', + maxDate: field.maxDate, + minDate: field.minDate }); } else if (field.range) { //2 text fields on the same line (e.g. port range: startPort - endPort) @@ -702,6 +700,10 @@ this.oldUnit = newUnit; }) + } else if (field.tagger) { + $name.hide(); + $value.hide(); + $input = $('
').taggerInForm().appendTo($formItem); } else { //text field $input = $('').attr({ name: key, @@ -717,10 +719,12 @@ $input.addClass("disallowSpecialCharacters"); } - if (field.validation != null) - $input.data('validation-rules', field.validation); - else - $input.data('validation-rules', {}); + if (!field.tagger) { // Tagger has it's own validation + if (field.validation != null) + $input.data('validation-rules', field.validation); + else + $input.data('validation-rules', {}); + } var fieldLabel = field.label; @@ -774,7 +778,7 @@ var complete = function($formContainer) { var $form = $formContainer.find('form'); - var data = cloudStack.serializeForm($form); + var data = $.extend(cloudStack.serializeForm($form), {'tags' : cloudStack.getTagsFromForm($form)}); if (!$formContainer.find('form').valid()) { // Ignore hidden field validation diff --git a/ui/scripts/ui/widgets/tagger.js b/ui/scripts/ui/widgets/tagger.js index a77d6c51830..7356dc055db 100644 --- a/ui/scripts/ui/widgets/tagger.js +++ b/ui/scripts/ui/widgets/tagger.js @@ -46,10 +46,7 @@ $keyField.append($keyLabel, $key); $valueField.append($valueLabel, $value); - $form.append( - $keyField, $valueField, - $submit - ); + $form.append($keyField, $valueField, $submit); $form.validate(); @@ -80,9 +77,11 @@ } }); - // Prevent input during submission - $key.attr('disabled', 'disabled'); - $value.attr('disabled', 'disabled'); + if (args.isAsyncSubmit) { + // Prevent input during submission + $key.attr('disabled', 'disabled'); + $value.attr('disabled', 'disabled'); + } return false; } : @@ -102,6 +101,8 @@ $label.append($key, '=', $value); $label.attr('title', title); + $label.attr('cloudstack_tag_key', _s(data.key)); + $label.attr('cloudstack_tag_value', _s(data.value)); $remove.click(function() { if (onRemove) onRemove($li, data); }); @@ -173,6 +174,7 @@ }; var $inputArea = elems.inputArea({ + isAsyncSubmit: true, onSubmit: function(args) { var data = args.data; var success = args.response.success; @@ -252,4 +254,53 @@ }); } }); + + $.widget('cloudStack.taggerInForm', { + _init: function(args) { + var $container = this.element.addClass('tagger'); + var $tagArea = $('
    ').addClass('tags'); + var $title = elems.info(_l('label.tags')).addClass('title inside-form'); + var $loading = $('
    ').addClass('loading-overlay'); + var $tags = {}; + + var onRemoveItem = function($item, data) { + $item.remove(); + if ($tags[data.key]) delete $tags[data.key]; + else { + cloudStack.dialog.notice({ + message: "Unexpected error occured in attempting deletion" + }); + } + }; + + var $inputArea = elems.inputArea({ + isAsyncSubmit: false, + onSubmit: function(args) { + var data = args.data; + if ($tags[data.key]) { + cloudStack.dialog.notice({ + message: "Key already present. Please delete previous and add again." + }); + } else { + var success = args.response.success; + var title = data.key + ' = ' + data.value; + elems.tagItem(title, onRemoveItem, data).appendTo($tagArea); + success(); + $tags[data.key] = data.value; + } + } + }); + + $container.append($title, $inputArea, $tagArea); + } + }); + + cloudStack.getTagsFromForm = function($form) { + var tagLabels = $($form).find('.tagger .tags .label'); + var tags = []; + $(tagLabels).each(function() { + tags.push({'key' : $(this).attr('cloudstack_tag_key'), 'value' : $(this).attr('cloudstack_tag_value')}); + }); + return tags; + }; }(jQuery, cloudStack)); From 95a509a1ed8f59e0d28ac6fd13e2b4112a999758 Mon Sep 17 00:00:00 2001 From: Anurag Awasthi Date: Thu, 27 Jun 2019 12:51:29 +0530 Subject: [PATCH 09/13] ui: Update config file with tables that users can see (#3429) This is a purely documentation change of config file. The cleanup was needed post merge of PR #3258 --- ui/config.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ui/config.js b/ui/config.js index f715360fd0d..27d8c39046a 100644 --- a/ui/config.js +++ b/ui/config.js @@ -29,10 +29,7 @@ cloudStackOptions = { "jp": "label.japanese.keyboard", "sc": "label.simplified.chinese.keyboard" }, - hiddenFields: { - "metrics.zones":[], // Options - "name", "state", "clusters", "cpuused", "cpuallocated", "memused", "memallocated" - "metrics.clusters": [], // Options - "name", "state", "hosts", "cpuused", "cpuallocated", "memused", "memallocated" - "metrics.hosts": [], // Options - "name", "state", "powerstate", "instances", "cpuused", "memused", "network" + hiddenFields: { // Fields to be hidden only for users in the tables below "metrics.storagepool": [], // Options - "name", "property", "disk", "metrics.instances": [], // Options - "name", "state", "ipaddress", "zonename", "cpuused", "memused", "network", "disk" "metrics.volumes": [] // Options - "name", "state", "vmname", "sizegb", "physicalsize", "utilization", "storagetype", "storage" From 23a8b74e4f29984cee2f8d4e5aaccfb7fd1ae382 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 27 Jun 2019 16:13:21 +0530 Subject: [PATCH 10/13] engine/schema: add guest-os support and mappings for XenServer 7.6 (#3427) This adds hypervisor capabilities and mappings for XenServer 7.6 Signed-off-by: Rohit Yadav --- .../src/main/resources/META-INF/db/schema-41200to41300.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql b/engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql index c0b0751a245..67b5f308041 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql @@ -23,6 +23,10 @@ INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '6.7', 128, 0, 13, 32, 1, 1); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'VMware', '6.7', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='VMware' AND hypervisor_version='6.5'; +-- XenServer 7.6 +INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, storage_motion_supported) values (UUID(), 'XenServer', '7.6.0', 500, 13, 1); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '7.6.0', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='7.5.0'; + -- DPDK client and server mode support ALTER TABLE `cloud`.`service_offering_details` CHANGE COLUMN `value` `value` TEXT NOT NULL; From d967993a0ab274e14ff7f214057d9b964a1b90a1 Mon Sep 17 00:00:00 2001 From: Anurag Awasthi Date: Thu, 27 Jun 2019 16:53:37 +0530 Subject: [PATCH 11/13] Remove additional line from config as storagepool isn't available for users --- ui/config.js | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/config.js b/ui/config.js index 27d8c39046a..0203eaaca8c 100644 --- a/ui/config.js +++ b/ui/config.js @@ -30,7 +30,6 @@ cloudStackOptions = { "sc": "label.simplified.chinese.keyboard" }, hiddenFields: { // Fields to be hidden only for users in the tables below - "metrics.storagepool": [], // Options - "name", "property", "disk", "metrics.instances": [], // Options - "name", "state", "ipaddress", "zonename", "cpuused", "memused", "network", "disk" "metrics.volumes": [] // Options - "name", "state", "vmname", "sizegb", "physicalsize", "utilization", "storagetype", "storage" } From 044e4e4b21372f498c5af14c5c2dfe7ff4113a0e Mon Sep 17 00:00:00 2001 From: Philipp Bankonier Date: Thu, 27 Jun 2019 15:32:40 +0200 Subject: [PATCH 12/13] ui: re-add custom css (#3431) This readds the custom.css file to overwrite the default cloudstack styles. Currently this is only possible by adding them to the custom.scss in the repository and compile them into the cloudstack3.css. Now there is an empty custom.css created when compiling the scss source. This custom.css can be overwritten with the custom styles afterwards. This functionality was broken by #3328. --- ui/css/custom.css | 23 +++++++++++++++++++++++ ui/css/src/scss/cloudstack3.scss | 2 -- ui/css/src/scss/custom.scss | 21 +++++++++++++++++++++ ui/css/src/scss/custom/custom.scss | 21 --------------------- ui/index.html | 1 + 5 files changed, 45 insertions(+), 23 deletions(-) create mode 100644 ui/css/custom.css create mode 100644 ui/css/src/scss/custom.scss delete mode 100644 ui/css/src/scss/custom/custom.scss diff --git a/ui/css/custom.css b/ui/css/custom.css new file mode 100644 index 00000000000..544d2ebbf85 --- /dev/null +++ b/ui/css/custom.css @@ -0,0 +1,23 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +* +* Use custom.css to override the default CloudStack styles +*/ + +/*# sourceMappingURL=src/sourcemaps/custom.css.map */ diff --git a/ui/css/src/scss/cloudstack3.scss b/ui/css/src/scss/cloudstack3.scss index 0d523cad937..fc6c367d36a 100644 --- a/ui/css/src/scss/cloudstack3.scss +++ b/ui/css/src/scss/cloudstack3.scss @@ -88,5 +88,3 @@ @import 'components/button-export'; @import 'components/jquery-ui'; @import 'components/token-input-facebook'; - -@import 'custom/custom'; diff --git a/ui/css/src/scss/custom.scss b/ui/css/src/scss/custom.scss new file mode 100644 index 00000000000..021362338d8 --- /dev/null +++ b/ui/css/src/scss/custom.scss @@ -0,0 +1,21 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +* +* Use custom.css to override the default CloudStack styles +*/ diff --git a/ui/css/src/scss/custom/custom.scss b/ui/css/src/scss/custom/custom.scss deleted file mode 100644 index 57ccdb3457e..00000000000 --- a/ui/css/src/scss/custom/custom.scss +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//* Use custom.scss to override the default CloudStack styles -//* Use it as your global @import file for many scss files. -//* Use lines like @import "database.scss"; inside here only. -//* Custom scss is imported on bottom of cloudstack3.scss diff --git a/ui/index.html b/ui/index.html index 1dddbd3cdc8..23fac1b7dd9 100644 --- a/ui/index.html +++ b/ui/index.html @@ -26,6 +26,7 @@ + From 541d280e3b84d7205faa4e0e3caae6d45098f3b3 Mon Sep 17 00:00:00 2001 From: Anurag Awasthi Date: Fri, 28 Jun 2019 17:05:25 +0530 Subject: [PATCH 13/13] ui: Bug fix for distinguishing between string and map type tags in forms (#3441) Some APIs consume 'tags' param as string and some consume as maps. Since each API can have at most one 'tags' param the extraction of map based tags should only happen when strings based tags are not extracted from the form serialization. --- ui/scripts/ui/dialog.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/scripts/ui/dialog.js b/ui/scripts/ui/dialog.js index de2709ef57f..d014ac99965 100644 --- a/ui/scripts/ui/dialog.js +++ b/ui/scripts/ui/dialog.js @@ -778,7 +778,14 @@ var complete = function($formContainer) { var $form = $formContainer.find('form'); - var data = $.extend(cloudStack.serializeForm($form), {'tags' : cloudStack.getTagsFromForm($form)}); + var data = cloudStack.serializeForm($form); + if (!data.tags) { + // Some APIs consume tags as a string (such as disk offering creation). + // The UI of those use a tagger that is not a custom cloudStack.tagger + // but rather a string. That case is handled by usual serialization. We + // only need to check extract tags when the string tags are not present. + $.extend(data, {'tags' : cloudStack.getTagsFromForm($form)}); + } if (!$formContainer.find('form').valid()) { // Ignore hidden field validation