From 8230f04a79097da90a8dc1accd4f4a840be102a2 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 6 Nov 2025 11:13:53 +0100 Subject: [PATCH 01/17] CKS: update cloud.kubernetes.cluster.network.offering to dynamic (#11847) --- .../com/cloud/kubernetes/cluster/KubernetesClusterService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 0c2338465de..ee7dc6a7fe2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -56,7 +56,7 @@ public interface KubernetesClusterService extends PluggableService, Configurable "cloud.kubernetes.cluster.network.offering", "DefaultNetworkOfferingforKubernetesService", "Name of the network offering that will be used to create isolated network in which Kubernetes cluster VMs will be launched", - false, + true, KubernetesServiceEnabled.key()); static final ConfigKey KubernetesClusterStartTimeout = new ConfigKey("Advanced", Long.class, "cloud.kubernetes.cluster.start.timeout", From b8ec941ec12b23ad0d1eb0561c6d449a6bfd088b Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 7 Nov 2025 14:01:11 +0530 Subject: [PATCH 02/17] uefi property typo (#11929) --- .../kvm/resource/LibvirtComputingResource.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 0aa094e56d9..aaa5c0c20f4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3281,25 +3281,25 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv setGuestLoader(bootMode, SECURE, guest, GuestDef.GUEST_LOADER_SECURE); setGuestLoader(bootMode, LEGACY, guest, GuestDef.GUEST_LOADER_LEGACY); - if (isUefiPropertieNotNull(GuestDef.GUEST_NVRAM_PATH)) { + if (isUefiPropertyNotNull(GuestDef.GUEST_NVRAM_PATH)) { guest.setNvram(uefiProperties.getProperty(GuestDef.GUEST_NVRAM_PATH)); } - if (isSecureBoot && isUefiPropertieNotNull(GuestDef.GUEST_NVRAM_TEMPLATE_SECURE) && SECURE.equalsIgnoreCase(bootMode)) { + if (isSecureBoot && isUefiPropertyNotNull(GuestDef.GUEST_NVRAM_TEMPLATE_SECURE) && SECURE.equalsIgnoreCase(bootMode)) { guest.setNvramTemplate(uefiProperties.getProperty(GuestDef.GUEST_NVRAM_TEMPLATE_SECURE)); - } else if (isUefiPropertieNotNull(GuestDef.GUEST_NVRAM_TEMPLATE_LEGACY)) { + } else if (isUefiPropertyNotNull(GuestDef.GUEST_NVRAM_TEMPLATE_LEGACY)) { guest.setNvramTemplate(uefiProperties.getProperty(GuestDef.GUEST_NVRAM_TEMPLATE_LEGACY)); } } - private void setGuestLoader(String bootMode, String mode, GuestDef guest, String propertie) { - if (isUefiPropertieNotNull(propertie) && mode.equalsIgnoreCase(bootMode)) { - guest.setLoader(uefiProperties.getProperty(propertie)); + private void setGuestLoader(String bootMode, String mode, GuestDef guest, String property) { + if (isUefiPropertyNotNull(property) && mode.equalsIgnoreCase(bootMode)) { + guest.setLoader(uefiProperties.getProperty(property)); } } - private boolean isUefiPropertieNotNull(String propertie) { - return uefiProperties.getProperty(propertie) != null; + private boolean isUefiPropertyNotNull(String property) { + return uefiProperties.getProperty(property) != null; } public boolean isGuestAarch64() { From 9c0efb707235ae906236620d0f132cadbfe25642 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 7 Nov 2025 14:07:11 +0530 Subject: [PATCH 03/17] DB setup: support db schema creation (with --schema-only) without force recreate option (#12004) --- setup/bindir/cloud-setup-databases.in | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/bindir/cloud-setup-databases.in b/setup/bindir/cloud-setup-databases.in index 8c453edda44..eb68c1e0181 100755 --- a/setup/bindir/cloud-setup-databases.in +++ b/setup/bindir/cloud-setup-databases.in @@ -199,6 +199,10 @@ for full help self.info("No mysql root user specified, will not create Cloud DB schema\n", None) return + if self.areCloudDatabasesCreated() and not self.options.schemaonly and not self.options.forcerecreate: + self.errorAndExit("Aborting script as the databases (cloud, cloud_usage) already exist.\n" \ + "Please use the --force-recreate parameter if you want to recreate the databases and schemas, or use --schema-only if you only want to create the schemas only.") + replacements = ( ("CREATE USER cloud identified by 'cloud';", "CREATE USER %s@`localhost` identified by '%s'; CREATE USER %s@`%%` identified by '%s';"%( @@ -239,10 +243,6 @@ for full help ("DROP USER 'cloud'@'%' ;", "DO NULL;") ) - if self.areCloudDatabasesCreated() and not self.options.forcerecreate: - self.errorAndExit("Aborting script as the databases (cloud, cloud_usage) already exist.\n" \ - "Please use the --force-recreate parameter if you want to recreate the schemas.") - scriptsToRun = ["create-database","create-schema", "create-database-premium","create-schema-premium"] if self.options.schemaonly: scriptsToRun = ["create-schema", "create-schema-premium"] @@ -617,11 +617,11 @@ for example: self.parser.add_option("-d", "--deploy-as", action="store", type="string", dest="rootcreds", default="", help="Colon-separated user name and password of a MySQL user with administrative privileges") self.parser.add_option("-s", "--schema-only", action="store_true", dest="schemaonly", default=False, - help="Creates the db schema without having to pass root credentials - " \ + help="Creates the db schema only without having to pass root credentials - " \ "Please note: The databases (cloud, cloud_usage) and user (cloud) has to be configured " \ "manually prior to running this script when using this flag.") self.parser.add_option("--force-recreate", action="store_true", dest="forcerecreate", default=False, - help="Force recreation of the existing DB schemas. This option is disabled by default." \ + help="Force recreation of the existing DB databases and schemas. This option is disabled by default." \ "Please note: The databases (cloud, cloud_usage) and its tables data will be lost and recreated.") self.parser.add_option("-a", "--auto", action="store", type="string", dest="serversetup", default="", From c5c3cc40c1e27ce80b764d94e311be7076960ba2 Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Fri, 7 Nov 2025 15:13:46 +0530 Subject: [PATCH 04/17] consider Instance in Starting state for listPodsByUserConcentration (#11845) --- engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java index cc8b9fc59a8..088a2b60b58 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java @@ -101,7 +101,7 @@ public class UserVmDaoImpl extends GenericDaoBase implements Use ReservationDao reservationDao; private static final String LIST_PODS_HAVING_VMS_FOR_ACCOUNT = - "SELECT pod_id FROM cloud.vm_instance WHERE data_center_id = ? AND account_id = ? AND pod_id IS NOT NULL AND (state = 'Running' OR state = 'Stopped') " + "SELECT pod_id FROM cloud.vm_instance WHERE data_center_id = ? AND account_id = ? AND pod_id IS NOT NULL AND state IN ('Starting', 'Running', 'Stopped') " + "GROUP BY pod_id HAVING count(id) > 0 ORDER BY count(id) DESC"; private static final String VM_DETAILS = "select vm_instance.id, " From 2954e9694752a27837c95e09047dc3b08135a449 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 7 Nov 2025 11:55:27 +0100 Subject: [PATCH 05/17] Veeam: get templateId from vm instance if vm is created from ISO (#10705) --- .../main/java/com/cloud/hypervisor/guru/VMwareGuru.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index c02513f4889..efe09194467 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -694,7 +694,12 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co updateTemplateRef(templateId, poolId, templatePath, templateSize); return templateId; } else { - return volumeVO.getTemplateId(); + Long templateId = volumeVO.getTemplateId(); + if (templateId == null && volumeVO.getInstanceId() != null) { + VMInstanceVO vmInstanceVO = vmDao.findByIdIncludingRemoved(volumeVO.getInstanceId()); + return vmInstanceVO.getTemplateId(); + } + return templateId; } } } From 2dd1e6d786f362adfffc7f45c49642d8e44184dd Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 7 Nov 2025 19:24:02 +0530 Subject: [PATCH 06/17] Enable UEFI on KVM hosts (by default), and configure with some default settings (#11740) --- agent/conf/uefi.properties.in | 24 +++++++++++++++++++ debian/cloudstack-agent.install | 1 + debian/cloudstack-agent.postinst | 2 +- debian/control | 2 +- packaging/debian/replace.properties | 5 ++++ packaging/el8/cloud.spec | 13 +++++++++- packaging/el8/replace.properties | 5 ++++ pom.xml | 4 ++++ .../cloud/server/ManagementServerImpl.java | 5 +++- systemvm/systemvm-agent-descriptor.xml | 1 + 10 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 agent/conf/uefi.properties.in diff --git a/agent/conf/uefi.properties.in b/agent/conf/uefi.properties.in new file mode 100644 index 00000000000..3c8866f634b --- /dev/null +++ b/agent/conf/uefi.properties.in @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Configuration file for UEFI + +guest.nvram.template.legacy=@GUESTNVRAMTEMPLATELEGACY@ +guest.loader.legacy=@GUESTLOADERLEGACY@ +guest.nvram.template.secure=@GUESTNVRAMTEMPLATESECURE@ +guest.loader.secure=@GUESTLOADERSECURE@ +guest.nvram.path=@GUESTNVRAMPATH@ diff --git a/debian/cloudstack-agent.install b/debian/cloudstack-agent.install index 58715e0746b..0b9e874cb42 100644 --- a/debian/cloudstack-agent.install +++ b/debian/cloudstack-agent.install @@ -16,6 +16,7 @@ # under the License. /etc/cloudstack/agent/agent.properties +/etc/cloudstack/agent/uefi.properties /etc/cloudstack/agent/environment.properties /etc/cloudstack/agent/log4j-cloud.xml /etc/default/cloudstack-agent diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index 758af6e068f..cd070c2f785 100755 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -23,7 +23,7 @@ case "$1" in configure) OLDCONFDIR="/etc/cloud/agent" NEWCONFDIR="/etc/cloudstack/agent" - CONFFILES="agent.properties log4j.xml log4j-cloud.xml" + CONFFILES="agent.properties uefi.properties log4j.xml log4j-cloud.xml" mkdir -m 0755 -p /usr/share/cloudstack-agent/tmp diff --git a/debian/control b/debian/control index 1292639ef30..78842e38ed2 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/packaging/debian/replace.properties b/packaging/debian/replace.properties index 5ea4a03b275..bd0c1488959 100644 --- a/packaging/debian/replace.properties +++ b/packaging/debian/replace.properties @@ -59,3 +59,8 @@ USAGELOG=/var/log/cloudstack/usage/usage.log USAGESYSCONFDIR=/etc/cloudstack/usage PACKAGE=cloudstack EXTENSIONSDEPLOYMENTMODE=production +GUESTNVRAMTEMPLATELEGACY=/usr/share/OVMF/OVMF_VARS_4M.fd +GUESTLOADERLEGACY=/usr/share/OVMF/OVMF_CODE_4M.fd +GUESTNVRAMTEMPLATESECURE=/usr/share/OVMF/OVMF_VARS_4M.ms.fd +GUESTLOADERSECURE=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd +GUESTNVRAMPATH=/var/lib/libvirt/qemu/nvram/ diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 7e97957473c..abfab23f705 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -115,6 +115,8 @@ Requires: ipset Requires: perl Requires: rsync Requires: cifs-utils +Requires: edk2-ovmf +Requires: swtpm Requires: (python3-libvirt or python3-libvirt-python) Requires: (qemu-img or qemu-tools) Requires: qemu-kvm @@ -356,6 +358,7 @@ install -D packaging/systemd/cloudstack-agent.service ${RPM_BUILD_ROOT}%{_unitdi install -D packaging/systemd/cloudstack-rolling-maintenance@.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-rolling-maintenance@.service install -D packaging/systemd/cloudstack-agent.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-agent install -D agent/target/transformed/agent.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/agent.properties +install -D agent/target/transformed/uefi.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/uefi.properties install -D agent/target/transformed/environment.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/environment.properties install -D agent/target/transformed/log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/log4j-cloud.xml install -D agent/target/transformed/cloud-setup-agent ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-agent @@ -523,7 +526,7 @@ mkdir -m 0755 -p /usr/share/cloudstack-agent/tmp /usr/bin/systemctl enable cloudstack-rolling-maintenance@p > /dev/null 2>&1 || true /usr/bin/systemctl enable --now rngd > /dev/null 2>&1 || true -# if saved configs from upgrade exist, copy them over +# if saved agent.properties from upgrade exist, copy them over if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/%{name}/agent/agent.properties %{_sysconfdir}/%{name}/agent/agent.properties.rpmnew cp -p %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/%{name}/agent @@ -531,6 +534,14 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi +# if saved uefi.properties from upgrade exist, copy them over +if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/uefi.properties" ]; then + mv %{_sysconfdir}/%{name}/agent/uefi.properties %{_sysconfdir}/%{name}/agent/uefi.properties.rpmnew + cp -p %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties %{_sysconfdir}/%{name}/agent + # make sure we only do this on the first install of this RPM, don't want to overwrite on a reinstall + mv %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties.rpmsave +fi + systemctl daemon-reload # Print help message diff --git a/packaging/el8/replace.properties b/packaging/el8/replace.properties index a6094b59c73..a5afab94ff2 100644 --- a/packaging/el8/replace.properties +++ b/packaging/el8/replace.properties @@ -58,3 +58,8 @@ USAGECLASSPATH= USAGELOG=/var/log/cloudstack/usage/usage.log USAGESYSCONFDIR=/etc/sysconfig EXTENSIONSDEPLOYMENTMODE=production +GUESTNVRAMTEMPLATELEGACY=/usr/share/edk2/ovmf/OVMF_VARS.fd +GUESTLOADERLEGACY=/usr/share/edk2/ovmf/OVMF_CODE.cc.fd +GUESTNVRAMTEMPLATESECURE=/usr/share/edk2/ovmf/OVMF_VARS.secboot.fd +GUESTLOADERSECURE=/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd +GUESTNVRAMPATH=/var/lib/libvirt/qemu/nvram/ diff --git a/pom.xml b/pom.xml index cc2cb759b1c..97d0d2645da 100644 --- a/pom.xml +++ b/pom.xml @@ -1038,15 +1038,19 @@ dist/console-proxy/js/jquery.js engine/schema/dist/** plugins/hypervisors/hyperv/conf/agent.properties + plugins/hypervisors/hyperv/conf/uefi.properties plugins/hypervisors/hyperv/DotNet/ServerResource/** scripts/installer/windows/acs_license.rtf scripts/vm/systemvm/id_rsa.cloud services/console-proxy/server/conf/agent.properties + services/console-proxy/server/conf/uefi.properties services/console-proxy/server/conf/environment.properties services/console-proxy/server/js/jquery.js services/secondary-storage/conf/agent.properties + services/secondary-storage/conf/uefi.properties services/secondary-storage/conf/environment.properties systemvm/agent/conf/agent.properties + systemvm/agent/conf/uefi.properties systemvm/agent/conf/environment.properties systemvm/agent/js/jquery.js systemvm/agent/js/jquery.flot.navigate.js diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 3f811c152f0..9e8fdb60694 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -1410,7 +1410,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe if (vmInstanceDetailVO != null && (ApiConstants.BootMode.LEGACY.toString().equalsIgnoreCase(vmInstanceDetailVO.getValue()) || ApiConstants.BootMode.SECURE.toString().equalsIgnoreCase(vmInstanceDetailVO.getValue()))) { - logger.info(" Live Migration of UEFI enabled VM : " + vm.getInstanceName() + " is not supported"); + logger.debug("{} VM is UEFI enabled, Checking for other UEFI enabled hosts as it can be live migrated to UEFI enabled host only.", vm.getInstanceName()); if (CollectionUtils.isEmpty(filteredHosts)) { filteredHosts = new ArrayList<>(allHosts); } @@ -1420,6 +1420,9 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe return new Pair<>(false, null); } filteredHosts.removeIf(host -> !uefiEnabledHosts.contains(host.getId())); + if (filteredHosts.isEmpty()) { + logger.warn("No UEFI enabled hosts are available for the live migration of VM {}", vm.getInstanceName()); + } return new Pair<>(!filteredHosts.isEmpty(), filteredHosts); } return new Pair<>(true, filteredHosts); diff --git a/systemvm/systemvm-agent-descriptor.xml b/systemvm/systemvm-agent-descriptor.xml index 8cf40a16276..1d6e338eb23 100644 --- a/systemvm/systemvm-agent-descriptor.xml +++ b/systemvm/systemvm-agent-descriptor.xml @@ -60,6 +60,7 @@ log4j-cloud.xml consoleproxy.properties agent.properties + uefi.properties From d26122bf22bb1a7e91cf8a3987157e266d89049d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 7 Nov 2025 16:13:10 +0100 Subject: [PATCH 07/17] Veeam: use pre-defined object mapper (#10715) --- .../cloudstack/backup/veeam/VeeamClient.java | 45 +++++++------------ .../backup/veeam/VeeamClientTest.java | 4 +- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index d911736090c..0a8b18fd65e 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -107,7 +107,8 @@ public class VeeamClient { private static final String REPOSITORY_REFERENCE = "RepositoryReference"; private static final String RESTORE_POINT_REFERENCE = "RestorePointReference"; private static final String BACKUP_FILE_REFERENCE = "BackupFileReference"; - private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + private static final ObjectMapper OBJECT_MAPPER = new XmlMapper(); private String veeamServerIp; @@ -127,6 +128,8 @@ public class VeeamClient { this.taskPollInterval = taskPollInterval; this.taskPollMaxRetry = taskPollMaxRetry; + OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + final RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) .setConnectionRequestTimeout(timeout * 1000) @@ -236,8 +239,7 @@ public class VeeamClient { private HttpResponse post(final String path, final Object obj) throws IOException { String xml = null; if (obj != null) { - XmlMapper xmlMapper = new XmlMapper(); - xml = xmlMapper.writer() + xml = OBJECT_MAPPER.writer() .with(ToXmlGenerator.Feature.WRITE_XML_DECLARATION) .writeValueAsString(obj); // Remove invalid/empty xmlns @@ -280,8 +282,7 @@ public class VeeamClient { try { final HttpResponse response = get("/hierarchyRoots"); checkResponseOK(response); - final ObjectMapper objectMapper = new XmlMapper(); - final EntityReferences references = objectMapper.readValue(response.getEntity().getContent(), EntityReferences.class); + final EntityReferences references = OBJECT_MAPPER.readValue(response.getEntity().getContent(), EntityReferences.class); for (final Ref ref : references.getRefs()) { if (ref.getName().equals(vmwareDcName) && ref.getType().equals(HIERARCHY_ROOT_REFERENCE)) { return ref.getUid(); @@ -300,8 +301,7 @@ public class VeeamClient { try { final HttpResponse response = get(String.format("/lookup?host=%s&type=Vm&name=%s", hierarchyId, vmName)); checkResponseOK(response); - final ObjectMapper objectMapper = new XmlMapper(); - final HierarchyItems items = objectMapper.readValue(response.getEntity().getContent(), HierarchyItems.class); + final HierarchyItems items = OBJECT_MAPPER.readValue(response.getEntity().getContent(), HierarchyItems.class); if (items == null || items.getItems() == null || items.getItems().isEmpty()) { throw new CloudRuntimeException("Could not find VM " + vmName + " in Veeam, please ask administrator to check Veeam B&R manager"); } @@ -319,14 +319,12 @@ public class VeeamClient { private Task parseTaskResponse(HttpResponse response) throws IOException { checkResponseOK(response); - final ObjectMapper objectMapper = new XmlMapper(); - return objectMapper.readValue(response.getEntity().getContent(), Task.class); + return OBJECT_MAPPER.readValue(response.getEntity().getContent(), Task.class); } protected RestoreSession parseRestoreSessionResponse(HttpResponse response) throws IOException { checkResponseOK(response); - final ObjectMapper objectMapper = new XmlMapper(); - return objectMapper.readValue(response.getEntity().getContent(), RestoreSession.class); + return OBJECT_MAPPER.readValue(response.getEntity().getContent(), RestoreSession.class); } private boolean checkTaskStatus(final HttpResponse response) throws IOException { @@ -413,8 +411,7 @@ public class VeeamClient { String repositoryName = getRepositoryNameFromJob(backupName); final HttpResponse response = get(String.format("/backupServers/%s/repositories", backupServerId)); checkResponseOK(response); - final ObjectMapper objectMapper = new XmlMapper(); - final EntityReferences references = objectMapper.readValue(response.getEntity().getContent(), EntityReferences.class); + final EntityReferences references = OBJECT_MAPPER.readValue(response.getEntity().getContent(), EntityReferences.class); for (final Ref ref : references.getRefs()) { if (ref.getType().equals(REPOSITORY_REFERENCE) && ref.getName().equals(repositoryName)) { return ref; @@ -450,8 +447,7 @@ public class VeeamClient { try { final HttpResponse response = get("/backups"); checkResponseOK(response); - final ObjectMapper objectMapper = new XmlMapper(); - final EntityReferences entityReferences = objectMapper.readValue(response.getEntity().getContent(), EntityReferences.class); + final EntityReferences entityReferences = OBJECT_MAPPER.readValue(response.getEntity().getContent(), EntityReferences.class); for (final Ref ref : entityReferences.getRefs()) { logger.debug("Veeam Backup found, name: " + ref.getName() + ", uid: " + ref.getUid() + ", type: " + ref.getType()); } @@ -466,8 +462,7 @@ public class VeeamClient { try { final HttpResponse response = get("/jobs"); checkResponseOK(response); - final ObjectMapper objectMapper = new XmlMapper(); - final EntityReferences entityReferences = objectMapper.readValue(response.getEntity().getContent(), EntityReferences.class); + final EntityReferences entityReferences = OBJECT_MAPPER.readValue(response.getEntity().getContent(), EntityReferences.class); final List policies = new ArrayList<>(); if (entityReferences == null || entityReferences.getRefs() == null) { return policies; @@ -489,9 +484,7 @@ public class VeeamClient { final HttpResponse response = get(String.format("/jobs/%s?format=Entity", jobId.replace("urn:veeam:Job:", ""))); checkResponseOK(response); - final ObjectMapper objectMapper = new XmlMapper(); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - return objectMapper.readValue(response.getEntity().getContent(), Job.class); + return OBJECT_MAPPER.readValue(response.getEntity().getContent(), Job.class); } catch (final IOException e) { logger.error("Failed to list Veeam jobs due to:", e); checkResponseTimeOut(e); @@ -571,9 +564,7 @@ public class VeeamClient { final String veeamVmRefId = lookupVM(hierarchyId, vmwareInstanceName); final HttpResponse response = get(String.format("/jobs/%s/includes", jobId)); checkResponseOK(response); - final ObjectMapper objectMapper = new XmlMapper(); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - final ObjectsInJob jobObjects = objectMapper.readValue(response.getEntity().getContent(), ObjectsInJob.class); + final ObjectsInJob jobObjects = OBJECT_MAPPER.readValue(response.getEntity().getContent(), ObjectsInJob.class); if (jobObjects == null || jobObjects.getObjects() == null) { logger.warn("No objects found in the Veeam job " + jobId); return false; @@ -715,8 +706,7 @@ public class VeeamClient { protected Map processHttpResponseForBackupMetrics(final InputStream content) { Map metrics = new HashMap<>(); try { - final ObjectMapper objectMapper = new XmlMapper(); - final BackupFiles backupFiles = objectMapper.readValue(content, BackupFiles.class); + final BackupFiles backupFiles = OBJECT_MAPPER.readValue(content, BackupFiles.class); if (backupFiles == null || CollectionUtils.isEmpty(backupFiles.getBackupFiles())) { throw new CloudRuntimeException("Could not get backup metrics via Veeam B&R API"); } @@ -885,8 +875,7 @@ public class VeeamClient { public List processHttpResponseForVmRestorePoints(InputStream content, String vmInternalName) { List vmRestorePointList = new ArrayList<>(); try { - final ObjectMapper objectMapper = new XmlMapper(); - final VmRestorePoints vmRestorePoints = objectMapper.readValue(content, VmRestorePoints.class); + final VmRestorePoints vmRestorePoints = OBJECT_MAPPER.readValue(content, VmRestorePoints.class); if (vmRestorePoints == null) { throw new CloudRuntimeException("Could not get VM restore points via Veeam B&R API"); } @@ -922,7 +911,7 @@ public class VeeamClient { } private Date formatDate(String date) throws ParseException { - return dateFormat.parse(StringUtils.substring(date, 0, 19)); + return DATE_FORMAT.parse(StringUtils.substring(date, 0, 19)); } public Pair restoreVMToDifferentLocation(String restorePointId, String hostIp, String dataStoreUuid) { diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index 63d6896bb85..f972399d2b9 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -475,7 +475,9 @@ public class VeeamClientTest { " xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\"\n" + " xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" + " xmlns=\"http://www.veeam.com/ent/v1.0\">\n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + From 15439ede7d463f13c877993c5e42060cf7589556 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 11 Nov 2025 03:29:54 -0500 Subject: [PATCH 08/17] UI: Update and reset domain level configuration (#11571) --- ui/src/components/view/SettingsTab.vue | 3 ++- ui/src/views/setting/ConfigurationTable.vue | 6 +++++- ui/src/views/setting/ConfigurationValue.vue | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/ui/src/components/view/SettingsTab.vue b/ui/src/components/view/SettingsTab.vue index be55d03c4b9..ab75bef8394 100644 --- a/ui/src/components/view/SettingsTab.vue +++ b/ui/src/components/view/SettingsTab.vue @@ -25,7 +25,8 @@ @search="handleSearch" /> + :config="items" + :resource="resource" /> diff --git a/ui/src/views/setting/ConfigurationTable.vue b/ui/src/views/setting/ConfigurationTable.vue index 56518d2570b..da05b9342a0 100644 --- a/ui/src/views/setting/ConfigurationTable.vue +++ b/ui/src/views/setting/ConfigurationTable.vue @@ -32,7 +32,7 @@ {{record.displaytext }} {{ ' (' + record.name + ')' }}
{{ record.description }} @@ -85,6 +85,10 @@ export default { pagesize: { type: Number, default: 20 + }, + resource: { + type: Object, + required: false } }, data () { diff --git a/ui/src/views/setting/ConfigurationValue.vue b/ui/src/views/setting/ConfigurationValue.vue index e438f0eb831..662e5ef142e 100644 --- a/ui/src/views/setting/ConfigurationValue.vue +++ b/ui/src/views/setting/ConfigurationValue.vue @@ -217,6 +217,10 @@ export default { actions: { type: Array, default: () => [] + }, + resource: { + type: Object, + required: false } }, data () { @@ -254,6 +258,12 @@ export default { this.setConfigData() }, watch: { + configrecord: { + handler () { + this.setConfigData() + }, + deep: true + } }, methods: { setConfigData () { @@ -280,6 +290,9 @@ export default { name: configrecord.name, value: newValue } + if (this.scopeKey === 'domainid' && !params[this.scopeKey]) { + params[this.scopeKey] = this.resource?.id + } postAPI('updateConfiguration', params).then(json => { this.editableValue = this.getEditableValue(json.updateconfigurationresponse.configuration) this.actualValue = this.editableValue @@ -315,6 +328,9 @@ export default { [this.scopeKey]: this.$route.params?.id, name: configrecord.name } + if (this.scopeKey === 'domainid' && !params[this.scopeKey]) { + params[this.scopeKey] = this.resource?.id + } postAPI('resetConfiguration', params).then(json => { this.editableValue = this.getEditableValue(json.resetconfigurationresponse.configuration) this.actualValue = this.editableValue From 81787b310eea6a8210b936e2e7a880144c7c4902 Mon Sep 17 00:00:00 2001 From: YoulongChen <30854794+YLChen-007@users.noreply.github.com> Date: Wed, 12 Nov 2025 15:36:19 +0800 Subject: [PATCH 09/17] fix API Request Parameters Logged Credential Masking in ApiServer (#12020) --- .../main/java/com/cloud/api/ApiServer.java | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/com/cloud/api/ApiServer.java b/server/src/main/java/com/cloud/api/ApiServer.java index 5e962cdb382..85d58ec0d53 100644 --- a/server/src/main/java/com/cloud/api/ApiServer.java +++ b/server/src/main/java/com/cloud/api/ApiServer.java @@ -39,6 +39,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Arrays; import java.util.Map; import java.util.Set; import java.util.TimeZone; @@ -244,6 +245,12 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer @Inject private MessageBus messageBus; + private static final Set sensitiveFields = new HashSet<>(Arrays.asList( + "password", "secretkey", "apikey", "token", + "sessionkey", "accesskey", "signature", + "authorization", "credential", "secret" + )); + private static final ConfigKey IntegrationAPIPort = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED , Integer.class , "integration.api.port" @@ -610,10 +617,23 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer logger.error("invalid request, no command sent"); if (logger.isTraceEnabled()) { logger.trace("dumping request parameters"); - for (final Object key : params.keySet()) { - final String keyStr = (String)key; - final String[] value = (String[])params.get(key); - logger.trace(" key: " + keyStr + ", value: " + ((value == null) ? "'null'" : value[0])); + + for (final Object key : params.keySet()) { + final String keyStr = (String) key; + final String[] value = (String[]) params.get(key); + + String lowerKeyStr = keyStr.toLowerCase(); + boolean isSensitive = sensitiveFields.stream() + .anyMatch(lowerKeyStr::contains); + + String logValue; + if (isSensitive) { + logValue = "******"; // mask sensitive values + } else { + logValue = (value == null) ? "'null'" : value[0]; + } + + logger.trace(" key: " + keyStr + ", value: " + logValue); } } throw new ServerApiException(ApiErrorCode.UNSUPPORTED_ACTION_ERROR, "Invalid request, no command sent"); From 671d8ad704bce04a5abef72cc8059bdfbe947bc9 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 12 Nov 2025 14:02:01 +0530 Subject: [PATCH 10/17] Track volume usage data at a vm granularity as well (#11531) Co-authored-by: Vishesh <8760112+vishesh92@users.noreply.github.com> --- .../java/com/cloud/event/UsageEventUtils.java | 12 ++ .../orchestration/VolumeOrchestrator.java | 4 +- .../java/com/cloud/event/UsageEventVO.java | 22 +++ .../cloud/event/dao/UsageEventDaoImpl.java | 8 +- .../java/com/cloud/usage/UsageVolumeVO.java | 14 +- .../cloud/usage/dao/UsageStorageDaoImpl.java | 2 + .../com/cloud/usage/dao/UsageVolumeDao.java | 6 +- .../cloud/usage/dao/UsageVolumeDaoImpl.java | 86 ++++----- .../META-INF/db/schema-42100to42200.sql | 7 + .../java/com/cloud/api/ApiResponseHelper.java | 3 + .../java/com/cloud/hypervisor/KVMGuru.java | 6 + .../cloud/storage/VolumeApiServiceImpl.java | 15 +- .../storage/listener/VolumeStateListener.java | 2 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 6 +- .../storage/VolumeApiServiceImplTest.java | 8 +- .../com/cloud/vm/UserVmManagerImplTest.java | 8 +- .../com/cloud/usage/UsageManagerImpl.java | 167 ++++++++++-------- .../cloud/usage/parser/VolumeUsageParser.java | 23 ++- 18 files changed, 243 insertions(+), 156 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java b/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java index 94fbb7a80af..1c88c7df124 100644 --- a/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java +++ b/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java @@ -94,6 +94,14 @@ public class UsageEventUtils { } + public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId, + Long size, String entityType, String entityUUID, Long vmId, boolean displayResource) { + if (displayResource) { + saveUsageEvent(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName); + } + publishUsageEvent(usageType, accountId, zoneId, entityType, entityUUID); + } + public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId, Long size, Long virtualSize, String entityType, String entityUUID, Map details) { saveUsageEvent(usageType, accountId, zoneId, resourceId, resourceName, offeringId, templateId, size, virtualSize, details); @@ -202,6 +210,10 @@ public class UsageEventUtils { s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, vmId, securityGroupId)); } + public static void saveUsageEvent(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) { + s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName)); + } + private static void publishUsageEvent(String usageEventType, Long accountId, Long zoneId, String resourceType, String resourceUUID) { String configKey = "publish.usage.events"; String value = s_configDao.getValue(configKey); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 2b759235ac8..430b7cbc5aa 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -903,7 +903,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati // Save usage event and update resource count for user vm volumes if (vm.getType() == VirtualMachine.Type.User) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offering.getId(), null, size, - Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume()); + Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume()); _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering); } DiskProfile diskProfile = toDiskProfile(vol, offering); @@ -981,7 +981,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati } UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offeringId, vol.getTemplateId(), size, - Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume()); + Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume()); _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering); } diff --git a/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java b/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java index 3fc9fda9487..41ecec0c7fb 100644 --- a/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java +++ b/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java @@ -75,6 +75,9 @@ public class UsageEventVO implements UsageEvent { @Column(name = "virtual_size") private Long virtualSize; + @Column(name = "vm_id") + private Long vmId; + public UsageEventVO() { } @@ -143,6 +146,18 @@ public class UsageEventVO implements UsageEvent { this.offeringId = securityGroupId; } + public UsageEventVO(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) { + this.type = usageType; + this.accountId = accountId; + this.zoneId = zoneId; + this.resourceId = resourceId; + this.offeringId = offeringId; + this.templateId = templateId; + this.size = size; + this.vmId = vmId; + this.resourceName = resourceName; + } + @Override public long getId() { return id; @@ -248,4 +263,11 @@ public class UsageEventVO implements UsageEvent { this.virtualSize = virtualSize; } + public Long getVmId() { + return vmId; + } + + public void setVmId(Long vmId) { + this.vmId = vmId; + } } diff --git a/engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java b/engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java index fdef509da5b..bce9c474e2d 100644 --- a/engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java @@ -45,11 +45,11 @@ public class UsageEventDaoImpl extends GenericDaoBase implem private final SearchBuilder latestEventsSearch; private final SearchBuilder IpeventsSearch; private static final String COPY_EVENTS = - "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) " - + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? "; + "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) " + + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? "; private static final String COPY_ALL_EVENTS = - "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) " - + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id <= ?"; + "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) " + + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id <= ?"; private static final String COPY_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) " + "SELECT id, usage_event_id, name, value FROM cloud.usage_event_details vmevtDetails WHERE vmevtDetails.usage_event_id > ? and vmevtDetails.usage_event_id <= ? "; private static final String COPY_ALL_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) " diff --git a/engine/schema/src/main/java/com/cloud/usage/UsageVolumeVO.java b/engine/schema/src/main/java/com/cloud/usage/UsageVolumeVO.java index 96abd2d69c0..6d5315e3346 100644 --- a/engine/schema/src/main/java/com/cloud/usage/UsageVolumeVO.java +++ b/engine/schema/src/main/java/com/cloud/usage/UsageVolumeVO.java @@ -59,6 +59,9 @@ public class UsageVolumeVO implements InternalIdentity { @Column(name = "size") private long size; + @Column(name = "vm_id") + private Long vmId; + @Column(name = "created") @Temporal(value = TemporalType.TIMESTAMP) private Date created = null; @@ -70,13 +73,14 @@ public class UsageVolumeVO implements InternalIdentity { protected UsageVolumeVO() { } - public UsageVolumeVO(long id, long zoneId, long accountId, long domainId, Long diskOfferingId, Long templateId, long size, Date created, Date deleted) { + public UsageVolumeVO(long id, long zoneId, long accountId, long domainId, Long diskOfferingId, Long templateId, Long vmId, long size, Date created, Date deleted) { this.volumeId = id; this.zoneId = zoneId; this.accountId = accountId; this.domainId = domainId; this.diskOfferingId = diskOfferingId; this.templateId = templateId; + this.vmId = vmId; this.size = size; this.created = created; this.deleted = deleted; @@ -126,4 +130,12 @@ public class UsageVolumeVO implements InternalIdentity { public long getVolumeId() { return volumeId; } + + public Long getVmId() { + return vmId; + } + + public void setVmId(Long vmId) { + this.vmId = vmId; + } } diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageStorageDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageStorageDaoImpl.java index 1da53349399..f863cd1e3a3 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageStorageDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageStorageDaoImpl.java @@ -57,6 +57,7 @@ public class UsageStorageDaoImpl extends GenericDaoBase im IdSearch.and("accountId", IdSearch.entity().getAccountId(), SearchCriteria.Op.EQ); IdSearch.and("id", IdSearch.entity().getEntityId(), SearchCriteria.Op.EQ); IdSearch.and("type", IdSearch.entity().getStorageType(), SearchCriteria.Op.EQ); + IdSearch.and("deleted", IdSearch.entity().getDeleted(), SearchCriteria.Op.NULL); IdSearch.done(); IdZoneSearch = createSearchBuilder(); @@ -74,6 +75,7 @@ public class UsageStorageDaoImpl extends GenericDaoBase im sc.setParameters("accountId", accountId); sc.setParameters("id", id); sc.setParameters("type", type); + sc.setParameters("deleted", null); return listBy(sc, null); } diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDao.java index 09590b73993..05287240f25 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDao.java @@ -23,9 +23,7 @@ import com.cloud.usage.UsageVolumeVO; import com.cloud.utils.db.GenericDao; public interface UsageVolumeDao extends GenericDao { - public void removeBy(long userId, long id); - - public void update(UsageVolumeVO usage); - public List getUsageRecords(Long accountId, Long domainId, Date startDate, Date endDate, boolean limit, int page); + + List listByVolumeId(long volumeId, long accountId); } diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDaoImpl.java index 4662a6f26ce..095070feac1 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDaoImpl.java @@ -18,81 +18,46 @@ package com.cloud.usage.dao; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.TimeZone; -import com.cloud.exception.CloudException; +import javax.annotation.PostConstruct; + import org.springframework.stereotype.Component; import com.cloud.usage.UsageVolumeVO; import com.cloud.utils.DateUtil; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; @Component public class UsageVolumeDaoImpl extends GenericDaoBase implements UsageVolumeDao { - protected static final String REMOVE_BY_USERID_VOLID = "DELETE FROM usage_volume WHERE account_id = ? AND volume_id = ?"; - protected static final String UPDATE_DELETED = "UPDATE usage_volume SET deleted = ? WHERE account_id = ? AND volume_id = ? and deleted IS NULL"; - protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted " + protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted " + "FROM usage_volume " + "WHERE account_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))"; - protected static final String GET_USAGE_RECORDS_BY_DOMAIN = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted " + protected static final String GET_USAGE_RECORDS_BY_DOMAIN = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted " + "FROM usage_volume " + "WHERE domain_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))"; - protected static final String GET_ALL_USAGE_RECORDS = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted " + protected static final String GET_ALL_USAGE_RECORDS = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted " + "FROM usage_volume " + "WHERE (deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?))"; + private SearchBuilder volumeSearch; public UsageVolumeDaoImpl() { } - @Override - public void removeBy(long accountId, long volId) { - TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); - try { - txn.start(); - try(PreparedStatement pstmt = txn.prepareStatement(REMOVE_BY_USERID_VOLID);) { - if (pstmt != null) { - pstmt.setLong(1, accountId); - pstmt.setLong(2, volId); - pstmt.executeUpdate(); - } - }catch (SQLException e) { - throw new CloudException("Error removing usageVolumeVO:"+e.getMessage(), e); - } - txn.commit(); - } catch (Exception e) { - txn.rollback(); - logger.warn("Error removing usageVolumeVO:"+e.getMessage(), e); - } finally { - txn.close(); - } - } - - @Override - public void update(UsageVolumeVO usage) { - TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); - PreparedStatement pstmt = null; - try { - txn.start(); - if (usage.getDeleted() != null) { - pstmt = txn.prepareAutoCloseStatement(UPDATE_DELETED); - pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), usage.getDeleted())); - pstmt.setLong(2, usage.getAccountId()); - pstmt.setLong(3, usage.getVolumeId()); - pstmt.executeUpdate(); - } - txn.commit(); - } catch (Exception e) { - txn.rollback(); - logger.warn("Error updating UsageVolumeVO", e); - } finally { - txn.close(); - } + @PostConstruct + protected void init() { + volumeSearch = createSearchBuilder(); + volumeSearch.and("accountId", volumeSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + volumeSearch.and("volumeId", volumeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); + volumeSearch.and("deleted", volumeSearch.entity().getDeleted(), SearchCriteria.Op.NULL); + volumeSearch.done(); } @Override @@ -150,11 +115,15 @@ public class UsageVolumeDaoImpl extends GenericDaoBase impl if (tId == 0) { tId = null; } - long size = Long.valueOf(rs.getLong(7)); + Long vmId = Long.valueOf(rs.getLong(7)); + if (vmId == 0) { + vmId = null; + } + long size = Long.valueOf(rs.getLong(8)); Date createdDate = null; Date deletedDate = null; - String createdTS = rs.getString(8); - String deletedTS = rs.getString(9); + String createdTS = rs.getString(9); + String deletedTS = rs.getString(10); if (createdTS != null) { createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS); @@ -163,7 +132,7 @@ public class UsageVolumeDaoImpl extends GenericDaoBase impl deletedDate = DateUtil.parseDateString(s_gmtTimeZone, deletedTS); } - usageRecords.add(new UsageVolumeVO(vId, zoneId, acctId, dId, doId, tId, size, createdDate, deletedDate)); + usageRecords.add(new UsageVolumeVO(vId, zoneId, acctId, dId, doId, tId, vmId, size, createdDate, deletedDate)); } } catch (Exception e) { txn.rollback(); @@ -174,4 +143,13 @@ public class UsageVolumeDaoImpl extends GenericDaoBase impl return usageRecords; } + + @Override + public List listByVolumeId(long volumeId, long accountId) { + SearchCriteria sc = volumeSearch.create(); + sc.setParameters("accountId", accountId); + sc.setParameters("volumeId", volumeId); + sc.setParameters("deleted", null); + return listBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index b523016aa3d..ecca3482d59 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -41,6 +41,13 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.ldap_configuration', 'uuid', 'VARCHA -- Populate uuid for existing rows where uuid is NULL or empty UPDATE `cloud`.`ldap_configuration` SET uuid = UUID() WHERE uuid IS NULL OR uuid = ''; +-- Add vm_id column to usage_event table for volume usage events +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.usage_event','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with volume usage events"'); +CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_event','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with volume usage events"'); + +-- Add vm_id column to cloud_usage.usage_volume table +CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_volume','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with the volume usage"'); + -- Add the column cross_zone_instance_creation to cloud.backup_repository. if enabled it means that new Instance can be created on all Zones from Backups on this Repository. CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_instance_creation', 'TINYINT(1) DEFAULT NULL COMMENT ''Backup Repository can be used for disaster recovery on another zone'''); diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 404cefb68fe..83b6e4d2bf1 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -4298,6 +4298,9 @@ public class ApiResponseHelper implements ResponseGenerator { if (volume != null) { builder.append("for ").append(volume.getName()).append(" (").append(volume.getUuid()).append(")"); } + if (vmInstance != null) { + builder.append(" attached to VM ").append(vmInstance.getHostName()).append(" (").append(vmInstance.getUuid()).append(")"); + } if (diskOff != null) { builder.append(" with disk offering ").append(diskOff.getName()).append(" (").append(diskOff.getUuid()).append(")"); } diff --git a/server/src/main/java/com/cloud/hypervisor/KVMGuru.java b/server/src/main/java/com/cloud/hypervisor/KVMGuru.java index 64881df4c82..2457c6d7a46 100644 --- a/server/src/main/java/com/cloud/hypervisor/KVMGuru.java +++ b/server/src/main/java/com/cloud/hypervisor/KVMGuru.java @@ -21,6 +21,8 @@ import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.configuration.ConfigurationManagerImpl; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; import com.cloud.host.HostVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; @@ -370,6 +372,8 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { _volumeDao.update(volume.getId(), volume); _volumeDao.attachVolume(volume.getId(), vm.getId(), getNextAvailableDeviceId(vmVolumes)); } + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), + volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), vm.getId(), volume.isDisplay()); } } catch (Exception e) { throw new RuntimeException("Could not restore VM " + vm.getName() + " due to : " + e.getMessage()); @@ -387,6 +391,8 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { _volumeDao.attachVolume(restoredVolume.getId(), vm.getId(), getNextAvailableDeviceId(vmVolumes)); restoredVolume.setState(Volume.State.Ready); _volumeDao.update(restoredVolume.getId(), restoredVolume); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, restoredVolume.getAccountId(), restoredVolume.getDataCenterId(), restoredVolume.getId(), restoredVolume.getName(), + restoredVolume.getDiskOfferingId(), restoredVolume.getTemplateId(), restoredVolume.getSize(), Volume.class.getName(), restoredVolume.getUuid(), vm.getId(), restoredVolume.isDisplay()); return true; } catch (Exception e) { restoredVolume.setDisplay(false); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index de6dd4ec67f..a2b87c84b4c 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -1004,7 +1004,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (snapshotId == null && displayVolume) { // for volume created from snapshot, create usage event after volume creation UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), diskOfferingId, null, size, - Volume.class.getName(), volume.getUuid(), displayVolume); + Volume.class.getName(), volume.getUuid(), volume.getInstanceId(), displayVolume); } if (volume != null && details != null) { @@ -1106,7 +1106,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic createdVolume = _volumeMgr.createVolumeFromSnapshot(volume, snapshot, vm); VolumeVO volumeVo = _volsDao.findById(createdVolume.getId()); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, createdVolume.getAccountId(), createdVolume.getDataCenterId(), createdVolume.getId(), createdVolume.getName(), - createdVolume.getDiskOfferingId(), null, createdVolume.getSize(), Volume.class.getName(), createdVolume.getUuid(), volumeVo.isDisplayVolume()); + createdVolume.getDiskOfferingId(), null, createdVolume.getSize(), Volume.class.getName(), createdVolume.getUuid(), volume.getInstanceId(), volumeVo.isDisplayVolume()); return volumeVo; } @@ -1903,7 +1903,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } UsageEventUtils .publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), offeringId, - volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.isDisplay()); + volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.getInstanceId(), volume.isDisplay()); logger.debug("Volume [{}] has been successfully recovered, thus a new usage event {} has been published.", volume, EventTypes.EVENT_VOLUME_CREATE); } @@ -2998,7 +2998,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (displayVolume) { // flag turned 1 equivalent to freshly created volume UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), volume.getDiskOfferingId(), - volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid()); + volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.getInstanceId(), displayVolume); } else { // flag turned 0 equivalent to deleting a volume UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), Volume.class.getName(), @@ -3259,6 +3259,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic handleTargetsForVMware(hostId, volumePool.getHostAddress(), volumePool.getPort(), volume.get_iScsiName()); } + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DETACH, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), + volume.getDiskOfferingId(), null, volume.getSize(), Volume.class.getName(), volume.getUuid(), null, volume.isDisplay()); return _volsDao.findById(volumeId); } else { @@ -4339,7 +4341,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic diskOfferingVO); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), - volume.getUuid(), volume.isDisplayVolume()); + volume.getUuid(), volume.getInstanceId(), volume.isDisplayVolume()); volService.moveVolumeOnSecondaryStorageToAnotherAccount(volume, oldAccount, newAccount); } @@ -4863,6 +4865,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (attached) { ev = Volume.Event.OperationSucceeded; logger.debug("Volume: {} successfully attached to VM: {}", volInfo.getVolume(), volInfo.getAttachedVM()); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, volumeToAttach.getAccountId(), volumeToAttach.getDataCenterId(), volumeToAttach.getId(), volumeToAttach.getName(), + volumeToAttach.getDiskOfferingId(), volumeToAttach.getTemplateId(), volumeToAttach.getSize(), Volume.class.getName(), volumeToAttach.getUuid(), vm.getId(), volumeToAttach.isDisplay()); + provideVMInfo(dataStore, vm.getId(), volInfo.getId()); } else { logger.debug("Volume: {} failed to attach to VM: {}", volInfo.getVolume(), volInfo.getAttachedVM()); diff --git a/server/src/main/java/com/cloud/storage/listener/VolumeStateListener.java b/server/src/main/java/com/cloud/storage/listener/VolumeStateListener.java index 961a7a7da43..0910c46705a 100644 --- a/server/src/main/java/com/cloud/storage/listener/VolumeStateListener.java +++ b/server/src/main/java/com/cloud/storage/listener/VolumeStateListener.java @@ -81,7 +81,7 @@ public class VolumeStateListener implements StateListener // For the Resize Volume Event, this publishes an event with an incorrect disk offering ID, so do nothing for now } else { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), vol.getDiskOfferingId(), null, vol.getSize(), - Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume()); + Volume.class.getName(), vol.getUuid(), instanceId, vol.isDisplayVolume()); } } else if (transition.getToState() == State.Destroy && vol.getVolumeType() != Volume.Type.ROOT) { //Do not Publish Usage Event for ROOT Disk as it would have been published already while destroying a VM UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 96c87c5376d..2e30b4ecbd8 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2408,6 +2408,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (Volume.State.Destroy.equals(volume.getState())) { _volumeService.recoverVolume(volume.getId()); _volsDao.attachVolume(volume.getId(), vmId, ROOT_DEVICE_ID); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), + volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), vmId, volume.isDisplay()); } else { _volumeService.publishVolumeCreationUsageEvent(volume); } @@ -8156,7 +8158,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir logger.trace("Generating a create volume event for volume [{}].", volume); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), - volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); + volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.getInstanceId(), volume.isDisplayVolume()); } } @@ -8959,6 +8961,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir handleManagedStorage(vm, root); _volsDao.attachVolume(newVol.getId(), vmId, newVol.getDeviceId()); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, newVol.getAccountId(), newVol.getDataCenterId(), newVol.getId(), newVol.getName(), + newVol.getDiskOfferingId(), newVol.getTemplateId(), newVol.getSize(), Volume.class.getName(), newVol.getUuid(), vmId, newVol.isDisplay()); // Detach, destroy and create the usage event for the old root volume. _volsDao.detachVolume(root.getId()); diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 79be3695fbd..0575b430ef1 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -1545,7 +1545,7 @@ public class VolumeApiServiceImplTest { volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock); usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(), - null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay())); + null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay())); } } @@ -1558,7 +1558,7 @@ public class VolumeApiServiceImplTest { volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock); usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(), - null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay())); + null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay())); } } @@ -1573,7 +1573,7 @@ public class VolumeApiServiceImplTest { volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock); usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(), - null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay())); + null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay())); } } @@ -1589,7 +1589,7 @@ public class VolumeApiServiceImplTest { volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock); usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(), - offeringMockId, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay())); + offeringMockId, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay())); } } diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index a21477aeb80..fe4ea0838f1 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -1120,10 +1120,12 @@ public class UserVmManagerImplTest { public void recoverRootVolumeTestDestroyState() { Mockito.doReturn(Volume.State.Destroy).when(volumeVOMock).getState(); - userVmManagerImpl.recoverRootVolume(volumeVOMock, vmId); + try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class)) { + userVmManagerImpl.recoverRootVolume(volumeVOMock, vmId); - Mockito.verify(volumeApiService).recoverVolume(volumeVOMock.getId()); - Mockito.verify(volumeDaoMock).attachVolume(volumeVOMock.getId(), vmId, UserVmManagerImpl.ROOT_DEVICE_ID); + Mockito.verify(volumeApiService).recoverVolume(volumeVOMock.getId()); + Mockito.verify(volumeDaoMock).attachVolume(volumeVOMock.getId(), vmId, UserVmManagerImpl.ROOT_DEVICE_ID); + } } @Test(expected = InvalidParameterValueException.class) diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 49d79999716..864ad35c922 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1008,7 +1008,12 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna private boolean isVolumeEvent(String eventType) { return eventType != null && - (eventType.equals(EventTypes.EVENT_VOLUME_CREATE) || eventType.equals(EventTypes.EVENT_VOLUME_DELETE) || eventType.equals(EventTypes.EVENT_VOLUME_RESIZE) || eventType.equals(EventTypes.EVENT_VOLUME_UPLOAD)); + (eventType.equals(EventTypes.EVENT_VOLUME_CREATE) || + eventType.equals(EventTypes.EVENT_VOLUME_DELETE) || + eventType.equals(EventTypes.EVENT_VOLUME_RESIZE) || + eventType.equals(EventTypes.EVENT_VOLUME_UPLOAD) || + eventType.equals(EventTypes.EVENT_VOLUME_ATTACH) || + eventType.equals(EventTypes.EVENT_VOLUME_DETACH)); } private boolean isTemplateEvent(String eventType) { @@ -1424,92 +1429,112 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna } } + private void deleteExistingSecondaryStorageUsageForVolume(long volId, long accountId, Date deletedDate) { + List storageVOs = _usageStorageDao.listById(accountId, volId, StorageTypes.VOLUME); + for (UsageStorageVO storageVO : storageVOs) { + logger.debug("Setting the volume with id: {} to 'deleted' in the usage_storage table for account: {}.", volId, accountId); + storageVO.setDeleted(deletedDate); + _usageStorageDao.update(storageVO); + } + } + + private void deleteExistingInstanceVolumeUsage(long volId, long accountId, Date deletedDate) { + List volumesVOs = _usageVolumeDao.listByVolumeId(volId, accountId); + for (UsageVolumeVO volumesVO : volumesVOs) { + if (volumesVO.getVmId() != null) { + logger.debug("Setting the volume with id: {} for instance id: {} to 'deleted' in the usage_volume table for account {}.", + volumesVO.getVolumeId(), volumesVO.getVmId(), accountId); + volumesVO.setDeleted(deletedDate); + _usageVolumeDao.update(volumesVO.getId(), volumesVO); + } + } + } + + private void deleteExistingVolumeUsage(long volId, long accountId, Date deletedDate) { + List volumesVOs = _usageVolumeDao.listByVolumeId(volId, accountId); + for (UsageVolumeVO volumesVO : volumesVOs) { + logger.debug("Setting the volume with id: {} to 'deleted' in the usage_volume table for account: {}.", volId, accountId); + volumesVO.setDeleted(deletedDate); + _usageVolumeDao.update(volumesVO.getId(), volumesVO); + } + } + private void createVolumeHelperEvent(UsageEventVO event) { long volId = event.getResourceId(); + Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId()); + List volumesVOs; + UsageVolumeVO volumeVO; - if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType())) { - //For volumes which are 'attached' successfully, set the 'deleted' column in the usage_storage table, + switch (event.getType()) { + case EventTypes.EVENT_VOLUME_CREATE: + //For volumes which are 'attached' successfully from uploaded state, set the 'deleted' column in the usage_storage table, //so that the secondary storage should stop accounting and only primary will be accounted. - SearchCriteria sc = _usageStorageDao.createSearchCriteria(); - sc.addAnd("entityId", SearchCriteria.Op.EQ, volId); - sc.addAnd("storageType", SearchCriteria.Op.EQ, StorageTypes.VOLUME); - List volumesVOs = _usageStorageDao.search(sc, null); - if (volumesVOs != null) { - if (volumesVOs.size() == 1) { - logger.debug("Setting the volume with id: " + volId + " to 'deleted' in the usage_storage table."); - volumesVOs.get(0).setDeleted(event.getCreateDate()); - _usageStorageDao.update(volumesVOs.get(0)); - } - } - } - if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType()) || EventTypes.EVENT_VOLUME_RESIZE.equals(event.getType())) { - SearchCriteria sc = _usageVolumeDao.createSearchCriteria(); - sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId()); - sc.addAnd("volumeId", SearchCriteria.Op.EQ, volId); - sc.addAnd("deleted", SearchCriteria.Op.NULL); - List volumesVOs = _usageVolumeDao.search(sc, null); + deleteExistingSecondaryStorageUsageForVolume(volId, event.getAccountId(), event.getCreateDate()); + + volumesVOs = _usageVolumeDao.listByVolumeId(volId, event.getAccountId()); if (volumesVOs.size() > 0) { //This is a safeguard to avoid double counting of volumes. logger.error("Found duplicate usage entry for volume: " + volId + " assigned to account: " + event.getAccountId() + "; marking as deleted..."); + deleteExistingVolumeUsage(volId, event.getAccountId(), event.getCreateDate()); } - //an entry exists if it is a resize volume event. marking the existing deleted and creating a new one in the case of resize. - for (UsageVolumeVO volumesVO : volumesVOs) { - if (logger.isDebugEnabled()) { - logger.debug("deleting volume: " + volumesVO.getId() + " from account: " + volumesVO.getAccountId()); - } - volumesVO.setDeleted(event.getCreateDate()); - _usageVolumeDao.update(volumesVO); - } - if (logger.isDebugEnabled()) { - logger.debug("create volume with id : " + volId + " for account: " + event.getAccountId()); - } - Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId()); - UsageVolumeVO volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), event.getSize(), event.getCreateDate(), null); + + logger.debug("Creating a new entry in usage_volume for volume with id: {} for account: {}", volId, event.getAccountId()); + volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), null, event.getSize(), event.getCreateDate(), null); _usageVolumeDao.persist(volumeVO); - } else if (EventTypes.EVENT_VOLUME_DELETE.equals(event.getType())) { - SearchCriteria sc = _usageVolumeDao.createSearchCriteria(); - sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId()); - sc.addAnd("volumeId", SearchCriteria.Op.EQ, volId); - sc.addAnd("deleted", SearchCriteria.Op.NULL); - List volumesVOs = _usageVolumeDao.search(sc, null); - if (volumesVOs.size() > 1) { - logger.warn("More that one usage entry for volume: " + volId + " assigned to account: " + event.getAccountId() + "; marking them all as deleted..."); + + if (event.getVmId() != null) { + volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), event.getVmId(), event.getSize(), event.getCreateDate(), null); + _usageVolumeDao.persist(volumeVO); } + break; + + case EventTypes.EVENT_VOLUME_RESIZE: + volumesVOs = _usageVolumeDao.listByVolumeId(volId, event.getAccountId()); for (UsageVolumeVO volumesVO : volumesVOs) { - if (logger.isDebugEnabled()) { - logger.debug("deleting volume: " + volumesVO.getId() + " from account: " + volumesVO.getAccountId()); - } - volumesVO.setDeleted(event.getCreateDate()); // there really shouldn't be more than one - _usageVolumeDao.update(volumesVO); - } - } else if (EventTypes.EVENT_VOLUME_UPLOAD.equals(event.getType())) { - //For Upload event add an entry to the usage_storage table. - SearchCriteria sc = _usageStorageDao.createSearchCriteria(); - sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId()); - sc.addAnd("entityId", SearchCriteria.Op.EQ, volId); - sc.addAnd("storageType", SearchCriteria.Op.EQ, StorageTypes.VOLUME); - sc.addAnd("deleted", SearchCriteria.Op.NULL); - List volumesVOs = _usageStorageDao.search(sc, null); - - if (volumesVOs.size() > 0) { - //This is a safeguard to avoid double counting of volumes. - logger.error("Found duplicate usage entry for volume: " + volId + " assigned to account: " + event.getAccountId() + "; marking as deleted..."); - } - for (UsageStorageVO volumesVO : volumesVOs) { - if (logger.isDebugEnabled()) { - logger.debug("deleting volume: " + volumesVO.getId() + " from account: " + volumesVO.getAccountId()); + String delete_msg = String.format("Setting the volume with id: %s to 'deleted' in the usage_volume table for account: %s.", volId, event.getAccountId()); + String create_msg = String.format("Creating a new entry in usage_volume for volume with id: %s after resize for account: %s", volId, event.getAccountId()); + Long vmId = volumesVO.getVmId(); + if (vmId != null) { + delete_msg = String.format("Setting the volume with id: %s for instance id: %s to 'deleted' in the usage_volume table for account: %s.", + volId, vmId, event.getAccountId()); + create_msg = String.format("Creating a new entry in usage_volume for volume with id: %s and instance id: %s after resize for account: %s", + volId, vmId, event.getAccountId()); } + logger.debug(delete_msg); volumesVO.setDeleted(event.getCreateDate()); - _usageStorageDao.update(volumesVO); - } + _usageVolumeDao.update(volumesVO.getId(), volumesVO); - if (logger.isDebugEnabled()) { - logger.debug("create volume with id : " + volId + " for account: " + event.getAccountId()); + logger.debug(create_msg); + volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), vmId, event.getSize(), event.getCreateDate(), null); + _usageVolumeDao.persist(volumeVO); } - Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId()); - UsageStorageVO volumeVO = new UsageStorageVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), StorageTypes.VOLUME, event.getTemplateId(), event.getSize(), event.getCreateDate(), null); - _usageStorageDao.persist(volumeVO); + break; + + case EventTypes.EVENT_VOLUME_DELETE: + deleteExistingVolumeUsage(volId, event.getAccountId(), event.getCreateDate()); + break; + + case EventTypes.EVENT_VOLUME_ATTACH: + deleteExistingInstanceVolumeUsage(event.getResourceId(), event.getAccountId(), event.getCreateDate()); + + logger.debug("Creating a new entry in usage_volume for volume with id: {}, and instance id: {} for account: {}", + volId, event.getVmId(), event.getAccountId()); + volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), event.getVmId(), event.getSize(), event.getCreateDate(), null); + _usageVolumeDao.persist(volumeVO); + break; + + case EventTypes.EVENT_VOLUME_DETACH: + deleteExistingInstanceVolumeUsage(event.getResourceId(), event.getAccountId(), event.getCreateDate()); + break; + + case EventTypes.EVENT_VOLUME_UPLOAD: + deleteExistingSecondaryStorageUsageForVolume(volId, event.getAccountId(), event.getCreateDate()); + + logger.debug("Creating a new entry in usage_storage for volume with id : {} for account: {}", volId, event.getAccountId()); + UsageStorageVO storageVO = new UsageStorageVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), StorageTypes.VOLUME, event.getTemplateId(), event.getSize(), event.getCreateDate(), null); + _usageStorageDao.persist(storageVO); + break; } } diff --git a/usage/src/main/java/com/cloud/usage/parser/VolumeUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/VolumeUsageParser.java index e834b713d42..0210b899e8c 100644 --- a/usage/src/main/java/com/cloud/usage/parser/VolumeUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/VolumeUsageParser.java @@ -73,12 +73,13 @@ public class VolumeUsageParser extends UsageParser { for (UsageVolumeVO usageVol : usageUsageVols) { long volId = usageVol.getVolumeId(); Long doId = usageVol.getDiskOfferingId(); + Long vmId = usageVol.getVmId(); long zoneId = usageVol.getZoneId(); Long templateId = usageVol.getTemplateId(); long size = usageVol.getSize(); - String key = volId + "-" + doId + "-" + size; + String key = volId + "-" + doId + "-" + vmId + "-" + size; - diskOfferingMap.put(key, new VolInfo(volId, zoneId, doId, templateId, size)); + diskOfferingMap.put(key, new VolInfo(volId, zoneId, doId, templateId, size, vmId)); Date volCreateDate = usageVol.getCreated(); Date volDeleteDate = usageVol.getDeleted(); @@ -110,7 +111,7 @@ public class VolumeUsageParser extends UsageParser { if (useTime > 0L) { VolInfo info = diskOfferingMap.get(volIdKey); createUsageRecord(UsageTypes.VOLUME, useTime, startDate, endDate, account, info.getVolumeId(), info.getZoneId(), info.getDiskOfferingId(), - info.getTemplateId(), info.getSize()); + info.getTemplateId(), info.getVmId(), info.getSize()); } } @@ -130,7 +131,7 @@ public class VolumeUsageParser extends UsageParser { } private void createUsageRecord(int type, long runningTime, Date startDate, Date endDate, AccountVO account, long volId, long zoneId, Long doId, - Long templateId, long size) { + Long templateId, Long vmId, long size) { // Our smallest increment is hourly for now logger.debug("Total running time {} ms", runningTime); @@ -152,7 +153,11 @@ public class VolumeUsageParser extends UsageParser { usageDesc += " (DiskOffering: " + doId + ")"; } - UsageVO usageRecord = new UsageVO(zoneId, account.getId(), account.getDomainId(), usageDesc, usageDisplay + " Hrs", type, new Double(usage), null, null, doId, templateId, volId, + if (vmId != null) { + usageDesc += " (VM: " + vmId + ")"; + } + + UsageVO usageRecord = new UsageVO(zoneId, account.getId(), account.getDomainId(), usageDesc, usageDisplay + " Hrs", type, new Double(usage), vmId, null, doId, templateId, volId, size, startDate, endDate); usageDao.persist(usageRecord); } @@ -163,13 +168,15 @@ public class VolumeUsageParser extends UsageParser { private Long diskOfferingId; private Long templateId; private long size; + private Long vmId; - public VolInfo(long volId, long zoneId, Long diskOfferingId, Long templateId, long size) { + public VolInfo(long volId, long zoneId, Long diskOfferingId, Long templateId, long size, Long vmId) { this.volId = volId; this.zoneId = zoneId; this.diskOfferingId = diskOfferingId; this.templateId = templateId; this.size = size; + this.vmId = vmId; } public long getZoneId() { @@ -191,5 +198,9 @@ public class VolumeUsageParser extends UsageParser { public long getSize() { return size; } + + public Long getVmId() { + return vmId; + } } } From f0a0936675c8a0462b5677f767c8fb56b161423a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Nov 2025 14:21:51 +0530 Subject: [PATCH 11/17] server: fix volume offering not updated after offering change (#12003) Signed-off-by: Abhishek Kumar --- .../src/main/java/com/cloud/storage/VolumeApiServiceImpl.java | 2 +- ui/public/locales/en.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 440da7e2c72..308991f77d5 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -1536,6 +1536,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } } + volume = _volsDao.findById(volumeId); if (newDiskOfferingId != null) { volume.setDiskOfferingId(newDiskOfferingId); _volumeMgr.saveVolumeDetails(newDiskOfferingId, volume.getId()); @@ -1550,7 +1551,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } // Update size if volume has same size as before, else it is already updated - volume = _volsDao.findById(volumeId); if (currentSize == volume.getSize() && currentSize != newSize) { volume.setSize(newSize); } else if (volume.getSize() != newSize) { diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index d18c0945d3b..8f2e1bb5c05 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -2845,7 +2845,7 @@ "message.change.offering.confirm": "Please confirm that you wish to change the service offering of this virtual Instance.", "message.change.offering.for.volume": "Successfully changed offering for the volume", "message.change.offering.for.volume.failed": "Change offering for the volume failed", -"message.change.offering.processing": "Changing offering for the volume...", +"message.change.offering.for.volume.processing": "Changing offering for the volume...", "message.change.password": "Please change your password.", "message.change.scope.failed": "Scope change failed", "message.change.scope.processing": "Scope change in progress", From e90e31d3861235553673c7789a252bc4ce512027 Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 12 Nov 2025 16:09:28 +0100 Subject: [PATCH 12/17] add isPerson check to query for AD (#11843) --- .../ldap/ADLdapUserManagerImpl.java | 18 ++- .../ldap/OpenLdapUserManagerImpl.java | 133 ++++++++---------- .../ldap/ADLdapUserManagerImplTest.java | 5 +- 3 files changed, 75 insertions(+), 81 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java index e96606dca2f..bf5d503e841 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java @@ -49,7 +49,7 @@ public class ADLdapUserManagerImpl extends OpenLdapUserManagerImpl implements Ld searchControls.setReturningAttributes(_ldapConfiguration.getReturnAttributes(domainId)); NamingEnumeration results = context.search(basedn, generateADGroupSearchFilter(groupName, domainId), searchControls); - final List users = new ArrayList(); + final List users = new ArrayList<>(); while (results.hasMoreElements()) { final SearchResult result = results.nextElement(); users.add(createUser(result, domainId)); @@ -58,10 +58,8 @@ public class ADLdapUserManagerImpl extends OpenLdapUserManagerImpl implements Ld } String generateADGroupSearchFilter(String groupName, Long domainId) { - final StringBuilder userObjectFilter = new StringBuilder(); - userObjectFilter.append("(objectClass="); - userObjectFilter.append(_ldapConfiguration.getUserObject(domainId)); - userObjectFilter.append(")"); + + final StringBuilder userObjectFilter = getUserObjectFilter(domainId); final StringBuilder memberOfFilter = new StringBuilder(); String groupCnName = _ldapConfiguration.getCommonNameAttribute() + "=" +groupName + "," + _ldapConfiguration.getBaseDn(domainId); @@ -75,10 +73,18 @@ public class ADLdapUserManagerImpl extends OpenLdapUserManagerImpl implements Ld result.append(memberOfFilter); result.append(")"); - logger.debug("group search filter = " + result); + logger.debug("group search filter = {}", result); return result.toString(); } + StringBuilder getUserObjectFilter(Long domainId) { + final StringBuilder userObjectFilter = new StringBuilder(); + userObjectFilter.append("(&(objectCategory=person)"); + userObjectFilter.append(super.getUserObjectFilter(domainId)); + userObjectFilter.append(")"); + return userObjectFilter; + } + protected boolean isUserDisabled(SearchResult result) throws NamingException { boolean isDisabledUser = false; String userAccountControl = LdapUtils.getAttributeValue(result.getAttributes(), _ldapConfiguration.getUserAccountControlAttribute()); diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java index d0b6bc4bd34..80d394d7478 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java @@ -75,23 +75,15 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { } private String generateSearchFilter(final String username, Long domainId) { - final StringBuilder userObjectFilter = new StringBuilder(); - userObjectFilter.append("(objectClass="); - userObjectFilter.append(_ldapConfiguration.getUserObject(domainId)); - userObjectFilter.append(")"); + final StringBuilder userObjectFilter = getUserObjectFilter(domainId); - final StringBuilder usernameFilter = new StringBuilder(); - usernameFilter.append("("); - usernameFilter.append(_ldapConfiguration.getUsernameAttribute(domainId)); - usernameFilter.append("="); - usernameFilter.append((username == null ? "*" : LdapUtils.escapeLDAPSearchFilter(username))); - usernameFilter.append(")"); + final StringBuilder usernameFilter = getUsernameFilter(username, domainId); String memberOfAttribute = getMemberOfAttribute(domainId); StringBuilder ldapGroupsFilter = new StringBuilder(); // this should get the trustmaps for this domain List ldapGroups = getMappedLdapGroups(domainId); - if (null != ldapGroups && ldapGroups.size() > 0) { + if (!ldapGroups.isEmpty()) { ldapGroupsFilter.append("(|"); for (String ldapGroup : ldapGroups) { ldapGroupsFilter.append(getMemberOfGroupString(ldapGroup, memberOfAttribute)); @@ -104,21 +96,35 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { if (null != pricipleGroup) { principleGroupFilter.append(getMemberOfGroupString(pricipleGroup, memberOfAttribute)); } - final StringBuilder result = new StringBuilder(); - result.append("(&"); - result.append(userObjectFilter); - result.append(usernameFilter); - result.append(ldapGroupsFilter); - result.append(principleGroupFilter); - result.append(")"); - String returnString = result.toString(); - if (logger.isTraceEnabled()) { - logger.trace("constructed ldap query: " + returnString); - } + String returnString = "(&" + + userObjectFilter + + usernameFilter + + ldapGroupsFilter + + principleGroupFilter + + ")"; + logger.trace("constructed ldap query: {}", returnString); return returnString; } + private StringBuilder getUsernameFilter(String username, Long domainId) { + final StringBuilder usernameFilter = new StringBuilder(); + usernameFilter.append("("); + usernameFilter.append(_ldapConfiguration.getUsernameAttribute(domainId)); + usernameFilter.append("="); + usernameFilter.append((username == null ? "*" : LdapUtils.escapeLDAPSearchFilter(username))); + usernameFilter.append(")"); + return usernameFilter; + } + + StringBuilder getUserObjectFilter(Long domainId) { + final StringBuilder userObjectFilter = new StringBuilder(); + userObjectFilter.append("(objectClass="); + userObjectFilter.append(_ldapConfiguration.getUserObject(domainId)); + userObjectFilter.append(")"); + return userObjectFilter; + } + private List getMappedLdapGroups(Long domainId) { List ldapGroups = new ArrayList<>(); // first get the trustmaps @@ -134,37 +140,31 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { private String getMemberOfGroupString(String group, String memberOfAttribute) { final StringBuilder memberOfFilter = new StringBuilder(); if (null != group) { - if(logger.isDebugEnabled()) { - logger.debug("adding search filter for '" + group + - "', using '" + memberOfAttribute + "'"); - } - memberOfFilter.append("(" + memberOfAttribute + "="); - memberOfFilter.append(group); - memberOfFilter.append(")"); + logger.debug("adding search filter for '{}', using '{}'", group, memberOfAttribute); + memberOfFilter.append("(") + .append(memberOfAttribute) + .append("=") + .append(group) + .append(")"); } return memberOfFilter.toString(); } private String generateGroupSearchFilter(final String groupName, Long domainId) { - final StringBuilder groupObjectFilter = new StringBuilder(); - groupObjectFilter.append("(objectClass="); - groupObjectFilter.append(_ldapConfiguration.getGroupObject(domainId)); - groupObjectFilter.append(")"); + String groupObjectFilter = "(objectClass=" + + _ldapConfiguration.getGroupObject(domainId) + + ")"; - final StringBuilder groupNameFilter = new StringBuilder(); - groupNameFilter.append("("); - groupNameFilter.append(_ldapConfiguration.getCommonNameAttribute()); - groupNameFilter.append("="); - groupNameFilter.append((groupName == null ? "*" : LdapUtils.escapeLDAPSearchFilter(groupName))); - groupNameFilter.append(")"); + String groupNameFilter = "(" + + _ldapConfiguration.getCommonNameAttribute() + + "=" + + (groupName == null ? "*" : LdapUtils.escapeLDAPSearchFilter(groupName)) + + ")"; - final StringBuilder result = new StringBuilder(); - result.append("(&"); - result.append(groupObjectFilter); - result.append(groupNameFilter); - result.append(")"); - - return result.toString(); + return "(&" + + groupObjectFilter + + groupNameFilter + + ")"; } @Override @@ -186,17 +186,9 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { basedn = _ldapConfiguration.getBaseDn(domainId); } - final StringBuilder userObjectFilter = new StringBuilder(); - userObjectFilter.append("(objectClass="); - userObjectFilter.append(_ldapConfiguration.getUserObject(domainId)); - userObjectFilter.append(")"); + final StringBuilder userObjectFilter = getUserObjectFilter(domainId); - final StringBuilder usernameFilter = new StringBuilder(); - usernameFilter.append("("); - usernameFilter.append(_ldapConfiguration.getUsernameAttribute(domainId)); - usernameFilter.append("="); - usernameFilter.append((username == null ? "*" : LdapUtils.escapeLDAPSearchFilter(username))); - usernameFilter.append(")"); + final StringBuilder usernameFilter = getUsernameFilter(username, domainId); final StringBuilder memberOfFilter = new StringBuilder(); if ("GROUP".equals(type)) { @@ -205,18 +197,17 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { memberOfFilter.append(")"); } - final StringBuilder searchQuery = new StringBuilder(); - searchQuery.append("(&"); - searchQuery.append(userObjectFilter); - searchQuery.append(usernameFilter); - searchQuery.append(memberOfFilter); - searchQuery.append(")"); + String searchQuery = "(&" + + userObjectFilter + + usernameFilter + + memberOfFilter + + ")"; - return searchUser(basedn, searchQuery.toString(), context, domainId); + return searchUser(basedn, searchQuery, context, domainId); } protected String getMemberOfAttribute(final Long domainId) { - return _ldapConfiguration.getUserMemberOfAttribute(domainId); + return LdapConfiguration.getUserMemberOfAttribute(domainId); } @Override @@ -243,7 +234,7 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { NamingEnumeration result = context.search(_ldapConfiguration.getBaseDn(domainId), generateGroupSearchFilter(groupName, domainId), controls); - final List users = new ArrayList(); + final List users = new ArrayList<>(); //Expecting only one result which has all the users if (result.hasMoreElements()) { Attribute attribute = result.nextElement().getAttributes().get(attributeName); @@ -254,7 +245,7 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { try{ users.add(getUserForDn(userdn, context, domainId)); } catch (NamingException e){ - logger.info("Userdn: " + userdn + " Not Found:: Exception message: " + e.getMessage()); + logger.info("Userdn: {} Not Found:: Exception message: {}", userdn, e.getMessage()); } } } @@ -286,17 +277,15 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { return false; } - public LdapUser searchUser(final String basedn, final String searchString, final LdapContext context, Long domainId) throws NamingException, IOException { + public LdapUser searchUser(final String basedn, final String searchString, final LdapContext context, Long domainId) throws NamingException { final SearchControls searchControls = new SearchControls(); searchControls.setSearchScope(_ldapConfiguration.getScope()); searchControls.setReturningAttributes(_ldapConfiguration.getReturnAttributes(domainId)); NamingEnumeration results = context.search(basedn, searchString, searchControls); - if(logger.isDebugEnabled()) { - logger.debug("searching user(s) with filter: \"" + searchString + "\""); - } - final List users = new ArrayList(); + logger.debug("searching user(s) with filter: \"{}\"", searchString); + final List users = new ArrayList<>(); while (results.hasMoreElements()) { final SearchResult result = results.nextElement(); users.add(createUser(result, domainId)); @@ -324,7 +313,7 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { byte[] cookie = null; int pageSize = _ldapConfiguration.getLdapPageSize(domainId); context.setRequestControls(new Control[]{new PagedResultsControl(pageSize, Control.NONCRITICAL)}); - final List users = new ArrayList(); + final List users = new ArrayList<>(); NamingEnumeration results; do { results = context.search(basedn, generateSearchFilter(username, domainId), searchControls); diff --git a/plugins/user-authenticators/ldap/src/test/java/org/apache/cloudstack/ldap/ADLdapUserManagerImplTest.java b/plugins/user-authenticators/ldap/src/test/java/org/apache/cloudstack/ldap/ADLdapUserManagerImplTest.java index 58b14ec3684..f2ac1dffaf9 100644 --- a/plugins/user-authenticators/ldap/src/test/java/org/apache/cloudstack/ldap/ADLdapUserManagerImplTest.java +++ b/plugins/user-authenticators/ldap/src/test/java/org/apache/cloudstack/ldap/ADLdapUserManagerImplTest.java @@ -54,9 +54,8 @@ public class ADLdapUserManagerImplTest { String [] groups = {"dev", "dev-hyd"}; for (String group: groups) { String result = adLdapUserManager.generateADGroupSearchFilter(group, 1L); - assertTrue(("(&(objectClass=user)(memberOf:1.2.840.113556.1.4.1941:=CN=" + group + ",DC=cloud,DC=citrix,DC=com))").equals(result)); + assertTrue(("(&(&(objectCategory=person)(objectClass=user))(memberOf:1.2.840.113556.1.4.1941:=CN=" + group + ",DC=cloud,DC=citrix,DC=com))").equals(result)); } - } @Test @@ -69,7 +68,7 @@ public class ADLdapUserManagerImplTest { String [] groups = {"dev", "dev-hyd"}; for (String group: groups) { String result = adLdapUserManager.generateADGroupSearchFilter(group, 1L); - assertTrue(("(&(objectClass=user)(memberOf=CN=" + group + ",DC=cloud,DC=citrix,DC=com))").equals(result)); + assertTrue(("(&(&(objectCategory=person)(objectClass=user))(memberOf=CN=" + group + ",DC=cloud,DC=citrix,DC=com))").equals(result)); } } From 028dd86945679cd77606afd12b3ed8035e98511e Mon Sep 17 00:00:00 2001 From: YoulongChen <30854794+YLChen-007@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:10:36 +0800 Subject: [PATCH 13/17] fixed Password Exposure in IPMI Tool Command Execution (#12028) --- .../cloudstack/utils/process/ProcessRunner.java | 11 ++++++++--- .../cloudstack/utils/process/ProcessRunnerTest.java | 12 ++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/utils/src/main/java/org/apache/cloudstack/utils/process/ProcessRunner.java b/utils/src/main/java/org/apache/cloudstack/utils/process/ProcessRunner.java index 430fa56aa68..e2d3be05772 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/process/ProcessRunner.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/process/ProcessRunner.java @@ -67,11 +67,13 @@ public final class ProcessRunner { public ProcessRunner(ExecutorService executor) { this.executor = executor; commandLogReplacements.add(new Ternary<>("ipmitool", "-P\\s+\\S+", "-P *****")); + commandLogReplacements.add(new Ternary<>("ipmitool", "(?i)password\\s+\\S+\\s+\\S+", "password **** ****")); } /** * Executes a process with provided list of commands with a max default timeout * of 5 minutes + * * @param commands list of string commands * @return returns process result */ @@ -82,6 +84,7 @@ public final class ProcessRunner { /** * Executes a process with provided list of commands with a given timeout that is less * than or equal to DEFAULT_MAX_TIMEOUT + * * @param commands list of string commands * @param timeOut timeout duration * @return returns process result @@ -109,14 +112,16 @@ public final class ProcessRunner { } }); try { - logger.debug("Waiting for a response from command [{}]. Defined timeout: [{}].", commandLog, timeOut.getStandardSeconds()); + logger.debug("Waiting for a response from command [{}]. Defined timeout: [{}].", commandLog, + timeOut.getStandardSeconds()); retVal = processFuture.get(timeOut.getStandardSeconds(), TimeUnit.SECONDS); } catch (ExecutionException e) { - logger.warn("Failed to complete the requested command [{}] due to execution error.", commands, e); + logger.warn("Failed to complete the requested command [{}] due to execution error.", commandLog, e); retVal = -2; stdError = e.getMessage(); } catch (TimeoutException e) { - logger.warn("Failed to complete the requested command [{}] within timeout. Defined timeout: [{}].", commandLog, timeOut.getStandardSeconds(), e); + logger.warn("Failed to complete the requested command [{}] within timeout. Defined timeout: [{}].", + commandLog, timeOut.getStandardSeconds(), e); retVal = -1; stdError = "Operation timed out, aborted."; } finally { diff --git a/utils/src/test/java/org/apache/cloudstack/utils/process/ProcessRunnerTest.java b/utils/src/test/java/org/apache/cloudstack/utils/process/ProcessRunnerTest.java index 6fc34ded259..0e594f2b0c9 100644 --- a/utils/src/test/java/org/apache/cloudstack/utils/process/ProcessRunnerTest.java +++ b/utils/src/test/java/org/apache/cloudstack/utils/process/ProcessRunnerTest.java @@ -60,4 +60,16 @@ public class ProcessRunnerTest { Assert.assertTrue(log.contains(password)); Assert.assertEquals(1, countSubstringOccurrences(log, password)); } + + @Test + public void testRemoveCommandSensitiveInfoForLoggingIpmiPasswordCommand() { + String userId = "3"; + String newPassword = "Sup3rSecr3t!"; + String command = String.format("/usr/bin/ipmitool user set password %s %s", userId, newPassword); + String log = processRunner.removeCommandSensitiveInfoForLogging(command); + + Assert.assertFalse(log.contains(userId)); + Assert.assertFalse(log.contains(newPassword)); + Assert.assertTrue(log.contains("password **** ****")); + } } From 21d844ba1c2928a5ee2656607702aa6bbb977c9b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 14 Nov 2025 15:13:42 +0530 Subject: [PATCH 14/17] ui: fix zone options for image instance deploy button (#12060) Signed-off-by: Abhishek Kumar --- .../view/ImageDeployInstanceButton.vue | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/ui/src/components/view/ImageDeployInstanceButton.vue b/ui/src/components/view/ImageDeployInstanceButton.vue index b2d4b55bc6a..2cdd5a0af46 100644 --- a/ui/src/components/view/ImageDeployInstanceButton.vue +++ b/ui/src/components/view/ImageDeployInstanceButton.vue @@ -71,9 +71,14 @@ export default { if (this.$route.meta.name === 'iso') { this.imageApi = 'listIsos' } - setTimeout(() => { - this.fetchData() - }, 100) + this.fetchData() + }, + watch: { + resource (newValue) { + if (newValue?.id) { + this.fetchData() + } + } }, computed: { allowed () { @@ -82,23 +87,22 @@ export default { } }, methods: { - arrayHasItems (array) { - return array !== null && array !== undefined && Array.isArray(array) && array.length > 0 - }, fetchData () { this.fetchResourceData() }, fetchResourceData () { - const params = {} - params.id = this.resource.id - params.templatefilter = 'executable' - params.listall = true - params.page = this.page - params.pagesize = this.pageSize + if (!this.resource || !this.resource.id) { + return + } + const params = { + id: this.resource.id, + templatefilter: 'executable', + listall: true + } this.dataSource = [] this.itemCount = 0 - this.fetchLoading = true + this.loading = true this.zones = [] getAPI(this.imageApi, params).then(json => { const imageResponse = json?.[this.imageApi.toLowerCase() + 'response']?.[this.$route.meta.name] || [] @@ -108,8 +112,8 @@ export default { })) }).catch(error => { this.$notifyError(error) - this.loading = false }).finally(() => { + this.loading = false if (this.zones.length !== 0) { this.$emit('update-zones', this.zones) } @@ -122,7 +126,8 @@ export default { } const zoneids = this.zones.map(z => z.id) this.loading = true - getAPI('listZones', { showicon: true, ids: zoneids.join(',') }).then(json => { + const params = { showicon: true, ids: zoneids.join(',') } + getAPI('listZones', params).then(json => { this.zones = json.listzonesresponse.zone || [] }).finally(() => { this.loading = false From dba889ea3efb05812290647515d4bad5318f84e0 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 24 Nov 2025 11:10:43 +0100 Subject: [PATCH 15/17] UI: fix list of zones if zone has icon (#12083) --- ui/src/views/compute/wizard/ZoneBlockRadioGroupSelect.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/views/compute/wizard/ZoneBlockRadioGroupSelect.vue b/ui/src/views/compute/wizard/ZoneBlockRadioGroupSelect.vue index 2250fc7977d..9c889d3aa5a 100644 --- a/ui/src/views/compute/wizard/ZoneBlockRadioGroupSelect.vue +++ b/ui/src/views/compute/wizard/ZoneBlockRadioGroupSelect.vue @@ -29,7 +29,7 @@