From 74e9146cfb4786745a7b35605b12f19b403c2738 Mon Sep 17 00:00:00 2001 From: dahn Date: Fri, 2 May 2025 13:52:00 +0200 Subject: [PATCH 01/18] check for custom offering and trim size (#10629) --- ui/src/views/storage/CreateVolume.vue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui/src/views/storage/CreateVolume.vue b/ui/src/views/storage/CreateVolume.vue index 3efe31a34e2..b8c5e307dec 100644 --- a/ui/src/views/storage/CreateVolume.vue +++ b/ui/src/views/storage/CreateVolume.vue @@ -269,6 +269,9 @@ export default { values.virtualmachineid = this.resource.id values.zoneid = this.resource.zoneid } + if (this.customDiskOffering) { + values.size = values.size.trim() + } if (this.createVolumeFromSnapshot) { values.snapshotid = this.resource.id } From d697cff8987bef6a93fe09ea9f9cd56801e19b4a Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 2 May 2025 21:04:17 +0530 Subject: [PATCH 02/18] Update dependency required for test_outofbandmanagement.py (#10805) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e82b777cdb..314239424a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,7 +232,7 @@ jobs: - name: Install Python dependencies run: | - python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid pynose + python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycryptodome mock flask netaddr pylint pycodestyle six astroid pynose - name: Install jacoco dependencies run: | From 1f8442eb69a169c17165e4b48bb1d4e7c7b39524 Mon Sep 17 00:00:00 2001 From: Imvedansh <113465074+Imvedansh@users.noreply.github.com> Date: Fri, 2 May 2025 21:06:46 +0530 Subject: [PATCH 03/18] Extra checks in UI when deleting accounts (#10760) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bernardo De Marco Gonçalves --- ui/public/locales/en.json | 9 +- ui/src/config/section/account.js | 5 +- ui/src/views/iam/DeleteAccount.vue | 176 ++++++++++++++++++++++ ui/src/views/iam/DeleteAccountWrapper.vue | 74 +++++++++ 4 files changed, 260 insertions(+), 4 deletions(-) create mode 100644 ui/src/views/iam/DeleteAccount.vue create mode 100644 ui/src/views/iam/DeleteAccountWrapper.vue diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 18df4d2a7bf..f4a8f59017e 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1,4 +1,5 @@ { +"message.delete.account.not.disabled": "Please disable the account before attempting to delete it.", "alert.service.domainrouter": "Domain router", "error.dedicate.cluster.failed": "Failed to dedicate cluster.", "error.dedicate.host.failed": "Failed to dedicate host.", @@ -855,6 +856,7 @@ "label.endipv6": "IPv6 end IP", "label.endpoint": "Endpoint", "label.endport": "End port", +"label.enter.account.name": "Enter the account name", "label.enter.code": "Enter 2FA code to verify", "label.enter.static.pin": "Enter static PIN to verify", "label.enter.token": "Enter token", @@ -2720,7 +2722,11 @@ "message.dedicating.host": "Dedicating host...", "message.dedicating.pod": "Dedicating pod...", "message.dedicating.zone": "Dedicating zone...", -"message.delete.account": "Please confirm that you want to delete this Account.", +"message.delete.account.confirm": "Please confirm that you want to delete this account by entering the name of the account below.", +"message.delete.account.failed": "Delete account failed", +"message.delete.account.processing": "Deleting account", +"message.delete.account.success": "Successfully deleted account", +"message.delete.account.warning": "Deleting this account will delete all of the instances, volumes and snapshots associated with the account.", "message.delete.acl.processing": "Removing ACL rule...", "message.delete.acl.rule": "Remove ACL rule", "message.delete.acl.rule.failed": "Failed to remove ACL rule.", @@ -2807,6 +2813,7 @@ "message.enabling.security.group.provider": "Enabling security group provider", "message.enter.valid.nic.ip": "Please enter a valid IP address for NIC", "message.error.access.key": "Please enter access key.", +"message.error.account.delete.name.mismatch": "Name entered doesn't match the account name.", "message.error.add.guest.network": "Either IPv4 fields or IPv6 fields need to be filled when adding a guest Network.", "message.error.add.interface.static.route": "Adding interface Static Route failed", "message.error.add.logical.router": "Adding Logical Router failed", diff --git a/ui/src/config/section/account.js b/ui/src/config/section/account.js index 28c0e3f556d..bdd71738660 100644 --- a/ui/src/config/section/account.js +++ b/ui/src/config/section/account.js @@ -225,11 +225,10 @@ export default { message: 'message.delete.account', dataView: true, disabled: (record, store) => { - return record.id !== 'undefined' && store.userInfo.accountid === record.id + return store.userInfo.accountid === record?.id }, - groupAction: true, popup: true, - groupMap: (selection) => { return selection.map(x => { return { id: x } }) } + component: shallowRef(defineAsyncComponent(() => import('@/views/iam/DeleteAccountWrapper.vue'))) } ] } diff --git a/ui/src/views/iam/DeleteAccount.vue b/ui/src/views/iam/DeleteAccount.vue new file mode 100644 index 00000000000..0e7c6139e4c --- /dev/null +++ b/ui/src/views/iam/DeleteAccount.vue @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/iam/DeleteAccountWrapper.vue b/ui/src/views/iam/DeleteAccountWrapper.vue new file mode 100644 index 00000000000..2c6a07ca3c6 --- /dev/null +++ b/ui/src/views/iam/DeleteAccountWrapper.vue @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + From 919c9797cc86dd135edc669840793c4e8a7a53b1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 6 May 2025 14:12:30 +0530 Subject: [PATCH 04/18] server: prevent duplicate HA works and alerts (#10624) Signed-off-by: Abhishek Kumar --- .../cloud/agent/manager/AgentManagerImpl.java | 18 +++++++--- .../cloud/ha/HighAvailabilityManagerImpl.java | 35 +++++++++++++++---- .../com/cloud/ha/dao/HighAvailabilityDao.java | 2 ++ .../cloud/ha/dao/HighAvailabilityDaoImpl.java | 14 +++++++- .../ha/HighAvailabilityManagerImplTest.java | 2 -- 5 files changed, 56 insertions(+), 15 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index d2e86fbc4b9..4b2578d20c4 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -38,9 +38,6 @@ import java.util.concurrent.locks.ReentrantLock; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.configuration.Config; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.db.GlobalLock; import org.apache.cloudstack.agent.lb.IndirectAgentLB; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -54,6 +51,7 @@ import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.apache.log4j.MDC; @@ -82,6 +80,7 @@ import com.cloud.agent.api.UnsupportedAnswer; import com.cloud.agent.transport.Request; import com.cloud.agent.transport.Response; import com.cloud.alert.AlertManager; +import com.cloud.configuration.Config; import com.cloud.configuration.ManagementServiceConfiguration; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenterVO; @@ -105,11 +104,13 @@ import com.cloud.resource.Discoverer; import com.cloud.resource.ResourceManager; import com.cloud.resource.ResourceState; import com.cloud.resource.ServerResource; +import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; +import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; @@ -124,7 +125,6 @@ import com.cloud.utils.nio.Link; import com.cloud.utils.nio.NioServer; import com.cloud.utils.nio.Task; import com.cloud.utils.time.InaccurateClock; -import org.apache.commons.lang3.StringUtils; /** * Implementation of the Agent Manager. This class controls the connection to the agents. @@ -208,6 +208,11 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected final ConfigKey CheckTxnBeforeSending = new ConfigKey("Developer", Boolean.class, "check.txn.before.sending.agent.commands", "false", "This parameter allows developers to enable a check to see if a transaction wraps commands that are sent to the resource. This is not to be enabled on production systems.", true); + public static final List HOST_DOWN_ALERT_UNSUPPORTED_HOST_TYPES = Arrays.asList( + Host.Type.SecondaryStorage, + Host.Type.ConsoleProxy + ); + @Override public boolean configure(final String name, final Map params) throws ConfigurationException { @@ -901,7 +906,10 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl if (determinedState == Status.Down) { final String message = "Host is down: " + host.getId() + "-" + host.getName() + ". Starting HA on the VMs"; s_logger.error(message); - if (host.getType() != Host.Type.SecondaryStorage && host.getType() != Host.Type.ConsoleProxy) { + if (Status.Down.equals(host.getStatus())) { + s_logger.debug(String.format("Skipping sending alert for %s as it already in %s state", + host, host.getStatus())); + } else if (!HOST_DOWN_ALERT_UNSUPPORTED_HOST_TYPES.contains(host.getType())) { _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Host down, " + host.getId(), message); } event = Status.Event.HostDown; diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index aa43e6b9161..b78b135adf9 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -21,6 +21,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -41,6 +42,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContext; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.log4j.NDC; @@ -71,7 +73,6 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.VpcVirtualNetworkApplianceService; import com.cloud.resource.ResourceManager; import com.cloud.server.ManagementServer; -import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageManager; @@ -223,6 +224,18 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur long _timeBetweenCleanups; String _haTag = null; + private boolean vmHasPendingHAJob(final List pendingHaWorks, final VMInstanceVO vm) { + Optional item = pendingHaWorks.stream() + .filter(h -> h.getInstanceId() == vm.getId()) + .reduce((first, second) -> second); + if (item.isPresent() && (item.get().getTimesTried() < _maxRetries || + !item.get().canScheduleNew(_timeBetweenFailures))) { + s_logger.debug(String.format("Skipping HA on %s as there is already a running HA job for it", vm)); + return true; + } + return false; + } + protected HighAvailabilityManagerImpl() { } @@ -265,28 +278,37 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur s_logger.warn("Scheduling restart for VMs on host " + host.getId() + "-" + host.getName()); final List vms = _instanceDao.listByHostId(host.getId()); + final List pendingHaWorks = _haDao.listPendingHAWorkForHost(host.getId()); final DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId()); // send an email alert that the host is down StringBuilder sb = null; List reorderedVMList = new ArrayList(); - if ((vms != null) && !vms.isEmpty()) { + int skippedHAVms = 0; + if (CollectionUtils.isNotEmpty(vms)) { sb = new StringBuilder(); sb.append(" Starting HA on the following VMs:"); // collect list of vm names for the alert email - for (int i = 0; i < vms.size(); i++) { - VMInstanceVO vm = vms.get(i); + for (VMInstanceVO vm : vms) { + if (vmHasPendingHAJob(pendingHaWorks, vm)) { + skippedHAVms++; + continue; + } if (vm.getType() == VirtualMachine.Type.User) { reorderedVMList.add(vm); } else { reorderedVMList.add(0, vm); } if (vm.isHaEnabled()) { - sb.append(" " + vm.getHostName()); + sb.append(" ").append(vm.getHostName()); } } } - + if (reorderedVMList.isEmpty() && skippedHAVms > 0 && skippedHAVms == vms.size()) { + s_logger.debug(String.format( + "Skipping sending alert for %s as it is suspected to be a duplicate of a recent alert", host)); + return; + } // send an email alert that the host is down, include VMs HostPodVO podVO = _podDao.findById(host.getPodId()); String hostDesc = "name: " + host.getName() + " (id:" + host.getId() + "), availability zone: " + dcVO.getName() + ", pod: " + podVO.getName(); @@ -294,7 +316,6 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur "Host [" + hostDesc + "] is down." + ((sb != null) ? sb.toString() : "")); for (VMInstanceVO vm : reorderedVMList) { - ServiceOfferingVO vmOffering = _serviceOfferingDao.findById(vm.getServiceOfferingId()); if (_itMgr.isRootVolumeOnLocalStorage(vm.getId())) { if (s_logger.isDebugEnabled()){ s_logger.debug("Skipping HA on vm " + vm + ", because it uses local storage. Its fate is tied to the host."); diff --git a/server/src/main/java/com/cloud/ha/dao/HighAvailabilityDao.java b/server/src/main/java/com/cloud/ha/dao/HighAvailabilityDao.java index e8a3e17f805..395b74e0464 100644 --- a/server/src/main/java/com/cloud/ha/dao/HighAvailabilityDao.java +++ b/server/src/main/java/com/cloud/ha/dao/HighAvailabilityDao.java @@ -85,4 +85,6 @@ public interface HighAvailabilityDao extends GenericDao { List listPendingHaWorkForVm(long vmId); List listPendingMigrationsForVm(long vmId); + + List listPendingHAWorkForHost(long hostId); } diff --git a/server/src/main/java/com/cloud/ha/dao/HighAvailabilityDaoImpl.java b/server/src/main/java/com/cloud/ha/dao/HighAvailabilityDaoImpl.java index c7284053fb2..7057ad84a42 100644 --- a/server/src/main/java/com/cloud/ha/dao/HighAvailabilityDaoImpl.java +++ b/server/src/main/java/com/cloud/ha/dao/HighAvailabilityDaoImpl.java @@ -19,7 +19,6 @@ package com.cloud.ha.dao; import java.util.Date; import java.util.List; - import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -260,4 +259,17 @@ public class HighAvailabilityDaoImpl extends GenericDaoBase impl return update(vo, sc); } + + @Override + public List listPendingHAWorkForHost(long hostId) { + SearchBuilder sb = createSearchBuilder(); + sb.and("hostId", sb.entity().getHostId(), Op.EQ); + sb.and("type", sb.entity().getWorkType(), Op.EQ); + sb.and("step", sb.entity().getStep(), Op.NIN); + SearchCriteria sc = sb.create(); + sc.setParameters("hostId", hostId); + sc.setParameters("type", WorkType.HA); + sc.setParameters("step", Step.Done, Step.Cancelled, Step.Error); + return listBy(sc); + } } diff --git a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java index 53e5a26849f..f542aee5d6d 100644 --- a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java +++ b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java @@ -62,7 +62,6 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.VpcVirtualNetworkApplianceService; import com.cloud.resource.ResourceManager; import com.cloud.server.ManagementServer; -import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.StorageManager; import com.cloud.storage.dao.GuestOSCategoryDao; @@ -214,7 +213,6 @@ public class HighAvailabilityManagerImplTest { Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(DataCenterVO.class)); Mockito.when(_haDao.findPreviousHA(Mockito.anyLong())).thenReturn(Arrays.asList(Mockito.mock(HaWorkVO.class))); Mockito.when(_haDao.persist((HaWorkVO)Mockito.anyObject())).thenReturn(Mockito.mock(HaWorkVO.class)); - Mockito.when(_serviceOfferingDao.findById(vm1.getServiceOfferingId())).thenReturn(Mockito.mock(ServiceOfferingVO.class)); highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); } From c1923a2dcb3c0902e21a4369ac1ee6e2b4dc1a5c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 7 May 2025 12:20:36 +0530 Subject: [PATCH 05/18] test: cleanup test_guest_os.py for multiple execution (#10818) Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_guest_os.py | 21 +++++++++++++++------ tools/marvin/marvin/lib/base.py | 15 +++++++++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/test/integration/smoke/test_guest_os.py b/test/integration/smoke/test_guest_os.py index c9d50a7c523..bb09b2b86a1 100644 --- a/test/integration/smoke/test_guest_os.py +++ b/test/integration/smoke/test_guest_os.py @@ -47,19 +47,14 @@ class TestGuestOS(cloudstackTestCase): cls.hypervisor = cls.get_hypervisor_type() - @classmethod def setUp(self): self.apiclient = self.testClient.getApiClient() #build cleanup list self.cleanup = [] - @classmethod def tearDown(self): - try: - cleanup_resources(self.apiclient, self.cleanup) - except Exception as e: - self.debug("Warning! Exception in tearDown: %s" % e) + super(TestGuestOS, self).tearDown() @classmethod def get_hypervisor_type(cls): @@ -95,6 +90,7 @@ class TestGuestOS(cloudstackTestCase): osdisplayname="testCentOS", oscategoryid=os_category.id ) + self.cleanup.append(self.guestos1) list_guestos = GuestOS.list(self.apiclient, id=self.guestos1.id, listall=True) self.assertNotEqual( len(list_guestos), @@ -112,6 +108,7 @@ class TestGuestOS(cloudstackTestCase): self.apiclient, id=self.guestos1.id ) + self.cleanup.remove(self.guestos1) @attr(tags=['advanced', 'simulator', 'basic', 'sg'], required_hardware=False) def test_CRUD_operations_guest_OS_mapping(self): @@ -127,6 +124,7 @@ class TestGuestOS(cloudstackTestCase): osdisplayname="testCentOS", oscategoryid=os_category.id ) + self.cleanup.append(self.guestos1) if self.hypervisor.hypervisor.lower() not in ["xenserver", "vmware"]: raise unittest.SkipTest("OS name check with hypervisor is supported only on XenServer and VMware") @@ -138,6 +136,7 @@ class TestGuestOS(cloudstackTestCase): hypervisorversion=self.hypervisor.hypervisorversion, osnameforhypervisor="testOSMappingName" ) + self.cleanup.append(self.guestosmapping1) list_guestos_mapping = GuestOsMapping.list(self.apiclient, id=self.guestosmapping1.id, listall=True) self.assertNotEqual( @@ -156,11 +155,13 @@ class TestGuestOS(cloudstackTestCase): self.apiclient, id=self.guestosmapping1.id ) + self.cleanup.remove(self.guestosmapping1) GuestOS.remove( self.apiclient, id=self.guestos1.id ) + self.cleanup.remove(self.guestos1) @attr(tags=['advanced', 'simulator', 'basic', 'sg'], required_hardware=False) def test_guest_OS_mapping_check_with_hypervisor(self): @@ -176,6 +177,7 @@ class TestGuestOS(cloudstackTestCase): osdisplayname="testOSname1", oscategoryid=os_category.id ) + self.cleanup.append(self.guestos1) if self.hypervisor.hypervisor.lower() not in ["xenserver", "vmware"]: raise unittest.SkipTest("OS name check with hypervisor is supported only on XenServer and VMware") @@ -193,6 +195,7 @@ class TestGuestOS(cloudstackTestCase): osnameforhypervisor=testosname, osmappingcheckenabled=True ) + self.cleanup.append(self.guestosmapping1) list_guestos_mapping = GuestOsMapping.list(self.apiclient, id=self.guestosmapping1.id, listall=True) self.assertNotEqual( @@ -211,11 +214,13 @@ class TestGuestOS(cloudstackTestCase): self.apiclient, id=self.guestosmapping1.id ) + self.cleanup.remove(self.guestosmapping1) GuestOS.remove( self.apiclient, id=self.guestos1.id ) + self.cleanup.remove(self.guestos1) @attr(tags=['advanced', 'simulator', 'basic', 'sg'], required_hardware=False) def test_guest_OS_mapping_check_with_hypervisor_failure(self): @@ -231,6 +236,7 @@ class TestGuestOS(cloudstackTestCase): osdisplayname="testOSname2", oscategoryid=os_category.id ) + self.cleanup.append(self.guestos1) if self.hypervisor.hypervisor.lower() not in ["xenserver", "vmware"]: raise unittest.SkipTest("OS name check with hypervisor is supported only on XenServer and VMware") @@ -246,10 +252,12 @@ class TestGuestOS(cloudstackTestCase): osnameforhypervisor=testosname, osmappingcheckenabled=True ) + self.cleanup.append(self.guestosmapping1) GuestOsMapping.remove( self.apiclient, id=self.guestosmapping1.id ) + self.cleanup.remove(self.guestosmapping1) self.fail("Since os mapping name is wrong, this API should fail") except CloudstackAPIException as e: self.debug("Addition guest OS mapping failed as expected %s " % e) @@ -257,4 +265,5 @@ class TestGuestOS(cloudstackTestCase): self.apiclient, id=self.guestos1.id ) + self.cleanup.remove(self.guestos1) return diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 557434ea2ee..3d232d887c8 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -4576,6 +4576,7 @@ class Project: def __init__(self, items): self.__dict__.update(items) + @classmethod def create(cls, apiclient, services, account=None, domainid=None, userid=None, accountid=None): """Create project""" @@ -6720,7 +6721,7 @@ class GuestOSCategory: class GuestOS: """Manage Guest OS""" - def __init__(self, items, services): + def __init__(self, items): self.__dict__.update(items) @classmethod @@ -6735,7 +6736,7 @@ class GuestOS: if details is not None: cmd.details = details - return (apiclient.addGuestOs(cmd)) + return GuestOS(apiclient.addGuestOs(cmd).__dict__) @classmethod def remove(cls, apiclient, id): @@ -6772,10 +6773,13 @@ class GuestOS: return (apiclient.listOsTypes(cmd)) + def delete(self, apiclient): + self.remove(apiclient, self.id) + class GuestOsMapping: """Manage Guest OS Mappings""" - def __init__(self, items, services): + def __init__(self, items): self.__dict__.update(items) @classmethod @@ -6793,7 +6797,7 @@ class GuestOsMapping: if forced is not None: cmd.forced = forced - return (apiclient.addGuestOsMapping(cmd)) + return GuestOsMapping(apiclient.addGuestOsMapping(cmd).__dict__) @classmethod def remove(cls, apiclient, id): @@ -6837,6 +6841,9 @@ class GuestOsMapping: return (apiclient.listGuestOsMapping(cmd)) + def delete(self, apiclient): + self.remove(apiclient, self.id) + class VMSchedule: def __init__(self, items): From f0838cdd30984818919c12663dec6effa26483c7 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Thu, 8 May 2025 12:18:14 +0530 Subject: [PATCH 06/18] [VMware] Update vlans with proper range before creating port group for dvSwitch (#10708) --- .../java/com/cloud/utils/StringUtils.java | 92 ++++++++++++++++++- .../vmware/mo/HypervisorHostHelper.java | 22 +++-- .../vmware/mo/HypervisorHostHelperTest.java | 56 ++++++++++- 3 files changed, 159 insertions(+), 11 deletions(-) diff --git a/utils/src/main/java/com/cloud/utils/StringUtils.java b/utils/src/main/java/com/cloud/utils/StringUtils.java index 01b6f833271..c6116123b40 100644 --- a/utils/src/main/java/com/cloud/utils/StringUtils.java +++ b/utils/src/main/java/com/cloud/utils/StringUtils.java @@ -28,6 +28,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -90,7 +92,7 @@ public class StringUtils extends org.apache.commons.lang3.StringUtils { /** * Converts a List of tags to a comma separated list - * @param tags + * @param tagsList * @return String containing a comma separated list of tags */ @@ -304,4 +306,92 @@ public class StringUtils extends org.apache.commons.lang3.StringUtils { return mapResult; } + + /** + * Converts the comma separated numbers to ranges for any consecutive numbers in the input with numbers (and ranges) + * Eg: "198,200-203,299,300,301,303,304,305,306,307,308,311,197" to "197-198,200-203,299-301,303-308,311" + * @param inputNumbersAndRanges + * @return String containing a converted ranges for any consecutive numbers + */ + public static String numbersToRange(String inputNumbersAndRanges) { + Set numberSet = new TreeSet<>(); + for (String inputNumber : inputNumbersAndRanges.split(",")) { + inputNumber = inputNumber.trim(); + if (inputNumber.contains("-")) { + String[] range = inputNumber.split("-"); + if (range.length == 2 && range[0] != null && range[1] != null) { + int start = NumbersUtil.parseInt(range[0], 0); + int end = NumbersUtil.parseInt(range[1], 0); + for (int i = start; i <= end; i++) { + numberSet.add(i); + } + } + } else { + numberSet.add(NumbersUtil.parseInt(inputNumber, 0)); + } + } + + StringBuilder result = new StringBuilder(); + if (!numberSet.isEmpty()) { + List numbers = new ArrayList<>(numberSet); + int startNumber = numbers.get(0); + int endNumber = startNumber; + + for (int i = 1; i < numbers.size(); i++) { + if (numbers.get(i) == endNumber + 1) { + endNumber = numbers.get(i); + } else { + appendRange(result, startNumber, endNumber); + startNumber = endNumber = numbers.get(i); + } + } + appendRange(result, startNumber, endNumber); + } + + return result.toString(); + } + + private static void appendRange(StringBuilder sb, int startNumber, int endNumber) { + if (sb.length() > 0) { + sb.append(","); + } + if (startNumber == endNumber) { + sb.append(startNumber); + } else { + sb.append(startNumber).append("-").append(endNumber); + } + } + + /** + * Converts the comma separated numbers and ranges to numbers + * Eg: "197-198,200-203,299-301,303-308,311" to "197,198,200,201,202,203,299,300,301,303,304,305,306,307,308,311" + * @param inputNumbersAndRanges + * @return String containing a converted numbers + */ + public static String rangeToNumbers(String inputNumbersAndRanges) { + Set numberSet = new TreeSet<>(); + for (String inputNumber : inputNumbersAndRanges.split(",")) { + inputNumber = inputNumber.trim(); + if (inputNumber.contains("-")) { + String[] range = inputNumber.split("-"); + int startNumber = Integer.parseInt(range[0]); + int endNumber = Integer.parseInt(range[1]); + for (int i = startNumber; i <= endNumber; i++) { + numberSet.add(i); + } + } else { + numberSet.add(Integer.parseInt(inputNumber)); + } + } + + StringBuilder result = new StringBuilder(); + for (int number : numberSet) { + if (result.length() > 0) { + result.append(","); + } + result.append(number); + } + + return result.toString(); + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index 44965e9321b..e43af246696 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -274,16 +274,18 @@ public class HypervisorHostHelper { } } - public static String composeCloudNetworkName(String prefix, String vlanId, String svlanId, Integer networkRateMbps, String vSwitchName) { + public static String composeCloudNetworkName(String prefix, String vlanId, String svlanId, Integer networkRateMbps, String vSwitchName, VirtualSwitchType vSwitchType) { StringBuffer sb = new StringBuffer(prefix); if (vlanId == null || UNTAGGED_VLAN_NAME.equalsIgnoreCase(vlanId)) { sb.append(".untagged"); } else { + if (vSwitchType != VirtualSwitchType.StandardVirtualSwitch && StringUtils.containsAny(vlanId, ",-")) { + vlanId = com.cloud.utils.StringUtils.numbersToRange(vlanId); + } sb.append(".").append(vlanId); if (svlanId != null) { sb.append(".").append("s" + svlanId); } - } if (networkRateMbps != null && networkRateMbps.intValue() > 0) @@ -293,7 +295,12 @@ public class HypervisorHostHelper { sb.append(".").append(VersioningContants.PORTGROUP_NAMING_VERSION); sb.append("-").append(vSwitchName); - return sb.toString(); + String networkName = sb.toString(); + if (networkName.length() > 80) { + // the maximum limit for a vSwitch name is 80 chars, applies to both standard and distributed virtual switches. + s_logger.warn(String.format("The network name: %s for the vSwitch %s of type %s, exceeds 80 chars", networkName, vSwitchName, vSwitchType)); + } + return networkName; } public static Map getValidatedVsmCredentials(VmwareContext context) throws Exception { @@ -595,7 +602,7 @@ public class HypervisorHostHelper { if (vlanId != null) { vlanId = vlanId.replace("vlan://", ""); } - networkName = composeCloudNetworkName(namePrefix, vlanId, secondaryvlanId, networkRateMbps, physicalNetwork); + networkName = composeCloudNetworkName(namePrefix, vlanId, secondaryvlanId, networkRateMbps, physicalNetwork, vSwitchType); if (vlanId != null && !UNTAGGED_VLAN_NAME.equalsIgnoreCase(vlanId) && !StringUtils.containsAny(vlanId, ",-")) { createGCTag = true; @@ -1167,8 +1174,9 @@ public class HypervisorHostHelper { if (vlanId == null && vlanRange != null && !vlanRange.isEmpty()) { s_logger.debug("Creating dvSwitch port vlan-trunk spec with range: " + vlanRange); VmwareDistributedVirtualSwitchTrunkVlanSpec trunkVlanSpec = new VmwareDistributedVirtualSwitchTrunkVlanSpec(); - for (final String vlanRangePart : vlanRange.split(",")) { - if (vlanRangePart == null || vlanRange.isEmpty()) { + String vlanRangeUpdated = com.cloud.utils.StringUtils.numbersToRange(vlanRange); + for (final String vlanRangePart : vlanRangeUpdated.split(",")) { + if (vlanRangePart == null || vlanRangePart.isEmpty()) { continue; } final NumericRange numericRange = new NumericRange(); @@ -1320,7 +1328,7 @@ public class HypervisorHostHelper { // No doubt about this, depending on vid=null to avoid lots of code below vid = null; } else { - networkName = composeCloudNetworkName(namePrefix, vlanId, null, networkRateMbps, vSwitchName); + networkName = composeCloudNetworkName(namePrefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.StandardVirtualSwitch); if (vlanId != null && !UNTAGGED_VLAN_NAME.equalsIgnoreCase(vlanId)) { createGCTag = true; diff --git a/vmware-base/src/test/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelperTest.java b/vmware-base/src/test/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelperTest.java index 1c888a05748..05fb9c9e6d3 100644 --- a/vmware-base/src/test/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelperTest.java +++ b/vmware-base/src/test/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelperTest.java @@ -557,7 +557,7 @@ public class HypervisorHostHelperTest { networkRateMbps = 200; prefix = "cloud.public"; vSwitchName = "vSwitch0"; - String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName); + String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName, VirtualSwitchType.StandardVirtualSwitch); assertEquals("cloud.public.100.200.1-vSwitch0", cloudNetworkName); } @@ -567,7 +567,7 @@ public class HypervisorHostHelperTest { networkRateMbps = null; prefix = "cloud.storage"; vSwitchName = "vSwitch1"; - String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName); + String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName, VirtualSwitchType.StandardVirtualSwitch); assertEquals("cloud.storage.untagged.0.1-vSwitch1", cloudNetworkName); } @@ -578,10 +578,60 @@ public class HypervisorHostHelperTest { networkRateMbps = 512; prefix = "cloud.guest"; vSwitchName = "vSwitch2"; - String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName); + String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName, VirtualSwitchType.StandardVirtualSwitch); assertEquals("cloud.guest.400.s123.512.1-vSwitch2", cloudNetworkName); } + @Test + public void testComposeCloudNetworkNameVlanRangeGuestTrafficDvSwitch() { + vlanId = "400-500"; + networkRateMbps = 512; + prefix = "cloud.guest"; + vSwitchName = "dvSwitch0"; + String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.VMwareDistributedVirtualSwitch); + assertEquals("cloud.guest.400-500.512.1-dvSwitch0", cloudNetworkName); + } + + @Test + public void testComposeCloudNetworkNameVlanNumbersGuestTrafficDvSwitch() { + vlanId = "3001,3002,3003,3004,3005,3006,3007,3008,3009,3010,3011,3012,3013,3014,3015,3016,3017,3018,3019,3020"; + networkRateMbps = 512; + prefix = "cloud.guest"; + vSwitchName = "dvSwitch0"; + String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.VMwareDistributedVirtualSwitch); + assertEquals("cloud.guest.3001-3020.512.1-dvSwitch0", cloudNetworkName); + } + + @Test + public void testComposeCloudNetworkNameVlanNumbersAndRangeGuestTrafficDvSwitch() { + vlanId = "3001,3004-3006,3007,3008,3009,3010,3011,3012,3013,3014,3015,3016,3017,3018,3020"; + networkRateMbps = 512; + prefix = "cloud.guest"; + vSwitchName = "dvSwitch0"; + String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.VMwareDistributedVirtualSwitch); + assertEquals("cloud.guest.3001,3004-3018,3020.512.1-dvSwitch0", cloudNetworkName); + } + + @Test + public void testComposeCloudNetworkNameUnorderedVlanNumbersAndRangeGuestTrafficDvSwitch() { + vlanId = "3018,3020,3011,3012,3004-3006,3007,3001,3008,3009,3010,3013,3014,3015,3016,3017"; + networkRateMbps = 512; + prefix = "cloud.guest"; + vSwitchName = "dvSwitch0"; + String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.VMwareDistributedVirtualSwitch); + assertEquals("cloud.guest.3001,3004-3018,3020.512.1-dvSwitch0", cloudNetworkName); + } + + @Test + public void testComposeCloudNetworkNameOverlappingVlanNumbersAndRangeGuestTrafficDvSwitch() { + vlanId = "3018,3020,3011,3012,3004-3006,3007,3001,3008,3009,3010,3013,3014,3015,3016,3017,3005-3008"; + networkRateMbps = 512; + prefix = "cloud.guest"; + vSwitchName = "dvSwitch0"; + String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.VMwareDistributedVirtualSwitch); + assertEquals("cloud.guest.3001,3004-3018,3020.512.1-dvSwitch0", cloudNetworkName); + } + @Test public void testOvfDomRewriter() { final String ovfString = "" + From c45ed060982b05765f5a48c184efc272fa8904f0 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 8 May 2025 09:17:00 +0200 Subject: [PATCH 07/18] test: fix test_hostha_simulator.py and test_outofbandmanagement.py (#10815) --- test/integration/smoke/test_hostha_simulator.py | 2 +- test/integration/smoke/test_outofbandmanagement.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/smoke/test_hostha_simulator.py b/test/integration/smoke/test_hostha_simulator.py index 1b3dd815a30..20d77e82743 100644 --- a/test/integration/smoke/test_hostha_simulator.py +++ b/test/integration/smoke/test_hostha_simulator.py @@ -457,7 +457,7 @@ class TestHostHA(cloudstackTestCase): retry_interval = 1 + (pingInterval * pingTimeout / 10) - res, _ = wait_until(retry_interval, 20, removeFakeMgmtServer, self.getFakeMsRunId()) + res, _ = wait_until(retry_interval, 100, removeFakeMgmtServer, self.getFakeMsRunId()) if not res: self.fail("Management server failed to turn down or remove fake mgmt server") diff --git a/test/integration/smoke/test_outofbandmanagement.py b/test/integration/smoke/test_outofbandmanagement.py index 79762719f69..3c2114dac9a 100644 --- a/test/integration/smoke/test_outofbandmanagement.py +++ b/test/integration/smoke/test_outofbandmanagement.py @@ -544,7 +544,7 @@ class TestOutOfBandManagement(cloudstackTestCase): retry_interval = 1 + (pingInterval * pingTimeout / 10) - res, _ = wait_until(retry_interval, 10, removeFakeMgmtServer, self.getFakeMsRunId()) + res, _ = wait_until(retry_interval, 100, removeFakeMgmtServer, self.getFakeMsRunId()) if not res: self.fail("Management server failed to turn down or remove fake mgmt server") From 2674da2ee9e667ce051b996832c0f3167672688c Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 8 May 2025 12:50:06 +0530 Subject: [PATCH 08/18] Nas BnR: Fix for restore not working correctly (#10785) --- .../LibvirtRestoreBackupCommandWrapper.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 49b67194356..141c1d5ea19 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -150,16 +150,16 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper Date: Thu, 8 May 2025 06:39:20 -0300 Subject: [PATCH 09/18] Add search bar on rules of roles (#10822) Co-authored-by: Vitor Hugo Homem Marzarotto --- ui/public/locales/en.json | 1 + ui/public/locales/pt_BR.json | 1 + ui/src/views/iam/RolePermissionTab.vue | 28 ++++++++++++++++++++++++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 721d8bacc29..e61e5ee4a6e 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1947,6 +1947,7 @@ "label.rolename": "Role", "label.roles": "Roles", "label.roletype": "Role Type", +"label.rolepermissiontab.searchbar": "Search Rule", "label.root.certificate": "Root certificate", "label.root.disk.size": "Root disk size (GB)", "label.rootdisk": "ROOT disk", diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index 65c657f69d8..307c7e8a1ae 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -1386,6 +1386,7 @@ "label.rolename": "Fun\u00e7\u00e3o", "label.roles": "Fun\u00e7\u00f5es", "label.roletype": "Tipo de fun\u00e7\u00e3o", +"label.rolepermissiontab.searchbar": "Pesquisa de regras", "label.root.certificate": "Certificado ra\u00edz", "label.root.disk.size": "Tamanho do disco ra\u00edz (GB)", "label.rootdisk": "Disco ra\u00edz", diff --git a/ui/src/views/iam/RolePermissionTab.vue b/ui/src/views/iam/RolePermissionTab.vue index 6d2bd71b284..18181d4d407 100644 --- a/ui/src/views/iam/RolePermissionTab.vue +++ b/ui/src/views/iam/RolePermissionTab.vue @@ -18,15 +18,24 @@