Merge branch '4.20' of https://github.com/apache/cloudstack

.github/workflows/ci.yml

@@ -236,7 +236,7 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid pynose
+          python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycryptodome mock flask netaddr pylint pycodestyle six astroid pynose
 
       - name: Install jacoco dependencies
         run: |

core/src/main/java/com/cloud/agent/api/GetVmIpAddressCommand.java

@@ -24,11 +24,13 @@ public class GetVmIpAddressCommand extends Command {
     String vmName;
     String vmNetworkCidr;
     boolean windows = false;
+    String macAddress;
 
-    public GetVmIpAddressCommand(String vmName, String vmNetworkCidr, boolean windows) {
+    public GetVmIpAddressCommand(String vmName, String vmNetworkCidr, boolean windows, String macAddress) {
         this.vmName = vmName;
         this.windows = windows;
         this.vmNetworkCidr = vmNetworkCidr;
+        this.macAddress = macAddress;
     }
 
     @Override
@@ -47,4 +49,8 @@ public class GetVmIpAddressCommand extends Command {
     public String getVmNetworkCidr() {
         return vmNetworkCidr;
     }
+
+    public String getMacAddress() {
+        return macAddress;
+    }
 }

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -247,6 +247,11 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     protected final ConfigKey<Boolean> CheckTxnBeforeSending = new ConfigKey<>("Developer", Boolean.class, "check.txn.before.sending.agent.commands", "false",
             "This parameter allows developers to enable a check to see if a transaction wraps commands that are sent to the resource.  This is not to be enabled on production systems.", true);
 
+    public static final List<Host.Type> HOST_DOWN_ALERT_UNSUPPORTED_HOST_TYPES = Arrays.asList(
+            Host.Type.SecondaryStorage,
+            Host.Type.ConsoleProxy
+    );
+
     @Override
     public boolean configure(final String name, final Map<String, Object> params) throws ConfigurationException {
 
@@ -1093,9 +1098,11 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
                 if (determinedState == Status.Down) {
                     final String message = String.format("Host %s is down. Starting HA on the VMs", host);
                     logger.error(message);
-                    if (host.getType() != Host.Type.SecondaryStorage && host.getType() != Host.Type.ConsoleProxy) {
+                    if (Status.Down.equals(host.getStatus())) {
-                        _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(),
+                        logger.debug(String.format("Skipping sending alert for %s as it already in %s state",
-                                host.getPodId(), String.format("Host down, %s", host), message);
+                                host, host.getStatus()));
+                    } else if (!HOST_DOWN_ALERT_UNSUPPORTED_HOST_TYPES.contains(host.getType())) {
+                        _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "Host down, " + host.getId(), message);
                     }
                     event = Status.Event.HostDown;
                 } else if (determinedState == Status.Up) {

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -595,7 +595,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 if (_networkOfferingDao.findByUniqueName(NetworkOffering.DefaultIsolatedNetworkOfferingForVpcNetworks) == null) {
                     offering = _configMgr.createNetworkOffering(NetworkOffering.DefaultIsolatedNetworkOfferingForVpcNetworks,
                             "Offering for Isolated VPC networks with Source Nat service enabled", TrafficType.Guest, null, false, Availability.Optional, null,
-                            defaultVPCOffProviders, true, Network.GuestType.Isolated, false, null, false, null, false, false, null, false, null, true, true, false, false, null, null, null,true, null, null, false);
+                            defaultVPCOffProviders, true, Network.GuestType.Isolated, false, null, true, null, false, false, null, false, null, true, true, false, false, null, null, null,true, null, null, false);
                 }
 
                 //#6 - default vpc offering with no LB service

engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDao.java

@@ -28,4 +28,6 @@ public interface UsageNetworksDao extends GenericDao<UsageNetworksVO, Long> {
     void remove(long networkId, Date removed);
 
     List<UsageNetworksVO> getUsageRecords(Long accountId, Date startDate, Date endDate);
+
+    List<UsageNetworksVO> listAll(long networkId);
 }

engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDaoImpl.java

@@ -19,6 +19,7 @@ package com.cloud.usage.dao;
 import com.cloud.usage.UsageNetworksVO;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.TransactionLegacy;
 
@@ -26,6 +27,7 @@ import org.springframework.stereotype.Component;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import javax.annotation.PostConstruct;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.util.ArrayList;
@@ -40,6 +42,14 @@ public class UsageNetworksDaoImpl extends GenericDaoBase<UsageNetworksVO, Long>
             " account_id = ? AND ((removed IS NULL AND created <= ?) OR (created BETWEEN ? AND ?) OR (removed BETWEEN ? AND ?) " +
             " OR ((created <= ?) AND (removed >= ?)))";
 
+    private SearchBuilder<UsageNetworksVO> usageNetworksSearch;
+
+    @PostConstruct
+    public void init() {
+        usageNetworksSearch = createSearchBuilder();
+        usageNetworksSearch.and("networkId", usageNetworksSearch.entity().getNetworkId(), SearchCriteria.Op.EQ);
+        usageNetworksSearch.done();
+    }
 
     @Override
     public void update(long networkId, long newNetworkOffering, String state) {
@@ -131,4 +141,11 @@ public class UsageNetworksDaoImpl extends GenericDaoBase<UsageNetworksVO, Long>
 
         return usageRecords;
     }
+
+    @Override
+    public List<UsageNetworksVO> listAll(long networkId) {
+        SearchCriteria<UsageNetworksVO> sc = usageNetworksSearch.create();
+        sc.setParameters("networkId", networkId);
+        return listBy(sc);
+    }
 }

engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDao.java

@@ -24,6 +24,10 @@ import java.util.List;
 
 public interface UsageVpcDao extends GenericDao<UsageVpcVO, Long> {
     void update(UsageVpcVO usage);
+
     void remove(long vpcId, Date removed);
+
     List<UsageVpcVO> getUsageRecords(Long accountId, Date startDate, Date endDate);
+
+    List<UsageVpcVO> listAll(long vpcId);
 }

engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java

@@ -19,10 +19,12 @@ package com.cloud.usage.dao;
 import com.cloud.usage.UsageVpcVO;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.TransactionLegacy;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.PostConstruct;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.util.ArrayList;
@@ -36,6 +38,15 @@ public class UsageVpcDaoImpl extends GenericDaoBase<UsageVpcVO, Long> implements
             " account_id = ? AND ((removed IS NULL AND created <= ?) OR (created BETWEEN ? AND ?) OR (removed BETWEEN ? AND ?) " +
             " OR ((created <= ?) AND (removed >= ?)))";
 
+    private SearchBuilder<UsageVpcVO> usageVpcSearch;
+
+    @PostConstruct
+    public void init() {
+        usageVpcSearch = createSearchBuilder();
+        usageVpcSearch.and("vpcId", usageVpcSearch.entity().getVpcId(), SearchCriteria.Op.EQ);
+        usageVpcSearch.done();
+    }
+
     @Override
     public void update(UsageVpcVO usage) {
         TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
@@ -124,4 +135,11 @@ public class UsageVpcDaoImpl extends GenericDaoBase<UsageVpcVO, Long> implements
 
         return usageRecords;
     }
+
+    @Override
+    public List<UsageVpcVO> listAll(long vpcId) {
+        SearchCriteria<UsageVpcVO> sc = usageVpcSearch.create();
+        sc.setParameters("vpcId", vpcId);
+        return listBy(sc);
+    }
 }

engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql

@@ -75,3 +75,6 @@ CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only Admin - Default', 'va
 
 CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support Admin - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
 CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support Admin - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');
+
+-- Re-apply VPC: update default network offering for vpc tier to conserve_mode=1 (#8309)
+UPDATE `cloud`.`network_offerings` SET conserve_mode=1 WHERE name='DefaultIsolatedNetworkOfferingForVpcNetworks';

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVmIpAddressCommandWrapper.java

@@ -66,10 +66,13 @@ public final class LibvirtGetVmIpAddressCommandWrapper extends CommandWrapper<Ge
 
         String sanitizedVmName = sanitizeBashCommandArgument(vmName);
         String networkCidr = command.getVmNetworkCidr();
+        String macAddress = command.getMacAddress();
 
-        ip = ipFromDomIf(sanitizedVmName, networkCidr);
+        init();
 
-        if (ip == null) {
+        ip = ipFromDomIf(sanitizedVmName, networkCidr, macAddress);
+
+        if (ip == null && networkCidr != null) {
             if(!command.isWindows()) {
                 ip = ipFromDhcpLeaseFile(sanitizedVmName, networkCidr);
             } else {
@@ -87,25 +90,17 @@ public final class LibvirtGetVmIpAddressCommandWrapper extends CommandWrapper<Ge
         return new Answer(command, result, ip);
     }
 
-    private String ipFromDomIf(String sanitizedVmName, String networkCidr) {
+    private String ipFromDomIf(String sanitizedVmName, String networkCidr, String macAddress) {
         String ip = null;
         List<String[]> commands = new ArrayList<>();
         commands.add(new String[]{virsh_path, "domifaddr", sanitizedVmName, "--source", "agent"});
         Pair<Integer,String> response = executePipedCommands(commands, 0);
         if (response != null) {
             String output = response.second();
-            String[] lines = output.split("\n");
+            Pair<String, String> ipAddresses = getIpAddresses(output, macAddress);
-            for (String line : lines) {
+            String ipv4 = ipAddresses.first();
-                if (line.contains("ipv4")) {
+            if (networkCidr == null || NetUtils.isIpWithInCidrRange(ipv4, networkCidr)) {
-                    String[] parts = line.split(" ");
+                ip = ipv4;
-                    String[] ipParts = parts[parts.length-1].split("/");
-                    if (ipParts.length > 1) {
-                        if (NetUtils.isIpWithInCidrRange(ipParts[0], networkCidr)) {
-                            ip = ipParts[0];
-                            break;
-                        }
-                    }
-                }
             }
         } else {
             logger.error("ipFromDomIf: Command execution failed for VM: " + sanitizedVmName);
@@ -113,6 +108,38 @@ public final class LibvirtGetVmIpAddressCommandWrapper extends CommandWrapper<Ge
         return ip;
     }
 
+    private Pair<String, String> getIpAddresses(String output, String macAddress) {
+        String ipv4 = null;
+        String ipv6 = null;
+        boolean found = false;
+        String[] lines = output.split("\n");
+        for (String line : lines) {
+            String[] parts = line.replaceAll(" +", " ").trim().split(" ");
+            if (parts.length < 4) {
+                continue;
+            }
+            String device = parts[0];
+            String mac = parts[1];
+            if (found) {
+                if (!device.equals("-") || !mac.equals("-")) {
+                    break;
+                }
+            } else if (!mac.equals(macAddress)) {
+                continue;
+            }
+            found = true;
+            String ipFamily = parts[2];
+            String ipPart = parts[3].split("/")[0];
+            if (ipFamily.equals("ipv4")) {
+                ipv4 = ipPart;
+            } else if (ipFamily.equals("ipv6")) {
+                ipv6 = ipPart;
+            }
+        }
+        logger.debug(String.format("Found ipv4: %s and ipv6: %s with mac address %s", ipv4, ipv6, macAddress));
+        return new Pair<>(ipv4, ipv6);
+    }
+
     private String ipFromDhcpLeaseFile(String sanitizedVmName, String networkCidr) {
         String ip = null;
         List<String[]> commands = new ArrayList<>();

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java

@@ -150,16 +150,16 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper<RestoreBa
         String mountDirectory = String.format("%s.%s",BACKUP_TEMP_FILE_PREFIX , randomChars);
         try {
             mountDirectory = Files.createTempDirectory(mountDirectory).toString();
-            String mountOpts = null;
-            if (Objects.nonNull(mountOptions)) {
-                mountOpts = mountOptions;
-                if ("cifs".equals(backupRepoType)) {
-                    mountOpts += ",nobrl";
-                }
-            }
             String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory);
-            if (Objects.nonNull(mountOpts)) {
+            if ("cifs".equals(backupRepoType)) {
-                mount += " -o " + mountOpts;
+                if (Objects.isNull(mountOptions) || mountOptions.trim().isEmpty()) {
+                    mountOptions = "nobrl";
+                } else {
+                    mountOptions += ",nobrl";
+                }
+            }
+            if (Objects.nonNull(mountOptions) && !mountOptions.trim().isEmpty()) {
+                mount += " -o " + mountOptions;
             }
             Script.runSimpleBashScript(mount);
         } catch (Exception e) {

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVmIpAddressCommandWrapperTest.java

@@ -66,6 +66,7 @@ public class LibvirtGetVmIpAddressCommandWrapperTest {
 
         when(getVmIpAddressCommand.getVmName()).thenReturn("validVmName");
         when(getVmIpAddressCommand.getVmNetworkCidr()).thenReturn("192.168.0.0/24");
+        when(getVmIpAddressCommand.getMacAddress()).thenReturn("02:0c:02:f9:00:80");
         when(getVmIpAddressCommand.isWindows()).thenReturn(false);
         when(Script.executePipedCommands(anyList(), anyLong())).thenReturn(new Pair<>(0, VIRSH_DOMIF_OUTPUT));
 
@@ -88,6 +89,7 @@ public class LibvirtGetVmIpAddressCommandWrapperTest {
 
         when(getVmIpAddressCommand.getVmName()).thenReturn("invalidVmName!");
         when(getVmIpAddressCommand.getVmNetworkCidr()).thenReturn("192.168.0.0/24");
+        when(getVmIpAddressCommand.getMacAddress()).thenReturn("02:0c:02:f9:00:80");
         when(getVmIpAddressCommand.isWindows()).thenReturn(false);
         when(Script.executePipedCommands(anyList(), anyLong())).thenReturn(new Pair<>(0, VIRSH_DOMIF_OUTPUT));
 
@@ -114,6 +116,7 @@ public class LibvirtGetVmIpAddressCommandWrapperTest {
 
             when(getVmIpAddressCommand.getVmName()).thenReturn("validVmName");
             when(getVmIpAddressCommand.getVmNetworkCidr()).thenReturn("192.168.0.0/24");
+            when(getVmIpAddressCommand.getMacAddress()).thenReturn("02:0c:02:f9:00:80");
             when(getVmIpAddressCommand.isWindows()).thenReturn(true);
             when(Script.executePipedCommands(anyList(), anyLong())).thenReturn(new Pair<>(0, "192.168.0.10"));
 

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/XenServerGuru.java

@@ -34,6 +34,7 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.commons.lang3.StringUtils;
 
+import com.cloud.agent.api.CleanupVMCommand;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.to.DataObjectType;
 import com.cloud.agent.api.to.DataStoreTO;
@@ -234,4 +235,12 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru,
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {MaxNumberOfVCPUSPerVM};
     }
+
+    @Override
+    public List<Command> finalizeExpunge(VirtualMachine vm) {
+        List<Command> commands = new ArrayList<>();
+        final CleanupVMCommand cleanupVMCommand = new CleanupVMCommand(vm.getInstanceName(), true);
+        commands.add(cleanupVMCommand);
+        return commands;
+    }
 }

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixCleanupVMCommandWrapper.java

@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.xenserver.resource.wrapper.xenbase;
+
+import java.util.Iterator;
+import java.util.Set;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CleanupVMCommand;
+import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.xensource.xenapi.Connection;
+import com.xensource.xenapi.Types;
+import com.xensource.xenapi.VM;
+
+@ResourceWrapper(handles =  CleanupVMCommand.class)
+public class CitrixCleanupVMCommandWrapper extends CommandWrapper<CleanupVMCommand, Answer, CitrixResourceBase> {
+
+    @Override
+    public Answer execute(final CleanupVMCommand command, final CitrixResourceBase citrixResourceBase) {
+        if (citrixResourceBase.isDestroyHaltedVms()) {
+            logger.debug(String.format("Cleanup VM is not needed for host with version %s",
+                    citrixResourceBase.getHost().getProductVersion()));
+            return new Answer(command);
+        }
+        final String vmName = command.getVmName();
+        try {
+            final Connection conn = citrixResourceBase.getConnection();
+            final Set<VM> vms = VM.getByNameLabel(conn, vmName);
+            if (vms.isEmpty()) {
+                return new Answer(command, true, "VM does not exist");
+            }
+            // destroy vm which is in HALTED state on this host
+            final Iterator<VM> iter = vms.iterator();
+            while (iter.hasNext()) {
+                final VM vm = iter.next();
+                final VM.Record vmr = vm.getRecord(conn);
+                if (!Types.VmPowerState.HALTED.equals(vmr.powerState)) {
+                    final String msg = String.format("VM %s is not in %s state", vmName, Types.VmPowerState.HALTED);
+                    logger.error(msg);
+                    return new Answer(command, false, msg);
+                }
+                if (citrixResourceBase.isRefNull(vmr.residentOn)) {
+                    continue;
+                }
+                if (vmr.residentOn.getUuid(conn).equals(citrixResourceBase.getHost().getUuid())) {
+                    continue;
+                }
+                iter.remove();
+            }
+            for (final VM vm : vms) {
+                citrixResourceBase.destroyVm(vm, conn, true);
+            }
+
+        } catch (final Exception e) {
+            final String msg = String.format("Clean up VM %s fail due to %s", vmName, e);
+            logger.error(msg, e);
+            return new Answer(command, false, e.getMessage());
+        }
+        return new Answer(command);
+    }
+}

plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixRequestWrapperTest.java

@@ -1934,7 +1934,7 @@ public class CitrixRequestWrapperTest {
         vmIpsMap.put("Test", "127.0.0.1");
         rec.networks = vmIpsMap;
 
-        final GetVmIpAddressCommand getVmIpAddrCmd = new GetVmIpAddressCommand("Test", "127.0.0.1/24", false);
+        final GetVmIpAddressCommand getVmIpAddrCmd = new GetVmIpAddressCommand("Test", "127.0.0.1/24", false, null);
 
         final CitrixRequestWrapper wrapper = CitrixRequestWrapper.getInstance();
         assertNotNull(wrapper);

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java

@@ -412,13 +412,14 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
             if (err == null && needResize) {
                 err = notifyQemuForTheNewSize(data, err, vol, payload);
             }
-
             if (err != null) {
                 // try restoring volume to its initial size
                 SpApiResponse response = StorPoolUtil.volumeUpdate(name, oldSize, true, oldMaxIops, conn);
                 if (response.getError() != null) {
                     logger.debug(String.format("Could not resize StorPool volume %s back to its original size. Error: %s", name, response.getError()));
                 }
+            } else {
+                updateVolumeWithTheNewSize(vol, payload);
             }
         } catch (Exception e) {
             logger.debug("sending resize command failed", e);
@@ -427,6 +428,17 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         return err;
     }
 
+    private void updateVolumeWithTheNewSize(VolumeObject vol, ResizeVolumePayload payload) {
+        vol.setSize(payload.newSize);
+        vol.update();
+        if (payload.newMaxIops != null) {
+            VolumeVO volume = volumeDao.findById(vol.getId());
+            volume.setMaxIops(payload.newMaxIops);
+            volumeDao.update(volume.getId(), volume);
+        }
+        updateStoragePool(vol.getPoolId(), payload.newSize - vol.getSize());
+    }
+
     private String notifyQemuForTheNewSize(DataObject data, String err, VolumeObject vol, ResizeVolumePayload payload)
             throws StorageUnavailableException {
         StoragePool pool = (StoragePool)data.getDataStore();
@@ -455,11 +467,24 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         }
         SpApiResponse resp = new SpApiResponse();
         if (tier != null || template != null) {
+            resp = updateVolumeByStorPoolQoS(payload, conn, name, tier, template);
+        } else {
+            resp = updateVolumeByOffering(vol, payload, conn, name);
+        }
+        if (resp.getError() != null) {
+            err = String.format("Could not resize StorPool volume %s. Error: %s", name, resp.getError());
+        }
+        return err;
+    }
+
+    private static SpApiResponse updateVolumeByStorPoolQoS(ResizeVolumePayload payload, SpConnectionDesc conn, String name, String tier, String template) {
         Map<String, String> tags = StorPoolHelper.addStorPoolTags(null, null, null, null, tier);
         StorPoolVolumeDef spVolume = new StorPoolVolumeDef(name, payload.newSize, tags, null, null, template, null, null,
                 payload.shrinkOk);
-            resp = StorPoolUtil.volumeUpdate(spVolume, conn);
+        return StorPoolUtil.volumeUpdate(spVolume, conn);
-        } else {
+    }
+
+    private static SpApiResponse updateVolumeByOffering(VolumeObject vol, ResizeVolumePayload payload, SpConnectionDesc conn, String name) {
         long maxIops = payload.newMaxIops == null ? Long.valueOf(0) : payload.newMaxIops;
 
         StorPoolVolumeDef spVolume = new StorPoolVolumeDef(name, payload.newSize, null, null, maxIops, null, null, null,
@@ -467,23 +492,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         StorPoolUtil.spLog(
                 "StorpoolPrimaryDataStoreDriverImpl.resize: name=%s, uuid=%s, oldSize=%d, newSize=%s, shrinkOk=%s, maxIops=%s",
                 name, vol.getUuid(), vol.getSize(), payload.newSize, payload.shrinkOk, maxIops);
-
+        return StorPoolUtil.volumeUpdate(spVolume, conn);
-            resp = StorPoolUtil.volumeUpdate(spVolume, conn);
-        }
-        if (resp.getError() != null) {
-            err = String.format("Could not resize StorPool volume %s. Error: %s", name, resp.getError());
-        } else {
-            vol.setSize(payload.newSize);
-            vol.update();
-            if (payload.newMaxIops != null) {
-                VolumeVO volume = volumeDao.findById(vol.getId());
-                volume.setMaxIops(payload.newMaxIops);
-                volumeDao.update(volume.getId(), volume);
-            }
-
-            updateStoragePool(vol.getPoolId(), payload.newSize - vol.getSize());
-        }
-        return err;
     }
 
     @Override

server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java

@@ -24,6 +24,7 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -45,6 +46,7 @@ import org.apache.cloudstack.managed.context.ManagedContext;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.management.ManagementServerHost;
 import org.apache.logging.log4j.ThreadContext;
+import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.alert.AlertManager;
@@ -73,7 +75,6 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.network.VpcVirtualNetworkApplianceService;
 import com.cloud.resource.ResourceManager;
 import com.cloud.server.ManagementServer;
-import com.cloud.service.ServiceOfferingVO;
 import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StorageManager;
@@ -236,6 +237,18 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur
     long _timeBetweenCleanups;
     String _haTag = null;
 
+    private boolean vmHasPendingHAJob(final List<HaWorkVO> pendingHaWorks, final VMInstanceVO vm) {
+        Optional<HaWorkVO> item = pendingHaWorks.stream()
+                .filter(h -> h.getInstanceId() == vm.getId())
+                .reduce((first, second) -> second);
+        if (item.isPresent() && (item.get().getTimesTried() < _maxRetries ||
+                !item.get().canScheduleNew(_timeBetweenFailures))) {
+            logger.debug(String.format("Skipping HA on %s as there is already a running HA job for it", vm));
+            return true;
+        }
+        return false;
+    }
+
     protected HighAvailabilityManagerImpl() {
     }
 
@@ -295,28 +308,37 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur
         logger.warn("Scheduling restart for VMs on host {}", host);
 
         final List<VMInstanceVO> vms = _instanceDao.listByHostId(host.getId());
+        final List<HaWorkVO> pendingHaWorks = _haDao.listPendingHAWorkForHost(host.getId());
         final DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
 
         // send an email alert that the host is down
         StringBuilder sb = null;
         List<VMInstanceVO> reorderedVMList = new ArrayList<VMInstanceVO>();
-        if ((vms != null) && !vms.isEmpty()) {
+        int skippedHAVms = 0;
+        if (CollectionUtils.isNotEmpty(vms)) {
             sb = new StringBuilder();
             sb.append("  Starting HA on the following VMs:");
             // collect list of vm names for the alert email
-            for (int i = 0; i < vms.size(); i++) {
+            for (VMInstanceVO vm : vms) {
-                VMInstanceVO vm = vms.get(i);
+                if (vmHasPendingHAJob(pendingHaWorks, vm)) {
+                    skippedHAVms++;
+                    continue;
+                }
                 if (vm.getType() == VirtualMachine.Type.User) {
                     reorderedVMList.add(vm);
                 } else {
                     reorderedVMList.add(0, vm);
                 }
                 if (vm.isHaEnabled()) {
-                    sb.append(" " + vm.getHostName());
+                    sb.append(" ").append(vm.getHostName());
                 }
             }
         }
-
+        if (reorderedVMList.isEmpty() && skippedHAVms > 0 && skippedHAVms == vms.size()) {
+            logger.debug(String.format(
+                    "Skipping sending alert for %s as it is suspected to be a duplicate of a recent alert", host));
+            return;
+        }
         // send an email alert that the host is down, include VMs
         HostPodVO podVO = _podDao.findById(host.getPodId());
         String hostDesc = "name: " + host.getName() + " (id:" + host.getId() + "), availability zone: " + dcVO.getName() + ", pod: " + podVO.getName();
@@ -324,7 +346,6 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur
                 "Host [" + hostDesc + "] is down." + ((sb != null) ? sb.toString() : ""));
 
         for (VMInstanceVO vm : reorderedVMList) {
-            ServiceOfferingVO vmOffering = _serviceOfferingDao.findById(vm.getServiceOfferingId());
             if (_itMgr.isRootVolumeOnLocalStorage(vm.getId())) {
                 if (logger.isDebugEnabled()){
                     logger.debug("Skipping HA on vm " + vm + ", because it uses local storage. Its fate is tied to the host.");

server/src/main/java/com/cloud/ha/dao/HighAvailabilityDao.java

@@ -85,6 +85,9 @@ public interface HighAvailabilityDao extends GenericDao<HaWorkVO, Long> {
     List<HaWorkVO> listPendingHaWorkForVm(long vmId);
 
     List<HaWorkVO> listPendingMigrationsForVm(long vmId);
+
+    List<HaWorkVO> listPendingHAWorkForHost(long hostId);
+
     int expungeByVmList(List<Long> vmIds, Long batchSize);
     void markPendingWorksAsInvestigating();
     void markServerPendingWorksAsInvestigating(long managementServerId);

server/src/main/java/com/cloud/ha/dao/HighAvailabilityDaoImpl.java

@@ -260,6 +260,19 @@ public class HighAvailabilityDaoImpl extends GenericDaoBase<HaWorkVO, Long> impl
         return update(vo, sc);
     }
 
+    @Override
+    public List<HaWorkVO> listPendingHAWorkForHost(long hostId) {
+        SearchBuilder<HaWorkVO> sb = createSearchBuilder();
+        sb.and("hostId", sb.entity().getHostId(), Op.EQ);
+        sb.and("type", sb.entity().getWorkType(), Op.EQ);
+        sb.and("step", sb.entity().getStep(), Op.NIN);
+        SearchCriteria<HaWorkVO> sc = sb.create();
+        sc.setParameters("hostId", hostId);
+        sc.setParameters("type", WorkType.HA);
+        sc.setParameters("step", Step.Done, Step.Cancelled, Step.Error);
+        return listBy(sc);
+    }
+
     @Override
     public int expungeByVmList(List<Long> vmIds, Long batchSize) {
         if (CollectionUtils.isEmpty(vmIds)) {

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -749,8 +749,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         boolean isWindows;
         Long hostId;
         String networkCidr;
+        String macAddress;
 
-        public VmIpAddrFetchThread(long vmId, String vmUuid, long nicId, String instanceName, boolean windows, Long hostId, String networkCidr) {
+        public VmIpAddrFetchThread(long vmId, long nicId, String instanceName, boolean windows, Long hostId, String networkCidr, String macAddress) {
             this.vmId = vmId;
             this.vmUuid = vmUuid;
             this.nicId = nicId;
@@ -758,11 +759,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             this.isWindows = windows;
             this.hostId = hostId;
             this.networkCidr = networkCidr;
+            this.macAddress = macAddress;
         }
 
         @Override
         protected void runInContext() {
-            GetVmIpAddressCommand cmd = new GetVmIpAddressCommand(vmName, networkCidr, isWindows);
+            GetVmIpAddressCommand cmd = new GetVmIpAddressCommand(vmName, networkCidr, isWindows, macAddress);
             boolean decrementCount = true;
 
             NicVO nic = _nicDao.findById(nicId);
@@ -2435,9 +2437,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     private void loadVmDetailsInMapForExternalDhcpIp() {
 
         List<NetworkVO> networks = _networkDao.listByGuestType(Network.GuestType.Shared);
+        networks.addAll(_networkDao.listByGuestType(Network.GuestType.L2));
 
         for (NetworkVO network: networks) {
-            if(_networkModel.isSharedNetworkWithoutServices(network.getId())) {
+            if (GuestType.L2.equals(network.getGuestType()) || _networkModel.isSharedNetworkWithoutServices(network.getId())) {
                 List<NicVO> nics = _nicDao.listByNetworkId(network.getId());
 
                 for (NicVO nic : nics) {
@@ -2685,9 +2688,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                             VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(userVm);
                             VirtualMachine vm = vmProfile.getVirtualMachine();
                             boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
-
+                            _vmIpFetchThreadExecutor.execute(new VmIpAddrFetchThread(vmId, nicId, vmInstance.getInstanceName(),
-                            _vmIpFetchThreadExecutor.execute(new VmIpAddrFetchThread(vmId, vmInstance.getUuid(), nicId, vmInstance.getInstanceName(),
+                                    isWindows, vm.getHostId(), network.getCidr(), nicVo.getMacAddress()));
-                                    isWindows, vm.getHostId(), network.getCidr()));
 
                         }
                     } catch (Exception e) {
@@ -3352,8 +3354,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             final List<NicVO> nics = _nicDao.listByVmId(vmId);
             for (NicVO nic : nics) {
                 Network network = _networkModel.getNetwork(nic.getNetworkId());
-                if (_networkModel.isSharedNetworkWithoutServices(network.getId())) {
+                if (GuestType.L2.equals(network.getGuestType()) || _networkModel.isSharedNetworkWithoutServices(network.getId())) {
-                    logger.debug("Adding vm {} nic {} into vmIdCountMap as part of vm reboot for vm ip fetch ", userVm, nic);
+                    logger.debug("Adding vm " +vmId +" nic id "+ nic.getId() +" into vmIdCountMap as part of vm " +
+                            "reboot for vm ip fetch ");
                     vmIdCountMap.put(nic.getId(), new VmAndCountDetails(nic.getInstanceId(), VmIpFetchTrialMax.value()));
                 }
             }
@@ -5366,7 +5369,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 final List<NicVO> nics = _nicDao.listByVmId(vm.getId());
                 for (NicVO nic : nics) {
                     Network network = _networkModel.getNetwork(nic.getNetworkId());
-                    if (_networkModel.isSharedNetworkWithoutServices(network.getId())) {
+                    if (GuestType.L2.equals(network.getGuestType()) || _networkModel.isSharedNetworkWithoutServices(network.getId())) {
                         vmIdCountMap.put(nic.getId(), new VmAndCountDetails(nic.getInstanceId(), VmIpFetchTrialMax.value()));
                     }
                 }

server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java

@@ -65,7 +65,6 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.network.VpcVirtualNetworkApplianceService;
 import com.cloud.resource.ResourceManager;
 import com.cloud.server.ManagementServer;
-import com.cloud.service.ServiceOfferingVO;
 import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.dao.GuestOSCategoryDao;
@@ -237,7 +236,6 @@ public class HighAvailabilityManagerImplTest {
         Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(DataCenterVO.class));
         Mockito.when(_haDao.findPreviousHA(Mockito.anyLong())).thenReturn(Arrays.asList(Mockito.mock(HaWorkVO.class)));
         Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));
-        Mockito.when(_serviceOfferingDao.findById(vm1.getServiceOfferingId())).thenReturn(Mockito.mock(ServiceOfferingVO.class));
 
         ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
         highAvailabilityManager.VmHaEnabled = haEnabled;

test/integration/component/test_affinity_groups_projects.py

@@ -99,9 +99,12 @@ class TestCreateAffinityGroup(cloudstackTestCase):
        cls.api_client = cls.testClient.getApiClient()
        cls.services = Services().services
 
+       cls._cleanup = []
+
        #Get Zone, Domain and templates
        cls.rootdomain = get_domain(cls.api_client)
        cls.domain = Domain.create(cls.api_client, cls.services["domain"])
+       cls._cleanup.append(cls.domain)
 
        cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests())
        cls.template = get_template(
@@ -177,21 +180,11 @@ class TestCreateAffinityGroup(cloudstackTestCase):
        self.cleanup = []
 
     def tearDown(self):
-       try:
+        super(TestCreateAffinityGroup, self).tearDownClass()
-#            #Clean up, terminate the created instance, volumes and snapshots
-          cleanup_resources(self.apiclient, self.cleanup)
-       except Exception as e:
-          raise Exception("Warning: Exception during cleanup : %s" % e)
-       return
 
     @classmethod
     def tearDownClass(cls):
-       try:
+        super(TestCreateAffinityGroup, cls).tearDownClass()
-          #Clean up, terminate the created templates
-          cls.domain.delete(cls.api_client, cleanup=True)
-          cleanup_resources(cls.api_client, cls._cleanup)
-       except Exception as e:
-          raise Exception("Warning: Exception during cleanup : %s" % e)
 
     def create_aff_grp(self, api_client=None, aff_grp=None, aff_grp_name=None, projectid=None):
 

test/integration/smoke/test_guest_os.py

@@ -47,19 +47,14 @@ class TestGuestOS(cloudstackTestCase):
 
         cls.hypervisor = cls.get_hypervisor_type()
 
-    @classmethod
     def setUp(self):
         self.apiclient = self.testClient.getApiClient()
 
         #build cleanup list
         self.cleanup = []
 
-    @classmethod
     def tearDown(self):
-        try:
+        super(TestGuestOS, self).tearDown()
-            cleanup_resources(self.apiclient, self.cleanup)
-        except Exception as e:
-            self.debug("Warning! Exception in tearDown: %s" % e)
 
     @classmethod
     def get_hypervisor_type(cls):
@@ -95,6 +90,7 @@ class TestGuestOS(cloudstackTestCase):
             osdisplayname="testCentOS",
             oscategoryid=os_category.id
         )
+        self.cleanup.append(self.guestos1)
         list_guestos = GuestOS.list(self.apiclient, id=self.guestos1.id, listall=True)
         self.assertNotEqual(
             len(list_guestos),
@@ -112,6 +108,7 @@ class TestGuestOS(cloudstackTestCase):
             self.apiclient,
             id=self.guestos1.id
         )
+        self.cleanup.remove(self.guestos1)
 
     @attr(tags=['advanced', 'simulator', 'basic', 'sg'], required_hardware=False)
     def test_CRUD_operations_guest_OS_mapping(self):
@@ -127,6 +124,7 @@ class TestGuestOS(cloudstackTestCase):
             osdisplayname="testCentOS",
             oscategoryid=os_category.id
         )
+        self.cleanup.append(self.guestos1)
 
         if self.hypervisor.hypervisor.lower() not in ["xenserver", "vmware"]:
             raise unittest.SkipTest("OS name check with hypervisor is supported only on XenServer and VMware")
@@ -138,6 +136,7 @@ class TestGuestOS(cloudstackTestCase):
             hypervisorversion=self.hypervisor.hypervisorversion,
             osnameforhypervisor="testOSMappingName"
         )
+        self.cleanup.append(self.guestosmapping1)
 
         list_guestos_mapping = GuestOsMapping.list(self.apiclient, id=self.guestosmapping1.id, listall=True)
         self.assertNotEqual(
@@ -156,11 +155,13 @@ class TestGuestOS(cloudstackTestCase):
             self.apiclient,
             id=self.guestosmapping1.id
         )
+        self.cleanup.remove(self.guestosmapping1)
 
         GuestOS.remove(
             self.apiclient,
             id=self.guestos1.id
         )
+        self.cleanup.remove(self.guestos1)
 
     @attr(tags=['advanced', 'simulator', 'basic', 'sg'], required_hardware=False)
     def test_guest_OS_mapping_check_with_hypervisor(self):
@@ -176,6 +177,7 @@ class TestGuestOS(cloudstackTestCase):
             osdisplayname="testOSname1",
             oscategoryid=os_category.id
         )
+        self.cleanup.append(self.guestos1)
 
         if self.hypervisor.hypervisor.lower() not in ["xenserver", "vmware"]:
             raise unittest.SkipTest("OS name check with hypervisor is supported only on XenServer and VMware")
@@ -193,6 +195,7 @@ class TestGuestOS(cloudstackTestCase):
             osnameforhypervisor=testosname,
             osmappingcheckenabled=True
         )
+        self.cleanup.append(self.guestosmapping1)
 
         list_guestos_mapping = GuestOsMapping.list(self.apiclient, id=self.guestosmapping1.id, listall=True)
         self.assertNotEqual(
@@ -211,11 +214,13 @@ class TestGuestOS(cloudstackTestCase):
             self.apiclient,
             id=self.guestosmapping1.id
         )
+        self.cleanup.remove(self.guestosmapping1)
 
         GuestOS.remove(
             self.apiclient,
             id=self.guestos1.id
         )
+        self.cleanup.remove(self.guestos1)
 
     @attr(tags=['advanced', 'simulator', 'basic', 'sg'], required_hardware=False)
     def test_guest_OS_mapping_check_with_hypervisor_failure(self):
@@ -231,6 +236,7 @@ class TestGuestOS(cloudstackTestCase):
             osdisplayname="testOSname2",
             oscategoryid=os_category.id
         )
+        self.cleanup.append(self.guestos1)
 
         if self.hypervisor.hypervisor.lower() not in ["xenserver", "vmware"]:
             raise unittest.SkipTest("OS name check with hypervisor is supported only on XenServer and VMware")
@@ -246,10 +252,12 @@ class TestGuestOS(cloudstackTestCase):
                 osnameforhypervisor=testosname,
                 osmappingcheckenabled=True
             )
+            self.cleanup.append(self.guestosmapping1)
             GuestOsMapping.remove(
                 self.apiclient,
                 id=self.guestosmapping1.id
             )
+            self.cleanup.remove(self.guestosmapping1)
             self.fail("Since os mapping name is wrong, this API should fail")
         except CloudstackAPIException as e:
             self.debug("Addition guest OS mapping failed as expected %s " % e)
@@ -257,4 +265,5 @@ class TestGuestOS(cloudstackTestCase):
             self.apiclient,
             id=self.guestos1.id
         )
+        self.cleanup.remove(self.guestos1)
         return

test/integration/smoke/test_hostha_simulator.py

@@ -457,7 +457,7 @@ class TestHostHA(cloudstackTestCase):
 
         retry_interval = 1 + (pingInterval * pingTimeout / 10)
 
-        res, _ = wait_until(retry_interval, 20, removeFakeMgmtServer, self.getFakeMsRunId())
+        res, _ = wait_until(retry_interval, 100, removeFakeMgmtServer, self.getFakeMsRunId())
         if not res:
             self.fail("Management server failed to turn down or remove fake mgmt server")
 

test/integration/smoke/test_outofbandmanagement.py

@@ -544,7 +544,7 @@ class TestOutOfBandManagement(cloudstackTestCase):
 
         retry_interval = 1 + (pingInterval * pingTimeout / 10)
 
-        res, _ = wait_until(retry_interval, 10, removeFakeMgmtServer, self.getFakeMsRunId())
+        res, _ = wait_until(retry_interval, 100, removeFakeMgmtServer, self.getFakeMsRunId())
         if not res:
             self.fail("Management server failed to turn down or remove fake mgmt server")
 

tools/marvin/marvin/lib/base.py

@@ -4576,6 +4576,7 @@ class Project:
     def __init__(self, items):
         self.__dict__.update(items)
 
+
     @classmethod
     def create(cls, apiclient, services, account=None, domainid=None, userid=None, accountid=None):
         """Create project"""
@@ -6720,7 +6721,7 @@ class GuestOSCategory:
 class GuestOS:
     """Manage Guest OS"""
 
-    def __init__(self, items, services):
+    def __init__(self, items):
         self.__dict__.update(items)
 
     @classmethod
@@ -6735,7 +6736,7 @@ class GuestOS:
         if details is not None:
             cmd.details = details
 
-        return (apiclient.addGuestOs(cmd))
+        return GuestOS(apiclient.addGuestOs(cmd).__dict__)
 
     @classmethod
     def remove(cls, apiclient, id):
@@ -6772,10 +6773,13 @@ class GuestOS:
 
         return (apiclient.listOsTypes(cmd))
 
+    def delete(self, apiclient):
+        self.remove(apiclient, self.id)
+
 class GuestOsMapping:
     """Manage Guest OS Mappings"""
 
-    def __init__(self, items, services):
+    def __init__(self, items):
         self.__dict__.update(items)
 
     @classmethod
@@ -6793,7 +6797,7 @@ class GuestOsMapping:
         if forced is not None:
             cmd.forced = forced
 
-        return (apiclient.addGuestOsMapping(cmd))
+        return GuestOsMapping(apiclient.addGuestOsMapping(cmd).__dict__)
 
     @classmethod
     def remove(cls, apiclient, id):
@@ -6837,6 +6841,9 @@ class GuestOsMapping:
 
         return (apiclient.listGuestOsMapping(cmd))
 
+    def delete(self, apiclient):
+        self.remove(apiclient, self.id)
+
 class VMSchedule:
 
     def __init__(self, items):

ui/public/locales/en.json

@@ -1,4 +1,5 @@
 {
+"message.delete.account.not.disabled": "Please disable the account before attempting to delete it.",
 "alert.service.domainrouter": "Domain router",
 "error.dedicate.bgp.peer.failed":"Failed to dedicate BGP peer",
 "error.dedicate.cluster.failed": "Failed to dedicate cluster.",
@@ -942,6 +943,7 @@
 "label.endipv6": "IPv6 end IP",
 "label.endpoint": "Endpoint",
 "label.endport": "End port",
+"label.enter.account.name": "Enter the account name",
 "label.enter.code": "Enter 2FA code to verify",
 "label.enter.static.pin": "Enter static PIN to verify",
 "label.enter.token": "Enter token",
@@ -1970,6 +1972,7 @@
 "label.rolename": "Role",
 "label.roles": "Roles",
 "label.roletype": "Role Type",
+"label.rolepermissiontab.searchbar": "Search Rule",
 "label.root.certificate": "Root certificate",
 "label.root.disk.size": "Root disk size (GB)",
 "label.rootdisk": "ROOT disk",
@@ -2980,7 +2983,11 @@
 "message.dedicating.host": "Dedicating host...",
 "message.dedicating.pod": "Dedicating pod...",
 "message.dedicating.zone": "Dedicating zone...",
-"message.delete.account": "Please confirm that you want to delete this Account.",
+"message.delete.account.confirm": "Please confirm that you want to delete this account by entering the name of the account below.",
+"message.delete.account.failed": "Delete account failed",
+"message.delete.account.processing": "Deleting account",
+"message.delete.account.success": "Successfully deleted account",
+"message.delete.account.warning": "Deleting this account will delete all of the instances, volumes and snapshots associated with the account.",
 "message.delete.acl.processing": "Removing ACL rule...",
 "message.delete.acl.rule": "Remove ACL rule",
 "message.delete.acl.rule.failed": "Failed to remove ACL rule.",
@@ -3073,6 +3080,7 @@
 "message.enabling.security.group.provider": "Enabling security group provider",
 "message.enter.valid.nic.ip": "Please enter a valid IP address for NIC",
 "message.error.access.key": "Please enter access key.",
+"message.error.account.delete.name.mismatch": "Name entered doesn't match the account name.",
 "message.error.add.guest.network": "Either IPv4 fields or IPv6 fields need to be filled when adding a guest Network.",
 "message.error.add.interface.static.route": "Adding interface Static Route failed",
 "message.error.add.logical.router": "Adding Logical Router failed",

ui/public/locales/pt_BR.json

@@ -1390,6 +1390,7 @@
 "label.rolename": "Fun\u00e7\u00e3o",
 "label.roles": "Fun\u00e7\u00f5es",
 "label.roletype": "Tipo de fun\u00e7\u00e3o",
+"label.rolepermissiontab.searchbar": "Pesquisa de regras",
 "label.root.certificate": "Certificado ra\u00edz",
 "label.root.disk.size": "Tamanho do disco ra\u00edz (GB)",
 "label.rootdisk": "Disco ra\u00edz",

ui/src/components/view/StatsTab.vue

@@ -522,7 +522,7 @@ export default {
         this.chartLabels.push(currentLabel)
 
         if (this.resourceIsVirtualMachine) {
-          cpuLine.data.push({ timestamp: currentLabel, stat: element.cpuused.split('%')[0] })
+          cpuLine.data.push({ timestamp: currentLabel, stat: element.cpuused.replace(',', '.').split('%')[0] })
 
           element.memoryusedkbs = element.memorykbs - element.memoryintfreekbs
           memFreeLinePercent.data.push({ timestamp: currentLabel, stat: this.calculateMemoryPercentage(false, element.memorykbs, element.memoryintfreekbs) })

ui/src/config/router.js

@@ -75,7 +75,6 @@ function generateRouterMap (section) {
           icon: child.icon,
           docHelp: vueProps.$applyDocHelpMappings(child.docHelp),
           permission: child.permission,
-          getApiToCall: child.getApiToCall,
           resourceType: child.resourceType,
           filters: child.filters,
           params: child.params ? child.params : {},

ui/src/config/section/account.js

@@ -228,11 +228,10 @@ export default {
       message: 'message.delete.account',
       dataView: true,
       disabled: (record, store) => {
-        return record.id !== 'undefined' && store.userInfo.accountid === record.id
+        return store.userInfo.accountid === record?.id
       },
-      groupAction: true,
       popup: true,
-      groupMap: (selection) => { return selection.map(x => { return { id: x } }) }
+      component: shallowRef(defineAsyncComponent(() => import('@/views/iam/DeleteAccountWrapper.vue')))
     }
   ]
 }

ui/src/config/section/compute.js

@@ -29,8 +29,7 @@ export default {
       title: 'label.instances',
       icon: 'cloud-server-outlined',
       docHelp: 'adminguide/virtual_machines.html',
-      permission: ['listVirtualMachines', 'listVirtualMachinesMetrics'],
+      permission: ['listVirtualMachinesMetrics'],
-      getApiToCall: () => store.getters.metrics ? 'listVirtualMachinesMetrics' : 'listVirtualMachines',
       resourceType: 'UserVm',
       params: () => {
         var params = { details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff' }

ui/src/config/section/infra/clusters.js

@@ -26,8 +26,8 @@ export default {
   permission: ['listClustersMetrics'],
   searchFilters: ['name', 'zoneid', 'podid', 'arch', 'hypervisor'],
   columns: () => {
-    const fields = ['name', 'state', 'allocationstate', 'clustertype', 'arch', 'hypervisortype', 'hosts']
+    const fields = ['name', 'allocationstate', 'clustertype', 'arch', 'hypervisortype']
-    const metricsFields = ['cpuused', 'cpumaxdeviation', 'cpuallocated', 'cputotal', 'memoryused', 'memorymaxdeviation', 'memoryallocated', 'memorytotal', 'drsimbalance']
+    const metricsFields = ['state', 'hosts', 'cpuused', 'cpumaxdeviation', 'cpuallocated', 'cputotal', 'memoryused', 'memorymaxdeviation', 'memoryallocated', 'memorytotal', 'drsimbalance']
     if (store.getters.metrics) {
       fields.push(...metricsFields)
     }

ui/src/config/section/infra/hosts.js

@@ -33,12 +33,10 @@ export default {
   params: { type: 'routing' },
   columns: () => {
     const fields = [
-      'name', 'state', 'resourcestate', 'ipaddress',
+      'name', 'state', 'resourcestate', 'ipaddress', 'arch', 'hypervisor',
-      'arch', 'hypervisor', 'instances',
+      { field: 'systeminstances', customTitle: 'system.vms' }, 'version'
-      { field: 'systeminstances', customTitle: 'system.vms' },
-      'powerstate', 'version'
     ]
-    const metricsFields = ['cpunumber', 'cputotalghz', 'cpuusedghz', 'cpuallocatedghz', 'memorytotalgb', 'memoryusedgb', 'memoryallocatedgb', 'networkread', 'networkwrite']
+    const metricsFields = ['instances', 'powerstate', 'cpunumber', 'cputotalghz', 'cpuusedghz', 'cpuallocatedghz', 'memorytotalgb', 'memoryusedgb', 'memoryallocatedgb', 'networkread', 'networkwrite']
     if (store.getters.metrics) {
       fields.push(...metricsFields)
     }

ui/src/config/section/infra/zones.js

@@ -26,8 +26,8 @@ export default {
   permission: ['listZonesMetrics'],
   searchFilters: ['name', 'domainid', 'tags'],
   columns: () => {
-    const fields = ['name', 'allocationstate', 'type', 'networktype', 'clusters']
+    const fields = ['name', 'allocationstate', 'type', 'networktype']
-    const metricsFields = ['cpuused', 'cpumaxdeviation', 'cpuallocated', 'cputotal', 'memoryused', 'memorymaxdeviation', 'memoryallocated', 'memorytotal']
+    const metricsFields = ['clusters', 'cpuused', 'cpumaxdeviation', 'cpuallocated', 'cputotal', 'memoryused', 'memorymaxdeviation', 'memoryallocated', 'memorytotal']
     if (store.getters.metrics) {
       fields.push(...metricsFields)
     }

ui/src/views/AutogenView.vue

@@ -829,7 +829,12 @@ export default {
       }
 
       if (this.$route && this.$route.meta && this.$route.meta.permission) {
-        this.apiName = (this.$route.meta.getApiToCall && this.$route.meta.getApiToCall()) || this.$route.meta.permission[0]
+        this.apiName = this.$route.meta.permission[0]
+        if (!store.getters.metrics && !this.dataView &&
+            this.apiName && this.apiName.endsWith('Metrics') &&
+            store.getters.apis[this.apiName.replace(/Metrics$/, '')]) {
+          this.apiName = this.apiName.replace(/Metrics$/, '')
+        }
         if (this.$route.meta.columns) {
           const columns = this.$route.meta.columns
           if (columns && typeof columns === 'function') {

ui/src/views/iam/DeleteAccount.vue

@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+<template>
+  <a-form
+    class="form"
+    :ref="formRef"
+    :model="form"
+    :rules="rules"
+    layout="vertical"
+    @finish="handleSubmit"
+    v-ctrl-enter="handleSubmit"
+   >
+    <div style="margin-bottom: 10px">
+      <a-alert type="warning">
+        <template #message>
+          <div v-html="$t('message.delete.account.warning')"></div>
+        </template>
+    </a-alert>
+    </div>
+    <div style="margin-bottom: 10px">
+      <a-alert>
+        <template #message>
+          <div v-html="$t('message.delete.account.confirm')"></div>
+        </template>
+      </a-alert>
+    </div>
+    <a-form-item name="name" ref="name">
+      <a-input
+        v-model:value="form.name"
+        :placeholder="$t('label.enter.account.name')"
+        style="width: 100%"/>
+    </a-form-item>
+    <p v-if="error" class="error">{{ error }}</p>
+    <div :span="24" class="actions">
+      <a-button @click="closeModal">{{ $t('label.cancel') }}</a-button>
+      <a-button type="primary" ref="submit" @click="handleSubmit">{{ $t('label.ok') }}</a-button>
+    </div>
+  </a-form>
+</template>
+
+<script>
+import { ref, reactive } from 'vue'
+import { api } from '@/api'
+
+export default {
+  name: 'DeleteAccount',
+  props: {
+    resource: {
+      type: Object,
+      required: true
+    }
+  },
+  data () {
+    return {
+      error: '',
+      isDeleting: false
+    }
+  },
+  created () {
+    this.initForm()
+  },
+  methods: {
+    initForm () {
+      this.formRef = ref()
+      this.form = reactive({})
+      this.rules = reactive({
+        name: [{ required: true, message: this.$t('label.required') }]
+      })
+    },
+    closeModal () {
+      this.$emit('close-action')
+    },
+    handleSubmit (e) {
+      e.preventDefault()
+      if (this.isDeleting) return // Prevent double submission
+      this.formRef.value.validate().then(async () => {
+        if (this.form.name !== this.resource.name) {
+          this.error = `${this.$t('message.error.account.delete.name.mismatch')}`
+          return
+        }
+        if (this.hasActiveResources) {
+          return
+        }
+        this.isDeleting = true
+        // Store the account ID and name before we close the modal
+        const accountId = this.resource.id
+        const accountName = this.resource.name
+        // Close the modal first
+        this.closeModal()
+        // Immediately navigate to the accounts page to avoid "unable to find account" errors
+        this.$router.push({ path: '/account' })
+          .then(() => {
+            // After successful navigation, start the deletion job
+            api('deleteAccount', {
+              id: accountId
+            }).then(response => {
+              this.$pollJob({
+                jobId: response.deleteaccountresponse.jobid,
+                title: this.$t('label.action.delete.account'),
+                description: accountId,
+                successMessage: `${this.$t('message.delete.account.success')} - ${accountName}`,
+                errorMessage: `${this.$t('message.delete.account.failed')} - ${accountName}`,
+                loadingMessage: `${this.$t('message.delete.account.processing')} - ${accountName}`,
+                catchMessage: this.$t('error.fetching.async.job.result')
+              })
+            }).catch(error => {
+              this.$notifyError(error)
+              this.isDeleting = false
+            })
+          })
+          .catch(err => {
+            console.error('Navigation failed:', err)
+            this.isDeleting = false
+            // If navigation fails, still try to delete the account
+            // but don't navigate afterwards
+            api('deleteAccount', {
+              id: accountId
+            }).then(response => {
+              this.$pollJob({
+                jobId: response.deleteaccountresponse.jobid,
+                title: this.$t('label.action.delete.account'),
+                description: accountId,
+                successMessage: `${this.$t('message.delete.account.success')} - ${accountName}`,
+                errorMessage: `${this.$t('message.delete.account.failed')} - ${accountName}`,
+                loadingMessage: `${this.$t('message.delete.account.processing')} - ${accountName}`,
+                catchMessage: this.$t('error.fetching.async.job.result')
+              })
+            }).catch(error => {
+              this.$notifyError(error)
+            })
+          })
+      }).catch((error) => {
+        this.formRef.value.scrollToField(error.errorFields[0].name)
+      })
+    }
+  }
+}
+</script>
+
+<style lang="scss" scoped>
+.form {
+  width: 80vw;
+  @media (min-width: 500px) {
+    width: 400px;
+  }
+}
+.actions {
+  display: flex;
+  justify-content: flex-end;
+  margin-top: 20px;
+  button {
+    &:not(:last-child) {
+      margin-right: 10px;
+    }
+  }
+}
+.error {
+  color: red;
+  margin-top: 10px;
+}
+</style>

ui/src/views/iam/DeleteAccountWrapper.vue

@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+<template>
+  <div>
+    <!-- Show error message if account is not disabled -->
+    <template v-if="resource.state !== 'disabled'">
+      <div style="margin-bottom: 10px">
+        <a-alert type="error">
+          <template #message>
+            <div>{{ $t('message.delete.account.not.disabled') }}</div>
+          </template>
+        </a-alert>
+      </div>
+      <div :span="24" class="actions">
+        <a-button type="primary" @click="closeModal">{{ $t('label.ok') }}</a-button>
+      </div>
+    </template>
+    <!-- Show delete form if account is disabled -->
+    <delete-account
+      v-else
+      :resource="resource"
+      @close-action="closeModal" />
+  </div>
+</template>
+
+<script>
+import DeleteAccount from './DeleteAccount.vue'
+
+export default {
+  name: 'DeleteAccountWrapper',
+  components: {
+    DeleteAccount
+  },
+  props: {
+    resource: {
+      type: Object,
+      required: true
+    }
+  },
+  methods: {
+    closeModal () {
+      this.$emit('close-action')
+    }
+  }
+}
+</script>
+
+<style lang="scss" scoped>
+.actions {
+  display: flex;
+  justify-content: flex-end;
+  margin-top: 20px;
+  button {
+    &:not(:last-child) {
+      margin-right: 10px;
+    }
+  }
+}
+</style>

ui/src/views/iam/RolePermissionTab.vue

@@ -18,15 +18,24 @@
 <template>
   <loading-outlined v-if="loadingTable" class="main-loading-spinner" />
   <div v-else>
-    <div style="width: 100%; display: flex; margin-bottom: 10px">
+    <div style="width: 100%; display: flex; margin-bottom: 20px">
       <a-button type="dashed" @click="exportRolePermissions" style="width: 100%">
         <template #icon><download-outlined /></template>
         {{ $t('label.export.rules') }}
       </a-button>
     </div>
+    <a-input-search
+      v-model:value="searchRule"
+      :placeholder="$t('label.rolepermissiontab.searchbar')"
+      background-color="gray"
+      style="width: 100%; margin-bottom: 10px; display: inline-block"
+      enter-button
+      @search="searchRulePermission"
+    />
     <div v-if="updateTable" class="loading-overlay">
       <loading-outlined />
     </div>
+
     <div
       class="rules-list ant-list ant-list-bordered"
       :class="{'rules-list--overflow-hidden' : updateTable}" >
@@ -137,7 +146,8 @@ export default {
       newRuleDescription: '',
       newRuleSelectError: false,
       drag: false,
-      apis: []
+      apis: [],
+      searchRule: ''
     }
   },
   created () {
@@ -172,6 +182,7 @@ export default {
       if (!this.resource.id) return
       api('listRolePermissions', { roleid: this.resource.id }).then(response => {
         this.rules = response.listrolepermissionsresponse.rolepermission
+        this.totalRules = this.rules
       }).catch(error => {
         this.$notifyError(error)
       }).finally(() => {
@@ -258,6 +269,19 @@ export default {
       hiddenElement.download = this.resource.name + '_' + this.resource.type + '.csv'
       hiddenElement.click()
       hiddenElement.remove()
+    },
+    searchRulePermission (searchValue) {
+      searchValue = searchValue.toLowerCase()
+      if (!searchValue) {
+        this.rules = this.totalRules
+      } else {
+        this.updateTable = true
+        const searchRules = this.totalRules.filter((rule) => rule.rule.toLowerCase().includes(searchValue))
+        this.rules = searchRules
+        setTimeout(() => {
+          this.updateTable = false
+        })
+      }
     }
   }
 }

ui/src/views/storage/CreateVolume.vue

@@ -389,6 +389,9 @@ export default {
           values.virtualmachineid = this.resource.id
           values.zoneid = this.resource.zoneid
         }
+        if (this.customDiskOffering) {
+          values.size = values.size.trim()
+        }
         if (this.createVolumeFromSnapshot) {
           values.snapshotid = this.resource.id
         }

usage/src/main/java/com/cloud/usage/UsageManagerImpl.java

@@ -2150,37 +2150,88 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
     }
 
     private void handleNetworkEvent(UsageEventVO event) {
-        Account account = _accountDao.findByIdIncludingRemoved(event.getAccountId());
+        String eventType = event.getType();
-        long domainId = account.getDomainId();
+        if (EventTypes.EVENT_NETWORK_DELETE.equals(eventType)) {
-        if (EventTypes.EVENT_NETWORK_DELETE.equals(event.getType())) {
+            removeNetworkHelperEntry(event);
-            usageNetworksDao.remove(event.getResourceId(), event.getCreateDate());
+        } else if (EventTypes.EVENT_NETWORK_CREATE.equals(eventType)) {
-        } else if (EventTypes.EVENT_NETWORK_CREATE.equals(event.getType())) {
+            createNetworkHelperEntry(event);
-            logger.debug("Marking existing helper entries for network [{}] as removed.", event.getResourceId());
+        } else if (EventTypes.EVENT_NETWORK_UPDATE.equals(eventType)) {
-            usageNetworksDao.remove(event.getResourceId(), event.getCreateDate());
+            updateNetworkHelperEntry(event);
-            logger.debug("Creating a helper entry for network [{}].", event.getResourceId());
-            UsageNetworksVO usageNetworksVO = new UsageNetworksVO(event.getResourceId(), event.getOfferingId(), event.getZoneId(), event.getAccountId(), domainId, Network.State.Allocated.name(), event.getCreateDate(), null);
-            usageNetworksDao.persist(usageNetworksVO);
-        } else if (EventTypes.EVENT_NETWORK_UPDATE.equals(event.getType())) {
-            usageNetworksDao.update(event.getResourceId(), event.getOfferingId(), event.getResourceType());
         } else {
-            logger.error("Unknown event type [{}] in Networks event parser. Skipping it.", event.getType());
+            logger.error(String.format("Unknown event type [%s] in Networks event parser. Skipping it.", eventType));
         }
     }
 
-    private void handleVpcEvent(UsageEventVO event) {
+    private void removeNetworkHelperEntry(UsageEventVO event) {
+        long networkId = event.getResourceId();
+        logger.debug(String.format("Removing helper entries of network [%s].", networkId));
+        usageNetworksDao.remove(networkId, event.getCreateDate());
+    }
+
+    private void createNetworkHelperEntry(UsageEventVO event) {
+        long networkId = event.getResourceId();
         Account account = _accountDao.findByIdIncludingRemoved(event.getAccountId());
         long domainId = account.getDomainId();
-        if (EventTypes.EVENT_VPC_DELETE.equals(event.getType())) {
+
-            usageVpcDao.remove(event.getResourceId(), event.getCreateDate());
+        List<UsageNetworksVO> entries = usageNetworksDao.listAll(networkId);
-        } else if (EventTypes.EVENT_VPC_CREATE.equals(event.getType())) {
+        if (!entries.isEmpty()) {
-            logger.debug("Marking existing helper entries for VPC [{}] as removed.", event.getResourceId());
+            logger.warn(String.format("Received a NETWORK.CREATE event for a network [%s] that already has helper entries; " +
-            usageVpcDao.remove(event.getResourceId(), event.getCreateDate());
+                    "therefore, we will not create a new one.", networkId));
-            logger.debug("Creating a helper entry for VPC [{}].", event.getResourceId());
+            return;
-            UsageVpcVO usageVPCVO = new UsageVpcVO(event.getResourceId(), event.getZoneId(), event.getAccountId(), domainId, Vpc.State.Enabled.name(), event.getCreateDate(), null);
-            usageVpcDao.persist(usageVPCVO);
-        } else {
-            logger.error("Unknown event type [{}] in VPC event parser. Skipping it.", event.getType());
         }
+
+        logger.debug(String.format("Creating a helper entry for network [%s].", networkId));
+        UsageNetworksVO usageNetworksVO = new UsageNetworksVO(networkId, event.getOfferingId(), event.getZoneId(),
+                event.getAccountId(), domainId, Network.State.Allocated.name(), event.getCreateDate(), null);
+        usageNetworksDao.persist(usageNetworksVO);
+    }
+
+    private void updateNetworkHelperEntry(UsageEventVO event) {
+        long networkId = event.getResourceId();
+        Account account = _accountDao.findByIdIncludingRemoved(event.getAccountId());
+        long domainId = account.getDomainId();
+
+        logger.debug(String.format("Marking previous helper entries of network [%s] as removed.", networkId));
+        usageNetworksDao.remove(networkId, event.getCreateDate());
+
+        logger.debug(String.format("Creating an updated helper entry for network [%s].", networkId));
+        UsageNetworksVO usageNetworksVO = new UsageNetworksVO(networkId, event.getOfferingId(), event.getZoneId(),
+                event.getAccountId(), domainId, event.getResourceType(), event.getCreateDate(), null);
+        usageNetworksDao.persist(usageNetworksVO);
+    }
+
+    private void handleVpcEvent(UsageEventVO event) {
+        String eventType = event.getType();
+        if (EventTypes.EVENT_VPC_DELETE.equals(eventType)) {
+            removeVpcHelperEntry(event);
+        } else if (EventTypes.EVENT_VPC_CREATE.equals(eventType)) {
+            createVpcHelperEntry(event);
+        } else {
+            logger.error(String.format("Unknown event type [%s] in VPC event parser. Skipping it.", eventType));
+        }
+    }
+
+    private void removeVpcHelperEntry(UsageEventVO event) {
+        long vpcId = event.getResourceId();
+        logger.debug(String.format("Removing helper entries of VPC [%s].", vpcId));
+        usageVpcDao.remove(vpcId, event.getCreateDate());
+    }
+
+    private void createVpcHelperEntry(UsageEventVO event) {
+        long vpcId = event.getResourceId();
+        Account account = _accountDao.findByIdIncludingRemoved(event.getAccountId());
+        long domainId = account.getDomainId();
+
+        List<UsageVpcVO> entries = usageVpcDao.listAll(vpcId);
+        if (!entries.isEmpty()) {
+            logger.warn(String.format("Active helper entries already exist for VPC [%s]; therefore, we will not create a new one.",
+                    vpcId));
+            return;
+        }
+
+        logger.debug(String.format("Creating a helper entry for VPC [%s].", vpcId));
+        UsageVpcVO usageVPCVO = new UsageVpcVO(vpcId, event.getZoneId(), event.getAccountId(), domainId, Vpc.State.Enabled.name(), event.getCreateDate(), null);
+        usageVpcDao.persist(usageVPCVO);
     }
 
     private class Heartbeat extends ManagedContextRunnable {

utils/src/main/java/com/cloud/utils/StringUtils.java

@@ -28,6 +28,8 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -90,7 +92,7 @@ public class StringUtils extends org.apache.commons.lang3.StringUtils {
 
     /**
      * Converts a List of tags to a comma separated list
-     * @param tags
+     * @param tagsList
      * @return String containing a comma separated list of tags
      */
 
@@ -304,4 +306,92 @@ public class StringUtils extends org.apache.commons.lang3.StringUtils {
 
         return mapResult;
     }
+
+    /**
+     * Converts the comma separated numbers to ranges for any consecutive numbers in the input with numbers (and ranges)
+     * Eg: "198,200-203,299,300,301,303,304,305,306,307,308,311,197" to "197-198,200-203,299-301,303-308,311"
+     * @param inputNumbersAndRanges
+     * @return String containing a converted ranges for any consecutive numbers
+     */
+    public static String numbersToRange(String inputNumbersAndRanges) {
+        Set<Integer> numberSet = new TreeSet<>();
+        for (String inputNumber : inputNumbersAndRanges.split(",")) {
+            inputNumber = inputNumber.trim();
+            if (inputNumber.contains("-")) {
+                String[] range = inputNumber.split("-");
+                if (range.length == 2 && range[0] != null && range[1] != null) {
+                    int start = NumbersUtil.parseInt(range[0], 0);
+                    int end = NumbersUtil.parseInt(range[1], 0);
+                    for (int i = start; i <= end; i++) {
+                        numberSet.add(i);
+                    }
+                }
+            } else {
+                numberSet.add(NumbersUtil.parseInt(inputNumber, 0));
+            }
+        }
+
+        StringBuilder result = new StringBuilder();
+        if (!numberSet.isEmpty()) {
+            List<Integer> numbers = new ArrayList<>(numberSet);
+            int startNumber = numbers.get(0);
+            int endNumber = startNumber;
+
+            for (int i = 1; i < numbers.size(); i++) {
+                if (numbers.get(i) == endNumber + 1) {
+                    endNumber = numbers.get(i);
+                } else {
+                    appendRange(result, startNumber, endNumber);
+                    startNumber = endNumber = numbers.get(i);
+                }
+            }
+            appendRange(result, startNumber, endNumber);
+        }
+
+        return result.toString();
+    }
+
+    private static void appendRange(StringBuilder sb, int startNumber, int endNumber) {
+        if (sb.length() > 0) {
+            sb.append(",");
+        }
+        if (startNumber == endNumber) {
+            sb.append(startNumber);
+        } else {
+            sb.append(startNumber).append("-").append(endNumber);
+        }
+    }
+
+    /**
+     * Converts the comma separated numbers and ranges to numbers
+     * Eg: "197-198,200-203,299-301,303-308,311" to "197,198,200,201,202,203,299,300,301,303,304,305,306,307,308,311"
+     * @param inputNumbersAndRanges
+     * @return String containing a converted numbers
+     */
+    public static String rangeToNumbers(String inputNumbersAndRanges) {
+        Set<Integer> numberSet = new TreeSet<>();
+        for (String inputNumber : inputNumbersAndRanges.split(",")) {
+            inputNumber = inputNumber.trim();
+            if (inputNumber.contains("-")) {
+                String[] range = inputNumber.split("-");
+                int startNumber = Integer.parseInt(range[0]);
+                int endNumber = Integer.parseInt(range[1]);
+                for (int i = startNumber; i <= endNumber; i++) {
+                    numberSet.add(i);
+                }
+            } else {
+                numberSet.add(Integer.parseInt(inputNumber));
+            }
+        }
+
+        StringBuilder result = new StringBuilder();
+        for (int number : numberSet) {
+            if (result.length() > 0) {
+                result.append(",");
+            }
+            result.append(number);
+        }
+
+        return result.toString();
+    }
 }

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java

@@ -276,16 +276,18 @@ public class HypervisorHostHelper {
         }
     }
 
-    public static String composeCloudNetworkName(String prefix, String vlanId, String svlanId, Integer networkRateMbps, String vSwitchName) {
+    public static String composeCloudNetworkName(String prefix, String vlanId, String svlanId, Integer networkRateMbps, String vSwitchName, VirtualSwitchType vSwitchType) {
         StringBuffer sb = new StringBuffer(prefix);
         if (vlanId == null || UNTAGGED_VLAN_NAME.equalsIgnoreCase(vlanId)) {
             sb.append(".untagged");
         } else {
+            if (vSwitchType != VirtualSwitchType.StandardVirtualSwitch && StringUtils.containsAny(vlanId, ",-")) {
+                vlanId = com.cloud.utils.StringUtils.numbersToRange(vlanId);
+            }
             sb.append(".").append(vlanId);
             if (svlanId != null) {
                 sb.append(".").append("s" + svlanId);
             }
-
         }
 
         if (networkRateMbps != null && networkRateMbps.intValue() > 0)
@@ -295,7 +297,12 @@ public class HypervisorHostHelper {
         sb.append(".").append(VersioningContants.PORTGROUP_NAMING_VERSION);
         sb.append("-").append(vSwitchName);
 
-        return sb.toString();
+        String networkName = sb.toString();
+        if (networkName.length() > 80) {
+            // the maximum limit for a vSwitch name is 80 chars, applies to both standard and distributed virtual switches.
+            LOGGER.warn(String.format("The network name: %s for the vSwitch %s of type %s, exceeds 80 chars", networkName, vSwitchName, vSwitchType));
+        }
+        return networkName;
     }
 
     public static Map<String, String> getValidatedVsmCredentials(VmwareContext context) throws Exception {
@@ -598,7 +605,7 @@ public class HypervisorHostHelper {
             if (vlanId != null) {
                 vlanId = vlanId.replace("vlan://", "");
             }
-            networkName = composeCloudNetworkName(namePrefix, vlanId, secondaryvlanId, networkRateMbps, physicalNetwork);
+            networkName = composeCloudNetworkName(namePrefix, vlanId, secondaryvlanId, networkRateMbps, physicalNetwork, vSwitchType);
 
             if (vlanId != null && !UNTAGGED_VLAN_NAME.equalsIgnoreCase(vlanId) && !StringUtils.containsAny(vlanId, ",-")) {
                 createGCTag = true;
@@ -1173,8 +1180,9 @@ public class HypervisorHostHelper {
         if (vlanId == null && vlanRange != null && !vlanRange.isEmpty()) {
             LOGGER.debug("Creating dvSwitch port vlan-trunk spec with range: " + vlanRange);
             VmwareDistributedVirtualSwitchTrunkVlanSpec trunkVlanSpec = new VmwareDistributedVirtualSwitchTrunkVlanSpec();
-            for (final String vlanRangePart : vlanRange.split(",")) {
+            String vlanRangeUpdated = com.cloud.utils.StringUtils.numbersToRange(vlanRange);
-                if (vlanRangePart == null || vlanRange.isEmpty()) {
+            for (final String vlanRangePart : vlanRangeUpdated.split(",")) {
+                if (vlanRangePart == null || vlanRangePart.isEmpty()) {
                     continue;
                 }
                 final NumericRange numericRange = new NumericRange();
@@ -1327,7 +1335,7 @@ public class HypervisorHostHelper {
             // No doubt about this, depending on vid=null to avoid lots of code below
             vid = null;
         } else {
-            networkName = composeCloudNetworkName(namePrefix, vlanId, null, networkRateMbps, vSwitchName);
+            networkName = composeCloudNetworkName(namePrefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.StandardVirtualSwitch);
 
             if (vlanId != null && !UNTAGGED_VLAN_NAME.equalsIgnoreCase(vlanId)) {
                 createGCTag = true;

vmware-base/src/test/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelperTest.java

@@ -581,7 +581,7 @@ public class HypervisorHostHelperTest {
         networkRateMbps = 200;
         prefix = "cloud.public";
         vSwitchName = "vSwitch0";
-        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName);
+        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName, VirtualSwitchType.StandardVirtualSwitch);
         assertEquals("cloud.public.100.200.1-vSwitch0", cloudNetworkName);
     }
 
@@ -591,7 +591,7 @@ public class HypervisorHostHelperTest {
         networkRateMbps = null;
         prefix = "cloud.storage";
         vSwitchName = "vSwitch1";
-        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName);
+        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName, VirtualSwitchType.StandardVirtualSwitch);
         assertEquals("cloud.storage.untagged.0.1-vSwitch1", cloudNetworkName);
     }
 
@@ -602,10 +602,60 @@ public class HypervisorHostHelperTest {
         networkRateMbps = 512;
         prefix = "cloud.guest";
         vSwitchName = "vSwitch2";
-        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName);
+        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, svlanId, networkRateMbps, vSwitchName, VirtualSwitchType.StandardVirtualSwitch);
         assertEquals("cloud.guest.400.s123.512.1-vSwitch2", cloudNetworkName);
     }
 
+    @Test
+    public void testComposeCloudNetworkNameVlanRangeGuestTrafficDvSwitch() {
+        vlanId = "400-500";
+        networkRateMbps = 512;
+        prefix = "cloud.guest";
+        vSwitchName = "dvSwitch0";
+        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.VMwareDistributedVirtualSwitch);
+        assertEquals("cloud.guest.400-500.512.1-dvSwitch0", cloudNetworkName);
+    }
+
+    @Test
+    public void testComposeCloudNetworkNameVlanNumbersGuestTrafficDvSwitch() {
+        vlanId = "3001,3002,3003,3004,3005,3006,3007,3008,3009,3010,3011,3012,3013,3014,3015,3016,3017,3018,3019,3020";
+        networkRateMbps = 512;
+        prefix = "cloud.guest";
+        vSwitchName = "dvSwitch0";
+        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.VMwareDistributedVirtualSwitch);
+        assertEquals("cloud.guest.3001-3020.512.1-dvSwitch0", cloudNetworkName);
+    }
+
+    @Test
+    public void testComposeCloudNetworkNameVlanNumbersAndRangeGuestTrafficDvSwitch() {
+        vlanId = "3001,3004-3006,3007,3008,3009,3010,3011,3012,3013,3014,3015,3016,3017,3018,3020";
+        networkRateMbps = 512;
+        prefix = "cloud.guest";
+        vSwitchName = "dvSwitch0";
+        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.VMwareDistributedVirtualSwitch);
+        assertEquals("cloud.guest.3001,3004-3018,3020.512.1-dvSwitch0", cloudNetworkName);
+    }
+
+    @Test
+    public void testComposeCloudNetworkNameUnorderedVlanNumbersAndRangeGuestTrafficDvSwitch() {
+        vlanId = "3018,3020,3011,3012,3004-3006,3007,3001,3008,3009,3010,3013,3014,3015,3016,3017";
+        networkRateMbps = 512;
+        prefix = "cloud.guest";
+        vSwitchName = "dvSwitch0";
+        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.VMwareDistributedVirtualSwitch);
+        assertEquals("cloud.guest.3001,3004-3018,3020.512.1-dvSwitch0", cloudNetworkName);
+    }
+
+    @Test
+    public void testComposeCloudNetworkNameOverlappingVlanNumbersAndRangeGuestTrafficDvSwitch() {
+        vlanId = "3018,3020,3011,3012,3004-3006,3007,3001,3008,3009,3010,3013,3014,3015,3016,3017,3005-3008";
+        networkRateMbps = 512;
+        prefix = "cloud.guest";
+        vSwitchName = "dvSwitch0";
+        String cloudNetworkName = HypervisorHostHelper.composeCloudNetworkName(prefix, vlanId, null, networkRateMbps, vSwitchName, VirtualSwitchType.VMwareDistributedVirtualSwitch);
+        assertEquals("cloud.guest.3001,3004-3018,3020.512.1-dvSwitch0", cloudNetworkName);
+    }
+
     @Test
     public void testOvfDomRewriter() {
         final String ovfString = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +