Merge branch '4.19' into 4.20

api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java

@@ -79,6 +79,14 @@ public class UnmanagedInstanceResponse extends BaseResponse {
     @Param(description = "the operating system of the virtual machine")
     private String operatingSystem;
 
+    @SerializedName(ApiConstants.BOOT_MODE)
+    @Param(description = "indicates the boot mode")
+    private String bootMode;
+
+    @SerializedName(ApiConstants.BOOT_TYPE)
+    @Param(description = "indicates the boot type")
+    private String bootType;
+
     @SerializedName(ApiConstants.DISK)
     @Param(description = "the list of disks associated with the virtual machine", responseObject = UnmanagedInstanceDiskResponse.class)
     private Set<UnmanagedInstanceDiskResponse> disks;
@@ -211,4 +219,20 @@ public class UnmanagedInstanceResponse extends BaseResponse {
     public void addNic(NicResponse nic) {
         this.nics.add(nic);
     }
+
+    public String getBootMode() {
+        return bootMode;
+    }
+
+    public void setBootMode(String bootMode) {
+        this.bootMode = bootMode;
+    }
+
+    public String getBootType() {
+        return bootType;
+    }
+
+    public void setBootType(String bootType) {
+        this.bootType = bootType;
+    }
 }

api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java

@@ -61,6 +61,9 @@ public class UnmanagedInstanceTO {
 
     private String vncPassword;
 
+    private String bootType;
+    private String bootMode;
+
     public String getName() {
         return name;
     }
@@ -196,6 +199,22 @@ public class UnmanagedInstanceTO {
                         this, "name", "internalCSName", "hostName", "clusterName"));
     }
 
+    public String getBootType() {
+        return bootType;
+    }
+
+    public void setBootType(String bootType) {
+        this.bootType = bootType;
+    }
+
+    public String getBootMode() {
+        return bootMode;
+    }
+
+    public void setBootMode(String bootMode) {
+        this.bootMode = bootMode;
+    }
+
     public static class Disk {
         private String diskId;
 

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -61,7 +61,11 @@ import org.apache.cloudstack.ca.CAManager;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
 import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
 import org.apache.cloudstack.framework.ca.Certificate;
 import org.apache.cloudstack.framework.config.ConfigKey;
@@ -411,6 +415,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     VmWorkJobDao vmWorkJobDao;
 
     private SingleCache<List<Long>> vmIdsInProgressCache;
+    DataStoreProviderManager dataStoreProviderManager;
 
     VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this);
 
@@ -1224,6 +1229,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                                 planChangedByVolume = true;
                             }
                         }
+                        DataStoreProvider storeProvider = dataStoreProviderManager.getDataStoreProvider(pool.getStorageProviderName());
+                        DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();
+                        if (storeDriver instanceof PrimaryDataStoreDriver) {
+                            ((PrimaryDataStoreDriver)storeDriver).detachVolumeFromAllStorageNodes(vol);
+                        }
                     }
                 }
 

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java

@@ -43,6 +43,7 @@ import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 import javax.persistence.EntityExistsException;
 
+import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
 import com.cloud.hypervisor.vmware.util.VmwareClient;
 import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd;
 import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd;
@@ -171,8 +172,11 @@ import com.cloud.vm.dao.VMInstanceDao;
 import com.vmware.pbm.PbmProfile;
 import com.vmware.vim25.AboutInfo;
 import com.vmware.vim25.ManagedObjectReference;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public class VmwareManagerImpl extends ManagerBase implements VmwareManager, VmwareStorageMount, Listener, VmwareDatacenterService, Configurable {
+    protected static Logger static_logger = LogManager.getLogger(VmwareManagerImpl.class);
 
     private static final long SECONDS_PER_MINUTE = 60;
     private static final int DEFAULT_PORTS_PER_DV_PORT_GROUP_VSPHERE4_x = 256;
@@ -1585,14 +1589,26 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
         return compatiblePools;
     }
 
-    @Override
+    private static class VcenterData {
-    public List<UnmanagedInstanceTO> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) {
+        public final String vcenter;
+        public final String datacenterName;
+        public final String username;
+        public final String password;
+
+        public VcenterData(String vcenter, String datacenterName, String username, String password) {
+            this.vcenter = vcenter;
+            this.datacenterName = datacenterName;
+            this.username = username;
+            this.password = password;
+        }
+    }
+
+    private VcenterData getVcenterData(ListVmwareDcVmsCmd cmd) {
         String vcenter = cmd.getVcenter();
         String datacenterName = cmd.getDatacenterName();
         String username = cmd.getUsername();
         String password = cmd.getPassword();
         Long existingVcenterId = cmd.getExistingVcenterId();
-        String keyword = cmd.getKeyword();
 
         if ((existingVcenterId == null && StringUtils.isBlank(vcenter)) ||
                 (existingVcenterId != null && StringUtils.isNotBlank(vcenter))) {
@@ -1613,34 +1629,69 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
             username = vmwareDc.getUser();
             password = vmwareDc.getPassword();
         }
+        VcenterData vmwaredc = new VcenterData(vcenter, datacenterName, username, password);
+        return vmwaredc;
+    }
+
+    private static VmwareContext getVmwareContext(String vcenter, String username, String password) throws Exception {
+        static_logger.debug(String.format("Connecting to the VMware vCenter %s", vcenter));
+        String serviceUrl = String.format("https://%s/sdk/vimService", vcenter);
+        VmwareClient vimClient = new VmwareClient(vcenter);
+        vimClient.connect(serviceUrl, username, password);
+        return new VmwareContext(vimClient, vcenter);
+    }
+
+    @Override
+    public List<UnmanagedInstanceTO> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) {
+        VcenterData vmwareDC = getVcenterData(cmd);
+        String vcenter = vmwareDC.vcenter;
+        String username = vmwareDC.username;
+        String password = vmwareDC.password;
+        String datacenterName = vmwareDC.datacenterName;
+        String keyword = cmd.getKeyword();
+        String esxiHostName = cmd.getHostName();
+        String virtualMachineName = cmd.getInstanceName();
 
         try {
             logger.debug(String.format("Connecting to the VMware datacenter %s at vCenter %s to retrieve VMs",
                     datacenterName, vcenter));
-            String serviceUrl = String.format("https://%s/sdk/vimService", vcenter);
+            VmwareContext context = getVmwareContext(vcenter, username, password);
-            VmwareClient vimClient = new VmwareClient(vcenter);
+            DatacenterMO dcMo = getDatacenterMO(context, vcenter, datacenterName);
-            vimClient.connect(serviceUrl, username, password);
-            VmwareContext context = new VmwareContext(vimClient, vcenter);
 
-            DatacenterMO dcMo = new DatacenterMO(context, datacenterName);
+            List<UnmanagedInstanceTO> instances;
-            ManagedObjectReference dcMor = dcMo.getMor();
+            if (StringUtils.isNotBlank(esxiHostName) && StringUtils.isNotBlank(virtualMachineName)) {
-            if (dcMor == null) {
+                ManagedObjectReference hostMor = dcMo.findHost(esxiHostName);
-                String msg = String.format("Unable to find VMware datacenter %s in vCenter %s",
+                if (hostMor == null) {
-                        datacenterName, vcenter);
+                    String errorMsg = String.format("Cannot find a host with name %s on vcenter %s", esxiHostName, vcenter);
-                logger.error(msg);
+                    logger.error(errorMsg);
-                throw new InvalidParameterValueException(msg);
+                    throw new CloudRuntimeException(errorMsg);
+                }
+                HostMO hostMO = new HostMO(context, hostMor);
+                VirtualMachineMO vmMo = hostMO.findVmOnHyperHost(virtualMachineName);
+                instances = Collections.singletonList(VmwareHelper.getUnmanagedInstance(hostMO, vmMo));
+            } else {
+                instances = dcMo.getAllVmsOnDatacenter(keyword);
             }
-            List<UnmanagedInstanceTO> instances = dcMo.getAllVmsOnDatacenter();
+            return instances;
-            return StringUtils.isBlank(keyword) ? instances :
-                    instances.stream().filter(x -> x.getName().toLowerCase().contains(keyword.toLowerCase())).collect(Collectors.toList());
         } catch (Exception e) {
-            String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s",
+            String errorMsg = String.format("Error retrieving VMs from the VMware VC %s datacenter %s: %s",
                     vcenter, datacenterName, e.getMessage());
             logger.error(errorMsg, e);
             throw new CloudRuntimeException(errorMsg);
         }
     }
 
+    private static DatacenterMO getDatacenterMO(VmwareContext context, String vcenter, String datacenterName) throws Exception {
+        DatacenterMO dcMo = new DatacenterMO(context, datacenterName);
+        ManagedObjectReference dcMor = dcMo.getMor();
+        if (dcMor == null) {
+            String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", datacenterName, vcenter);
+            static_logger.error(msg);
+            throw new InvalidParameterValueException(msg);
+        }
+        return dcMo;
+    }
+
     @Override
     public boolean hasNexusVSM(Long clusterId) {
         ClusterVSMMapVO vsmMapVo = null;
@@ -1693,7 +1744,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
     }
 
     /**
-     * This task is to cleanup templates from primary storage that are otherwise not cleaned by the {@link com.cloud.storage.StorageManagerImpl.StorageGarbageCollector}.
+     * This task is to cleanup templates from primary storage that are otherwise not cleaned by the {code}StorageGarbageCollector{code} from {@link com.cloud.storage.StorageManagerImpl}.
      * it is called at regular intervals when storage.template.cleanup.enabled == true
      * It collect all templates that
      * - are deleted from cloudstack

plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java

@@ -70,6 +70,12 @@ public class ListVmwareDcVmsCmd extends BaseListCmd {
     @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.")
     private String password;
 
+    @Parameter(name = ApiConstants.HOST_NAME, type = CommandType.STRING, description = "Name of the host on vCenter. Must be set along with the instancename parameter")
+    private String hostName;
+
+    @Parameter(name = ApiConstants.INSTANCE_NAME, type = CommandType.STRING, description = "Name of the VM on vCenter. Must be set along with the hostname parameter")
+    private String instanceName;
+
     public String getVcenter() {
         return vcenter;
     }
@@ -86,10 +92,18 @@ public class ListVmwareDcVmsCmd extends BaseListCmd {
         return datacenterName;
     }
 
+    public String getHostName() {
+        return hostName;
+    }
+
     public Long getExistingVcenterId() {
         return existingVcenterId;
     }
 
+    public String getInstanceName() {
+        return instanceName;
+    }
+
     @Override
     public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
         checkParameters();
@@ -125,6 +139,11 @@ public class ListVmwareDcVmsCmd extends BaseListCmd {
             throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
                     "Please set all the information for a vCenter IP/Name, datacenter, username and password");
         }
+        if ((StringUtils.isNotBlank(instanceName) && StringUtils.isBlank(hostName)) ||
+                (StringUtils.isBlank(instanceName) && StringUtils.isNotBlank(hostName))) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+                    "Please set the hostname parameter along with the instancename parameter");
+        }
     }
 
     @Override

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -5267,6 +5267,8 @@ public class ApiResponseHelper implements ResponseGenerator {
         response.setMemory(instance.getMemory());
         response.setOperatingSystemId(instance.getOperatingSystemId());
         response.setOperatingSystem(instance.getOperatingSystem());
+        response.setBootMode(instance.getBootMode());
+        response.setBootType(instance.getBootType());
         response.setObjectName("unmanagedinstance");
 
         if (instance.getDisks() != null) {

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -4554,14 +4554,24 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         String endIP = cmd.getEndIp();
         final String newVlanGateway = cmd.getGateway();
         final String newVlanNetmask = cmd.getNetmask();
+        Long networkId = cmd.getNetworkID();
+        Long physicalNetworkId = cmd.getPhysicalNetworkId();
+
+        // Verify that network exists
+        Network network = getNetwork(networkId);
+        if (network != null) {
+            zoneId = network.getDataCenterId();
+            physicalNetworkId = network.getPhysicalNetworkId();
+        }
+
         String vlanId = cmd.getVlan();
+        vlanId = verifyAndUpdateVlanId(vlanId, network);
+
         // TODO decide if we should be forgiving or demand a valid and complete URI
         if (!(vlanId == null || "".equals(vlanId) || vlanId.startsWith(BroadcastDomainType.Vlan.scheme()))) {
             vlanId = BroadcastDomainType.Vlan.toUri(vlanId).toString();
         }
         final Boolean forVirtualNetwork = cmd.isForVirtualNetwork();
-        Long networkId = cmd.getNetworkID();
-        Long physicalNetworkId = cmd.getPhysicalNetworkId();
         final String accountName = cmd.getAccountName();
         final Long projectId = cmd.getProjectId();
         final Long domainId = cmd.getDomainId();
@@ -4634,18 +4644,6 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             }
         }
 
-        // Verify that network exists
-        Network network = null;
-        if (networkId != null) {
-            network = _networkDao.findById(networkId);
-            if (network == null) {
-                throw new InvalidParameterValueException("Unable to find network by id " + networkId);
-            } else {
-                zoneId = network.getDataCenterId();
-                physicalNetworkId = network.getPhysicalNetworkId();
-            }
-        }
-
         // Verify that zone exists
         final DataCenterVO zone = _zoneDao.findById(zoneId);
         if (zone == null) {
@@ -4784,6 +4782,32 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 ip6Cidr, domain, vlanOwner, network, sameSubnet, cmd.isForNsx());
     }
 
+    private Network getNetwork(Long networkId) {
+        if (networkId == null) {
+            return null;
+        }
+
+        Network network = _networkDao.findById(networkId);
+        if (network == null) {
+            throw new InvalidParameterValueException("Unable to find network by id " + networkId);
+        }
+
+        return network;
+    }
+
+    private String verifyAndUpdateVlanId(String vlanId, Network network) {
+        if (!StringUtils.isBlank(vlanId)) {
+            return vlanId;
+        }
+
+        if (network == null || network.getTrafficType() != TrafficType.Guest) {
+            return Vlan.UNTAGGED;
+        }
+
+        boolean connectivityWithoutVlan = isConnectivityWithoutVlan(network);
+        return getNetworkVlanId(network, connectivityWithoutVlan);
+    }
+
     private Vlan commitVlan(final Long zoneId, final Long podId, final String startIP, final String endIP, final String newVlanGatewayFinal, final String newVlanNetmaskFinal,
             final String vlanId, final Boolean forVirtualNetwork, final Boolean forSystemVms, final Long networkId, final Long physicalNetworkId, final String startIPv6, final String endIPv6,
             final String ip6Gateway, final String ip6Cidr, final Domain domain, final Account vlanOwner, final Network network, final Pair<Boolean, Pair<String, String>> sameSubnet, boolean forNsx) {
@@ -4992,28 +5016,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         // same as network's vlan
         // 2) if vlan is missing, default it to the guest network's vlan
         if (network.getTrafficType() == TrafficType.Guest) {
-            String networkVlanId = null;
+            boolean connectivityWithoutVlan = isConnectivityWithoutVlan(network);
-            boolean connectivityWithoutVlan = false;
+            String networkVlanId = getNetworkVlanId(network, connectivityWithoutVlan);
-            if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Connectivity)) {
-                Map<Capability, String> connectivityCapabilities = _networkModel.getNetworkServiceCapabilities(network.getId(), Service.Connectivity);
-                connectivityWithoutVlan = MapUtils.isNotEmpty(connectivityCapabilities) && connectivityCapabilities.containsKey(Capability.NoVlan);
-            }
-
-            final URI uri = network.getBroadcastUri();
-            if (connectivityWithoutVlan) {
-                networkVlanId = network.getBroadcastDomainType().toUri(network.getUuid()).toString();
-            } else if (uri != null) {
-                // Do not search for the VLAN tag when the network doesn't support VLAN
-               if (uri.toString().startsWith("vlan")) {
-                    final String[] vlan = uri.toString().split("vlan:\\/\\/");
-                    networkVlanId = vlan[1];
-                    // For pvlan
-                    if (network.getBroadcastDomainType() != BroadcastDomainType.Vlan) {
-                        networkVlanId = networkVlanId.split("-")[0];
-                    }
-               }
-            }
-
             if (vlanId != null && !connectivityWithoutVlan) {
                 // if vlan is specified, throw an error if it's not equal to
                 // network's vlanId
@@ -5145,6 +5149,36 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         return vlan;
     }
 
+    private boolean isConnectivityWithoutVlan(Network network) {
+        boolean connectivityWithoutVlan = false;
+        if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Connectivity)) {
+            Map<Capability, String> connectivityCapabilities = _networkModel.getNetworkServiceCapabilities(network.getId(), Service.Connectivity);
+            connectivityWithoutVlan = MapUtils.isNotEmpty(connectivityCapabilities) && connectivityCapabilities.containsKey(Capability.NoVlan);
+        }
+        return connectivityWithoutVlan;
+    }
+
+    private String getNetworkVlanId(Network network, boolean connectivityWithoutVlan) {
+        String networkVlanId = null;
+        if (connectivityWithoutVlan) {
+            return network.getBroadcastDomainType().toUri(network.getUuid()).toString();
+        }
+
+        final URI uri = network.getBroadcastUri();
+        if (uri != null) {
+            // Do not search for the VLAN tag when the network doesn't support VLAN
+            if (uri.toString().startsWith("vlan")) {
+                final String[] vlan = uri.toString().split("vlan:\\/\\/");
+                networkVlanId = vlan[1];
+                // For pvlan
+                if (network.getBroadcastDomainType() != BroadcastDomainType.Vlan) {
+                    networkVlanId = networkVlanId.split("-")[0];
+                }
+            }
+        }
+        return networkVlanId;
+    }
+
     private void checkZoneVlanIpOverlap(DataCenterVO zone, Network network, String newCidr, String vlanId, String vlanGateway, String vlanNetmask, String startIP, String endIP) {
         // Throw an exception if this subnet overlaps with subnet on other VLAN,
         // if this is ip range extension, gateway, network mask should be same and ip range should not overlap

server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java

@@ -918,7 +918,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         } else if (intervalTypeStr != null && volumeId != null) {
             Type type = SnapshotVO.getSnapshotType(intervalTypeStr);
             if (type == null) {
-                throw new InvalidParameterValueException("Unsupported snapstho interval type " + intervalTypeStr);
+                throw new InvalidParameterValueException("Unsupported snapshot interval type " + intervalTypeStr);
             }
             sc.setParameters("snapshotTypeEQ", type.ordinal());
         } else {

server/src/main/java/com/cloud/usage/UsageServiceImpl.java

@@ -26,8 +26,6 @@ import java.util.TimeZone;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.domain.Domain;
-import com.cloud.utils.DateUtil;
 import org.apache.cloudstack.api.command.admin.usage.GenerateUsageRecordsCmd;
 import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd;
 import org.apache.cloudstack.api.command.admin.usage.RemoveRawUsageRecordsCmd;
@@ -42,6 +40,7 @@ import org.jetbrains.annotations.NotNull;
 import org.springframework.stereotype.Component;
 
 import com.cloud.configuration.Config;
+import com.cloud.domain.Domain;
 import com.cloud.domain.DomainVO;
 import com.cloud.domain.dao.DomainDao;
 import com.cloud.exception.InvalidParameterValueException;
@@ -58,6 +57,8 @@ import com.cloud.network.rules.PortForwardingRuleVO;
 import com.cloud.network.rules.dao.PortForwardingRulesDao;
 import com.cloud.network.security.SecurityGroupVO;
 import com.cloud.network.security.dao.SecurityGroupDao;
+import com.cloud.offerings.NetworkOfferingVO;
+import com.cloud.offerings.dao.NetworkOfferingDao;
 import com.cloud.projects.Project;
 import com.cloud.projects.ProjectManager;
 import com.cloud.storage.SnapshotVO;
@@ -72,6 +73,7 @@ import com.cloud.user.Account;
 import com.cloud.user.AccountService;
 import com.cloud.user.AccountVO;
 import com.cloud.user.dao.AccountDao;
+import com.cloud.utils.DateUtil;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.Manager;
 import com.cloud.utils.component.ManagerBase;
@@ -121,6 +123,8 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag
     private IPAddressDao _ipDao;
     @Inject
     private HostDao _hostDao;
+    @Inject
+    private NetworkOfferingDao _networkOfferingDao;
 
     public UsageServiceImpl() {
     }
@@ -245,6 +249,7 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag
             }
 
             Long usageDbId = null;
+            boolean offeringExistsForNetworkOfferingType = false;
 
             switch (usageType.intValue()) {
                 case UsageTypes.NETWORK_BYTES_RECEIVED:
@@ -318,13 +323,19 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag
                         usageDbId = ip.getId();
                     }
                     break;
+                case UsageTypes.NETWORK_OFFERING:
+                    NetworkOfferingVO networkOffering = _networkOfferingDao.findByUuidIncludingRemoved(usageId);
+                    if (networkOffering != null) {
+                        offeringExistsForNetworkOfferingType = true;
+                        sc.addAnd("offeringId", SearchCriteria.Op.EQ, networkOffering.getId());
+                    }
                 default:
                     break;
             }
 
             if (usageDbId != null) {
                 sc.addAnd("usageId", SearchCriteria.Op.EQ, usageDbId);
-            } else {
+            } else if (!offeringExistsForNetworkOfferingType) {
                 // return an empty list if usageId was not found
                 return new Pair<List<? extends Usage>, Integer>(new ArrayList<Usage>(), new Integer(0));
             }

server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java

@@ -55,6 +55,8 @@ import com.cloud.network.dao.FirewallRulesDao;
 import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.IPAddressVO;
 import com.cloud.network.dao.Ipv6GuestPrefixSubnetNetworkMapDao;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
 import com.cloud.network.dao.PhysicalNetworkDao;
 import com.cloud.network.dao.PhysicalNetworkVO;
 import com.cloud.offering.DiskOffering;
@@ -196,6 +198,8 @@ public class ConfigurationManagerTest {
     @Mock
     HostPodDao _podDao;
     @Mock
+    NetworkDao _networkDao;
+    @Mock
     PhysicalNetworkDao _physicalNetworkDao;
     @Mock
     ImageStoreDao _imageStoreDao;
@@ -1331,6 +1335,8 @@ public class ConfigurationManagerTest {
     public void testWrongIpv6CreateVlanAndPublicIpRange() {
         CreateVlanIpRangeCmd cmd = Mockito.mock(CreateVlanIpRangeCmd.class);
         Mockito.when(cmd.getIp6Cidr()).thenReturn("fd17:5:8a43:e2a4:c000::/66");
+        NetworkVO network = Mockito.mock(NetworkVO.class);
+        Mockito.when(_networkDao.findById(Mockito.anyLong())).thenReturn(network);
         try {
             configurationMgr.createVlanAndPublicIpRange(cmd);
         } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException e) {

services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java

@@ -1089,6 +1089,12 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
 
         try {
             _itMgr.expunge(ssvm.getUuid());
+            ssvm.setPublicIpAddress(null);
+            ssvm.setPublicMacAddress(null);
+            ssvm.setPublicNetmask(null);
+            ssvm.setPrivateMacAddress(null);
+            ssvm.setPrivateIpAddress(null);
+            _secStorageVmDao.update(ssvm.getId(), ssvm);
             _secStorageVmDao.remove(ssvm.getId());
             HostVO host = _hostDao.findByTypeNameAndZoneId(ssvm.getDataCenterId(), ssvm.getHostName(), Host.Type.SecondaryStorageVM);
             if (host != null) {
@@ -1373,7 +1379,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
     @Override
     public void finalizeExpunge(VirtualMachine vm) {
         SecondaryStorageVmVO ssvm = _secStorageVmDao.findByUuid(vm.getUuid());
-
+        ssvm.setPrivateMacAddress(null);
         ssvm.setPublicIpAddress(null);
         ssvm.setPublicMacAddress(null);
         ssvm.setPublicNetmask(null);

ui/.env.qa

@@ -0,0 +1 @@
+CS_URL=https://qa.cloudstack.cloud/simulator/pr/10580

ui/public/locales/en.json

@@ -2086,6 +2086,7 @@
 "label.sharewith": "Share with",
 "label.showing": "Showing",
 "label.show.usage.records": "Show usage records",
+"label.showing.results.for": "Showing results for \"%x\"",
 "label.shrinkok": "Shrink OK",
 "label.shutdown": "Shutdown",
 "label.shutdown.provider": "Shutdown provider",

ui/src/components/header/ProjectMenu.vue

@@ -17,112 +17,75 @@
 
 <template>
   <span class="header-notice-opener">
-    <a-select
+    <infinite-scroll-select
-      v-if="!isDisabled()"
+      v-if="!isDisabled"
+      v-model:value="selectedProjectId"
       class="project-select"
-      :loading="loading"
+      api="listProjects"
-      v-model:value="projectSelected"
+      :apiParams="projectsApiParams"
-      :filterOption="filterProject"
+      resourceType="project"
-      @change="changeProject"
+      :defaultOption="defaultOption"
-      @focus="fetchData"
+      defaultIcon="project-outlined"
-      showSearch>
+      :pageSize="100"
-
+      @change-option="changeProject" />
-      <a-select-option
-        v-for="(project, index) in projects"
-        :key="index"
-        :label="project.displaytext || project.name">
-        <span>
-          <resource-icon v-if="project.icon && project.icon.base64image" :image="project.icon.base64image" size="1x" style="margin-right: 5px"/>
-          <project-outlined v-else style="margin-right: 5px" />
-          {{ project.displaytext || project.name }}
-        </span>
-      </a-select-option>
-    </a-select>
   </span>
 </template>
 
 <script>
-import store from '@/store'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect'
-import { api } from '@/api'
-import _ from 'lodash'
-import ResourceIcon from '@/components/view/ResourceIcon'
 
 export default {
   name: 'ProjectMenu',
   components: {
-    ResourceIcon
+    InfiniteScrollSelect
   },
   data () {
     return {
-      projects: [],
+      selectedProjectId: null,
       loading: false
     }
   },
   created () {
-    this.fetchData()
+    this.selectedProjectId = this.$store.getters?.project?.id || this.defaultOption.id
+    this.$store.dispatch('ToggleTheme', this.selectedProjectId ? 'dark' : 'light')
   },
   computed: {
-    projectSelected () {
+    isDisabled () {
-      let projectIndex = 0
+      return !('listProjects' in this.$store.getters.apis)
-      if (this.$store.getters?.project?.id) {
+    },
-        projectIndex = this.projects.findIndex(project => project.id === this.$store.getters.project.id)
+    defaultOption () {
-        this.$store.dispatch('ToggleTheme', projectIndex === undefined ? 'light' : 'dark')
+      return { id: 0, name: this.$t('label.default.view') }
+    },
+    projectsApiParams () {
+      return {
+        details: 'min',
+        listall: true
       }
-
+    }
-      return projectIndex
+  },
+  mounted () {
+    this.unwatchProject = this.$store.watch(
+      (state, getters) => getters.project?.id,
+      (newId) => {
+        this.selectedProjectId = newId
+      }
+    )
+  },
+  beforeUnmount () {
+    if (this.unwatchProject) {
+      this.unwatchProject()
     }
   },
   methods: {
-    fetchData () {
+    changeProject (project) {
-      if (this.isDisabled()) {
-        return
-      }
-      var page = 1
-      const projects = []
-      const getNextPage = () => {
-        this.loading = true
-        api('listProjects', { listAll: true, page: page, pageSize: 500, details: 'min', showIcon: true }).then(json => {
-          if (json?.listprojectsresponse?.project) {
-            projects.push(...json.listprojectsresponse.project)
-          }
-          if (projects.length < json.listprojectsresponse.count) {
-            page++
-            getNextPage()
-          }
-        }).finally(() => {
-          this.loading = false
-          this.$store.commit('RELOAD_ALL_PROJECTS', projects)
-        })
-      }
-      getNextPage()
-    },
-    isDisabled () {
-      return !Object.prototype.hasOwnProperty.call(store.getters.apis, 'listProjects')
-    },
-    changeProject (index) {
-      const project = this.projects[index]
       this.$store.dispatch('ProjectView', project.id)
       this.$store.dispatch('SetProject', project)
-      this.$store.dispatch('ToggleTheme', project.id === undefined ? 'light' : 'dark')
+      this.$store.dispatch('ToggleTheme', project.id ? 'dark' : 'light')
       this.$message.success(`${this.$t('message.switch.to')} "${project.displaytext || project.name}"`)
       if (this.$route.name !== 'dashboard') {
         this.$router.push({ name: 'dashboard' })
       }
-    },
-    filterProject (input, option) {
-      return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
     }
-  },
-  mounted () {
-    this.$store.watch(
-      (state, getters) => getters.allProjects,
-      (newValue, oldValue) => {
-        if (oldValue !== newValue && newValue !== undefined) {
-          this.projects = _.orderBy(newValue, ['displaytext'], ['asc'])
-          this.projects.unshift({ name: this.$t('label.default.view') })
-        }
-      }
-    )
   }
 }
 </script>

ui/src/components/widgets/InfiniteScrollSelect.vue

@@ -0,0 +1,298 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+<!--
+  InfiniteScrollSelect.vue
+
+  A reusable select component that supports:
+  - Infinite scrolling with paginated API
+  - Dynamic search filtering. Needs minimum
+  - Deduplicated option loading
+  - Auto-fetching of preselected value if not present in the initial result
+
+  Usage Example:
+
+  <infinite-scroll-select
+    v-model:value="form.account"
+    api="listAccounts"
+    :apiParams="accountsApiParams"
+    resourceType="account"
+    optionValueKey="name"
+    optionLabelKey="name"
+    @change-option-value="handleAccountNameChange" />
+
+  Props:
+  - api (String, required): API command name (e.g., 'listAccounts')
+  - apiParams (Object, optional): Additional parameters passed to the API
+  - resourceType (String, required): The key in the API response containing the resource array (e.g., 'account')
+  - optionValueKey (String, optional): Property to use as the value for options (e.g., 'name'). Default is 'id'
+  - optionLabelKey (String, optional): Property to use as the label for options (e.g., 'name'). Default is 'name'
+  - defaultOption (Object, optional): Preselected object to include initially
+  - showIcon (Boolean, optional): Whether to show icon for the options. Default is true
+  - defaultIcon (String, optional): Icon to be shown when there is no resource icon for the option. Default is 'cloud-outlined'
+
+  Events:
+  - @change-option-value (Function): Emits the selected option value(s) when value(s) changes. Do not use @change as it will give warnings and may not work
+  - @change-option (Function): Emits the selected option object when value changes. Works only when mode is not multiple
+
+  Features:
+  - Debounced remote filtering
+  - Custom dropdown footer/header (e.g., clear search button)
+  - Handles preselection and fetches missing option automatically
+-->
+<template>
+  <a-select
+    :filter-option="false"
+    :loading="loading"
+    show-search
+    placeholder="Select"
+    @search="onSearchTimed"
+    @popupScroll="onScroll"
+    @change="onChange"
+  >
+    <template #dropdownRender="{ menuNode: menu }">
+      <v-nodes :vnodes="menu" />
+      <div v-if="!!searchQuery">
+        <a-divider style="margin: 4px 0" />
+        <div class="select-list-footer">
+          <span>{{ formattedSearchFooterMessage }}</span>
+          <close-outlined
+            @mousedown="e => e.preventDefault()"
+            @click="onSearch()" />
+        </div>
+      </div>
+    </template>
+    <a-select-option v-for="option in options" :key="option.id" :value="option[optionValueKey]">
+      <span>
+        <span v-if="showIcon">
+          <resource-icon v-if="option.icon && option.icon.base64image" :image="option.icon.base64image" size="1x" style="margin-right: 5px"/>
+          <render-icon v-else :icon="defaultIcon" style="margin-right: 5px" />
+        </span>
+        <span>{{ option[optionLabelKey] }}</span>
+      </span>
+    </a-select-option>
+  </a-select>
+</template>
+
+<script>
+import { api } from '@/api'
+import ResourceIcon from '@/components/view/ResourceIcon'
+
+export default {
+  name: 'InfiniteScrollSelect',
+  components: {
+    ResourceIcon,
+    VNodes: (_, { attrs }) => {
+      return attrs.vnodes
+    }
+  },
+  props: {
+    api: {
+      type: String,
+      required: true
+    },
+    apiParams: {
+      type: Object,
+      required: null
+    },
+    resourceType: {
+      type: String,
+      required: true
+    },
+    optionValueKey: {
+      type: String,
+      default: 'id'
+    },
+    optionLabelKey: {
+      type: String,
+      default: 'name'
+    },
+    defaultOption: {
+      type: Object,
+      default: null
+    },
+    showIcon: {
+      type: Boolean,
+      default: true
+    },
+    defaultIcon: {
+      type: String,
+      default: 'cloud-outlined'
+    },
+    pageSize: {
+      type: Number,
+      default: null
+    }
+  },
+  data () {
+    return {
+      options: [],
+      page: 1,
+      totalCount: null,
+      loading: false,
+      searchQuery: '',
+      searchTimer: null,
+      scrollHandlerAttached: false,
+      preselectedOptionValue: null,
+      successiveFetches: 0
+    }
+  },
+  created () {
+    this.addDefaultOptionIfNeeded(true)
+  },
+  mounted () {
+    this.preselectedOptionValue = this.$attrs.value
+    this.fetchItems()
+  },
+  computed: {
+    maxSuccessiveFetches () {
+      return 10
+    },
+    computedPageSize () {
+      return this.pageSize || this.$store.getters.defaultListViewPageSize
+    },
+    formattedSearchFooterMessage () {
+      return `${this.$t('label.showing.results.for').replace('%x', this.searchQuery)}`
+    }
+  },
+  watch: {
+    apiParams () {
+      this.onSearch()
+    }
+  },
+  emits: ['change-option-value', 'change-option'],
+  methods: {
+    async fetchItems () {
+      if (this.successiveFetches === 0 && this.loading) return
+      this.loading = true
+      const params = {
+        page: this.page,
+        pagesize: this.computedPageSize
+      }
+      if (this.searchQuery && this.searchQuery.length > 0) {
+        params.keyword = this.searchQuery
+      }
+      if (this.apiParams) {
+        Object.assign(params, this.apiParams)
+      }
+      if (this.showIcon) {
+        params.showicon = true
+      }
+      api(this.api, params).then(json => {
+        const response = json[this.api.toLowerCase() + 'response'] || {}
+        if (this.totalCount === null) {
+          this.totalCount = response.count || 0
+        }
+        const newOpts = response[this.resourceType] || []
+        const existingOptions = new Set(this.options.map(o => o[this.optionValueKey]))
+        newOpts.forEach(opt => {
+          if (!existingOptions.has(opt[this.optionValueKey])) {
+            this.options.push(opt)
+          }
+        })
+        this.page++
+        this.checkAndFetchPreselectedOption()
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
+        if (this.successiveFetches === 0) {
+          this.loading = false
+        }
+      })
+    },
+    checkAndFetchPreselectedOption () {
+      if (!this.preselectedOptionValue ||
+        (Array.isArray(this.preselectedOptionValue) && this.preselectedOptionValue.length === 0) ||
+        this.successiveFetches >= this.maxSuccessiveFetches) {
+        this.resetPreselectedOptionValue()
+        return
+      }
+      const matchValue = Array.isArray(this.preselectedOptionValue) ? this.preselectedOptionValue[0] : this.preselectedOptionValue
+      const match = this.options.find(entry => entry[this.optionValueKey] === matchValue)
+      if (!match) {
+        this.successiveFetches++
+        if (this.options.length < this.totalCount) {
+          this.fetchItems()
+        } else {
+          this.resetPreselectedOptionValue()
+        }
+        return
+      }
+      if (Array.isArray(this.preselectedOptionValue) && this.preselectedOptionValue.length > 1) {
+        this.preselectedOptionValue = this.preselectedOptionValue.filter(o => o !== match)
+      } else {
+        this.resetPreselectedOptionValue()
+      }
+    },
+    addDefaultOptionIfNeeded () {
+      if (this.defaultOption) {
+        this.options.push(this.defaultOption)
+      }
+    },
+    resetPreselectedOptionValue () {
+      this.preselectedOptionValue = null
+      this.successiveFetches = 0
+    },
+    onSearchTimed (value) {
+      clearTimeout(this.searchTimer)
+      this.searchTimer = setTimeout(() => {
+        this.onSearch(value)
+      }, 500)
+    },
+    onSearch (value) {
+      this.searchQuery = value
+      this.page = 1
+      this.totalCount = null
+      this.options = []
+      if (!this.searchQuery) {
+        this.addDefaultOptionIfNeeded()
+      }
+      this.fetchItems()
+    },
+    onScroll (e) {
+      const nearBottom = e.target.scrollTop + e.target.clientHeight >= e.target.scrollHeight - 10
+      const hasMore = this.options.length < this.totalCount
+      if (nearBottom && hasMore && !this.loading) {
+        this.fetchItems()
+      }
+    },
+    onChange (value) {
+      this.resetPreselectedOptionValue()
+      this.$emit('change-option-value', value)
+      if (Array.isArray(value)) {
+        return
+      }
+      if (value === undefined || value == null) {
+        this.$emit('change-option', undefined)
+        return
+      }
+      const match = this.options.find(entry => entry[this.optionValueKey] === value)
+      if (match) {
+        this.$emit('change-option', match)
+      }
+    }
+  }
+}
+</script>
+
+<style lang="less" scoped>
+  .select-list-footer {
+    margin: 4px 10px;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+  }
+</style>

ui/src/views/compute/EditVM.vue

@@ -202,7 +202,7 @@ export default {
     },
     fetchZoneDetails () {
       api('listZones', {
-        zoneid: this.resource.zoneid
+        id: this.resource.zoneid
       }).then(response => {
         const zone = response?.listzonesresponse?.zone || []
         this.securityGroupsEnabled = zone?.[0]?.securitygroupsenabled || this.$store.getters.showSecurityGroups
@@ -336,10 +336,8 @@ export default {
         params.name = values.name
         params.displayname = values.displayname
         params.ostypeid = values.ostypeid
-        if (this.securityGroupsEnabled) {
+        if (this.securityGroupsEnabled && Array.isArray(values.securitygroupids) && values.securitygroupids.length > 0) {
-          if (values.securitygroupids) {
+          params.securitygroupids = values.securitygroupids
-            params.securitygroupids = values.securitygroupids
-          }
         }
         if (values.isdynamicallyscalable !== undefined) {
           params.isdynamicallyscalable = values.isdynamicallyscalable

ui/src/views/network/CreateNetworkPermission.vue

@@ -29,47 +29,27 @@
             <template #label>
               <tooltip-label :title="$t('label.account')" :tooltip="apiParams.accountids.description"/>
             </template>
-            <a-select
+            <infinite-scroll-select
               v-model:value="form.accountids"
               mode="multiple"
-              :loading="accountLoading"
               :placeholder="apiParams.accountids.description"
-              showSearch
+              api="listAccounts"
-              optionFilterProp="label"
+              :apiParams="accountsApiParams"
-              :filterOption="(input, option) => {
+              resourceType="account"
-                return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
+              defaultIcon="team-outlined" />
-              }" >
-              <a-select-option v-for="(opt, optIndex) in accounts" :key="optIndex" :label="opt.name || opt.description">
-                <span>
-                  <resource-icon v-if="opt.icon" :image="opt.icon.base64image" size="1x" style="margin-right: 5px"/>
-                  <global-outlined style="margin-right: 5px" />
-                  {{ opt.name || opt.description }}
-                </span>
-              </a-select-option>
-            </a-select>
           </a-form-item>
           <a-form-item v-if="isAdminOrDomainAdmin()" name="projectids" ref="projectids">
             <template #label>
               <tooltip-label :title="$t('label.project')" :tooltip="apiParams.projectids.description"/>
             </template>
-            <a-select
+            <infinite-scroll-select
               v-model:value="form.projectids"
               mode="multiple"
-              :loading="projectLoading"
               :placeholder="apiParams.projectids.description"
-              showSearch
+              api="listProjects"
-              optionFilterProp="label"
+              :apiParams="projectsApiParams"
-              :filterOption="(input, option) => {
+              resourceType="project"
-                return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
+              defaultIcon="project-outlined" />
-              }" >
-              <a-select-option v-for="(opt, optIndex) in projects" :key="optIndex" :label="opt.name || opt.description">
-                <span>
-                  <resource-icon v-if="opt.icon" :image="opt.icon.base64image" size="1x" style="margin-right: 5px"/>
-                  <global-outlined style="margin-right: 5px" />
-                  {{ opt.name || opt.description }}
-                </span>
-              </a-select-option>
-            </a-select>
           </a-form-item>
           <a-form-item v-if="!isAdminOrDomainAdmin()">
             <template #label>
@@ -106,12 +86,14 @@ import { isAdminOrDomainAdmin } from '@/role'
 import { ref, reactive, toRaw } from 'vue'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect'
 
 export default {
   name: 'CreateNetworkPermissions',
   components: {
     TooltipLabel,
-    ResourceIcon
+    ResourceIcon,
+    InfiniteScrollSelect
   },
   props: {
     resource: {
@@ -121,11 +103,7 @@ export default {
   },
   data () {
     return {
-      loading: false,
+      loading: false
-      accountLoading: false,
-      projectLoading: false,
-      accounts: [],
-      projects: []
     }
   },
   created () {
@@ -133,45 +111,24 @@ export default {
     this.form = reactive({})
     this.rules = reactive({})
     this.apiParams = this.$getApiParams('createNetworkPermissions')
-    this.fetchData()
+  },
+  computed: {
+    accountsApiParams () {
+      return {
+        details: 'min',
+        domainid: this.resource.domainid
+      }
+    },
+    projectsApiParams () {
+      return {
+        details: 'min'
+      }
+    }
   },
   methods: {
     isAdminOrDomainAdmin () {
       return isAdminOrDomainAdmin()
     },
-    async fetchData () {
-      this.fetchAccountData()
-      this.fetchProjectData()
-    },
-    fetchAccountData () {
-      this.accounts = []
-      const params = {}
-      params.showicon = true
-      params.details = 'min'
-      params.domainid = this.resource.domainid
-      this.accountLoading = true
-      api('listAccounts', params).then(json => {
-        const listaccounts = json.listaccountsresponse.account || []
-        this.accounts = listaccounts
-      }).finally(() => {
-        this.accountLoading = false
-      })
-    },
-    fetchProjectData () {
-      this.projects = []
-      const params = {}
-      params.listall = true
-      params.showicon = true
-      params.details = 'min'
-      params.domainid = this.resource.domainid
-      this.projectLoading = true
-      api('listProjects', params).then(json => {
-        const listProjects = json.listprojectsresponse.project || []
-        this.projects = listProjects
-      }).finally(() => {
-        this.projectLoading = false
-      })
-    },
     handleSubmit (e) {
       e.preventDefault()
       if (this.loading) return
@@ -179,31 +136,12 @@ export default {
         const values = toRaw(this.form)
         const params = {}
         params.networkid = this.resource.id
-        var accountIndexes = values.accountids
+        if (values.accountids && values.accountids.length > 0) {
-        var accountId = null
+          params.accountids = values.accountids.join(',')
-        if (accountIndexes && accountIndexes.length > 0) {
-          var accountIds = []
-          for (var i = 0; i < accountIndexes.length; i++) {
-            accountIds = accountIds.concat(this.accounts[accountIndexes[i]].id)
-          }
-          accountId = accountIds.join(',')
         }
-        if (accountId) {
+        if (values.projectids && values.projectids.length > 0) {
-          params.accountids = accountId
+          params.projectids = values.projectids.join(',')
         }
-        var projectIndexes = values.projectids
-        var projectId = null
-        if (projectIndexes && projectIndexes.length > 0) {
-          var projectIds = []
-          for (var j = 0; j < projectIndexes.length; j++) {
-            projectIds = projectIds.concat(this.projects[projectIndexes[j]].id)
-          }
-          projectId = projectIds.join(',')
-        }
-        if (projectId) {
-          params.projectids = projectId
-        }
-
         if (values.accounts && values.accounts.length > 0) {
           params.accounts = values.accounts
         }

ui/src/views/tools/ManageInstances.vue

@@ -1316,6 +1316,31 @@ export default {
         this.fetchInstances()
       }
     },
+    fetchVmwareInstanceForKVMMigration (vmname, hostname) {
+      const params = {}
+      if (this.isMigrateFromVmware && this.selectedVmwareVcenter) {
+        if (this.selectedVmwareVcenter.vcenter) {
+          params.datacentername = this.selectedVmwareVcenter.datacentername
+          params.vcenter = this.selectedVmwareVcenter.vcenter
+          params.username = this.selectedVmwareVcenter.username
+          params.password = this.selectedVmwareVcenter.password
+        } else {
+          params.existingvcenterid = this.selectedVmwareVcenter.existingvcenterid
+        }
+        params.instancename = vmname
+        params.hostname = hostname
+      }
+      api('listVmwareDcVms', params).then(json => {
+        const response = json.listvmwaredcvmsresponse
+        this.selectedUnmanagedInstance = response.unmanagedinstance[0]
+        this.selectedUnmanagedInstance.ostypename = this.selectedUnmanagedInstance.osdisplayname
+        this.selectedUnmanagedInstance.state = this.selectedUnmanagedInstance.powerstate
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
+        this.loading = false
+      })
+    },
     onManageInstanceAction () {
       this.selectedUnmanagedInstance = {}
       if (this.unmanagedInstances.length > 0 &&
@@ -1333,6 +1358,9 @@ export default {
           }
         })
         this.showUnmanageForm = false
+      } else if (this.isMigrateFromVmware) {
+        this.fetchVmwareInstanceForKVMMigration(this.selectedUnmanagedInstance.name, this.selectedUnmanagedInstance.hostname)
+        this.showUnmanageForm = true
       } else {
         this.showUnmanageForm = true
       }

ui/src/views/tools/SelectVmwareVcenter.vue

@@ -227,6 +227,8 @@ export default {
       } else {
         params.existingvcenterid = this.selectedExistingVcenterId
       }
+      params.page = 1
+      params.pagesize = 10
       api('listVmwareDcVms', params).then(json => {
         const obj = {
           params: params,
@@ -265,6 +267,11 @@ export default {
         this.loading = false
       })
     },
+    onSelectExternalVmwareDatacenter (value) {
+      if (this.vcenterSelectedOption === 'new' && !(this.vcenter === '' || this.datacentername === '' || this.username === '' || this.password === '')) {
+        this.listVmwareDatacenterVms()
+      }
+    },
     onSelectExistingVmwareDatacenter (value) {
       this.selectedExistingVcenterId = value
     },

utils/src/main/java/com/cloud/utils/script/Script.java

@@ -49,6 +49,7 @@ import org.joda.time.Duration;
 
 import com.cloud.utils.Pair;
 import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.script.OutputInterpreter.TimedOutLogger;
 
@@ -156,25 +157,15 @@ public class Script implements Callable<String> {
         boolean obscureParam = false;
         for (int i = 0; i < command.length; i++) {
             String cmd = command[i];
-            if (obscureParam) {
+            if (StringUtils.isNotEmpty(cmd) && cmd.startsWith("vi://")) {
-                builder.append("******").append(" ");
+                String[] tokens = cmd.split("@");
-                obscureParam = false;
+                if (tokens.length >= 2) {
-            } else {
+                    builder.append("vi://").append("******@").append(tokens[1]).append(" ");
-                builder.append(command[i]).append(" ");
+                } else {
+                    builder.append("vi://").append("******").append(" ");
+                }
+                continue;
             }
-
-            if ("-y".equals(cmd) || "-z".equals(cmd)) {
-                obscureParam = true;
-                _passwordCommand = true;
-            }
-        }
-        return builder.toString();
-    }
-
-    protected String buildCommandLine(List<String> command) {
-        StringBuilder builder = new StringBuilder();
-        boolean obscureParam = false;
-        for (String cmd : command) {
             if (obscureParam) {
                 builder.append("******").append(" ");
                 obscureParam = false;

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/BaseMO.java

@@ -20,17 +20,35 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
 import com.cloud.hypervisor.vmware.util.VmwareContext;
+import com.cloud.utils.Pair;
+
+import com.vmware.vim25.DynamicProperty;
+import com.vmware.vim25.ObjectContent;
+import com.vmware.vim25.VirtualMachineBootOptions;
+import com.vmware.vim25.VirtualMachinePowerState;
+import org.apache.cloudstack.vm.UnmanagedInstanceTO;
+import org.apache.commons.lang3.StringUtils;
 
 import com.vmware.vim25.CustomFieldDef;
 import com.vmware.vim25.CustomFieldStringValue;
 import com.vmware.vim25.ManagedObjectReference;
 
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 public class BaseMO {
     protected static Logger logger = LogManager.getLogger(BaseMO.class);
 
     protected VmwareContext _context;
     protected ManagedObjectReference _mor;
 
+    protected static String[] propertyPathsForUnmanagedVmsThinListing = new String[] {"name", "config.template",
+            "runtime.powerState", "config.guestId", "config.guestFullName", "runtime.host",
+            "config.bootOptions", "config.firmware"};
+
     private String _name;
 
     public BaseMO(VmwareContext context, ManagedObjectReference mor) {
@@ -154,4 +172,93 @@ public class BaseMO {
 
         return cfmMo.getCustomFieldKey(morType, fieldName);
     }
+
+    private static UnmanagedInstanceTO.PowerState convertPowerState(VirtualMachinePowerState powerState) {
+        return powerState == VirtualMachinePowerState.POWERED_ON ? UnmanagedInstanceTO.PowerState.PowerOn :
+                powerState == VirtualMachinePowerState.POWERED_OFF ? UnmanagedInstanceTO.PowerState.PowerOff : UnmanagedInstanceTO.PowerState.PowerUnknown;
+    }
+
+    protected List<UnmanagedInstanceTO> convertVmsObjectContentsToUnmanagedInstances(List<ObjectContent> ocs, String keyword) throws Exception {
+        Map<String, Pair<String, String>> hostClusterNamesMap = new HashMap<>();
+        List<UnmanagedInstanceTO> vms = new ArrayList<>();
+        if (ocs != null) {
+            for (ObjectContent oc : ocs) {
+                List<DynamicProperty> objProps = oc.getPropSet();
+                if (objProps != null) {
+                    UnmanagedInstanceTO vm = createUnmanagedInstanceTOFromThinListingDynamicProperties(
+                            objProps, keyword, hostClusterNamesMap);
+                    if (vm != null) {
+                        vms.add(vm);
+                    }
+                }
+            }
+        }
+        if (vms.size() > 0) {
+            vms.sort(Comparator.comparing(UnmanagedInstanceTO::getName));
+        }
+        return vms;
+    }
+
+    private UnmanagedInstanceTO createUnmanagedInstanceTOFromThinListingDynamicProperties(List<DynamicProperty> objProps,
+                                                                                            String keyword,
+                                                                                            Map<String, Pair<String, String>> hostClusterNamesMap) throws Exception {
+        UnmanagedInstanceTO vm = new UnmanagedInstanceTO();
+        String vmName;
+        boolean isTemplate = false;
+        boolean excludeByKeyword = false;
+
+        for (DynamicProperty objProp : objProps) {
+            if (objProp.getName().equals("name")) {
+                vmName = (String) objProp.getVal();
+                if (StringUtils.isNotBlank(keyword) && !vmName.contains(keyword)) {
+                    excludeByKeyword = true;
+                }
+                vm.setName(vmName);
+            } else if (objProp.getName().equals("config.template")) {
+                isTemplate = (Boolean) objProp.getVal();
+            } else if (objProp.getName().equals("runtime.powerState")) {
+                VirtualMachinePowerState powerState = (VirtualMachinePowerState) objProp.getVal();
+                vm.setPowerState(convertPowerState(powerState));
+            } else if (objProp.getName().equals("config.guestFullName")) {
+                vm.setOperatingSystem((String) objProp.getVal());
+            } else if (objProp.getName().equals("config.guestId")) {
+                vm.setOperatingSystemId((String) objProp.getVal());
+            } else if (objProp.getName().equals("config.bootOptions")) {
+                VirtualMachineBootOptions bootOptions = (VirtualMachineBootOptions) objProp.getVal();
+                String bootMode = "LEGACY";
+                if (bootOptions != null && bootOptions.isEfiSecureBootEnabled()) {
+                    bootMode = "SECURE";
+                }
+                vm.setBootMode(bootMode);
+            } else if (objProp.getName().equals("config.firmware")) {
+                String firmware = (String) objProp.getVal();
+                vm.setBootType(firmware.equalsIgnoreCase("efi") ? "UEFI" : "BIOS");
+            } else if (objProp.getName().equals("runtime.host")) {
+                ManagedObjectReference hostMor = (ManagedObjectReference) objProp.getVal();
+                setUnmanagedInstanceTOHostAndCluster(vm, hostMor, hostClusterNamesMap);
+            }
+        }
+        if (isTemplate || excludeByKeyword) {
+            return null;
+        }
+        return vm;
+    }
+
+    private void setUnmanagedInstanceTOHostAndCluster(UnmanagedInstanceTO vm, ManagedObjectReference hostMor,
+                                                      Map<String, Pair<String, String>> hostClusterNamesMap) throws Exception {
+        if (hostMor != null && StringUtils.isNotBlank(hostMor.getValue())) {
+            String hostMorValue = hostMor.getValue();
+            Pair<String, String> hostClusterPair;
+            if (hostClusterNamesMap.containsKey(hostMorValue)) {
+                hostClusterPair = hostClusterNamesMap.get(hostMorValue);
+            } else {
+                HostMO hostMO = new HostMO(_context, hostMor);
+                ClusterMO clusterMO = new ClusterMO(_context, hostMO.getHyperHostCluster());
+                hostClusterPair = new Pair<>(hostMO.getHostName(), clusterMO.getName());
+                hostClusterNamesMap.put(hostMorValue, hostClusterPair);
+            }
+            vm.setHostName(hostClusterPair.first());
+            vm.setClusterName(hostClusterPair.second());
+        }
+    }
 }

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java

@@ -21,7 +21,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import com.cloud.hypervisor.vmware.util.VmwareHelper;
 import org.apache.cloudstack.vm.UnmanagedInstanceTO;
 import org.apache.commons.collections.CollectionUtils;
 
@@ -98,7 +97,7 @@ public class DatacenterMO extends BaseMO {
         int key = cfmMo.getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_UUID);
         assert (key != 0);
 
-        List<VirtualMachineMO> list = new ArrayList<VirtualMachineMO>();
+        List<VirtualMachineMO> list = new ArrayList<>();
 
         List<ObjectContent> ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name", String.format("value[%d]", key)});
         if (ocs != null && ocs.size() > 0) {
@@ -159,28 +158,9 @@ public class DatacenterMO extends BaseMO {
         return null;
     }
 
-    public List<UnmanagedInstanceTO> getAllVmsOnDatacenter() throws Exception {
+    public List<UnmanagedInstanceTO> getAllVmsOnDatacenter(String keyword) throws Exception {
-        List<UnmanagedInstanceTO> vms = new ArrayList<>();
+        List<ObjectContent> ocs = getVmPropertiesOnDatacenterVmFolder(propertyPathsForUnmanagedVmsThinListing);
-        List<ObjectContent> ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name"});
+        return convertVmsObjectContentsToUnmanagedInstances(ocs, keyword);
-        if (ocs != null) {
-            for (ObjectContent oc : ocs) {
-                ManagedObjectReference vmMor = oc.getObj();
-                if (vmMor != null) {
-                    VirtualMachineMO vmMo = new VirtualMachineMO(_context, vmMor);
-                    try {
-                        if (!vmMo.isTemplate()) {
-                            HostMO hostMO = vmMo.getRunningHost();
-                            UnmanagedInstanceTO unmanagedInstance = VmwareHelper.getUnmanagedInstance(hostMO, vmMo);
-                            vms.add(unmanagedInstance);
-                        }
-                    } catch (Exception e) {
-                        logger.debug(String.format("Unexpected error checking unmanaged instance %s, excluding it: %s", vmMo.getVmName(), e.getMessage()), e);
-                    }
-                }
-            }
-        }
-
-        return vms;
     }
 
     public List<HostMO> getAllHostsOnDatacenter() throws Exception {
@@ -275,7 +255,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
@@ -301,7 +281,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
@@ -336,7 +316,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
@@ -364,7 +344,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         List<ObjectContent> ocs = context.getService().retrieveProperties(context.getPropertyCollector(), pfSpecArr);
@@ -375,7 +355,7 @@ public class DatacenterMO extends BaseMO {
         assert (ocs.get(0).getPropSet().get(0).getVal() != null);
 
         String dcName = ocs.get(0).getPropSet().get(0).getVal().toString();
-        return new Pair<DatacenterMO, String>(new DatacenterMO(context, ocs.get(0).getObj()), dcName);
+        return new Pair<>(new DatacenterMO(context, ocs.get(0).getObj()), dcName);
     }
 
     public ManagedObjectReference getDvPortGroupMor(String dvPortGroupName) throws Exception {
@@ -396,7 +376,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         List<ObjectContent> ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
@@ -443,7 +423,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         List<ObjectContent> ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
@@ -490,7 +470,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         List<ObjectContent> ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);

vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java

@@ -59,6 +59,8 @@ import com.vmware.vim25.NasDatastoreInfo;
 import com.vmware.vim25.VMwareDVSPortSetting;
 import com.vmware.vim25.VirtualDeviceFileBackingInfo;
 import com.vmware.vim25.VirtualIDEController;
+import com.vmware.vim25.VirtualMachineBootOptions;
+import com.vmware.vim25.VirtualMachineConfigInfo;
 import com.vmware.vim25.VirtualMachineConfigSummary;
 import com.vmware.vim25.VirtualMachineGuestOsIdentifier;
 import com.vmware.vim25.VirtualMachineToolsStatus;
@@ -811,6 +813,17 @@ public class VmwareHelper {
                 instance.setCpuSpeed(configSummary.getCpuReservation());
                 instance.setMemory(configSummary.getMemorySizeMB());
             }
+            VirtualMachineConfigInfo configInfo = vmMo.getConfigInfo();
+            if (configInfo != null) {
+                String firmware = configInfo.getFirmware();
+                instance.setBootType(firmware.equalsIgnoreCase("efi") ? "UEFI" : "BIOS");
+                VirtualMachineBootOptions bootOptions = configInfo.getBootOptions();
+                String bootMode = "LEGACY";
+                if (bootOptions != null && bootOptions.isEfiSecureBootEnabled()) {
+                    bootMode = "SECURE";
+                }
+                instance.setBootMode(bootMode);
+            }
 
             try {
                 ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), hyperHost.getHyperHostCluster());