Merge remote-tracking branch 'apache/4.20'

api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java

@@ -79,6 +79,14 @@ public class UnmanagedInstanceResponse extends BaseResponse {
     @Param(description = "the operating system of the virtual machine")
     private String operatingSystem;
 
+    @SerializedName(ApiConstants.BOOT_MODE)
+    @Param(description = "indicates the boot mode")
+    private String bootMode;
+
+    @SerializedName(ApiConstants.BOOT_TYPE)
+    @Param(description = "indicates the boot type")
+    private String bootType;
+
     @SerializedName(ApiConstants.DISK)
     @Param(description = "the list of disks associated with the virtual machine", responseObject = UnmanagedInstanceDiskResponse.class)
     private Set<UnmanagedInstanceDiskResponse> disks;
@@ -211,4 +219,20 @@ public class UnmanagedInstanceResponse extends BaseResponse {
     public void addNic(NicResponse nic) {
         this.nics.add(nic);
     }
+
+    public String getBootMode() {
+        return bootMode;
+    }
+
+    public void setBootMode(String bootMode) {
+        this.bootMode = bootMode;
+    }
+
+    public String getBootType() {
+        return bootType;
+    }
+
+    public void setBootType(String bootType) {
+        this.bootType = bootType;
+    }
 }

api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java

@@ -61,6 +61,9 @@ public class UnmanagedInstanceTO {
 
     private String vncPassword;
 
+    private String bootType;
+    private String bootMode;
+
     public String getName() {
         return name;
     }
@@ -196,6 +199,22 @@ public class UnmanagedInstanceTO {
                         this, "name", "internalCSName", "hostName", "clusterName"));
     }
 
+    public String getBootType() {
+        return bootType;
+    }
+
+    public void setBootType(String bootType) {
+        this.bootType = bootType;
+    }
+
+    public String getBootMode() {
+        return bootMode;
+    }
+
+    public void setBootMode(String bootMode) {
+        this.bootMode = bootMode;
+    }
+
     public static class Disk {
         private String diskId;
 

engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -210,7 +210,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         scanDirectAgentToLoad();
     }
 
-    private void scanDirectAgentToLoad() {
+    protected void scanDirectAgentToLoad() {
         logger.trace("Begin scanning directly connected hosts");
 
         // for agents that are self-managed, threshold to be considered as disconnected after pingtimeout
@@ -231,11 +231,21 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                                 logger.info("{} is detected down, but we have a forward attache running, disconnect this one before launching the host", host);
                                 removeAgent(agentattache, Status.Disconnected);
                             } else {
-                                continue;
+                                logger.debug("Host {} status is {} but has an AgentAttache which is not forForward, try to load directly", host, host.getStatus());
+                                Status hostStatus = investigate(agentattache);
+                                if (Status.Up == hostStatus) {
+                                    /* Got ping response from host, bring it back */
+                                    logger.info("After investigation, Agent for host {} is determined to be up and running", host);
+                                    agentStatusTransitTo(host, Event.Ping, _nodeId);
+                                } else {
+                                    logger.debug("After investigation, AgentAttache is not null but host status is {}, try to load directly {}", hostStatus, host);
+                                    loadDirectlyConnectedHost(host, false);
+                                }
                             }
+                        } else {
+                            logger.debug("AgentAttache is null, loading directly connected {}", host);
+                            loadDirectlyConnectedHost(host, false);
                         }
-                        logger.debug("Loading directly connected {}", host);
-                        loadDirectlyConnectedHost(host, false);
                     } catch (final Throwable e) {
                         logger.warn(" can not load directly connected {} due to ", host, e);
                     }
@@ -381,20 +391,20 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             return;
         }
         if (!result) {
-                throw new CloudRuntimeException("Failed to propagate agent change request event:" + Event.ShutdownRequested + " to host:" + hostId);
+                throw new CloudRuntimeException(String.format("Failed to propagate agent change request event: %s to host: %s", Event.ShutdownRequested, hostId));
         }
     }
 
     public void notifyNodesInCluster(final AgentAttache attache) {
         logger.debug("Notifying other nodes of to disconnect");
-        final Command[] cmds = new Command[] {new ChangeAgentCommand(attache.getId(), Event.AgentDisconnected)};
+        final Command[] cmds = new Command[]{new ChangeAgentCommand(attache.getId(), Event.AgentDisconnected)};
         _clusterMgr.broadcast(attache.getId(), _gson.toJson(cmds));
     }
 
     // notifies MS peers to schedule a host scan task immediately, triggered during addHost operation
     public void notifyNodesInClusterToScheduleHostScanTask() {
         logger.debug("Notifying other MS nodes to run host scan task");
-        final Command[] cmds = new Command[] {new ScheduleHostScanTaskCommand()};
+        final Command[] cmds = new Command[]{new ScheduleHostScanTaskCommand()};
         _clusterMgr.broadcast(0, _gson.toJson(cmds));
     }
 
@@ -435,7 +445,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             }
             try {
                 logD(bytes, "Routing to peer");
-                Link.write(ch, new ByteBuffer[] {ByteBuffer.wrap(bytes)}, sslEngine);
+                Link.write(ch, new ByteBuffer[]{ByteBuffer.wrap(bytes)}, sslEngine);
                 return true;
             } catch (final IOException e) {
                 try {
@@ -954,7 +964,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                         if (!_agentToTransferIds.isEmpty()) {
                             logger.debug("Found {} agents to transfer", _agentToTransferIds.size());
                             // for (Long hostId : _agentToTransferIds) {
-                            for (final Iterator<Long> iterator = _agentToTransferIds.iterator(); iterator.hasNext();) {
+                            for (final Iterator<Long> iterator = _agentToTransferIds.iterator(); iterator.hasNext(); ) {
                                 final Long hostId = iterator.next();
                                 final AgentAttache attache = findAttache(hostId);
 
@@ -1095,7 +1105,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             return;
         }
 
-        final ClusteredAgentAttache forwardAttache = (ClusteredAgentAttache)attache;
+        final ClusteredAgentAttache forwardAttache = (ClusteredAgentAttache) attache;
 
         if (success) {
 
@@ -1146,10 +1156,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         }
 
         synchronized (_agents) {
-            final ClusteredDirectAgentAttache attache = (ClusteredDirectAgentAttache)_agents.get(hostId);
+            final ClusteredDirectAgentAttache attache = (ClusteredDirectAgentAttache) _agents.get(hostId);
             if (attache != null && attache.getQueueSize() == 0 && attache.getNonRecurringListenersSize() == 0) {
                 handleDisconnectWithoutInvestigation(attache, Event.StartAgentRebalance, true, true);
-                final ClusteredAgentAttache forwardAttache = (ClusteredAgentAttache)createAttache(host);
+                final ClusteredAgentAttache forwardAttache = (ClusteredAgentAttache) createAttache(host);
                 if (forwardAttache == null) {
                     logger.warn("Unable to create a forward attache for the host {} as a part of rebalance process", host);
                     return false;
@@ -1253,7 +1263,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             }
 
             if (cmds.length == 1 && cmds[0] instanceof ChangeAgentCommand) { // intercepted
-                final ChangeAgentCommand cmd = (ChangeAgentCommand)cmds[0];
+                final ChangeAgentCommand cmd = (ChangeAgentCommand) cmds[0];
 
                 logger.debug("Intercepting command for agent change: agent {} event: {}", cmd.getAgentId(), cmd.getEvent());
                 boolean result;
@@ -1270,7 +1280,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 answers[0] = new ChangeAgentAnswer(cmd, result);
                 return _gson.toJson(answers);
             } else if (cmds.length == 1 && cmds[0] instanceof TransferAgentCommand) {
-                final TransferAgentCommand cmd = (TransferAgentCommand)cmds[0];
+                final TransferAgentCommand cmd = (TransferAgentCommand) cmds[0];
 
                 logger.debug("Intercepting command for agent rebalancing: agent: {}, event: {}, connection transfer: {}", cmd.getAgentId(), cmd.getEvent(), cmd.isConnectionTransfer());
                 boolean result;
@@ -1289,7 +1299,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 answers[0] = new Answer(cmd, result, null);
                 return _gson.toJson(answers);
             } else if (cmds.length == 1 && cmds[0] instanceof PropagateResourceEventCommand) {
-                final PropagateResourceEventCommand cmd = (PropagateResourceEventCommand)cmds[0];
+                final PropagateResourceEventCommand cmd = (PropagateResourceEventCommand) cmds[0];
 
                 logger.debug("Intercepting command to propagate event {} for host {} ({})", () -> cmd.getEvent().name(), cmd::getHostId, () -> _hostDao.findById(cmd.getHostId()));
 
@@ -1306,10 +1316,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 answers[0] = new Answer(cmd, result, null);
                 return _gson.toJson(answers);
             } else if (cmds.length == 1 && cmds[0] instanceof ScheduleHostScanTaskCommand) {
-                final ScheduleHostScanTaskCommand cmd = (ScheduleHostScanTaskCommand)cmds[0];
+                final ScheduleHostScanTaskCommand cmd = (ScheduleHostScanTaskCommand) cmds[0];
                 return handleScheduleHostScanTaskCommand(cmd);
             } else if (cmds.length == 1 && cmds[0] instanceof BaseShutdownManagementServerHostCommand) {
-                final BaseShutdownManagementServerHostCommand cmd = (BaseShutdownManagementServerHostCommand)cmds[0];
+                final BaseShutdownManagementServerHostCommand cmd = (BaseShutdownManagementServerHostCommand) cmds[0];
                 return handleShutdownManagementServerHostCommand(cmd);
             }
 
@@ -1362,7 +1372,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 try {
                     managementServerMaintenanceManager.prepareForShutdown();
                     return "Successfully prepared for shutdown";
-                } catch(CloudRuntimeException e) {
+                } catch (CloudRuntimeException e) {
                     return e.getMessage();
                 }
             }
@@ -1371,7 +1381,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 try {
                     managementServerMaintenanceManager.triggerShutdown();
                     return "Successfully triggered shutdown";
-                } catch(CloudRuntimeException e) {
+                } catch (CloudRuntimeException e) {
                     return e.getMessage();
                 }
             }
@@ -1380,7 +1390,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 try {
                     managementServerMaintenanceManager.cancelShutdown();
                     return "Successfully cancelled shutdown";
-                } catch(CloudRuntimeException e) {
+                } catch (CloudRuntimeException e) {
                     return e.getMessage();
                 }
             }

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -61,7 +61,11 @@ import org.apache.cloudstack.ca.CAManager;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
 import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
 import org.apache.cloudstack.framework.ca.Certificate;
@@ -413,6 +417,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     ResourceCleanupService resourceCleanupService;
     @Inject
     VmWorkJobDao vmWorkJobDao;
+    @Inject
+    DataStoreProviderManager dataStoreProviderManager;
 
     private SingleCache<List<Long>> vmIdsInProgressCache;
 
@@ -1238,6 +1244,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                                 planChangedByVolume = true;
                             }
                         }
+                        DataStoreProvider storeProvider = dataStoreProviderManager.getDataStoreProvider(pool.getStorageProviderName());
+                        if (storeProvider != null) {
+                            DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();
+                            if (storeDriver instanceof PrimaryDataStoreDriver) {
+                                ((PrimaryDataStoreDriver)storeDriver).detachVolumeFromAllStorageNodes(vol);
+                            }
+                        }
                     }
                 }
 

engine/orchestration/src/test/java/com/cloud/agent/manager/ClusteredAgentManagerImplTest.java

@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.agent.manager;
+
+import com.cloud.configuration.ManagementServiceConfiguration;
+import com.cloud.ha.HighAvailabilityManagerImpl;
+import com.cloud.host.HostVO;
+import com.cloud.host.Status;
+import com.cloud.host.dao.HostDao;
+import com.cloud.resource.ResourceManagerImpl;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ClusteredAgentManagerImplTest {
+
+    private HostDao _hostDao;
+    @Mock
+    ManagementServiceConfiguration _mgmtServiceConf;
+
+    @Before
+    public void setUp() throws Exception {
+        _hostDao = mock(HostDao.class);
+    }
+
+    @Test
+    public void scanDirectAgentToLoadNoHostsTest() {
+        ClusteredAgentManagerImpl clusteredAgentManagerImpl = mock(ClusteredAgentManagerImpl.class);
+        clusteredAgentManagerImpl._hostDao = _hostDao;
+        clusteredAgentManagerImpl.scanDirectAgentToLoad();
+        verify(clusteredAgentManagerImpl, never()).findAttache(anyLong());
+        verify(clusteredAgentManagerImpl, never()).loadDirectlyConnectedHost(any(), anyBoolean());
+    }
+
+    @Test
+    public void scanDirectAgentToLoadHostWithoutAttacheTest() {
+        // Arrange
+        ClusteredAgentManagerImpl clusteredAgentManagerImpl = Mockito.spy(ClusteredAgentManagerImpl.class);
+        HostVO hostVO = mock(HostVO.class);
+        clusteredAgentManagerImpl._hostDao = _hostDao;
+        clusteredAgentManagerImpl.mgmtServiceConf = _mgmtServiceConf;
+        clusteredAgentManagerImpl._resourceMgr = mock(ResourceManagerImpl.class);
+        when(_mgmtServiceConf.getTimeout()).thenReturn(16000L);
+        when(hostVO.getId()).thenReturn(1L);
+        List hosts = new ArrayList<>();
+        hosts.add(hostVO);
+        when(_hostDao.findAndUpdateDirectAgentToLoad(anyLong(), anyLong(), anyLong())).thenReturn(hosts);
+        AgentAttache agentAttache = mock(AgentAttache.class);
+        doReturn(Boolean.TRUE).when(clusteredAgentManagerImpl).loadDirectlyConnectedHost(hostVO, false);
+        clusteredAgentManagerImpl.scanDirectAgentToLoad();
+        verify(clusteredAgentManagerImpl).loadDirectlyConnectedHost(hostVO, false);
+    }
+
+    @Test
+    public void scanDirectAgentToLoadHostWithForwardAttacheTest() {
+        ClusteredAgentManagerImpl clusteredAgentManagerImpl = Mockito.spy(ClusteredAgentManagerImpl.class);
+        HostVO hostVO = mock(HostVO.class);
+        clusteredAgentManagerImpl._hostDao = _hostDao;
+        clusteredAgentManagerImpl.mgmtServiceConf = _mgmtServiceConf;
+        when(_mgmtServiceConf.getTimeout()).thenReturn(16000L);
+        when(hostVO.getId()).thenReturn(1L);
+        List hosts = new ArrayList<>();
+        hosts.add(hostVO);
+        when(_hostDao.findAndUpdateDirectAgentToLoad(anyLong(), anyLong(), anyLong())).thenReturn(hosts);
+        AgentAttache agentAttache = mock(AgentAttache.class);
+        when(agentAttache.forForward()).thenReturn(Boolean.TRUE);
+        when(clusteredAgentManagerImpl.findAttache(1L)).thenReturn(agentAttache);
+
+        clusteredAgentManagerImpl.scanDirectAgentToLoad();
+        verify(clusteredAgentManagerImpl).removeAgent(agentAttache, Status.Disconnected);
+    }
+
+    @Test
+    public void scanDirectAgentToLoadHostWithNonForwardAttacheTest() {
+        // Arrange
+        ClusteredAgentManagerImpl clusteredAgentManagerImpl = Mockito.spy(new ClusteredAgentManagerImpl());
+        HostVO hostVO = mock(HostVO.class);
+        clusteredAgentManagerImpl._hostDao = _hostDao;
+        clusteredAgentManagerImpl.mgmtServiceConf = _mgmtServiceConf;
+        clusteredAgentManagerImpl._haMgr = mock(HighAvailabilityManagerImpl.class);
+        when(_mgmtServiceConf.getTimeout()).thenReturn(16000L);
+        when(hostVO.getId()).thenReturn(0L);
+        List hosts = new ArrayList<>();
+        hosts.add(hostVO);
+        when(_hostDao.findAndUpdateDirectAgentToLoad(anyLong(), anyLong(), anyLong())).thenReturn(hosts);
+
+        AgentAttache agentAttache = mock(AgentAttache.class);
+        when(agentAttache.forForward()).thenReturn(Boolean.FALSE);
+        when(clusteredAgentManagerImpl.findAttache(0L)).thenReturn(agentAttache);
+        doReturn(Boolean.TRUE).when(clusteredAgentManagerImpl).agentStatusTransitTo(hostVO, Status.Event.Ping, clusteredAgentManagerImpl._nodeId);
+        doReturn(Status.Up).when(clusteredAgentManagerImpl).investigate(agentAttache);
+
+        clusteredAgentManagerImpl.scanDirectAgentToLoad();
+        verify(clusteredAgentManagerImpl).investigate(agentAttache);
+        verify(clusteredAgentManagerImpl).agentStatusTransitTo(hostVO, Status.Event.Ping, clusteredAgentManagerImpl._nodeId);
+    }
+
+    @Test
+    public void scanDirectAgentToLoadHostWithNonForwardAttacheAndDisconnectedTest() {
+        ClusteredAgentManagerImpl clusteredAgentManagerImpl = Mockito.spy(ClusteredAgentManagerImpl.class);
+        HostVO hostVO = mock(HostVO.class);
+        clusteredAgentManagerImpl._hostDao = _hostDao;
+        clusteredAgentManagerImpl.mgmtServiceConf = _mgmtServiceConf;
+        clusteredAgentManagerImpl._haMgr = mock(HighAvailabilityManagerImpl.class);
+        clusteredAgentManagerImpl._resourceMgr = mock(ResourceManagerImpl.class);
+        when(_mgmtServiceConf.getTimeout()).thenReturn(16000L);
+        when(hostVO.getId()).thenReturn(0L);
+        List hosts = new ArrayList<>();
+        hosts.add(hostVO);
+        when(_hostDao.findAndUpdateDirectAgentToLoad(anyLong(), anyLong(), anyLong())).thenReturn(hosts);
+        AgentAttache agentAttache = mock(AgentAttache.class);
+        when(agentAttache.forForward()).thenReturn(Boolean.FALSE);
+        when(clusteredAgentManagerImpl.findAttache(0L)).thenReturn(agentAttache);
+        doReturn(Boolean.TRUE).when(clusteredAgentManagerImpl).loadDirectlyConnectedHost(hostVO, false);
+        clusteredAgentManagerImpl.scanDirectAgentToLoad();
+        verify(clusteredAgentManagerImpl).investigate(agentAttache);
+        verify(clusteredAgentManagerImpl).loadDirectlyConnectedHost(hostVO, false);
+    }
+}

engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java

@@ -80,7 +80,7 @@ public class DefaultEndPointSelector implements EndPointSelector {
     private final String findOneHostOnPrimaryStorage = "select t.id from "
                             + "(select h.id, cd.value, hd.value as " + VOL_ENCRYPT_COLUMN_NAME + " "
                             + "from host h join storage_pool_host_ref s on h.id = s.host_id  "
-                            + "join cluster c on c.id=h.cluster_id "
+                            + "join cluster c on c.id=h.cluster_id and c.allocation_state = 'Enabled'"
                             + "left join cluster_details cd on c.id=cd.cluster_id and cd.name='" + CapacityManager.StorageOperationsExcludeCluster.key() + "' "
                             + "left join host_details hd on h.id=hd.host_id and hd.name='" + HOST_VOLUME_ENCRYPTION + "' "
                             + "where h.status = 'Up' and h.type = 'Routing' and h.resource_state = 'Enabled' and s.pool_id = ? ";

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java

@@ -328,6 +328,11 @@ public class VolumeServiceImpl implements VolumeService {
         } else {
             vo.processEvent(Event.OperationFailed);
             errMsg = result.getResult();
+            VolumeVO volume = volDao.findById(vo.getId());
+            if (volume != null && volume.getState() == State.Allocated && volume.getPodId() != null) {
+                volume.setPoolId(null);
+                volDao.update(volume.getId(), volume);
+            }
         }
         VolumeApiResult volResult = new VolumeApiResult((VolumeObject)vo);
         if (errMsg != null) {
@@ -1255,6 +1260,10 @@ public class VolumeServiceImpl implements VolumeService {
         }
 
         if (volume.getState() == State.Allocated) { // Possible states here: Allocated, Ready & Creating
+            if (volume.getPodId() != null) {
+                volume.setPoolId(null);
+                volDao.update(volume.getId(), volume);
+            }
             return;
         }
 
@@ -2494,7 +2503,7 @@ public class VolumeServiceImpl implements VolumeService {
         try {
             volume.processEvent(Event.ResizeRequested);
         } catch (Exception e) {
-            logger.debug("Failed to change state to resize", e);
+            logger.debug("Failed to change volume state to resize", e);
             result.setResult(e.toString());
             future.complete(result);
             return future;
@@ -2506,10 +2515,8 @@ public class VolumeServiceImpl implements VolumeService {
         try {
             volume.getDataStore().getDriver().resize(volume, caller);
         } catch (Exception e) {
-            logger.debug("Failed to change state to resize", e);
+            logger.debug("Failed to resize volume", e);
-
             result.setResult(e.toString());
-
             future.complete(result);
         }
 
@@ -2553,7 +2560,7 @@ public class VolumeServiceImpl implements VolumeService {
             try {
                 volume.processEvent(Event.OperationFailed);
             } catch (Exception e) {
-                logger.debug("Failed to change state", e);
+                logger.debug("Failed to change volume state (after resize failure)", e);
             }
             VolumeApiResult res = new VolumeApiResult(volume);
             res.setResult(result.getResult());
@@ -2564,13 +2571,8 @@ public class VolumeServiceImpl implements VolumeService {
         try {
             volume.processEvent(Event.OperationSuccessed);
         } catch (Exception e) {
-            logger.debug("Failed to change state", e);
+            logger.debug("Failed to change volume state (after resize success)", e);
-            VolumeApiResult res = new VolumeApiResult(volume);
-            res.setResult(result.getResult());
-            future.complete(res);
-            return null;
         }
-
         VolumeApiResult res = new VolumeApiResult(volume);
         future.complete(res);
 

plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java

@@ -25,6 +25,7 @@ import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.ScopeType;
+import com.cloud.storage.Storage;
 import com.cloud.storage.StoragePoolHostVO;
 import com.cloud.storage.Volume;
 import com.cloud.storage.VolumeVO;
@@ -278,6 +279,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co
         restoredVolume.setPoolId(dataStore.getPoolId());
         restoredVolume.setPath(restoredVolume.getUuid());
         restoredVolume.setState(Volume.State.Copying);
+        restoredVolume.setFormat(Storage.ImageFormat.QCOW2);
         restoredVolume.setSize(backedUpVolumeSize);
         restoredVolume.setDiskOfferingId(volume.getDiskOfferingId());
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java

@@ -45,7 +45,7 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper<RestoreBa
     private static final String MOUNT_COMMAND = "sudo mount -t %s %s %s";
     private static final String UMOUNT_COMMAND = "sudo umount %s";
     private static final String FILE_PATH_PLACEHOLDER = "%s/%s";
-    private static final String ATTACH_DISK_COMMAND = " virsh attach-disk %s %s %s --cache none";
+    private static final String ATTACH_DISK_COMMAND = " virsh attach-disk %s %s %s --driver qemu --subdriver qcow2 --cache none";
     private static final String CURRRENT_DEVICE = "virsh domblklist --domain %s | tail -n 3 | head -n 1 | awk '{print $1}'";
     private static final String RSYNC_COMMAND = "rsync -az %s %s";
 

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java

@@ -43,6 +43,7 @@ import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 import javax.persistence.EntityExistsException;
 
+import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
 import com.cloud.hypervisor.vmware.util.VmwareClient;
 import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd;
 import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd;
@@ -171,8 +172,11 @@ import com.cloud.vm.dao.VMInstanceDao;
 import com.vmware.pbm.PbmProfile;
 import com.vmware.vim25.AboutInfo;
 import com.vmware.vim25.ManagedObjectReference;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public class VmwareManagerImpl extends ManagerBase implements VmwareManager, VmwareStorageMount, Listener, VmwareDatacenterService, Configurable {
+    protected static Logger static_logger = LogManager.getLogger(VmwareManagerImpl.class);
 
     private static final long SECONDS_PER_MINUTE = 60;
     private static final int DEFAULT_PORTS_PER_DV_PORT_GROUP_VSPHERE4_x = 256;
@@ -1585,14 +1589,26 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
         return compatiblePools;
     }
 
-    @Override
+    private static class VcenterData {
-    public List<UnmanagedInstanceTO> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) {
+        public final String vcenter;
+        public final String datacenterName;
+        public final String username;
+        public final String password;
+
+        public VcenterData(String vcenter, String datacenterName, String username, String password) {
+            this.vcenter = vcenter;
+            this.datacenterName = datacenterName;
+            this.username = username;
+            this.password = password;
+        }
+    }
+
+    private VcenterData getVcenterData(ListVmwareDcVmsCmd cmd) {
         String vcenter = cmd.getVcenter();
         String datacenterName = cmd.getDatacenterName();
         String username = cmd.getUsername();
         String password = cmd.getPassword();
         Long existingVcenterId = cmd.getExistingVcenterId();
-        String keyword = cmd.getKeyword();
 
         if ((existingVcenterId == null && StringUtils.isBlank(vcenter)) ||
                 (existingVcenterId != null && StringUtils.isNotBlank(vcenter))) {
@@ -1613,34 +1629,69 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
             username = vmwareDc.getUser();
             password = vmwareDc.getPassword();
         }
+        VcenterData vmwaredc = new VcenterData(vcenter, datacenterName, username, password);
+        return vmwaredc;
+    }
+
+    private static VmwareContext getVmwareContext(String vcenter, String username, String password) throws Exception {
+        static_logger.debug(String.format("Connecting to the VMware vCenter %s", vcenter));
+        String serviceUrl = String.format("https://%s/sdk/vimService", vcenter);
+        VmwareClient vimClient = new VmwareClient(vcenter);
+        vimClient.connect(serviceUrl, username, password);
+        return new VmwareContext(vimClient, vcenter);
+    }
+
+    @Override
+    public List<UnmanagedInstanceTO> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) {
+        VcenterData vmwareDC = getVcenterData(cmd);
+        String vcenter = vmwareDC.vcenter;
+        String username = vmwareDC.username;
+        String password = vmwareDC.password;
+        String datacenterName = vmwareDC.datacenterName;
+        String keyword = cmd.getKeyword();
+        String esxiHostName = cmd.getHostName();
+        String virtualMachineName = cmd.getInstanceName();
 
         try {
             logger.debug(String.format("Connecting to the VMware datacenter %s at vCenter %s to retrieve VMs",
                     datacenterName, vcenter));
-            String serviceUrl = String.format("https://%s/sdk/vimService", vcenter);
+            VmwareContext context = getVmwareContext(vcenter, username, password);
-            VmwareClient vimClient = new VmwareClient(vcenter);
+            DatacenterMO dcMo = getDatacenterMO(context, vcenter, datacenterName);
-            vimClient.connect(serviceUrl, username, password);
-            VmwareContext context = new VmwareContext(vimClient, vcenter);
 
-            DatacenterMO dcMo = new DatacenterMO(context, datacenterName);
+            List<UnmanagedInstanceTO> instances;
-            ManagedObjectReference dcMor = dcMo.getMor();
+            if (StringUtils.isNotBlank(esxiHostName) && StringUtils.isNotBlank(virtualMachineName)) {
-            if (dcMor == null) {
+                ManagedObjectReference hostMor = dcMo.findHost(esxiHostName);
-                String msg = String.format("Unable to find VMware datacenter %s in vCenter %s",
+                if (hostMor == null) {
-                        datacenterName, vcenter);
+                    String errorMsg = String.format("Cannot find a host with name %s on vcenter %s", esxiHostName, vcenter);
-                logger.error(msg);
+                    logger.error(errorMsg);
-                throw new InvalidParameterValueException(msg);
+                    throw new CloudRuntimeException(errorMsg);
+                }
+                HostMO hostMO = new HostMO(context, hostMor);
+                VirtualMachineMO vmMo = hostMO.findVmOnHyperHost(virtualMachineName);
+                instances = Collections.singletonList(VmwareHelper.getUnmanagedInstance(hostMO, vmMo));
+            } else {
+                instances = dcMo.getAllVmsOnDatacenter(keyword);
             }
-            List<UnmanagedInstanceTO> instances = dcMo.getAllVmsOnDatacenter();
+            return instances;
-            return StringUtils.isBlank(keyword) ? instances :
-                    instances.stream().filter(x -> x.getName().toLowerCase().contains(keyword.toLowerCase())).collect(Collectors.toList());
         } catch (Exception e) {
-            String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s",
+            String errorMsg = String.format("Error retrieving VMs from the VMware VC %s datacenter %s: %s",
                     vcenter, datacenterName, e.getMessage());
             logger.error(errorMsg, e);
             throw new CloudRuntimeException(errorMsg);
         }
     }
 
+    private static DatacenterMO getDatacenterMO(VmwareContext context, String vcenter, String datacenterName) throws Exception {
+        DatacenterMO dcMo = new DatacenterMO(context, datacenterName);
+        ManagedObjectReference dcMor = dcMo.getMor();
+        if (dcMor == null) {
+            String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", datacenterName, vcenter);
+            static_logger.error(msg);
+            throw new InvalidParameterValueException(msg);
+        }
+        return dcMo;
+    }
+
     @Override
     public boolean hasNexusVSM(Long clusterId) {
         ClusterVSMMapVO vsmMapVo = null;
@@ -1693,7 +1744,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
     }
 
     /**
-     * This task is to cleanup templates from primary storage that are otherwise not cleaned by the {@link com.cloud.storage.StorageManagerImpl.StorageGarbageCollector}.
+     * This task is to cleanup templates from primary storage that are otherwise not cleaned by the {code}StorageGarbageCollector{code} from {@link com.cloud.storage.StorageManagerImpl}.
      * it is called at regular intervals when storage.template.cleanup.enabled == true
      * It collect all templates that
      * - are deleted from cloudstack

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -2042,7 +2042,6 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
         VirtualMachineDefinedProfileSpec diskProfileSpec = null;
         VirtualMachineDefinedProfileSpec vmProfileSpec = null;
 
-
         DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo();
         boolean deployAsIs = deployAsIsInfo != null;
 
@@ -2086,7 +2085,6 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
             }
 
             VirtualMachineDiskInfoBuilder diskInfoBuilder = null;
-            VirtualDevice[] nicDevices = null;
             VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
             DiskControllerType systemVmScsiControllerType = DiskControllerType.lsilogic;
             int firstScsiControllerBusNum = 0;
@@ -2103,7 +2101,6 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
                 diskDatastores = vmMo.getAllDiskDatastores();
                 diskInfoBuilder = vmMo.getDiskInfoBuilder();
                 hasSnapshot = vmMo.hasSnapshot();
-                nicDevices = vmMo.getNicDevices();
 
                 tearDownVmDevices(vmMo, hasSnapshot, deployAsIs);
                 ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType,
@@ -2119,17 +2116,20 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
                     }
 
                     takeVmFromOtherHyperHost(hyperHost, vmInternalCSName);
+                    vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
 
-                    if (getVmPowerState(vmMo) != PowerState.PowerOff)
+                    if (vmMo != null) {
-                        vmMo.safePowerOff(_shutdownWaitMs);
+                        if (getVmPowerState(vmMo) != PowerState.PowerOff)
+                            vmMo.safePowerOff(_shutdownWaitMs);
 
-                    diskInfoBuilder = vmMo.getDiskInfoBuilder();
+                        diskInfoBuilder = vmMo.getDiskInfoBuilder();
-                    hasSnapshot = vmMo.hasSnapshot();
+                        hasSnapshot = vmMo.hasSnapshot();
-                    diskDatastores = vmMo.getAllDiskDatastores();
+                        diskDatastores = vmMo.getAllDiskDatastores();
 
-                    tearDownVmDevices(vmMo, hasSnapshot, deployAsIs);
+                        tearDownVmDevices(vmMo, hasSnapshot, deployAsIs);
-                    ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType,
+                        ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType,
-                            numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs);
+                                numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs);
+                    }
                 } else {
                     // If a VM with the same name is found in a different cluster in the DC, unregister the old VM and configure a new VM (cold-migration).
                     VirtualMachineMO existingVmInDc = dcMo.findVm(vmInternalCSName);
@@ -2146,7 +2146,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
                         vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName);
                         if (vmMo == null) {
                             logger.info("Cloned deploy-as-is VM " + vmInternalCSName + " is not in this host, relocating it");
-                            vmMo = takeVmFromOtherHyperHost(hyperHost, vmInternalCSName);
+                            takeVmFromOtherHyperHost(hyperHost, vmInternalCSName);
                         }
                     } else {
                         DiskTO rootDisk = null;
@@ -2256,11 +2256,11 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
                 vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm());
             }
 
-            if(!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()){
+            if (!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) {
                 logger.warn("hotadd of memory is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName);
             }
 
-            if(!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()){
+            if (!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) {
                 logger.warn("hotadd of cpu is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName);
             }
 
@@ -2593,7 +2593,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
 
             Map<String, Map<String, String>> iqnToData = new HashMap<>();
 
-            postDiskConfigBeforeStart(vmMo, vmSpec, sortedDisks, ideControllerKey, scsiControllerKey, iqnToData, hyperHost, context);
+            postDiskConfigBeforeStart(vmMo, vmSpec, sortedDisks, iqnToData, hyperHost, context);
 
             //
             // Power-on VM
@@ -2731,14 +2731,24 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
     }
 
     private boolean powerOnVM(final VirtualMachineMO vmMo, final String vmInternalCSName, final String vmNameOnVcenter) throws Exception {
-        int retry = 20;
+        final int retry = 20;
-        while (retry-- > 0) {
+        int retryAttempt = 0;
+        while (++retryAttempt <= retry) {
             try {
+                logger.debug(String.format("VM %s, powerOn attempt #%d", vmInternalCSName, retryAttempt));
                 return vmMo.powerOn();
             } catch (Exception e) {
                 logger.info(String.format("Got exception while power on VM %s with hostname %s", vmInternalCSName, vmNameOnVcenter), e);
-                if (e.getMessage() != null && e.getMessage().contains("File system specific implementation of Ioctl[file] failed")) {
+                if (e.getMessage() != null &&
+                        (e.getMessage().contains("File system specific implementation of Ioctl[file] failed") ||
+                                e.getMessage().contains("Unable to access file") ||
+                                e.getMessage().contains("it is locked"))) {
                     logger.debug(String.format("Failed to power on VM %s with hostname %s. Retrying", vmInternalCSName, vmNameOnVcenter));
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException ie) {
+                        logger.debug(String.format("Waiting to power on VM %s been interrupted: ", vmInternalCSName));
+                    }
                 } else {
                     throw e;
                 }
@@ -3292,7 +3302,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
 
     int getReservedMemoryMb(VirtualMachineTO vmSpec) {
         if (vmSpec.getDetails().get(VMwareGuru.VmwareReserveMemory.key()).equalsIgnoreCase("true")) {
-            if(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null){
+            if (vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null) {
                 float reservedMemory = (vmSpec.getMaxRam() * Float.parseFloat(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION)));
                 return (int) (reservedMemory / ResourceType.bytesToMiB);
             }
@@ -3630,18 +3640,18 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
 
     private VirtualMachineDiskInfo getMatchingExistingDisk(VirtualMachineDiskInfoBuilder diskInfoBuilder, DiskTO vol, VmwareHypervisorHost hyperHost, VmwareContext context)
             throws Exception {
-        if (diskInfoBuilder != null) {
+        if (diskInfoBuilder == null) {
-            VolumeObjectTO volume = (VolumeObjectTO) vol.getData();
-            String chainInfo = volume.getChainInfo();
-            Map<String, String> details = vol.getDetails();
-            boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED));
-            String iScsiName = details.get(DiskTO.IQN);
-            String datastoreUUID = volume.getDataStore().getUuid();
-
-            return getMatchingExistingDiskWithVolumeDetails(diskInfoBuilder, volume.getPath(), chainInfo, isManaged, iScsiName, datastoreUUID, hyperHost, context);
-        } else {
             return null;
         }
+
+        VolumeObjectTO volume = (VolumeObjectTO) vol.getData();
+        String chainInfo = volume.getChainInfo();
+        Map<String, String> details = vol.getDetails();
+        boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED));
+        String iScsiName = details.get(DiskTO.IQN);
+        String datastoreUUID = volume.getDataStore().getUuid();
+
+        return getMatchingExistingDiskWithVolumeDetails(diskInfoBuilder, volume.getPath(), chainInfo, isManaged, iScsiName, datastoreUUID, hyperHost, context);
     }
 
     private String getDiskController(VirtualMachineMO vmMo, VirtualMachineDiskInfo matchingExistingDisk, DiskTO vol, Pair<String, String> controllerInfo, boolean deployAsIs) throws Exception {
@@ -3666,34 +3676,36 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
         return VmwareHelper.getControllerBasedOnDiskType(controllerInfo, vol);
     }
 
-    private void postDiskConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, DiskTO[] sortedDisks, int ideControllerKey,
+    private void postDiskConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, DiskTO[] sortedDisks,
-                                           int scsiControllerKey, Map<String, Map<String, String>> iqnToData, VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception {
+                                           Map<String, Map<String, String>> iqnToData, VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception {
         VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder();
 
         for (DiskTO vol : sortedDisks) {
             if (vol.getType() == Volume.Type.ISO)
                 continue;
 
-            VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData();
-
             VirtualMachineDiskInfo diskInfo = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context);
-            assert (diskInfo != null);
+            if (diskInfo == null) {
+                continue;
+            }
 
             String[] diskChain = diskInfo.getDiskChain();
-            assert (diskChain.length > 0);
+            if (diskChain.length <= 0) {
-
+                continue;
-            Map<String, String> details = vol.getDetails();
-            boolean managed = false;
-
-            if (details != null) {
-                managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED));
             }
 
             DatastoreFile file = new DatastoreFile(diskChain[0]);
 
+            boolean managed = false;
+            Map<String, String> details = vol.getDetails();
+            if (details != null) {
+                managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED));
+            }
+
+            VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData();
+
             if (managed) {
                 DatastoreFile originalFile = new DatastoreFile(volumeTO.getPath());
-
                 if (!file.getFileBaseName().equalsIgnoreCase(originalFile.getFileBaseName())) {
                     if (logger.isInfoEnabled())
                         logger.info("Detected disk-chain top file change on volume: " + volumeTO.getId() + " " + volumeTO.getPath() + " -> " + diskChain[0]);
@@ -3706,7 +3718,6 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
             }
 
             VolumeObjectTO volInSpec = getVolumeInSpec(vmSpec, volumeTO);
-
             if (volInSpec != null) {
                 if (managed) {
                     Map<String, String> data = new HashMap<>();
@@ -3871,20 +3882,20 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
             if (diskInfo != null) {
                 logger.info("Found existing disk info from volume path: " + volume.getPath());
                 return dsMo;
-            } else {
+            }
-                String chainInfo = volume.getChainInfo();
+
-                if (chainInfo != null) {
+            String chainInfo = volume.getChainInfo();
-                    VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class);
+            if (chainInfo != null) {
-                    if (infoInChain != null) {
+                VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class);
-                        String[] disks = infoInChain.getDiskChain();
+                if (infoInChain != null) {
-                        if (disks.length > 0) {
+                    String[] disks = infoInChain.getDiskChain();
-                            for (String diskPath : disks) {
+                    if (disks.length > 0) {
-                                DatastoreFile file = new DatastoreFile(diskPath);
+                        for (String diskPath : disks) {
-                                diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName);
+                            DatastoreFile file = new DatastoreFile(diskPath);
-                                if (diskInfo != null) {
+                            diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName);
-                                    logger.info("Found existing disk from chain info: " + diskPath);
+                            if (diskInfo != null) {
-                                    return dsMo;
+                                logger.info("Found existing disk from chain info: " + diskPath);
-                                }
+                                return dsMo;
                             }
                         }
                     }
@@ -4747,7 +4758,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
             Map<Integer, Long> volumeDeviceKey = new HashMap<>();
             if (cmd instanceof MigrateVolumeCommand) { // Else device keys will be found in relocateVirtualMachine
                 MigrateVolumeCommand mcmd = (MigrateVolumeCommand) cmd;
-                addVolumeDiskmapping(vmMo, volumeDeviceKey, mcmd.getVolumePath(), mcmd.getVolumeId());
+                addVolumeDiskMapping(vmMo, volumeDeviceKey, mcmd.getVolumePath(), mcmd.getVolumeId());
                 if (logger.isTraceEnabled()) {
                     for (Integer diskId: volumeDeviceKey.keySet()) {
                         logger.trace(String.format("Disk to migrate has disk id %d and volumeId %d", diskId, volumeDeviceKey.get(diskId)));
@@ -4765,9 +4776,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
 
     Answer createAnswerForCmd(VirtualMachineMO vmMo, List<VolumeObjectTO> volumeObjectToList, Command cmd, Map<Integer, Long> volumeDeviceKey) throws Exception {
         List<VolumeObjectTO> volumeToList;
-        VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder();
         VirtualDisk[] disks = vmMo.getAllDiskDevice();
-        Answer answer;
         if (logger.isTraceEnabled()) {
             logger.trace(String.format("creating answer for %s", cmd.getClass().getSimpleName()));
         }
@@ -4784,7 +4793,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
         return new Answer(cmd, false, null);
     }
 
-    private void addVolumeDiskmapping(VirtualMachineMO vmMo, Map<Integer, Long> volumeDeviceKey, String volumePath, long volumeId) throws Exception {
+    private void addVolumeDiskMapping(VirtualMachineMO vmMo, Map<Integer, Long> volumeDeviceKey, String volumePath, long volumeId) throws Exception {
         if (logger.isDebugEnabled()) {
             logger.debug(String.format("locating disk for volume (%d) using path %s", volumeId, volumePath));
         }
@@ -4919,7 +4928,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
             VmwareHypervisorHost dsHost = hyperHostInTargetCluster == null ? hyperHost : hyperHostInTargetCluster;
             String targetDsName = cmd.getTargetPool().getUuid();
             morDestinationDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(dsHost, targetDsName);
-            if(morDestinationDS == null) {
+            if (morDestinationDS == null) {
                 String msg = "Unable to find the target datastore: " + targetDsName + " on host: " + dsHost.getHyperHostName();
                 logger.error(msg);
                 throw new CloudRuntimeException(msg);
@@ -5886,6 +5895,11 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes
             logger.debug(msg);
             return new Answer(cmd, true, msg);
         } catch (Exception e) {
+            if (e.getMessage().contains("was not found")) {
+                String msg = String.format("%s - VM [%s] file(s) not found, cleanup not needed .", e.getMessage(), cmd.getVmName());
+                logger.debug(msg);
+                return new Answer(cmd, true, msg);
+            }
             return new Answer(cmd, false, createLogMessageException(e, cmd));
         }
     }

plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java

@@ -20,6 +20,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import com.vmware.vim25.ManagedObjectReference;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.logging.log4j.Logger;
@@ -193,7 +194,7 @@ public class VmwareStorageLayoutHelper implements Configurable {
             if (ds.fileExists(vmdkFullCloneModeLegacyPair[i])) {
                 LOGGER.info("sync " + vmdkFullCloneModeLegacyPair[i] + "->" + vmdkFullCloneModePair[i]);
 
-                ds.moveDatastoreFile(vmdkFullCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[i], dcMo.getMor(), true);
+                moveDatastoreFile(ds, vmdkFullCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[i], dcMo.getMor(), true);
             }
         }
 
@@ -201,13 +202,13 @@ public class VmwareStorageLayoutHelper implements Configurable {
             if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[i])) {
                 LOGGER.info("sync " + vmdkLinkedCloneModeLegacyPair[i] + "->" + vmdkLinkedCloneModePair[i]);
 
-                ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[i], dcMo.getMor(), true);
+                moveDatastoreFile(ds, vmdkLinkedCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[i], dcMo.getMor(), true);
             }
         }
 
         if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[0])) {
             LOGGER.info("sync " + vmdkLinkedCloneModeLegacyPair[0] + "->" + vmdkLinkedCloneModePair[0]);
-            ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[0], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[0], dcMo.getMor(), true);
+            moveDatastoreFile(ds, vmdkLinkedCloneModeLegacyPair[0], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[0], dcMo.getMor(), true);
         }
 
         // Note: we will always return a path
@@ -242,14 +243,14 @@ public class VmwareStorageLayoutHelper implements Configurable {
                 String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, String.format("%s-%s",vmdkName, linkedCloneExtension));
 
                 LOGGER.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath);
-                ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true);
+                moveDatastoreFile(ds, companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true);
             }
         }
 
         // move the identity VMDK file the last
         String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, vmdkName + ".vmdk");
         LOGGER.info("Fixup folder-synchronization. move " + fileDsFullPath + " -> " + targetPath);
-        ds.moveDatastoreFile(fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true);
+        moveDatastoreFile(ds, fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true);
 
         try {
             if (folderName != null) {
@@ -287,7 +288,7 @@ public class VmwareStorageLayoutHelper implements Configurable {
                     DatastoreFile targetFile = new DatastoreFile(file.getDatastoreName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, file.getFileName());
                     if (!targetFile.getPath().equalsIgnoreCase(file.getPath())) {
                         LOGGER.info("Move " + file.getPath() + " -> " + targetFile.getPath());
-                        dsMo.moveDatastoreFile(file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true);
+                        moveDatastoreFile(dsMo, file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true);
 
                         List<String> vSphereFileExtensions = new ArrayList<>(Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*")));
                         // add flat file format to the above list
@@ -297,7 +298,7 @@ public class VmwareStorageLayoutHelper implements Configurable {
                             String pairTargetFilePath = targetFile.getCompanionPath(String.format("%s-%s", file.getFileBaseName(), linkedCloneExtension));
                             if (dsMo.fileExists(pairSrcFilePath)) {
                                 LOGGER.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath);
-                                dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true);
+                                moveDatastoreFile(dsMo, pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true);
                             }
                         }
                     }
@@ -429,6 +430,31 @@ public class VmwareStorageLayoutHelper implements Configurable {
         return dsMo.searchFileInSubFolders(volumePath + ".vmdk", false, null);
     }
 
+    public static boolean moveDatastoreFile(final DatastoreMO dsMo, String srcFilePath, ManagedObjectReference morSrcDc, ManagedObjectReference morDestDs,
+                                                        String destFilePath, ManagedObjectReference morDestDc, boolean forceOverwrite) throws Exception {
+        final int retry = 20;
+        int retryAttempt = 0;
+        while (++retryAttempt <= retry) {
+            try {
+                LOGGER.debug(String.format("Move datastore file %s, attempt #%d", srcFilePath, retryAttempt));
+                return dsMo.moveDatastoreFile(srcFilePath, morSrcDc, morDestDs, destFilePath, morDestDc, forceOverwrite);
+            } catch (Exception e) {
+                LOGGER.info(String.format("Got exception while moving datastore file %s ", srcFilePath), e);
+                if (e.getMessage() != null && e.getMessage().contains("Unable to access file")) {
+                    LOGGER.debug(String.format("Failed to move datastore file %s. Retrying", srcFilePath));
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException ie) {
+                        LOGGER.debug(String.format("Waiting to move datastore file %s been interrupted: ", srcFilePath));
+                    }
+                } else {
+                    throw e;
+                }
+            }
+        }
+        return false;
+    }
+
     @Override
     public String getConfigComponentName() {
         return VmwareStorageLayoutHelper.class.getSimpleName();

plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java

@@ -682,9 +682,9 @@ public class VmwareStorageProcessor implements StorageProcessor {
                     String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairManagedDatastorePath(dsMo, null,
                             managedStoragePoolRootVolumeName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, false);
 
-                    dsMo.moveDatastoreFile(vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true);
+                    VmwareStorageLayoutHelper.moveDatastoreFile(dsMo, vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true);
                     for (int i=1; i<vmwareLayoutFilePair.length; i++) {
-                        dsMo.moveDatastoreFile(vmwareLayoutFilePair[i], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[i], dcMo.getMor(), true);
+                        VmwareStorageLayoutHelper.moveDatastoreFile(dsMo, vmwareLayoutFilePair[i], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[i], dcMo.getMor(), true);
                     }
 
                     String folderToDelete = dsMo.getDatastorePath(managedStoragePoolRootVolumeName, true);
@@ -814,7 +814,7 @@ public class VmwareStorageProcessor implements StorageProcessor {
                         existingVm.detachAllDisksAndDestroy();
                     }
                     logger.info("ROOT Volume from deploy-as-is template, cloning template");
-                    cloneVMFromTemplate(hyperHost, template.getPath(), vmName, primaryStore.getUuid());
+                    cloneVMFromTemplate(hyperHost, template, volume, vmName, primaryStore.getUuid());
                 } else {
                     logger.info("ROOT Volume from deploy-as-is template, volume already created at this point");
                 }
@@ -945,7 +945,7 @@ public class VmwareStorageProcessor implements StorageProcessor {
             String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag);
 
             for (int i = 0; i < vmwareLayoutFilePair.length; i++) {
-                dsMo.moveDatastoreFile(vmwareLayoutFilePair[i], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[i], dcMo.getMor(), true);
+                VmwareStorageLayoutHelper.moveDatastoreFile(dsMo, vmwareLayoutFilePair[i], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[i], dcMo.getMor(), true);
             }
 
             logger.info("detach disks from volume-wrapper VM and destroy {}", vmdkName);
@@ -1222,10 +1222,10 @@ public class VmwareStorageProcessor implements StorageProcessor {
             // Get VMDK filename
             String templateVMDKName = "";
             File[] files = new File(installFullPath).listFiles();
-            if(files != null) {
+            if (files != null) {
                 for(File file : files) {
                     String fileName = file.getName();
-                    if(fileName.toLowerCase().startsWith(templateUniqueName) && fileName.toLowerCase().endsWith(".vmdk")) {
+                    if (fileName.toLowerCase().startsWith(templateUniqueName) && fileName.toLowerCase().endsWith(".vmdk")) {
                         templateVMDKName += fileName;
                         break;
                     }
@@ -1856,16 +1856,16 @@ public class VmwareStorageProcessor implements StorageProcessor {
             CopyCmdAnswer answer = null;
 
             try {
-                if(vmName != null) {
+                if (vmName != null) {
                     vmMo = hyperHost.findVmOnHyperHost(vmName);
                     if (vmMo == null) {
-                        if(logger.isDebugEnabled()) {
+                        if (logger.isDebugEnabled()) {
                             logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter");
                         }
                         vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
                     }
                 }
-                if(vmMo == null) {
+                if (vmMo == null) {
                     dsMo = new DatastoreMO(hyperHost.getContext(), morDs);
                     workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo);
                     vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName, null);
@@ -1899,10 +1899,10 @@ public class VmwareStorageProcessor implements StorageProcessor {
                     String secondaryMountPoint = mountService.getMountPoint(secondaryStorageUrl, _nfsVersion);
                     String snapshotDir =  destSnapshot.getPath() + "/" + snapshotBackupUuid;
                     File[] files = new File(secondaryMountPoint + "/" + snapshotDir).listFiles();
-                    if(files != null) {
+                    if (files != null) {
                         for(File file : files) {
                             String fileName = file.getName();
-                            if(fileName.toLowerCase().startsWith(snapshotBackupUuid) && fileName.toLowerCase().endsWith(".vmdk")) {
+                            if (fileName.toLowerCase().startsWith(snapshotBackupUuid) && fileName.toLowerCase().endsWith(".vmdk")) {
                                 physicalSize = new File(secondaryMountPoint + "/" + snapshotDir + "/" + fileName).length();
                                 break;
                             }
@@ -3651,7 +3651,7 @@ public class VmwareStorageProcessor implements StorageProcessor {
             }
             workerVm.tagAsWorkerVM();
 
-            if(!primaryDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) {
+            if (!primaryDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) {
                 HypervisorHostHelper.createBaseFolderInDatastore(primaryDsMo, primaryDsMo.getDataCenterMor());
                 workerVm.moveAllVmDiskFiles(primaryDsMo, HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, false);
             }
@@ -3811,8 +3811,9 @@ public class VmwareStorageProcessor implements StorageProcessor {
     /**
      * Return the cloned VM from the template
      */
-    public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, String templateName, String cloneName, String templatePrimaryStoreUuid) {
+    public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, TemplateObjectTO template, VolumeObjectTO volume, String cloneName, String templatePrimaryStoreUuid) {
         try {
+            String templateName = template.getPath();
             VmwareContext context = hyperHost.getContext();
             DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter());
             VirtualMachineMO templateMo = dcMo.findVm(templateName);
@@ -3826,6 +3827,9 @@ public class VmwareStorageProcessor implements StorageProcessor {
                 throw new CloudRuntimeException("Unable to find datastore in vSphere");
             }
             logger.info("Cloning VM " + cloneName + " from template " + templateName + " into datastore " + templatePrimaryStoreUuid);
+            if (template.getSize() != null) {
+                _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag;
+            }
             if (!_fullCloneFlag) {
                 createVMLinkedClone(templateMo, dcMo, cloneName, morDatastore, morPool, null);
             } else {

plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java

@@ -70,6 +70,12 @@ public class ListVmwareDcVmsCmd extends BaseListCmd {
     @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.")
     private String password;
 
+    @Parameter(name = ApiConstants.HOST_NAME, type = CommandType.STRING, description = "Name of the host on vCenter. Must be set along with the instancename parameter")
+    private String hostName;
+
+    @Parameter(name = ApiConstants.INSTANCE_NAME, type = CommandType.STRING, description = "Name of the VM on vCenter. Must be set along with the hostname parameter")
+    private String instanceName;
+
     public String getVcenter() {
         return vcenter;
     }
@@ -86,10 +92,18 @@ public class ListVmwareDcVmsCmd extends BaseListCmd {
         return datacenterName;
     }
 
+    public String getHostName() {
+        return hostName;
+    }
+
     public Long getExistingVcenterId() {
         return existingVcenterId;
     }
 
+    public String getInstanceName() {
+        return instanceName;
+    }
+
     @Override
     public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
         checkParameters();
@@ -125,6 +139,11 @@ public class ListVmwareDcVmsCmd extends BaseListCmd {
             throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
                     "Please set all the information for a vCenter IP/Name, datacenter, username and password");
         }
+        if ((StringUtils.isNotBlank(instanceName) && StringUtils.isBlank(hostName)) ||
+                (StringUtils.isBlank(instanceName) && StringUtils.isNotBlank(hostName))) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+                    "Please set the hostname parameter along with the instancename parameter");
+        }
     }
 
     @Override

plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java

@@ -453,9 +453,18 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri
         boolean encryptionRequired = anyVolumeRequiresEncryption(vol);
         long [] endpointsToRunResize = resizeParameter.hosts;
 
+        CreateCmdResult result = new CreateCmdResult(null, null);
+
         // if hosts are provided, they are where the VM last ran. We can use that.
         if (endpointsToRunResize == null || endpointsToRunResize.length == 0) {
             EndPoint ep = epSelector.select(data, encryptionRequired);
+            if (ep == null) {
+                String errMsg = String.format(NO_REMOTE_ENDPOINT_WITH_ENCRYPTION, encryptionRequired);
+                logger.error(errMsg);
+                result.setResult(errMsg);
+                callback.complete(result);
+                return;
+            }
             endpointsToRunResize = new long[] {ep.getId()};
         }
         ResizeVolumeCommand resizeCmd = new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), vol.getSize(),
@@ -463,7 +472,6 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri
         if (pool.getParent() != 0) {
             resizeCmd.setContextParam(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString());
         }
-        CreateCmdResult result = new CreateCmdResult(null, null);
         try {
             ResizeVolumeAnswer answer = (ResizeVolumeAnswer) storageMgr.sendToPool(pool, endpointsToRunResize, resizeCmd);
             if (answer != null && answer.getResult()) {
@@ -480,7 +488,6 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri
                 logger.debug("return a null answer, mark it as failed for unknown reason");
                 result.setResult("return a null answer, mark it as failed for unknown reason");
             }
-
         } catch (Exception e) {
             logger.debug("sending resize command failed", e);
             result.setResult(e.toString());

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorHostListener.java

@@ -18,7 +18,6 @@ package org.apache.cloudstack.storage.datastore.provider;
 
 import com.cloud.exception.StorageConflictException;
 import com.cloud.host.HostVO;
-import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 
 public class LinstorHostListener extends DefaultHostListener {
     @Override
@@ -28,7 +27,6 @@ public class LinstorHostListener extends DefaultHostListener {
             host.setParent(host.getName());
             hostDao.update(host.getId(), host);
         }
-        StoragePoolVO pool = primaryStoreDao.findById(poolId);
+        return super.hostConnect(hostId, poolId);
-        return super.hostConnect(host, pool);
     }
 }

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -5291,6 +5291,8 @@ public class ApiResponseHelper implements ResponseGenerator {
         response.setMemory(instance.getMemory());
         response.setOperatingSystemId(instance.getOperatingSystemId());
         response.setOperatingSystem(instance.getOperatingSystem());
+        response.setBootMode(instance.getBootMode());
+        response.setBootType(instance.getBootType());
         response.setObjectName("unmanagedinstance");
 
         if (instance.getDisks() != null) {

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -4569,14 +4569,24 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         String endIP = cmd.getEndIp();
         final String newVlanGateway = cmd.getGateway();
         final String newVlanNetmask = cmd.getNetmask();
+        Long networkId = cmd.getNetworkID();
+        Long physicalNetworkId = cmd.getPhysicalNetworkId();
+
+        // Verify that network exists
+        Network network = getNetwork(networkId);
+        if (network != null) {
+            zoneId = network.getDataCenterId();
+            physicalNetworkId = network.getPhysicalNetworkId();
+        }
+
         String vlanId = cmd.getVlan();
+        vlanId = verifyAndUpdateVlanId(vlanId, network);
+
         // TODO decide if we should be forgiving or demand a valid and complete URI
         if (!(vlanId == null || "".equals(vlanId) || vlanId.startsWith(BroadcastDomainType.Vlan.scheme()))) {
             vlanId = BroadcastDomainType.Vlan.toUri(vlanId).toString();
         }
         final Boolean forVirtualNetwork = cmd.isForVirtualNetwork();
-        Long networkId = cmd.getNetworkID();
-        Long physicalNetworkId = cmd.getPhysicalNetworkId();
         final String accountName = cmd.getAccountName();
         final Long projectId = cmd.getProjectId();
         final Long domainId = cmd.getDomainId();
@@ -4649,18 +4659,6 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             }
         }
 
-        // Verify that network exists
-        Network network = null;
-        if (networkId != null) {
-            network = _networkDao.findById(networkId);
-            if (network == null) {
-                throw new InvalidParameterValueException("Unable to find network by id " + networkId);
-            } else {
-                zoneId = network.getDataCenterId();
-                physicalNetworkId = network.getPhysicalNetworkId();
-            }
-        }
-
         // Verify that zone exists
         final DataCenterVO zone = _zoneDao.findById(zoneId);
         if (zone == null) {
@@ -4799,6 +4797,32 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 ip6Cidr, domain, vlanOwner, network, sameSubnet, cmd.isForNsx());
     }
 
+    private Network getNetwork(Long networkId) {
+        if (networkId == null) {
+            return null;
+        }
+
+        Network network = _networkDao.findById(networkId);
+        if (network == null) {
+            throw new InvalidParameterValueException("Unable to find network by id " + networkId);
+        }
+
+        return network;
+    }
+
+    private String verifyAndUpdateVlanId(String vlanId, Network network) {
+        if (!StringUtils.isBlank(vlanId)) {
+            return vlanId;
+        }
+
+        if (network == null || network.getTrafficType() != TrafficType.Guest) {
+            return Vlan.UNTAGGED;
+        }
+
+        boolean connectivityWithoutVlan = isConnectivityWithoutVlan(network);
+        return getNetworkVlanId(network, connectivityWithoutVlan);
+    }
+
     private Vlan commitVlan(final Long zoneId, final Long podId, final String startIP, final String endIP, final String newVlanGatewayFinal, final String newVlanNetmaskFinal,
             final String vlanId, final Boolean forVirtualNetwork, final Boolean forSystemVms, final Long networkId, final Long physicalNetworkId, final String startIPv6, final String endIPv6,
             final String ip6Gateway, final String ip6Cidr, final Domain domain, final Account vlanOwner, final Network network, final Pair<Boolean, Pair<String, String>> sameSubnet, boolean forNsx) {
@@ -5007,28 +5031,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         // same as network's vlan
         // 2) if vlan is missing, default it to the guest network's vlan
         if (network.getTrafficType() == TrafficType.Guest) {
-            String networkVlanId = null;
+            boolean connectivityWithoutVlan = isConnectivityWithoutVlan(network);
-            boolean connectivityWithoutVlan = false;
+            String networkVlanId = getNetworkVlanId(network, connectivityWithoutVlan);
-            if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Connectivity)) {
-                Map<Capability, String> connectivityCapabilities = _networkModel.getNetworkServiceCapabilities(network.getId(), Service.Connectivity);
-                connectivityWithoutVlan = MapUtils.isNotEmpty(connectivityCapabilities) && connectivityCapabilities.containsKey(Capability.NoVlan);
-            }
-
-            final URI uri = network.getBroadcastUri();
-            if (connectivityWithoutVlan) {
-                networkVlanId = network.getBroadcastDomainType().toUri(network.getUuid()).toString();
-            } else if (uri != null) {
-                // Do not search for the VLAN tag when the network doesn't support VLAN
-               if (uri.toString().startsWith("vlan")) {
-                    final String[] vlan = uri.toString().split("vlan:\\/\\/");
-                    networkVlanId = vlan[1];
-                    // For pvlan
-                    if (network.getBroadcastDomainType() != BroadcastDomainType.Vlan) {
-                        networkVlanId = networkVlanId.split("-")[0];
-                    }
-               }
-            }
-
             if (vlanId != null && !connectivityWithoutVlan) {
                 // if vlan is specified, throw an error if it's not equal to
                 // network's vlanId
@@ -5160,6 +5164,36 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         return vlan;
     }
 
+    private boolean isConnectivityWithoutVlan(Network network) {
+        boolean connectivityWithoutVlan = false;
+        if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Connectivity)) {
+            Map<Capability, String> connectivityCapabilities = _networkModel.getNetworkServiceCapabilities(network.getId(), Service.Connectivity);
+            connectivityWithoutVlan = MapUtils.isNotEmpty(connectivityCapabilities) && connectivityCapabilities.containsKey(Capability.NoVlan);
+        }
+        return connectivityWithoutVlan;
+    }
+
+    private String getNetworkVlanId(Network network, boolean connectivityWithoutVlan) {
+        String networkVlanId = null;
+        if (connectivityWithoutVlan) {
+            return network.getBroadcastDomainType().toUri(network.getUuid()).toString();
+        }
+
+        final URI uri = network.getBroadcastUri();
+        if (uri != null) {
+            // Do not search for the VLAN tag when the network doesn't support VLAN
+            if (uri.toString().startsWith("vlan")) {
+                final String[] vlan = uri.toString().split("vlan:\\/\\/");
+                networkVlanId = vlan[1];
+                // For pvlan
+                if (network.getBroadcastDomainType() != BroadcastDomainType.Vlan) {
+                    networkVlanId = networkVlanId.split("-")[0];
+                }
+            }
+        }
+        return networkVlanId;
+    }
+
     private void checkZoneVlanIpOverlap(DataCenterVO zone, Network network, String newCidr, String vlanId, String vlanGateway, String vlanNetmask, String startIP, String endIP) {
         // Throw an exception if this subnet overlaps with subnet on other VLAN,
         // if this is ip range extension, gateway, network mask should be same and ip range should not overlap

server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java

@@ -1030,7 +1030,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         } else if (intervalTypeStr != null && volumeId != null) {
             Type type = SnapshotVO.getSnapshotType(intervalTypeStr);
             if (type == null) {
-                throw new InvalidParameterValueException("Unsupported snapstho interval type " + intervalTypeStr);
+                throw new InvalidParameterValueException("Unsupported snapshot interval type " + intervalTypeStr);
             }
             sc.setParameters("snapshotTypeEQ", type.ordinal());
         } else {

server/src/main/java/com/cloud/usage/UsageServiceImpl.java

@@ -26,8 +26,6 @@ import java.util.TimeZone;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.domain.Domain;
-import com.cloud.utils.DateUtil;
 import org.apache.cloudstack.api.command.admin.usage.GenerateUsageRecordsCmd;
 import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd;
 import org.apache.cloudstack.api.command.admin.usage.RemoveRawUsageRecordsCmd;
@@ -42,6 +40,7 @@ import org.jetbrains.annotations.NotNull;
 import org.springframework.stereotype.Component;
 
 import com.cloud.configuration.Config;
+import com.cloud.domain.Domain;
 import com.cloud.domain.DomainVO;
 import com.cloud.domain.dao.DomainDao;
 import com.cloud.exception.InvalidParameterValueException;
@@ -58,6 +57,8 @@ import com.cloud.network.rules.PortForwardingRuleVO;
 import com.cloud.network.rules.dao.PortForwardingRulesDao;
 import com.cloud.network.security.SecurityGroupVO;
 import com.cloud.network.security.dao.SecurityGroupDao;
+import com.cloud.offerings.NetworkOfferingVO;
+import com.cloud.offerings.dao.NetworkOfferingDao;
 import com.cloud.projects.Project;
 import com.cloud.projects.ProjectManager;
 import com.cloud.storage.SnapshotVO;
@@ -72,6 +73,7 @@ import com.cloud.user.Account;
 import com.cloud.user.AccountService;
 import com.cloud.user.AccountVO;
 import com.cloud.user.dao.AccountDao;
+import com.cloud.utils.DateUtil;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.Manager;
 import com.cloud.utils.component.ManagerBase;
@@ -121,6 +123,8 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag
     private IPAddressDao _ipDao;
     @Inject
     private HostDao _hostDao;
+    @Inject
+    private NetworkOfferingDao _networkOfferingDao;
 
     public UsageServiceImpl() {
     }
@@ -245,6 +249,7 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag
             }
 
             Long usageDbId = null;
+            boolean offeringExistsForNetworkOfferingType = false;
 
             switch (usageType.intValue()) {
                 case UsageTypes.NETWORK_BYTES_RECEIVED:
@@ -318,13 +323,19 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag
                         usageDbId = ip.getId();
                     }
                     break;
+                case UsageTypes.NETWORK_OFFERING:
+                    NetworkOfferingVO networkOffering = _networkOfferingDao.findByUuidIncludingRemoved(usageId);
+                    if (networkOffering != null) {
+                        offeringExistsForNetworkOfferingType = true;
+                        sc.addAnd("offeringId", SearchCriteria.Op.EQ, networkOffering.getId());
+                    }
                 default:
                     break;
             }
 
             if (usageDbId != null) {
                 sc.addAnd("usageId", SearchCriteria.Op.EQ, usageDbId);
-            } else {
+            } else if (!offeringExistsForNetworkOfferingType) {
                 // return an empty list if usageId was not found
                 return new Pair<List<? extends Usage>, Integer>(new ArrayList<Usage>(), new Integer(0));
             }

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -8444,6 +8444,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
 
                         getRootVolumeSizeForVmRestore(newVol, template, userVm, diskOffering, details, true);
                         volumeMgr.saveVolumeDetails(newVol.getDiskOfferingId(), newVol.getId());
+                        newVol = _volsDao.findById(newVol.getId());
 
                         // 1. Save usage event and update resource count for user vm volumes
                         try {
@@ -8543,7 +8544,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
 
     Long getRootVolumeSizeForVmRestore(Volume vol, VMTemplateVO template, UserVmVO userVm, DiskOffering diskOffering, Map<String, String> details, boolean update) {
         VolumeVO resizedVolume = (VolumeVO) vol;
-
         Long size = null;
         if (template != null && template.getSize() != null) {
             UserVmDetailVO vmRootDiskSizeDetail = userVmDetailsDao.findDetail(userVm.getId(), VmDetailConstants.ROOT_DISK_SIZE);

server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java

@@ -55,6 +55,8 @@ import com.cloud.network.dao.FirewallRulesDao;
 import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.IPAddressVO;
 import com.cloud.network.dao.Ipv6GuestPrefixSubnetNetworkMapDao;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
 import com.cloud.network.dao.PhysicalNetworkDao;
 import com.cloud.network.dao.PhysicalNetworkVO;
 import com.cloud.offering.DiskOffering;
@@ -196,6 +198,8 @@ public class ConfigurationManagerTest {
     @Mock
     HostPodDao _podDao;
     @Mock
+    NetworkDao _networkDao;
+    @Mock
     PhysicalNetworkDao _physicalNetworkDao;
     @Mock
     ImageStoreDao _imageStoreDao;
@@ -1331,6 +1335,8 @@ public class ConfigurationManagerTest {
     public void testWrongIpv6CreateVlanAndPublicIpRange() {
         CreateVlanIpRangeCmd cmd = Mockito.mock(CreateVlanIpRangeCmd.class);
         Mockito.when(cmd.getIp6Cidr()).thenReturn("fd17:5:8a43:e2a4:c000::/66");
+        NetworkVO network = Mockito.mock(NetworkVO.class);
+        Mockito.when(_networkDao.findById(Mockito.anyLong())).thenReturn(network);
         try {
             configurationMgr.createVlanAndPublicIpRange(cmd);
         } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException e) {

services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java

@@ -1089,6 +1089,12 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
 
         try {
             _itMgr.expunge(ssvm.getUuid());
+            ssvm.setPublicIpAddress(null);
+            ssvm.setPublicMacAddress(null);
+            ssvm.setPublicNetmask(null);
+            ssvm.setPrivateMacAddress(null);
+            ssvm.setPrivateIpAddress(null);
+            _secStorageVmDao.update(ssvm.getId(), ssvm);
             _secStorageVmDao.remove(ssvm.getId());
             HostVO host = _hostDao.findByTypeNameAndZoneId(ssvm.getDataCenterId(), ssvm.getHostName(), Host.Type.SecondaryStorageVM);
             if (host != null) {
@@ -1373,7 +1379,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
     @Override
     public void finalizeExpunge(VirtualMachine vm) {
         SecondaryStorageVmVO ssvm = _secStorageVmDao.findByUuid(vm.getUuid());
-
+        ssvm.setPrivateMacAddress(null);
         ssvm.setPublicIpAddress(null);
         ssvm.setPublicMacAddress(null);
         ssvm.setPublicNetmask(null);

test/integration/smoke/test_restore_vm.py

@@ -148,9 +148,13 @@ class TestRestoreVM(cloudstackTestCase):
         self.assertEqual(root_vol.state, 'Ready', "Volume should be in Ready state")
         self.assertEqual(root_vol.size, 16 * 1024 * 1024 * 1024, "Size of volume and custom disk size should match")
 
-        old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id)[0]
+        if self.hypervisor.lower() == "vmware":
-        self.assertEqual(old_root_vol.state, "Destroy", "Old volume should be in Destroy state")
+            old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id)
-        Volume.delete(old_root_vol, self.apiclient)
+            self.assertEqual(old_root_vol, None, "Old volume should be deleted")
+        else:
+            old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id)[0]
+            self.assertEqual(old_root_vol.state, "Destroy", "Old volume should be in Destroy state")
+            Volume.delete(old_root_vol, self.apiclient)
 
     @attr(tags=["advanced", "basic"], required_hardware="false")
     def test_04_restore_vm_allocated_root(self):

test/integration/smoke/test_ssvm.py

@@ -988,6 +988,9 @@ class TestSSVMs(cloudstackTestCase):
 
         # Private IP Address of System VMs are allowed to change after reboot - CLOUDSTACK-7745
 
+        # Agent in Up state for a while after reboot, wait for the agent to Disconnect and back Up.
+        time.sleep(60)
+
         # Wait for the agent to be up
         self.waitForSystemVMAgent(cpvm_response.name)
 
@@ -1103,6 +1106,9 @@ class TestSSVMs(cloudstackTestCase):
             "Check whether CPVM is running or not"
         )
 
+        # Agent in Up state for a while after reboot, wait for the agent to Disconnect and back Up.
+        time.sleep(60)
+
         # Wait for the agent to be up
         self.waitForSystemVMAgent(cpvm_response.name)
 

tools/marvin/marvin/config/test_data.py

@@ -1080,7 +1080,7 @@ test_data = {
             "format": "vhd",
             "hypervisor": "xenserver",
             "ostype": "Other Linux (64-bit)",
-            "url": "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64-azure.vhd.tar.gz",
+            "url": "https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64-azure.vhd.tar.gz",
             "requireshvm": "True",
             "ispublic": "True",
             "isextractable": "True"
@@ -1091,10 +1091,10 @@ test_data = {
             "format": "ova",
             "hypervisor": "vmware",
             "ostype": "Other Linux (64-bit)",
-            "url": "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.ova",
+            "url": "https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64.ova",
             "requireshvm": "True",
             "ispublic": "True",
-            "deployasis": "True"
+            "deployasis": "False"
         },
     },
     "test_ovf_templates": [

ui/.env.qa

@@ -0,0 +1 @@
+CS_URL=https://qa.cloudstack.cloud/simulator/pr/10580

ui/public/locales/en.json

@@ -2119,6 +2119,7 @@
 "label.sharewith": "Share with",
 "label.showing": "Showing",
 "label.show.usage.records": "Show usage records",
+"label.showing.results.for": "Showing results for \"%x\"",
 "label.shrinkok": "Shrink OK",
 "label.shutdown": "Shutdown",
 "label.shutdown.provider": "Shutdown provider",

ui/src/components/header/ProjectMenu.vue

@@ -17,112 +17,75 @@
 
 <template>
   <span class="header-notice-opener">
-    <a-select
+    <infinite-scroll-select
-      v-if="!isDisabled()"
+      v-if="!isDisabled"
+      v-model:value="selectedProjectId"
       class="project-select"
-      :loading="loading"
+      api="listProjects"
-      v-model:value="projectSelected"
+      :apiParams="projectsApiParams"
-      :filterOption="filterProject"
+      resourceType="project"
-      @change="changeProject"
+      :defaultOption="defaultOption"
-      @focus="fetchData"
+      defaultIcon="project-outlined"
-      showSearch>
+      :pageSize="100"
-
+      @change-option="changeProject" />
-      <a-select-option
-        v-for="(project, index) in projects"
-        :key="index"
-        :label="project.displaytext || project.name">
-        <span>
-          <resource-icon v-if="project.icon && project.icon.base64image" :image="project.icon.base64image" size="1x" style="margin-right: 5px"/>
-          <project-outlined v-else style="margin-right: 5px" />
-          {{ project.displaytext || project.name }}
-        </span>
-      </a-select-option>
-    </a-select>
   </span>
 </template>
 
 <script>
-import store from '@/store'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect'
-import { api } from '@/api'
-import _ from 'lodash'
-import ResourceIcon from '@/components/view/ResourceIcon'
 
 export default {
   name: 'ProjectMenu',
   components: {
-    ResourceIcon
+    InfiniteScrollSelect
   },
   data () {
     return {
-      projects: [],
+      selectedProjectId: null,
       loading: false
     }
   },
   created () {
-    this.fetchData()
+    this.selectedProjectId = this.$store.getters?.project?.id || this.defaultOption.id
+    this.$store.dispatch('ToggleTheme', this.selectedProjectId ? 'dark' : 'light')
   },
   computed: {
-    projectSelected () {
+    isDisabled () {
-      let projectIndex = 0
+      return !('listProjects' in this.$store.getters.apis)
-      if (this.$store.getters?.project?.id) {
+    },
-        projectIndex = this.projects.findIndex(project => project.id === this.$store.getters.project.id)
+    defaultOption () {
-        this.$store.dispatch('ToggleTheme', projectIndex === undefined ? 'light' : 'dark')
+      return { id: 0, name: this.$t('label.default.view') }
+    },
+    projectsApiParams () {
+      return {
+        details: 'min',
+        listall: true
       }
-
+    }
-      return projectIndex
+  },
+  mounted () {
+    this.unwatchProject = this.$store.watch(
+      (state, getters) => getters.project?.id,
+      (newId) => {
+        this.selectedProjectId = newId
+      }
+    )
+  },
+  beforeUnmount () {
+    if (this.unwatchProject) {
+      this.unwatchProject()
     }
   },
   methods: {
-    fetchData () {
+    changeProject (project) {
-      if (this.isDisabled()) {
-        return
-      }
-      var page = 1
-      const projects = []
-      const getNextPage = () => {
-        this.loading = true
-        api('listProjects', { listAll: true, page: page, pageSize: 500, details: 'min', showIcon: true }).then(json => {
-          if (json?.listprojectsresponse?.project) {
-            projects.push(...json.listprojectsresponse.project)
-          }
-          if (projects.length < json.listprojectsresponse.count) {
-            page++
-            getNextPage()
-          }
-        }).finally(() => {
-          this.loading = false
-          this.$store.commit('RELOAD_ALL_PROJECTS', projects)
-        })
-      }
-      getNextPage()
-    },
-    isDisabled () {
-      return !Object.prototype.hasOwnProperty.call(store.getters.apis, 'listProjects')
-    },
-    changeProject (index) {
-      const project = this.projects[index]
       this.$store.dispatch('ProjectView', project.id)
       this.$store.dispatch('SetProject', project)
-      this.$store.dispatch('ToggleTheme', project.id === undefined ? 'light' : 'dark')
+      this.$store.dispatch('ToggleTheme', project.id ? 'dark' : 'light')
       this.$message.success(`${this.$t('message.switch.to')} "${project.displaytext || project.name}"`)
       if (this.$route.name !== 'dashboard') {
         this.$router.push({ name: 'dashboard' })
       }
-    },
-    filterProject (input, option) {
-      return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
     }
-  },
-  mounted () {
-    this.$store.watch(
-      (state, getters) => getters.allProjects,
-      (newValue, oldValue) => {
-        if (oldValue !== newValue && newValue !== undefined) {
-          this.projects = _.orderBy(newValue, ['displaytext'], ['asc'])
-          this.projects.unshift({ name: this.$t('label.default.view') })
-        }
-      }
-    )
   }
 }
 </script>

ui/src/components/widgets/InfiniteScrollSelect.vue

@@ -0,0 +1,298 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+<!--
+  InfiniteScrollSelect.vue
+
+  A reusable select component that supports:
+  - Infinite scrolling with paginated API
+  - Dynamic search filtering. Needs minimum
+  - Deduplicated option loading
+  - Auto-fetching of preselected value if not present in the initial result
+
+  Usage Example:
+
+  <infinite-scroll-select
+    v-model:value="form.account"
+    api="listAccounts"
+    :apiParams="accountsApiParams"
+    resourceType="account"
+    optionValueKey="name"
+    optionLabelKey="name"
+    @change-option-value="handleAccountNameChange" />
+
+  Props:
+  - api (String, required): API command name (e.g., 'listAccounts')
+  - apiParams (Object, optional): Additional parameters passed to the API
+  - resourceType (String, required): The key in the API response containing the resource array (e.g., 'account')
+  - optionValueKey (String, optional): Property to use as the value for options (e.g., 'name'). Default is 'id'
+  - optionLabelKey (String, optional): Property to use as the label for options (e.g., 'name'). Default is 'name'
+  - defaultOption (Object, optional): Preselected object to include initially
+  - showIcon (Boolean, optional): Whether to show icon for the options. Default is true
+  - defaultIcon (String, optional): Icon to be shown when there is no resource icon for the option. Default is 'cloud-outlined'
+
+  Events:
+  - @change-option-value (Function): Emits the selected option value(s) when value(s) changes. Do not use @change as it will give warnings and may not work
+  - @change-option (Function): Emits the selected option object when value changes. Works only when mode is not multiple
+
+  Features:
+  - Debounced remote filtering
+  - Custom dropdown footer/header (e.g., clear search button)
+  - Handles preselection and fetches missing option automatically
+-->
+<template>
+  <a-select
+    :filter-option="false"
+    :loading="loading"
+    show-search
+    placeholder="Select"
+    @search="onSearchTimed"
+    @popupScroll="onScroll"
+    @change="onChange"
+  >
+    <template #dropdownRender="{ menuNode: menu }">
+      <v-nodes :vnodes="menu" />
+      <div v-if="!!searchQuery">
+        <a-divider style="margin: 4px 0" />
+        <div class="select-list-footer">
+          <span>{{ formattedSearchFooterMessage }}</span>
+          <close-outlined
+            @mousedown="e => e.preventDefault()"
+            @click="onSearch()" />
+        </div>
+      </div>
+    </template>
+    <a-select-option v-for="option in options" :key="option.id" :value="option[optionValueKey]">
+      <span>
+        <span v-if="showIcon">
+          <resource-icon v-if="option.icon && option.icon.base64image" :image="option.icon.base64image" size="1x" style="margin-right: 5px"/>
+          <render-icon v-else :icon="defaultIcon" style="margin-right: 5px" />
+        </span>
+        <span>{{ option[optionLabelKey] }}</span>
+      </span>
+    </a-select-option>
+  </a-select>
+</template>
+
+<script>
+import { api } from '@/api'
+import ResourceIcon from '@/components/view/ResourceIcon'
+
+export default {
+  name: 'InfiniteScrollSelect',
+  components: {
+    ResourceIcon,
+    VNodes: (_, { attrs }) => {
+      return attrs.vnodes
+    }
+  },
+  props: {
+    api: {
+      type: String,
+      required: true
+    },
+    apiParams: {
+      type: Object,
+      required: null
+    },
+    resourceType: {
+      type: String,
+      required: true
+    },
+    optionValueKey: {
+      type: String,
+      default: 'id'
+    },
+    optionLabelKey: {
+      type: String,
+      default: 'name'
+    },
+    defaultOption: {
+      type: Object,
+      default: null
+    },
+    showIcon: {
+      type: Boolean,
+      default: true
+    },
+    defaultIcon: {
+      type: String,
+      default: 'cloud-outlined'
+    },
+    pageSize: {
+      type: Number,
+      default: null
+    }
+  },
+  data () {
+    return {
+      options: [],
+      page: 1,
+      totalCount: null,
+      loading: false,
+      searchQuery: '',
+      searchTimer: null,
+      scrollHandlerAttached: false,
+      preselectedOptionValue: null,
+      successiveFetches: 0
+    }
+  },
+  created () {
+    this.addDefaultOptionIfNeeded(true)
+  },
+  mounted () {
+    this.preselectedOptionValue = this.$attrs.value
+    this.fetchItems()
+  },
+  computed: {
+    maxSuccessiveFetches () {
+      return 10
+    },
+    computedPageSize () {
+      return this.pageSize || this.$store.getters.defaultListViewPageSize
+    },
+    formattedSearchFooterMessage () {
+      return `${this.$t('label.showing.results.for').replace('%x', this.searchQuery)}`
+    }
+  },
+  watch: {
+    apiParams () {
+      this.onSearch()
+    }
+  },
+  emits: ['change-option-value', 'change-option'],
+  methods: {
+    async fetchItems () {
+      if (this.successiveFetches === 0 && this.loading) return
+      this.loading = true
+      const params = {
+        page: this.page,
+        pagesize: this.computedPageSize
+      }
+      if (this.searchQuery && this.searchQuery.length > 0) {
+        params.keyword = this.searchQuery
+      }
+      if (this.apiParams) {
+        Object.assign(params, this.apiParams)
+      }
+      if (this.showIcon) {
+        params.showicon = true
+      }
+      api(this.api, params).then(json => {
+        const response = json[this.api.toLowerCase() + 'response'] || {}
+        if (this.totalCount === null) {
+          this.totalCount = response.count || 0
+        }
+        const newOpts = response[this.resourceType] || []
+        const existingOptions = new Set(this.options.map(o => o[this.optionValueKey]))
+        newOpts.forEach(opt => {
+          if (!existingOptions.has(opt[this.optionValueKey])) {
+            this.options.push(opt)
+          }
+        })
+        this.page++
+        this.checkAndFetchPreselectedOption()
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
+        if (this.successiveFetches === 0) {
+          this.loading = false
+        }
+      })
+    },
+    checkAndFetchPreselectedOption () {
+      if (!this.preselectedOptionValue ||
+        (Array.isArray(this.preselectedOptionValue) && this.preselectedOptionValue.length === 0) ||
+        this.successiveFetches >= this.maxSuccessiveFetches) {
+        this.resetPreselectedOptionValue()
+        return
+      }
+      const matchValue = Array.isArray(this.preselectedOptionValue) ? this.preselectedOptionValue[0] : this.preselectedOptionValue
+      const match = this.options.find(entry => entry[this.optionValueKey] === matchValue)
+      if (!match) {
+        this.successiveFetches++
+        if (this.options.length < this.totalCount) {
+          this.fetchItems()
+        } else {
+          this.resetPreselectedOptionValue()
+        }
+        return
+      }
+      if (Array.isArray(this.preselectedOptionValue) && this.preselectedOptionValue.length > 1) {
+        this.preselectedOptionValue = this.preselectedOptionValue.filter(o => o !== match)
+      } else {
+        this.resetPreselectedOptionValue()
+      }
+    },
+    addDefaultOptionIfNeeded () {
+      if (this.defaultOption) {
+        this.options.push(this.defaultOption)
+      }
+    },
+    resetPreselectedOptionValue () {
+      this.preselectedOptionValue = null
+      this.successiveFetches = 0
+    },
+    onSearchTimed (value) {
+      clearTimeout(this.searchTimer)
+      this.searchTimer = setTimeout(() => {
+        this.onSearch(value)
+      }, 500)
+    },
+    onSearch (value) {
+      this.searchQuery = value
+      this.page = 1
+      this.totalCount = null
+      this.options = []
+      if (!this.searchQuery) {
+        this.addDefaultOptionIfNeeded()
+      }
+      this.fetchItems()
+    },
+    onScroll (e) {
+      const nearBottom = e.target.scrollTop + e.target.clientHeight >= e.target.scrollHeight - 10
+      const hasMore = this.options.length < this.totalCount
+      if (nearBottom && hasMore && !this.loading) {
+        this.fetchItems()
+      }
+    },
+    onChange (value) {
+      this.resetPreselectedOptionValue()
+      this.$emit('change-option-value', value)
+      if (Array.isArray(value)) {
+        return
+      }
+      if (value === undefined || value == null) {
+        this.$emit('change-option', undefined)
+        return
+      }
+      const match = this.options.find(entry => entry[this.optionValueKey] === value)
+      if (match) {
+        this.$emit('change-option', match)
+      }
+    }
+  }
+}
+</script>
+
+<style lang="less" scoped>
+  .select-list-footer {
+    margin: 4px 10px;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+  }
+</style>

ui/src/views/AutogenView.vue

@@ -919,7 +919,7 @@ export default {
       if (['listVirtualMachinesMetrics'].includes(this.apiName) && this.dataView) {
         delete params.details
         delete params.isvnf
-        params.details = 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp'
+        params.details = 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff'
       }
 
       this.loading = true

ui/src/views/compute/EditVM.vue

@@ -202,7 +202,7 @@ export default {
     },
     fetchZoneDetails () {
       api('listZones', {
-        zoneid: this.resource.zoneid
+        id: this.resource.zoneid
       }).then(response => {
         const zone = response?.listzonesresponse?.zone || []
         this.securityGroupsEnabled = zone?.[0]?.securitygroupsenabled || this.$store.getters.showSecurityGroups
@@ -336,10 +336,8 @@ export default {
         params.name = values.name
         params.displayname = values.displayname
         params.ostypeid = values.ostypeid
-        if (this.securityGroupsEnabled) {
+        if (this.securityGroupsEnabled && Array.isArray(values.securitygroupids) && values.securitygroupids.length > 0) {
-          if (values.securitygroupids) {
+          params.securitygroupids = values.securitygroupids
-            params.securitygroupids = values.securitygroupids
-          }
         }
         if (values.isdynamicallyscalable !== undefined) {
           params.isdynamicallyscalable = values.isdynamicallyscalable

ui/src/views/network/CreateNetworkPermission.vue

@@ -29,47 +29,27 @@
             <template #label>
               <tooltip-label :title="$t('label.account')" :tooltip="apiParams.accountids.description"/>
             </template>
-            <a-select
+            <infinite-scroll-select
               v-model:value="form.accountids"
               mode="multiple"
-              :loading="accountLoading"
               :placeholder="apiParams.accountids.description"
-              showSearch
+              api="listAccounts"
-              optionFilterProp="label"
+              :apiParams="accountsApiParams"
-              :filterOption="(input, option) => {
+              resourceType="account"
-                return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
+              defaultIcon="team-outlined" />
-              }" >
-              <a-select-option v-for="(opt, optIndex) in accounts" :key="optIndex" :label="opt.name || opt.description">
-                <span>
-                  <resource-icon v-if="opt.icon" :image="opt.icon.base64image" size="1x" style="margin-right: 5px"/>
-                  <global-outlined style="margin-right: 5px" />
-                  {{ opt.name || opt.description }}
-                </span>
-              </a-select-option>
-            </a-select>
           </a-form-item>
           <a-form-item v-if="isAdminOrDomainAdmin()" name="projectids" ref="projectids">
             <template #label>
               <tooltip-label :title="$t('label.project')" :tooltip="apiParams.projectids.description"/>
             </template>
-            <a-select
+            <infinite-scroll-select
               v-model:value="form.projectids"
               mode="multiple"
-              :loading="projectLoading"
               :placeholder="apiParams.projectids.description"
-              showSearch
+              api="listProjects"
-              optionFilterProp="label"
+              :apiParams="projectsApiParams"
-              :filterOption="(input, option) => {
+              resourceType="project"
-                return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
+              defaultIcon="project-outlined" />
-              }" >
-              <a-select-option v-for="(opt, optIndex) in projects" :key="optIndex" :label="opt.name || opt.description">
-                <span>
-                  <resource-icon v-if="opt.icon" :image="opt.icon.base64image" size="1x" style="margin-right: 5px"/>
-                  <global-outlined style="margin-right: 5px" />
-                  {{ opt.name || opt.description }}
-                </span>
-              </a-select-option>
-            </a-select>
           </a-form-item>
           <a-form-item v-if="!isAdminOrDomainAdmin()">
             <template #label>
@@ -106,12 +86,14 @@ import { isAdminOrDomainAdmin } from '@/role'
 import { ref, reactive, toRaw } from 'vue'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect'
 
 export default {
   name: 'CreateNetworkPermissions',
   components: {
     TooltipLabel,
-    ResourceIcon
+    ResourceIcon,
+    InfiniteScrollSelect
   },
   props: {
     resource: {
@@ -121,11 +103,7 @@ export default {
   },
   data () {
     return {
-      loading: false,
+      loading: false
-      accountLoading: false,
-      projectLoading: false,
-      accounts: [],
-      projects: []
     }
   },
   created () {
@@ -133,45 +111,24 @@ export default {
     this.form = reactive({})
     this.rules = reactive({})
     this.apiParams = this.$getApiParams('createNetworkPermissions')
-    this.fetchData()
+  },
+  computed: {
+    accountsApiParams () {
+      return {
+        details: 'min',
+        domainid: this.resource.domainid
+      }
+    },
+    projectsApiParams () {
+      return {
+        details: 'min'
+      }
+    }
   },
   methods: {
     isAdminOrDomainAdmin () {
       return isAdminOrDomainAdmin()
     },
-    async fetchData () {
-      this.fetchAccountData()
-      this.fetchProjectData()
-    },
-    fetchAccountData () {
-      this.accounts = []
-      const params = {}
-      params.showicon = true
-      params.details = 'min'
-      params.domainid = this.resource.domainid
-      this.accountLoading = true
-      api('listAccounts', params).then(json => {
-        const listaccounts = json.listaccountsresponse.account || []
-        this.accounts = listaccounts
-      }).finally(() => {
-        this.accountLoading = false
-      })
-    },
-    fetchProjectData () {
-      this.projects = []
-      const params = {}
-      params.listall = true
-      params.showicon = true
-      params.details = 'min'
-      params.domainid = this.resource.domainid
-      this.projectLoading = true
-      api('listProjects', params).then(json => {
-        const listProjects = json.listprojectsresponse.project || []
-        this.projects = listProjects
-      }).finally(() => {
-        this.projectLoading = false
-      })
-    },
     handleSubmit (e) {
       e.preventDefault()
       if (this.loading) return
@@ -179,31 +136,12 @@ export default {
         const values = toRaw(this.form)
         const params = {}
         params.networkid = this.resource.id
-        var accountIndexes = values.accountids
+        if (values.accountids && values.accountids.length > 0) {
-        var accountId = null
+          params.accountids = values.accountids.join(',')
-        if (accountIndexes && accountIndexes.length > 0) {
-          var accountIds = []
-          for (var i = 0; i < accountIndexes.length; i++) {
-            accountIds = accountIds.concat(this.accounts[accountIndexes[i]].id)
-          }
-          accountId = accountIds.join(',')
         }
-        if (accountId) {
+        if (values.projectids && values.projectids.length > 0) {
-          params.accountids = accountId
+          params.projectids = values.projectids.join(',')
         }
-        var projectIndexes = values.projectids
-        var projectId = null
-        if (projectIndexes && projectIndexes.length > 0) {
-          var projectIds = []
-          for (var j = 0; j < projectIndexes.length; j++) {
-            projectIds = projectIds.concat(this.projects[projectIndexes[j]].id)
-          }
-          projectId = projectIds.join(',')
-        }
-        if (projectId) {
-          params.projectids = projectId
-        }
-
         if (values.accounts && values.accounts.length > 0) {
           params.accounts = values.accounts
         }

ui/src/views/tools/ManageInstances.vue

@@ -1316,6 +1316,31 @@ export default {
         this.fetchInstances()
       }
     },
+    fetchVmwareInstanceForKVMMigration (vmname, hostname) {
+      const params = {}
+      if (this.isMigrateFromVmware && this.selectedVmwareVcenter) {
+        if (this.selectedVmwareVcenter.vcenter) {
+          params.datacentername = this.selectedVmwareVcenter.datacentername
+          params.vcenter = this.selectedVmwareVcenter.vcenter
+          params.username = this.selectedVmwareVcenter.username
+          params.password = this.selectedVmwareVcenter.password
+        } else {
+          params.existingvcenterid = this.selectedVmwareVcenter.existingvcenterid
+        }
+        params.instancename = vmname
+        params.hostname = hostname
+      }
+      api('listVmwareDcVms', params).then(json => {
+        const response = json.listvmwaredcvmsresponse
+        this.selectedUnmanagedInstance = response.unmanagedinstance[0]
+        this.selectedUnmanagedInstance.ostypename = this.selectedUnmanagedInstance.osdisplayname
+        this.selectedUnmanagedInstance.state = this.selectedUnmanagedInstance.powerstate
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
+        this.loading = false
+      })
+    },
     onManageInstanceAction () {
       this.selectedUnmanagedInstance = {}
       if (this.unmanagedInstances.length > 0 &&
@@ -1333,6 +1358,9 @@ export default {
           }
         })
         this.showUnmanageForm = false
+      } else if (this.isMigrateFromVmware) {
+        this.fetchVmwareInstanceForKVMMigration(this.selectedUnmanagedInstance.name, this.selectedUnmanagedInstance.hostname)
+        this.showUnmanageForm = true
       } else {
         this.showUnmanageForm = true
       }

ui/src/views/tools/SelectVmwareVcenter.vue

@@ -227,6 +227,8 @@ export default {
       } else {
         params.existingvcenterid = this.selectedExistingVcenterId
       }
+      params.page = 1
+      params.pagesize = 10
       api('listVmwareDcVms', params).then(json => {
         const obj = {
           params: params,
@@ -265,6 +267,11 @@ export default {
         this.loading = false
       })
     },
+    onSelectExternalVmwareDatacenter (value) {
+      if (this.vcenterSelectedOption === 'new' && !(this.vcenter === '' || this.datacentername === '' || this.username === '' || this.password === '')) {
+        this.listVmwareDatacenterVms()
+      }
+    },
     onSelectExistingVmwareDatacenter (value) {
       this.selectedExistingVcenterId = value
     },

utils/src/main/java/com/cloud/utils/script/Script.java

@@ -50,6 +50,7 @@ import org.joda.time.Duration;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.Pair;
 import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.script.OutputInterpreter.TimedOutLogger;
 
@@ -157,25 +158,15 @@ public class Script implements Callable<String> {
         boolean obscureParam = false;
         for (int i = 0; i < command.length; i++) {
             String cmd = command[i];
-            if (obscureParam) {
+            if (StringUtils.isNotEmpty(cmd) && cmd.startsWith("vi://")) {
-                builder.append("******").append(" ");
+                String[] tokens = cmd.split("@");
-                obscureParam = false;
+                if (tokens.length >= 2) {
-            } else {
+                    builder.append("vi://").append("******@").append(tokens[1]).append(" ");
-                builder.append(command[i]).append(" ");
+                } else {
+                    builder.append("vi://").append("******").append(" ");
+                }
+                continue;
             }
-
-            if ("-y".equals(cmd) || "-z".equals(cmd)) {
-                obscureParam = true;
-                _passwordCommand = true;
-            }
-        }
-        return builder.toString();
-    }
-
-    protected String buildCommandLine(List<String> command) {
-        StringBuilder builder = new StringBuilder();
-        boolean obscureParam = false;
-        for (String cmd : command) {
             if (obscureParam) {
                 builder.append("******").append(" ");
                 obscureParam = false;

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/BaseMO.java

@@ -20,17 +20,35 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
 import com.cloud.hypervisor.vmware.util.VmwareContext;
+import com.cloud.utils.Pair;
+
+import com.vmware.vim25.DynamicProperty;
+import com.vmware.vim25.ObjectContent;
+import com.vmware.vim25.VirtualMachineBootOptions;
+import com.vmware.vim25.VirtualMachinePowerState;
+import org.apache.cloudstack.vm.UnmanagedInstanceTO;
+import org.apache.commons.lang3.StringUtils;
 
 import com.vmware.vim25.CustomFieldDef;
 import com.vmware.vim25.CustomFieldStringValue;
 import com.vmware.vim25.ManagedObjectReference;
 
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 public class BaseMO {
     protected static Logger logger = LogManager.getLogger(BaseMO.class);
 
     protected VmwareContext _context;
     protected ManagedObjectReference _mor;
 
+    protected static String[] propertyPathsForUnmanagedVmsThinListing = new String[] {"name", "config.template",
+            "runtime.powerState", "config.guestId", "config.guestFullName", "runtime.host",
+            "config.bootOptions", "config.firmware"};
+
     private String _name;
 
     public BaseMO(VmwareContext context, ManagedObjectReference mor) {
@@ -154,4 +172,93 @@ public class BaseMO {
 
         return cfmMo.getCustomFieldKey(morType, fieldName);
     }
+
+    private static UnmanagedInstanceTO.PowerState convertPowerState(VirtualMachinePowerState powerState) {
+        return powerState == VirtualMachinePowerState.POWERED_ON ? UnmanagedInstanceTO.PowerState.PowerOn :
+                powerState == VirtualMachinePowerState.POWERED_OFF ? UnmanagedInstanceTO.PowerState.PowerOff : UnmanagedInstanceTO.PowerState.PowerUnknown;
+    }
+
+    protected List<UnmanagedInstanceTO> convertVmsObjectContentsToUnmanagedInstances(List<ObjectContent> ocs, String keyword) throws Exception {
+        Map<String, Pair<String, String>> hostClusterNamesMap = new HashMap<>();
+        List<UnmanagedInstanceTO> vms = new ArrayList<>();
+        if (ocs != null) {
+            for (ObjectContent oc : ocs) {
+                List<DynamicProperty> objProps = oc.getPropSet();
+                if (objProps != null) {
+                    UnmanagedInstanceTO vm = createUnmanagedInstanceTOFromThinListingDynamicProperties(
+                            objProps, keyword, hostClusterNamesMap);
+                    if (vm != null) {
+                        vms.add(vm);
+                    }
+                }
+            }
+        }
+        if (vms.size() > 0) {
+            vms.sort(Comparator.comparing(UnmanagedInstanceTO::getName));
+        }
+        return vms;
+    }
+
+    private UnmanagedInstanceTO createUnmanagedInstanceTOFromThinListingDynamicProperties(List<DynamicProperty> objProps,
+                                                                                            String keyword,
+                                                                                            Map<String, Pair<String, String>> hostClusterNamesMap) throws Exception {
+        UnmanagedInstanceTO vm = new UnmanagedInstanceTO();
+        String vmName;
+        boolean isTemplate = false;
+        boolean excludeByKeyword = false;
+
+        for (DynamicProperty objProp : objProps) {
+            if (objProp.getName().equals("name")) {
+                vmName = (String) objProp.getVal();
+                if (StringUtils.isNotBlank(keyword) && !vmName.contains(keyword)) {
+                    excludeByKeyword = true;
+                }
+                vm.setName(vmName);
+            } else if (objProp.getName().equals("config.template")) {
+                isTemplate = (Boolean) objProp.getVal();
+            } else if (objProp.getName().equals("runtime.powerState")) {
+                VirtualMachinePowerState powerState = (VirtualMachinePowerState) objProp.getVal();
+                vm.setPowerState(convertPowerState(powerState));
+            } else if (objProp.getName().equals("config.guestFullName")) {
+                vm.setOperatingSystem((String) objProp.getVal());
+            } else if (objProp.getName().equals("config.guestId")) {
+                vm.setOperatingSystemId((String) objProp.getVal());
+            } else if (objProp.getName().equals("config.bootOptions")) {
+                VirtualMachineBootOptions bootOptions = (VirtualMachineBootOptions) objProp.getVal();
+                String bootMode = "LEGACY";
+                if (bootOptions != null && bootOptions.isEfiSecureBootEnabled()) {
+                    bootMode = "SECURE";
+                }
+                vm.setBootMode(bootMode);
+            } else if (objProp.getName().equals("config.firmware")) {
+                String firmware = (String) objProp.getVal();
+                vm.setBootType(firmware.equalsIgnoreCase("efi") ? "UEFI" : "BIOS");
+            } else if (objProp.getName().equals("runtime.host")) {
+                ManagedObjectReference hostMor = (ManagedObjectReference) objProp.getVal();
+                setUnmanagedInstanceTOHostAndCluster(vm, hostMor, hostClusterNamesMap);
+            }
+        }
+        if (isTemplate || excludeByKeyword) {
+            return null;
+        }
+        return vm;
+    }
+
+    private void setUnmanagedInstanceTOHostAndCluster(UnmanagedInstanceTO vm, ManagedObjectReference hostMor,
+                                                      Map<String, Pair<String, String>> hostClusterNamesMap) throws Exception {
+        if (hostMor != null && StringUtils.isNotBlank(hostMor.getValue())) {
+            String hostMorValue = hostMor.getValue();
+            Pair<String, String> hostClusterPair;
+            if (hostClusterNamesMap.containsKey(hostMorValue)) {
+                hostClusterPair = hostClusterNamesMap.get(hostMorValue);
+            } else {
+                HostMO hostMO = new HostMO(_context, hostMor);
+                ClusterMO clusterMO = new ClusterMO(_context, hostMO.getHyperHostCluster());
+                hostClusterPair = new Pair<>(hostMO.getHostName(), clusterMO.getName());
+                hostClusterNamesMap.put(hostMorValue, hostClusterPair);
+            }
+            vm.setHostName(hostClusterPair.first());
+            vm.setClusterName(hostClusterPair.second());
+        }
+    }
 }

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java

@@ -21,7 +21,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import com.cloud.hypervisor.vmware.util.VmwareHelper;
 import org.apache.cloudstack.vm.UnmanagedInstanceTO;
 import org.apache.commons.collections.CollectionUtils;
 
@@ -98,7 +97,7 @@ public class DatacenterMO extends BaseMO {
         int key = cfmMo.getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_UUID);
         assert (key != 0);
 
-        List<VirtualMachineMO> list = new ArrayList<VirtualMachineMO>();
+        List<VirtualMachineMO> list = new ArrayList<>();
 
         List<ObjectContent> ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name", String.format("value[%d]", key)});
         if (ocs != null && ocs.size() > 0) {
@@ -159,28 +158,9 @@ public class DatacenterMO extends BaseMO {
         return null;
     }
 
-    public List<UnmanagedInstanceTO> getAllVmsOnDatacenter() throws Exception {
+    public List<UnmanagedInstanceTO> getAllVmsOnDatacenter(String keyword) throws Exception {
-        List<UnmanagedInstanceTO> vms = new ArrayList<>();
+        List<ObjectContent> ocs = getVmPropertiesOnDatacenterVmFolder(propertyPathsForUnmanagedVmsThinListing);
-        List<ObjectContent> ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name"});
+        return convertVmsObjectContentsToUnmanagedInstances(ocs, keyword);
-        if (ocs != null) {
-            for (ObjectContent oc : ocs) {
-                ManagedObjectReference vmMor = oc.getObj();
-                if (vmMor != null) {
-                    VirtualMachineMO vmMo = new VirtualMachineMO(_context, vmMor);
-                    try {
-                        if (!vmMo.isTemplate()) {
-                            HostMO hostMO = vmMo.getRunningHost();
-                            UnmanagedInstanceTO unmanagedInstance = VmwareHelper.getUnmanagedInstance(hostMO, vmMo);
-                            vms.add(unmanagedInstance);
-                        }
-                    } catch (Exception e) {
-                        logger.debug(String.format("Unexpected error checking unmanaged instance %s, excluding it: %s", vmMo.getVmName(), e.getMessage()), e);
-                    }
-                }
-            }
-        }
-
-        return vms;
     }
 
     public List<HostMO> getAllHostsOnDatacenter() throws Exception {
@@ -275,7 +255,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
@@ -301,7 +281,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
@@ -336,7 +316,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
@@ -364,7 +344,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         List<ObjectContent> ocs = context.getService().retrieveProperties(context.getPropertyCollector(), pfSpecArr);
@@ -375,7 +355,7 @@ public class DatacenterMO extends BaseMO {
         assert (ocs.get(0).getPropSet().get(0).getVal() != null);
 
         String dcName = ocs.get(0).getPropSet().get(0).getVal().toString();
-        return new Pair<DatacenterMO, String>(new DatacenterMO(context, ocs.get(0).getObj()), dcName);
+        return new Pair<>(new DatacenterMO(context, ocs.get(0).getObj()), dcName);
     }
 
     public ManagedObjectReference getDvPortGroupMor(String dvPortGroupName) throws Exception {
@@ -396,7 +376,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         List<ObjectContent> ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
@@ -443,7 +423,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         List<ObjectContent> ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
@@ -490,7 +470,7 @@ public class DatacenterMO extends BaseMO {
         PropertyFilterSpec pfSpec = new PropertyFilterSpec();
         pfSpec.getPropSet().add(pSpec);
         pfSpec.getObjectSet().add(oSpec);
-        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<>();
         pfSpecArr.add(pfSpec);
 
         List<ObjectContent> ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java

@@ -2333,7 +2333,7 @@ public class VirtualMachineMO extends BaseMO {
                     vmdkDescriptor = getVmdkFileInfo(fileItem.first());
 
                     logger.info("Move VM disk file " + srcFile.getPath() + " to " + destFile.getPath());
-                    srcDsMo.moveDatastoreFile(fileItem.first(), dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true);
+                    moveDatastoreFile(srcDsMo, fileItem.first(), dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true);
 
                     if (vmdkDescriptor != null) {
                         String vmdkBaseFileName = vmdkDescriptor.first().getBaseFileName();
@@ -2341,13 +2341,38 @@ public class VirtualMachineMO extends BaseMO {
                         destFile = new DatastoreFile(destDsMo.getName(), destDsDir, vmdkBaseFileName);
 
                         logger.info("Move VM disk file " + baseFilePath + " to " + destFile.getPath());
-                        srcDsMo.moveDatastoreFile(baseFilePath, dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true);
+                        moveDatastoreFile(srcDsMo, baseFilePath, dcMo.getMor(), destDsMo.getMor(), destFile.getPath(), dcMo.getMor(), true);
                     }
                 }
             }
         }
     }
 
+    private boolean moveDatastoreFile(final DatastoreMO dsMo, String srcFilePath, ManagedObjectReference morSrcDc, ManagedObjectReference morDestDs,
+                                   String destFilePath, ManagedObjectReference morDestDc, boolean forceOverwrite) throws Exception {
+        final int retry = 20;
+        int retryAttempt = 0;
+        while (++retryAttempt <= retry) {
+            try {
+                logger.debug(String.format("Move datastore file %s, attempt #%d", srcFilePath, retryAttempt));
+                return dsMo.moveDatastoreFile(srcFilePath, morSrcDc, morDestDs, destFilePath, morDestDc, forceOverwrite);
+            } catch (Exception e) {
+                logger.info(String.format("Got exception while moving datastore file %s ", srcFilePath), e);
+                if (e.getMessage() != null && e.getMessage().contains("Unable to access file")) {
+                    logger.debug(String.format("Failed to move datastore file %s. Retrying", srcFilePath));
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException ie) {
+                        logger.debug(String.format("Waiting to move datastore file %s been interrupted: ", srcFilePath));
+                    }
+                } else {
+                    throw e;
+                }
+            }
+        }
+        return false;
+    }
+
     public int getPvScsiDeviceControllerKeyNoException() throws Exception {
         List<VirtualDevice> devices = (List<VirtualDevice>)_context.getVimClient().
                 getDynamicProperty(_mor, "config.hardware.device");

vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java

@@ -59,6 +59,8 @@ import com.vmware.vim25.NasDatastoreInfo;
 import com.vmware.vim25.VMwareDVSPortSetting;
 import com.vmware.vim25.VirtualDeviceFileBackingInfo;
 import com.vmware.vim25.VirtualIDEController;
+import com.vmware.vim25.VirtualMachineBootOptions;
+import com.vmware.vim25.VirtualMachineConfigInfo;
 import com.vmware.vim25.VirtualMachineConfigSummary;
 import com.vmware.vim25.VirtualMachineGuestOsIdentifier;
 import com.vmware.vim25.VirtualMachineToolsStatus;
@@ -811,6 +813,17 @@ public class VmwareHelper {
                 instance.setCpuSpeed(configSummary.getCpuReservation());
                 instance.setMemory(configSummary.getMemorySizeMB());
             }
+            VirtualMachineConfigInfo configInfo = vmMo.getConfigInfo();
+            if (configInfo != null) {
+                String firmware = configInfo.getFirmware();
+                instance.setBootType(firmware.equalsIgnoreCase("efi") ? "UEFI" : "BIOS");
+                VirtualMachineBootOptions bootOptions = configInfo.getBootOptions();
+                String bootMode = "LEGACY";
+                if (bootOptions != null && bootOptions.isEfiSecureBootEnabled()) {
+                    bootMode = "SECURE";
+                }
+                instance.setBootMode(bootMode);
+            }
 
             try {
                 ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), hyperHost.getHyperHostCluster());