diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java index 7a26b178591..c1156f5f23a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java @@ -79,6 +79,14 @@ public class UnmanagedInstanceResponse extends BaseResponse { @Param(description = "the operating system of the virtual machine") private String operatingSystem; + @SerializedName(ApiConstants.BOOT_MODE) + @Param(description = "indicates the boot mode") + private String bootMode; + + @SerializedName(ApiConstants.BOOT_TYPE) + @Param(description = "indicates the boot type") + private String bootType; + @SerializedName(ApiConstants.DISK) @Param(description = "the list of disks associated with the virtual machine", responseObject = UnmanagedInstanceDiskResponse.class) private Set disks; @@ -211,4 +219,20 @@ public class UnmanagedInstanceResponse extends BaseResponse { public void addNic(NicResponse nic) { this.nics.add(nic); } + + public String getBootMode() { + return bootMode; + } + + public void setBootMode(String bootMode) { + this.bootMode = bootMode; + } + + public String getBootType() { + return bootType; + } + + public void setBootType(String bootType) { + this.bootType = bootType; + } } diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java index 3d5646f68c9..bba97dff71c 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java @@ -61,6 +61,9 @@ public class UnmanagedInstanceTO { private String vncPassword; + private String bootType; + private String bootMode; + public String getName() { return name; } @@ -196,6 +199,22 @@ public class UnmanagedInstanceTO { this, "name", "internalCSName", "hostName", "clusterName")); } + public String getBootType() { + return bootType; + } + + public void setBootType(String bootType) { + this.bootType = bootType; + } + + public String getBootMode() { + return bootMode; + } + + public void setBootMode(String bootMode) { + this.bootMode = bootMode; + } + public static class Disk { private String diskId; diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index 40899c7d8f4..8795c8d428f 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -210,7 +210,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust scanDirectAgentToLoad(); } - private void scanDirectAgentToLoad() { + protected void scanDirectAgentToLoad() { logger.trace("Begin scanning directly connected hosts"); // for agents that are self-managed, threshold to be considered as disconnected after pingtimeout @@ -231,11 +231,21 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust logger.info("{} is detected down, but we have a forward attache running, disconnect this one before launching the host", host); removeAgent(agentattache, Status.Disconnected); } else { - continue; + logger.debug("Host {} status is {} but has an AgentAttache which is not forForward, try to load directly", host, host.getStatus()); + Status hostStatus = investigate(agentattache); + if (Status.Up == hostStatus) { + /* Got ping response from host, bring it back */ + logger.info("After investigation, Agent for host {} is determined to be up and running", host); + agentStatusTransitTo(host, Event.Ping, _nodeId); + } else { + logger.debug("After investigation, AgentAttache is not null but host status is {}, try to load directly {}", hostStatus, host); + loadDirectlyConnectedHost(host, false); + } } + } else { + logger.debug("AgentAttache is null, loading directly connected {}", host); + loadDirectlyConnectedHost(host, false); } - logger.debug("Loading directly connected {}", host); - loadDirectlyConnectedHost(host, false); } catch (final Throwable e) { logger.warn(" can not load directly connected {} due to ", host, e); } @@ -381,20 +391,20 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust return; } if (!result) { - throw new CloudRuntimeException("Failed to propagate agent change request event:" + Event.ShutdownRequested + " to host:" + hostId); + throw new CloudRuntimeException(String.format("Failed to propagate agent change request event: %s to host: %s", Event.ShutdownRequested, hostId)); } } public void notifyNodesInCluster(final AgentAttache attache) { logger.debug("Notifying other nodes of to disconnect"); - final Command[] cmds = new Command[] {new ChangeAgentCommand(attache.getId(), Event.AgentDisconnected)}; + final Command[] cmds = new Command[]{new ChangeAgentCommand(attache.getId(), Event.AgentDisconnected)}; _clusterMgr.broadcast(attache.getId(), _gson.toJson(cmds)); } // notifies MS peers to schedule a host scan task immediately, triggered during addHost operation public void notifyNodesInClusterToScheduleHostScanTask() { logger.debug("Notifying other MS nodes to run host scan task"); - final Command[] cmds = new Command[] {new ScheduleHostScanTaskCommand()}; + final Command[] cmds = new Command[]{new ScheduleHostScanTaskCommand()}; _clusterMgr.broadcast(0, _gson.toJson(cmds)); } @@ -435,7 +445,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust } try { logD(bytes, "Routing to peer"); - Link.write(ch, new ByteBuffer[] {ByteBuffer.wrap(bytes)}, sslEngine); + Link.write(ch, new ByteBuffer[]{ByteBuffer.wrap(bytes)}, sslEngine); return true; } catch (final IOException e) { try { @@ -954,7 +964,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust if (!_agentToTransferIds.isEmpty()) { logger.debug("Found {} agents to transfer", _agentToTransferIds.size()); // for (Long hostId : _agentToTransferIds) { - for (final Iterator iterator = _agentToTransferIds.iterator(); iterator.hasNext();) { + for (final Iterator iterator = _agentToTransferIds.iterator(); iterator.hasNext(); ) { final Long hostId = iterator.next(); final AgentAttache attache = findAttache(hostId); @@ -1095,7 +1105,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust return; } - final ClusteredAgentAttache forwardAttache = (ClusteredAgentAttache)attache; + final ClusteredAgentAttache forwardAttache = (ClusteredAgentAttache) attache; if (success) { @@ -1146,10 +1156,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust } synchronized (_agents) { - final ClusteredDirectAgentAttache attache = (ClusteredDirectAgentAttache)_agents.get(hostId); + final ClusteredDirectAgentAttache attache = (ClusteredDirectAgentAttache) _agents.get(hostId); if (attache != null && attache.getQueueSize() == 0 && attache.getNonRecurringListenersSize() == 0) { handleDisconnectWithoutInvestigation(attache, Event.StartAgentRebalance, true, true); - final ClusteredAgentAttache forwardAttache = (ClusteredAgentAttache)createAttache(host); + final ClusteredAgentAttache forwardAttache = (ClusteredAgentAttache) createAttache(host); if (forwardAttache == null) { logger.warn("Unable to create a forward attache for the host {} as a part of rebalance process", host); return false; @@ -1253,7 +1263,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust } if (cmds.length == 1 && cmds[0] instanceof ChangeAgentCommand) { // intercepted - final ChangeAgentCommand cmd = (ChangeAgentCommand)cmds[0]; + final ChangeAgentCommand cmd = (ChangeAgentCommand) cmds[0]; logger.debug("Intercepting command for agent change: agent {} event: {}", cmd.getAgentId(), cmd.getEvent()); boolean result; @@ -1270,7 +1280,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust answers[0] = new ChangeAgentAnswer(cmd, result); return _gson.toJson(answers); } else if (cmds.length == 1 && cmds[0] instanceof TransferAgentCommand) { - final TransferAgentCommand cmd = (TransferAgentCommand)cmds[0]; + final TransferAgentCommand cmd = (TransferAgentCommand) cmds[0]; logger.debug("Intercepting command for agent rebalancing: agent: {}, event: {}, connection transfer: {}", cmd.getAgentId(), cmd.getEvent(), cmd.isConnectionTransfer()); boolean result; @@ -1289,7 +1299,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust answers[0] = new Answer(cmd, result, null); return _gson.toJson(answers); } else if (cmds.length == 1 && cmds[0] instanceof PropagateResourceEventCommand) { - final PropagateResourceEventCommand cmd = (PropagateResourceEventCommand)cmds[0]; + final PropagateResourceEventCommand cmd = (PropagateResourceEventCommand) cmds[0]; logger.debug("Intercepting command to propagate event {} for host {} ({})", () -> cmd.getEvent().name(), cmd::getHostId, () -> _hostDao.findById(cmd.getHostId())); @@ -1306,10 +1316,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust answers[0] = new Answer(cmd, result, null); return _gson.toJson(answers); } else if (cmds.length == 1 && cmds[0] instanceof ScheduleHostScanTaskCommand) { - final ScheduleHostScanTaskCommand cmd = (ScheduleHostScanTaskCommand)cmds[0]; + final ScheduleHostScanTaskCommand cmd = (ScheduleHostScanTaskCommand) cmds[0]; return handleScheduleHostScanTaskCommand(cmd); } else if (cmds.length == 1 && cmds[0] instanceof BaseShutdownManagementServerHostCommand) { - final BaseShutdownManagementServerHostCommand cmd = (BaseShutdownManagementServerHostCommand)cmds[0]; + final BaseShutdownManagementServerHostCommand cmd = (BaseShutdownManagementServerHostCommand) cmds[0]; return handleShutdownManagementServerHostCommand(cmd); } @@ -1362,7 +1372,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust try { managementServerMaintenanceManager.prepareForShutdown(); return "Successfully prepared for shutdown"; - } catch(CloudRuntimeException e) { + } catch (CloudRuntimeException e) { return e.getMessage(); } } @@ -1371,7 +1381,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust try { managementServerMaintenanceManager.triggerShutdown(); return "Successfully triggered shutdown"; - } catch(CloudRuntimeException e) { + } catch (CloudRuntimeException e) { return e.getMessage(); } } @@ -1380,7 +1390,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust try { managementServerMaintenanceManager.cancelShutdown(); return "Successfully cancelled shutdown"; - } catch(CloudRuntimeException e) { + } catch (CloudRuntimeException e) { return e.getMessage(); } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index e2e6f3c6f9d..13475579f77 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -61,7 +61,11 @@ import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.framework.ca.Certificate; @@ -413,6 +417,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac ResourceCleanupService resourceCleanupService; @Inject VmWorkJobDao vmWorkJobDao; + @Inject + DataStoreProviderManager dataStoreProviderManager; private SingleCache> vmIdsInProgressCache; @@ -1238,6 +1244,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac planChangedByVolume = true; } } + DataStoreProvider storeProvider = dataStoreProviderManager.getDataStoreProvider(pool.getStorageProviderName()); + if (storeProvider != null) { + DataStoreDriver storeDriver = storeProvider.getDataStoreDriver(); + if (storeDriver instanceof PrimaryDataStoreDriver) { + ((PrimaryDataStoreDriver)storeDriver).detachVolumeFromAllStorageNodes(vol); + } + } } } diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/ClusteredAgentManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/ClusteredAgentManagerImplTest.java new file mode 100644 index 00000000000..5e4678f6222 --- /dev/null +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/ClusteredAgentManagerImplTest.java @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.manager; + +import com.cloud.configuration.ManagementServiceConfiguration; +import com.cloud.ha.HighAvailabilityManagerImpl; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.resource.ResourceManagerImpl; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ClusteredAgentManagerImplTest { + + private HostDao _hostDao; + @Mock + ManagementServiceConfiguration _mgmtServiceConf; + + @Before + public void setUp() throws Exception { + _hostDao = mock(HostDao.class); + } + + @Test + public void scanDirectAgentToLoadNoHostsTest() { + ClusteredAgentManagerImpl clusteredAgentManagerImpl = mock(ClusteredAgentManagerImpl.class); + clusteredAgentManagerImpl._hostDao = _hostDao; + clusteredAgentManagerImpl.scanDirectAgentToLoad(); + verify(clusteredAgentManagerImpl, never()).findAttache(anyLong()); + verify(clusteredAgentManagerImpl, never()).loadDirectlyConnectedHost(any(), anyBoolean()); + } + + @Test + public void scanDirectAgentToLoadHostWithoutAttacheTest() { + // Arrange + ClusteredAgentManagerImpl clusteredAgentManagerImpl = Mockito.spy(ClusteredAgentManagerImpl.class); + HostVO hostVO = mock(HostVO.class); + clusteredAgentManagerImpl._hostDao = _hostDao; + clusteredAgentManagerImpl.mgmtServiceConf = _mgmtServiceConf; + clusteredAgentManagerImpl._resourceMgr = mock(ResourceManagerImpl.class); + when(_mgmtServiceConf.getTimeout()).thenReturn(16000L); + when(hostVO.getId()).thenReturn(1L); + List hosts = new ArrayList<>(); + hosts.add(hostVO); + when(_hostDao.findAndUpdateDirectAgentToLoad(anyLong(), anyLong(), anyLong())).thenReturn(hosts); + AgentAttache agentAttache = mock(AgentAttache.class); + doReturn(Boolean.TRUE).when(clusteredAgentManagerImpl).loadDirectlyConnectedHost(hostVO, false); + clusteredAgentManagerImpl.scanDirectAgentToLoad(); + verify(clusteredAgentManagerImpl).loadDirectlyConnectedHost(hostVO, false); + } + + @Test + public void scanDirectAgentToLoadHostWithForwardAttacheTest() { + ClusteredAgentManagerImpl clusteredAgentManagerImpl = Mockito.spy(ClusteredAgentManagerImpl.class); + HostVO hostVO = mock(HostVO.class); + clusteredAgentManagerImpl._hostDao = _hostDao; + clusteredAgentManagerImpl.mgmtServiceConf = _mgmtServiceConf; + when(_mgmtServiceConf.getTimeout()).thenReturn(16000L); + when(hostVO.getId()).thenReturn(1L); + List hosts = new ArrayList<>(); + hosts.add(hostVO); + when(_hostDao.findAndUpdateDirectAgentToLoad(anyLong(), anyLong(), anyLong())).thenReturn(hosts); + AgentAttache agentAttache = mock(AgentAttache.class); + when(agentAttache.forForward()).thenReturn(Boolean.TRUE); + when(clusteredAgentManagerImpl.findAttache(1L)).thenReturn(agentAttache); + + clusteredAgentManagerImpl.scanDirectAgentToLoad(); + verify(clusteredAgentManagerImpl).removeAgent(agentAttache, Status.Disconnected); + } + + @Test + public void scanDirectAgentToLoadHostWithNonForwardAttacheTest() { + // Arrange + ClusteredAgentManagerImpl clusteredAgentManagerImpl = Mockito.spy(new ClusteredAgentManagerImpl()); + HostVO hostVO = mock(HostVO.class); + clusteredAgentManagerImpl._hostDao = _hostDao; + clusteredAgentManagerImpl.mgmtServiceConf = _mgmtServiceConf; + clusteredAgentManagerImpl._haMgr = mock(HighAvailabilityManagerImpl.class); + when(_mgmtServiceConf.getTimeout()).thenReturn(16000L); + when(hostVO.getId()).thenReturn(0L); + List hosts = new ArrayList<>(); + hosts.add(hostVO); + when(_hostDao.findAndUpdateDirectAgentToLoad(anyLong(), anyLong(), anyLong())).thenReturn(hosts); + + AgentAttache agentAttache = mock(AgentAttache.class); + when(agentAttache.forForward()).thenReturn(Boolean.FALSE); + when(clusteredAgentManagerImpl.findAttache(0L)).thenReturn(agentAttache); + doReturn(Boolean.TRUE).when(clusteredAgentManagerImpl).agentStatusTransitTo(hostVO, Status.Event.Ping, clusteredAgentManagerImpl._nodeId); + doReturn(Status.Up).when(clusteredAgentManagerImpl).investigate(agentAttache); + + clusteredAgentManagerImpl.scanDirectAgentToLoad(); + verify(clusteredAgentManagerImpl).investigate(agentAttache); + verify(clusteredAgentManagerImpl).agentStatusTransitTo(hostVO, Status.Event.Ping, clusteredAgentManagerImpl._nodeId); + } + + @Test + public void scanDirectAgentToLoadHostWithNonForwardAttacheAndDisconnectedTest() { + ClusteredAgentManagerImpl clusteredAgentManagerImpl = Mockito.spy(ClusteredAgentManagerImpl.class); + HostVO hostVO = mock(HostVO.class); + clusteredAgentManagerImpl._hostDao = _hostDao; + clusteredAgentManagerImpl.mgmtServiceConf = _mgmtServiceConf; + clusteredAgentManagerImpl._haMgr = mock(HighAvailabilityManagerImpl.class); + clusteredAgentManagerImpl._resourceMgr = mock(ResourceManagerImpl.class); + when(_mgmtServiceConf.getTimeout()).thenReturn(16000L); + when(hostVO.getId()).thenReturn(0L); + List hosts = new ArrayList<>(); + hosts.add(hostVO); + when(_hostDao.findAndUpdateDirectAgentToLoad(anyLong(), anyLong(), anyLong())).thenReturn(hosts); + AgentAttache agentAttache = mock(AgentAttache.class); + when(agentAttache.forForward()).thenReturn(Boolean.FALSE); + when(clusteredAgentManagerImpl.findAttache(0L)).thenReturn(agentAttache); + doReturn(Boolean.TRUE).when(clusteredAgentManagerImpl).loadDirectlyConnectedHost(hostVO, false); + clusteredAgentManagerImpl.scanDirectAgentToLoad(); + verify(clusteredAgentManagerImpl).investigate(agentAttache); + verify(clusteredAgentManagerImpl).loadDirectlyConnectedHost(hostVO, false); + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index 6febb258259..da6b1cfede3 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -80,7 +80,7 @@ public class DefaultEndPointSelector implements EndPointSelector { private final String findOneHostOnPrimaryStorage = "select t.id from " + "(select h.id, cd.value, hd.value as " + VOL_ENCRYPT_COLUMN_NAME + " " + "from host h join storage_pool_host_ref s on h.id = s.host_id " - + "join cluster c on c.id=h.cluster_id " + + "join cluster c on c.id=h.cluster_id and c.allocation_state = 'Enabled'" + "left join cluster_details cd on c.id=cd.cluster_id and cd.name='" + CapacityManager.StorageOperationsExcludeCluster.key() + "' " + "left join host_details hd on h.id=hd.host_id and hd.name='" + HOST_VOLUME_ENCRYPTION + "' " + "where h.status = 'Up' and h.type = 'Routing' and h.resource_state = 'Enabled' and s.pool_id = ? "; diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index f296a32e2c0..3a7b7da7768 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -328,6 +328,11 @@ public class VolumeServiceImpl implements VolumeService { } else { vo.processEvent(Event.OperationFailed); errMsg = result.getResult(); + VolumeVO volume = volDao.findById(vo.getId()); + if (volume != null && volume.getState() == State.Allocated && volume.getPodId() != null) { + volume.setPoolId(null); + volDao.update(volume.getId(), volume); + } } VolumeApiResult volResult = new VolumeApiResult((VolumeObject)vo); if (errMsg != null) { @@ -1255,6 +1260,10 @@ public class VolumeServiceImpl implements VolumeService { } if (volume.getState() == State.Allocated) { // Possible states here: Allocated, Ready & Creating + if (volume.getPodId() != null) { + volume.setPoolId(null); + volDao.update(volume.getId(), volume); + } return; } @@ -2494,7 +2503,7 @@ public class VolumeServiceImpl implements VolumeService { try { volume.processEvent(Event.ResizeRequested); } catch (Exception e) { - logger.debug("Failed to change state to resize", e); + logger.debug("Failed to change volume state to resize", e); result.setResult(e.toString()); future.complete(result); return future; @@ -2506,10 +2515,8 @@ public class VolumeServiceImpl implements VolumeService { try { volume.getDataStore().getDriver().resize(volume, caller); } catch (Exception e) { - logger.debug("Failed to change state to resize", e); - + logger.debug("Failed to resize volume", e); result.setResult(e.toString()); - future.complete(result); } @@ -2553,7 +2560,7 @@ public class VolumeServiceImpl implements VolumeService { try { volume.processEvent(Event.OperationFailed); } catch (Exception e) { - logger.debug("Failed to change state", e); + logger.debug("Failed to change volume state (after resize failure)", e); } VolumeApiResult res = new VolumeApiResult(volume); res.setResult(result.getResult()); @@ -2564,13 +2571,8 @@ public class VolumeServiceImpl implements VolumeService { try { volume.processEvent(Event.OperationSuccessed); } catch (Exception e) { - logger.debug("Failed to change state", e); - VolumeApiResult res = new VolumeApiResult(volume); - res.setResult(result.getResult()); - future.complete(res); - return null; + logger.debug("Failed to change volume state (after resize success)", e); } - VolumeApiResult res = new VolumeApiResult(volume); future.complete(res); diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 1e3b5aaa8e5..fa9d2b58629 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -25,6 +25,7 @@ import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; @@ -278,6 +279,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co restoredVolume.setPoolId(dataStore.getPoolId()); restoredVolume.setPath(restoredVolume.getUuid()); restoredVolume.setState(Volume.State.Copying); + restoredVolume.setFormat(Storage.ImageFormat.QCOW2); restoredVolume.setSize(backedUpVolumeSize); restoredVolume.setDiskOfferingId(volume.getDiskOfferingId()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 141c1d5ea19..2810f98c935 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -45,7 +45,7 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper listVMsInDatacenter(ListVmwareDcVmsCmd cmd) { + private static class VcenterData { + public final String vcenter; + public final String datacenterName; + public final String username; + public final String password; + + public VcenterData(String vcenter, String datacenterName, String username, String password) { + this.vcenter = vcenter; + this.datacenterName = datacenterName; + this.username = username; + this.password = password; + } + } + + private VcenterData getVcenterData(ListVmwareDcVmsCmd cmd) { String vcenter = cmd.getVcenter(); String datacenterName = cmd.getDatacenterName(); String username = cmd.getUsername(); String password = cmd.getPassword(); Long existingVcenterId = cmd.getExistingVcenterId(); - String keyword = cmd.getKeyword(); if ((existingVcenterId == null && StringUtils.isBlank(vcenter)) || (existingVcenterId != null && StringUtils.isNotBlank(vcenter))) { @@ -1613,34 +1629,69 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw username = vmwareDc.getUser(); password = vmwareDc.getPassword(); } + VcenterData vmwaredc = new VcenterData(vcenter, datacenterName, username, password); + return vmwaredc; + } + + private static VmwareContext getVmwareContext(String vcenter, String username, String password) throws Exception { + static_logger.debug(String.format("Connecting to the VMware vCenter %s", vcenter)); + String serviceUrl = String.format("https://%s/sdk/vimService", vcenter); + VmwareClient vimClient = new VmwareClient(vcenter); + vimClient.connect(serviceUrl, username, password); + return new VmwareContext(vimClient, vcenter); + } + + @Override + public List listVMsInDatacenter(ListVmwareDcVmsCmd cmd) { + VcenterData vmwareDC = getVcenterData(cmd); + String vcenter = vmwareDC.vcenter; + String username = vmwareDC.username; + String password = vmwareDC.password; + String datacenterName = vmwareDC.datacenterName; + String keyword = cmd.getKeyword(); + String esxiHostName = cmd.getHostName(); + String virtualMachineName = cmd.getInstanceName(); try { logger.debug(String.format("Connecting to the VMware datacenter %s at vCenter %s to retrieve VMs", datacenterName, vcenter)); - String serviceUrl = String.format("https://%s/sdk/vimService", vcenter); - VmwareClient vimClient = new VmwareClient(vcenter); - vimClient.connect(serviceUrl, username, password); - VmwareContext context = new VmwareContext(vimClient, vcenter); + VmwareContext context = getVmwareContext(vcenter, username, password); + DatacenterMO dcMo = getDatacenterMO(context, vcenter, datacenterName); - DatacenterMO dcMo = new DatacenterMO(context, datacenterName); - ManagedObjectReference dcMor = dcMo.getMor(); - if (dcMor == null) { - String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", - datacenterName, vcenter); - logger.error(msg); - throw new InvalidParameterValueException(msg); + List instances; + if (StringUtils.isNotBlank(esxiHostName) && StringUtils.isNotBlank(virtualMachineName)) { + ManagedObjectReference hostMor = dcMo.findHost(esxiHostName); + if (hostMor == null) { + String errorMsg = String.format("Cannot find a host with name %s on vcenter %s", esxiHostName, vcenter); + logger.error(errorMsg); + throw new CloudRuntimeException(errorMsg); + } + HostMO hostMO = new HostMO(context, hostMor); + VirtualMachineMO vmMo = hostMO.findVmOnHyperHost(virtualMachineName); + instances = Collections.singletonList(VmwareHelper.getUnmanagedInstance(hostMO, vmMo)); + } else { + instances = dcMo.getAllVmsOnDatacenter(keyword); } - List instances = dcMo.getAllVmsOnDatacenter(); - return StringUtils.isBlank(keyword) ? instances : - instances.stream().filter(x -> x.getName().toLowerCase().contains(keyword.toLowerCase())).collect(Collectors.toList()); + return instances; } catch (Exception e) { - String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s", + String errorMsg = String.format("Error retrieving VMs from the VMware VC %s datacenter %s: %s", vcenter, datacenterName, e.getMessage()); logger.error(errorMsg, e); throw new CloudRuntimeException(errorMsg); } } + private static DatacenterMO getDatacenterMO(VmwareContext context, String vcenter, String datacenterName) throws Exception { + DatacenterMO dcMo = new DatacenterMO(context, datacenterName); + ManagedObjectReference dcMor = dcMo.getMor(); + if (dcMor == null) { + String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", datacenterName, vcenter); + static_logger.error(msg); + throw new InvalidParameterValueException(msg); + } + return dcMo; + } + @Override public boolean hasNexusVSM(Long clusterId) { ClusterVSMMapVO vsmMapVo = null; @@ -1693,7 +1744,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } /** - * This task is to cleanup templates from primary storage that are otherwise not cleaned by the {@link com.cloud.storage.StorageManagerImpl.StorageGarbageCollector}. + * This task is to cleanup templates from primary storage that are otherwise not cleaned by the {code}StorageGarbageCollector{code} from {@link com.cloud.storage.StorageManagerImpl}. * it is called at regular intervals when storage.template.cleanup.enabled == true * It collect all templates that * - are deleted from cloudstack diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index cb1cc909e06..478ccf1d2af 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2042,7 +2042,6 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes VirtualMachineDefinedProfileSpec diskProfileSpec = null; VirtualMachineDefinedProfileSpec vmProfileSpec = null; - DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); boolean deployAsIs = deployAsIsInfo != null; @@ -2086,7 +2085,6 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes } VirtualMachineDiskInfoBuilder diskInfoBuilder = null; - VirtualDevice[] nicDevices = null; VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); DiskControllerType systemVmScsiControllerType = DiskControllerType.lsilogic; int firstScsiControllerBusNum = 0; @@ -2103,7 +2101,6 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes diskDatastores = vmMo.getAllDiskDatastores(); diskInfoBuilder = vmMo.getDiskInfoBuilder(); hasSnapshot = vmMo.hasSnapshot(); - nicDevices = vmMo.getNicDevices(); tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, @@ -2119,17 +2116,20 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes } takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); + vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); - if (getVmPowerState(vmMo) != PowerState.PowerOff) - vmMo.safePowerOff(_shutdownWaitMs); + if (vmMo != null) { + if (getVmPowerState(vmMo) != PowerState.PowerOff) + vmMo.safePowerOff(_shutdownWaitMs); - diskInfoBuilder = vmMo.getDiskInfoBuilder(); - hasSnapshot = vmMo.hasSnapshot(); - diskDatastores = vmMo.getAllDiskDatastores(); + diskInfoBuilder = vmMo.getDiskInfoBuilder(); + hasSnapshot = vmMo.hasSnapshot(); + diskDatastores = vmMo.getAllDiskDatastores(); - tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); - ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, - numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs); + tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); + ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, + numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs); + } } else { // If a VM with the same name is found in a different cluster in the DC, unregister the old VM and configure a new VM (cold-migration). VirtualMachineMO existingVmInDc = dcMo.findVm(vmInternalCSName); @@ -2146,7 +2146,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); if (vmMo == null) { logger.info("Cloned deploy-as-is VM " + vmInternalCSName + " is not in this host, relocating it"); - vmMo = takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); + takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); } } else { DiskTO rootDisk = null; @@ -2256,11 +2256,11 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()); } - if(!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()){ + if (!vmMo.isMemoryHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) { logger.warn("hotadd of memory is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName); } - if(!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()){ + if (!vmMo.isCpuHotAddSupported(guestOsId) && vmSpec.isEnableDynamicallyScaleVm()) { logger.warn("hotadd of cpu is not supported, dynamic scaling feature can not be applied to vm: " + vmInternalCSName); } @@ -2593,7 +2593,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes Map> iqnToData = new HashMap<>(); - postDiskConfigBeforeStart(vmMo, vmSpec, sortedDisks, ideControllerKey, scsiControllerKey, iqnToData, hyperHost, context); + postDiskConfigBeforeStart(vmMo, vmSpec, sortedDisks, iqnToData, hyperHost, context); // // Power-on VM @@ -2731,14 +2731,24 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes } private boolean powerOnVM(final VirtualMachineMO vmMo, final String vmInternalCSName, final String vmNameOnVcenter) throws Exception { - int retry = 20; - while (retry-- > 0) { + final int retry = 20; + int retryAttempt = 0; + while (++retryAttempt <= retry) { try { + logger.debug(String.format("VM %s, powerOn attempt #%d", vmInternalCSName, retryAttempt)); return vmMo.powerOn(); } catch (Exception e) { logger.info(String.format("Got exception while power on VM %s with hostname %s", vmInternalCSName, vmNameOnVcenter), e); - if (e.getMessage() != null && e.getMessage().contains("File system specific implementation of Ioctl[file] failed")) { + if (e.getMessage() != null && + (e.getMessage().contains("File system specific implementation of Ioctl[file] failed") || + e.getMessage().contains("Unable to access file") || + e.getMessage().contains("it is locked"))) { logger.debug(String.format("Failed to power on VM %s with hostname %s. Retrying", vmInternalCSName, vmNameOnVcenter)); + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + logger.debug(String.format("Waiting to power on VM %s been interrupted: ", vmInternalCSName)); + } } else { throw e; } @@ -3292,7 +3302,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes int getReservedMemoryMb(VirtualMachineTO vmSpec) { if (vmSpec.getDetails().get(VMwareGuru.VmwareReserveMemory.key()).equalsIgnoreCase("true")) { - if(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null){ + if (vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION) != null) { float reservedMemory = (vmSpec.getMaxRam() * Float.parseFloat(vmSpec.getDetails().get(VmDetailConstants.RAM_RESERVATION))); return (int) (reservedMemory / ResourceType.bytesToMiB); } @@ -3630,18 +3640,18 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes private VirtualMachineDiskInfo getMatchingExistingDisk(VirtualMachineDiskInfoBuilder diskInfoBuilder, DiskTO vol, VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception { - if (diskInfoBuilder != null) { - VolumeObjectTO volume = (VolumeObjectTO) vol.getData(); - String chainInfo = volume.getChainInfo(); - Map details = vol.getDetails(); - boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED)); - String iScsiName = details.get(DiskTO.IQN); - String datastoreUUID = volume.getDataStore().getUuid(); - - return getMatchingExistingDiskWithVolumeDetails(diskInfoBuilder, volume.getPath(), chainInfo, isManaged, iScsiName, datastoreUUID, hyperHost, context); - } else { + if (diskInfoBuilder == null) { return null; } + + VolumeObjectTO volume = (VolumeObjectTO) vol.getData(); + String chainInfo = volume.getChainInfo(); + Map details = vol.getDetails(); + boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED)); + String iScsiName = details.get(DiskTO.IQN); + String datastoreUUID = volume.getDataStore().getUuid(); + + return getMatchingExistingDiskWithVolumeDetails(diskInfoBuilder, volume.getPath(), chainInfo, isManaged, iScsiName, datastoreUUID, hyperHost, context); } private String getDiskController(VirtualMachineMO vmMo, VirtualMachineDiskInfo matchingExistingDisk, DiskTO vol, Pair controllerInfo, boolean deployAsIs) throws Exception { @@ -3666,34 +3676,36 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes return VmwareHelper.getControllerBasedOnDiskType(controllerInfo, vol); } - private void postDiskConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, DiskTO[] sortedDisks, int ideControllerKey, - int scsiControllerKey, Map> iqnToData, VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception { + private void postDiskConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO vmSpec, DiskTO[] sortedDisks, + Map> iqnToData, VmwareHypervisorHost hyperHost, VmwareContext context) throws Exception { VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); for (DiskTO vol : sortedDisks) { if (vol.getType() == Volume.Type.ISO) continue; - VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); - VirtualMachineDiskInfo diskInfo = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context); - assert (diskInfo != null); + if (diskInfo == null) { + continue; + } String[] diskChain = diskInfo.getDiskChain(); - assert (diskChain.length > 0); - - Map details = vol.getDetails(); - boolean managed = false; - - if (details != null) { - managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); + if (diskChain.length <= 0) { + continue; } DatastoreFile file = new DatastoreFile(diskChain[0]); + boolean managed = false; + Map details = vol.getDetails(); + if (details != null) { + managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); + } + + VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); + if (managed) { DatastoreFile originalFile = new DatastoreFile(volumeTO.getPath()); - if (!file.getFileBaseName().equalsIgnoreCase(originalFile.getFileBaseName())) { if (logger.isInfoEnabled()) logger.info("Detected disk-chain top file change on volume: " + volumeTO.getId() + " " + volumeTO.getPath() + " -> " + diskChain[0]); @@ -3706,7 +3718,6 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes } VolumeObjectTO volInSpec = getVolumeInSpec(vmSpec, volumeTO); - if (volInSpec != null) { if (managed) { Map data = new HashMap<>(); @@ -3871,20 +3882,20 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes if (diskInfo != null) { logger.info("Found existing disk info from volume path: " + volume.getPath()); return dsMo; - } else { - String chainInfo = volume.getChainInfo(); - if (chainInfo != null) { - VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class); - if (infoInChain != null) { - String[] disks = infoInChain.getDiskChain(); - if (disks.length > 0) { - for (String diskPath : disks) { - DatastoreFile file = new DatastoreFile(diskPath); - diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName); - if (diskInfo != null) { - logger.info("Found existing disk from chain info: " + diskPath); - return dsMo; - } + } + + String chainInfo = volume.getChainInfo(); + if (chainInfo != null) { + VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class); + if (infoInChain != null) { + String[] disks = infoInChain.getDiskChain(); + if (disks.length > 0) { + for (String diskPath : disks) { + DatastoreFile file = new DatastoreFile(diskPath); + diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName); + if (diskInfo != null) { + logger.info("Found existing disk from chain info: " + diskPath); + return dsMo; } } } @@ -4747,7 +4758,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes Map volumeDeviceKey = new HashMap<>(); if (cmd instanceof MigrateVolumeCommand) { // Else device keys will be found in relocateVirtualMachine MigrateVolumeCommand mcmd = (MigrateVolumeCommand) cmd; - addVolumeDiskmapping(vmMo, volumeDeviceKey, mcmd.getVolumePath(), mcmd.getVolumeId()); + addVolumeDiskMapping(vmMo, volumeDeviceKey, mcmd.getVolumePath(), mcmd.getVolumeId()); if (logger.isTraceEnabled()) { for (Integer diskId: volumeDeviceKey.keySet()) { logger.trace(String.format("Disk to migrate has disk id %d and volumeId %d", diskId, volumeDeviceKey.get(diskId))); @@ -4765,9 +4776,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes Answer createAnswerForCmd(VirtualMachineMO vmMo, List volumeObjectToList, Command cmd, Map volumeDeviceKey) throws Exception { List volumeToList; - VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); VirtualDisk[] disks = vmMo.getAllDiskDevice(); - Answer answer; if (logger.isTraceEnabled()) { logger.trace(String.format("creating answer for %s", cmd.getClass().getSimpleName())); } @@ -4784,7 +4793,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes return new Answer(cmd, false, null); } - private void addVolumeDiskmapping(VirtualMachineMO vmMo, Map volumeDeviceKey, String volumePath, long volumeId) throws Exception { + private void addVolumeDiskMapping(VirtualMachineMO vmMo, Map volumeDeviceKey, String volumePath, long volumeId) throws Exception { if (logger.isDebugEnabled()) { logger.debug(String.format("locating disk for volume (%d) using path %s", volumeId, volumePath)); } @@ -4919,7 +4928,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes VmwareHypervisorHost dsHost = hyperHostInTargetCluster == null ? hyperHost : hyperHostInTargetCluster; String targetDsName = cmd.getTargetPool().getUuid(); morDestinationDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(dsHost, targetDsName); - if(morDestinationDS == null) { + if (morDestinationDS == null) { String msg = "Unable to find the target datastore: " + targetDsName + " on host: " + dsHost.getHyperHostName(); logger.error(msg); throw new CloudRuntimeException(msg); @@ -5886,6 +5895,11 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes logger.debug(msg); return new Answer(cmd, true, msg); } catch (Exception e) { + if (e.getMessage().contains("was not found")) { + String msg = String.format("%s - VM [%s] file(s) not found, cleanup not needed .", e.getMessage(), cmd.getVmName()); + logger.debug(msg); + return new Answer(cmd, true, msg); + } return new Answer(cmd, false, createLogMessageException(e, cmd)); } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java index ab9754a7c9e..69572a3cd17 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import com.vmware.vim25.ManagedObjectReference; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.logging.log4j.Logger; @@ -193,7 +194,7 @@ public class VmwareStorageLayoutHelper implements Configurable { if (ds.fileExists(vmdkFullCloneModeLegacyPair[i])) { LOGGER.info("sync " + vmdkFullCloneModeLegacyPair[i] + "->" + vmdkFullCloneModePair[i]); - ds.moveDatastoreFile(vmdkFullCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[i], dcMo.getMor(), true); + moveDatastoreFile(ds, vmdkFullCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[i], dcMo.getMor(), true); } } @@ -201,13 +202,13 @@ public class VmwareStorageLayoutHelper implements Configurable { if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[i])) { LOGGER.info("sync " + vmdkLinkedCloneModeLegacyPair[i] + "->" + vmdkLinkedCloneModePair[i]); - ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[i], dcMo.getMor(), true); + moveDatastoreFile(ds, vmdkLinkedCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[i], dcMo.getMor(), true); } } if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[0])) { LOGGER.info("sync " + vmdkLinkedCloneModeLegacyPair[0] + "->" + vmdkLinkedCloneModePair[0]); - ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[0], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[0], dcMo.getMor(), true); + moveDatastoreFile(ds, vmdkLinkedCloneModeLegacyPair[0], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[0], dcMo.getMor(), true); } // Note: we will always return a path @@ -242,14 +243,14 @@ public class VmwareStorageLayoutHelper implements Configurable { String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, String.format("%s-%s",vmdkName, linkedCloneExtension)); LOGGER.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); - ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + moveDatastoreFile(ds, companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); } } // move the identity VMDK file the last String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, vmdkName + ".vmdk"); LOGGER.info("Fixup folder-synchronization. move " + fileDsFullPath + " -> " + targetPath); - ds.moveDatastoreFile(fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + moveDatastoreFile(ds, fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); try { if (folderName != null) { @@ -287,7 +288,7 @@ public class VmwareStorageLayoutHelper implements Configurable { DatastoreFile targetFile = new DatastoreFile(file.getDatastoreName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, file.getFileName()); if (!targetFile.getPath().equalsIgnoreCase(file.getPath())) { LOGGER.info("Move " + file.getPath() + " -> " + targetFile.getPath()); - dsMo.moveDatastoreFile(file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true); + moveDatastoreFile(dsMo, file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true); List vSphereFileExtensions = new ArrayList<>(Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"))); // add flat file format to the above list @@ -297,7 +298,7 @@ public class VmwareStorageLayoutHelper implements Configurable { String pairTargetFilePath = targetFile.getCompanionPath(String.format("%s-%s", file.getFileBaseName(), linkedCloneExtension)); if (dsMo.fileExists(pairSrcFilePath)) { LOGGER.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath); - dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); + moveDatastoreFile(dsMo, pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); } } } @@ -429,6 +430,31 @@ public class VmwareStorageLayoutHelper implements Configurable { return dsMo.searchFileInSubFolders(volumePath + ".vmdk", false, null); } + public static boolean moveDatastoreFile(final DatastoreMO dsMo, String srcFilePath, ManagedObjectReference morSrcDc, ManagedObjectReference morDestDs, + String destFilePath, ManagedObjectReference morDestDc, boolean forceOverwrite) throws Exception { + final int retry = 20; + int retryAttempt = 0; + while (++retryAttempt <= retry) { + try { + LOGGER.debug(String.format("Move datastore file %s, attempt #%d", srcFilePath, retryAttempt)); + return dsMo.moveDatastoreFile(srcFilePath, morSrcDc, morDestDs, destFilePath, morDestDc, forceOverwrite); + } catch (Exception e) { + LOGGER.info(String.format("Got exception while moving datastore file %s ", srcFilePath), e); + if (e.getMessage() != null && e.getMessage().contains("Unable to access file")) { + LOGGER.debug(String.format("Failed to move datastore file %s. Retrying", srcFilePath)); + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + LOGGER.debug(String.format("Waiting to move datastore file %s been interrupted: ", srcFilePath)); + } + } else { + throw e; + } + } + } + return false; + } + @Override public String getConfigComponentName() { return VmwareStorageLayoutHelper.class.getSimpleName(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index c99d7d4d707..e4b3282defb 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -682,9 +682,9 @@ public class VmwareStorageProcessor implements StorageProcessor { String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairManagedDatastorePath(dsMo, null, managedStoragePoolRootVolumeName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, false); - dsMo.moveDatastoreFile(vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true); + VmwareStorageLayoutHelper.moveDatastoreFile(dsMo, vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true); for (int i=1; i template.getSize() ? true : _fullCloneFlag; + } if (!_fullCloneFlag) { createVMLinkedClone(templateMo, dcMo, cloneName, morDatastore, morPool, null); } else { diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java index 4dd1b4beb09..3c7b233f7c9 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java @@ -70,6 +70,12 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.") private String password; + @Parameter(name = ApiConstants.HOST_NAME, type = CommandType.STRING, description = "Name of the host on vCenter. Must be set along with the instancename parameter") + private String hostName; + + @Parameter(name = ApiConstants.INSTANCE_NAME, type = CommandType.STRING, description = "Name of the VM on vCenter. Must be set along with the hostname parameter") + private String instanceName; + public String getVcenter() { return vcenter; } @@ -86,10 +92,18 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { return datacenterName; } + public String getHostName() { + return hostName; + } + public Long getExistingVcenterId() { return existingVcenterId; } + public String getInstanceName() { + return instanceName; + } + @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { checkParameters(); @@ -125,6 +139,11 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Please set all the information for a vCenter IP/Name, datacenter, username and password"); } + if ((StringUtils.isNotBlank(instanceName) && StringUtils.isBlank(hostName)) || + (StringUtils.isBlank(instanceName) && StringUtils.isNotBlank(hostName))) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Please set the hostname parameter along with the instancename parameter"); + } } @Override diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java index 0461ab8aa0d..d2f0076f95f 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java @@ -453,9 +453,18 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri boolean encryptionRequired = anyVolumeRequiresEncryption(vol); long [] endpointsToRunResize = resizeParameter.hosts; + CreateCmdResult result = new CreateCmdResult(null, null); + // if hosts are provided, they are where the VM last ran. We can use that. if (endpointsToRunResize == null || endpointsToRunResize.length == 0) { EndPoint ep = epSelector.select(data, encryptionRequired); + if (ep == null) { + String errMsg = String.format(NO_REMOTE_ENDPOINT_WITH_ENCRYPTION, encryptionRequired); + logger.error(errMsg); + result.setResult(errMsg); + callback.complete(result); + return; + } endpointsToRunResize = new long[] {ep.getId()}; } ResizeVolumeCommand resizeCmd = new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), vol.getSize(), @@ -463,7 +472,6 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri if (pool.getParent() != 0) { resizeCmd.setContextParam(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString()); } - CreateCmdResult result = new CreateCmdResult(null, null); try { ResizeVolumeAnswer answer = (ResizeVolumeAnswer) storageMgr.sendToPool(pool, endpointsToRunResize, resizeCmd); if (answer != null && answer.getResult()) { @@ -480,7 +488,6 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri logger.debug("return a null answer, mark it as failed for unknown reason"); result.setResult("return a null answer, mark it as failed for unknown reason"); } - } catch (Exception e) { logger.debug("sending resize command failed", e); result.setResult(e.toString()); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorHostListener.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorHostListener.java index da458002f6d..534431ed681 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorHostListener.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/provider/LinstorHostListener.java @@ -18,7 +18,6 @@ package org.apache.cloudstack.storage.datastore.provider; import com.cloud.exception.StorageConflictException; import com.cloud.host.HostVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; public class LinstorHostListener extends DefaultHostListener { @Override @@ -28,7 +27,6 @@ public class LinstorHostListener extends DefaultHostListener { host.setParent(host.getName()); hostDao.update(host.getId(), host); } - StoragePoolVO pool = primaryStoreDao.findById(poolId); - return super.hostConnect(host, pool); + return super.hostConnect(hostId, poolId); } } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 443bba8e05b..652480259c9 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -5291,6 +5291,8 @@ public class ApiResponseHelper implements ResponseGenerator { response.setMemory(instance.getMemory()); response.setOperatingSystemId(instance.getOperatingSystemId()); response.setOperatingSystem(instance.getOperatingSystem()); + response.setBootMode(instance.getBootMode()); + response.setBootType(instance.getBootType()); response.setObjectName("unmanagedinstance"); if (instance.getDisks() != null) { diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 9fc7fc589e5..79165293ff6 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -4569,14 +4569,24 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati String endIP = cmd.getEndIp(); final String newVlanGateway = cmd.getGateway(); final String newVlanNetmask = cmd.getNetmask(); + Long networkId = cmd.getNetworkID(); + Long physicalNetworkId = cmd.getPhysicalNetworkId(); + + // Verify that network exists + Network network = getNetwork(networkId); + if (network != null) { + zoneId = network.getDataCenterId(); + physicalNetworkId = network.getPhysicalNetworkId(); + } + String vlanId = cmd.getVlan(); + vlanId = verifyAndUpdateVlanId(vlanId, network); + // TODO decide if we should be forgiving or demand a valid and complete URI if (!(vlanId == null || "".equals(vlanId) || vlanId.startsWith(BroadcastDomainType.Vlan.scheme()))) { vlanId = BroadcastDomainType.Vlan.toUri(vlanId).toString(); } final Boolean forVirtualNetwork = cmd.isForVirtualNetwork(); - Long networkId = cmd.getNetworkID(); - Long physicalNetworkId = cmd.getPhysicalNetworkId(); final String accountName = cmd.getAccountName(); final Long projectId = cmd.getProjectId(); final Long domainId = cmd.getDomainId(); @@ -4649,18 +4659,6 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } - // Verify that network exists - Network network = null; - if (networkId != null) { - network = _networkDao.findById(networkId); - if (network == null) { - throw new InvalidParameterValueException("Unable to find network by id " + networkId); - } else { - zoneId = network.getDataCenterId(); - physicalNetworkId = network.getPhysicalNetworkId(); - } - } - // Verify that zone exists final DataCenterVO zone = _zoneDao.findById(zoneId); if (zone == null) { @@ -4799,6 +4797,32 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati ip6Cidr, domain, vlanOwner, network, sameSubnet, cmd.isForNsx()); } + private Network getNetwork(Long networkId) { + if (networkId == null) { + return null; + } + + Network network = _networkDao.findById(networkId); + if (network == null) { + throw new InvalidParameterValueException("Unable to find network by id " + networkId); + } + + return network; + } + + private String verifyAndUpdateVlanId(String vlanId, Network network) { + if (!StringUtils.isBlank(vlanId)) { + return vlanId; + } + + if (network == null || network.getTrafficType() != TrafficType.Guest) { + return Vlan.UNTAGGED; + } + + boolean connectivityWithoutVlan = isConnectivityWithoutVlan(network); + return getNetworkVlanId(network, connectivityWithoutVlan); + } + private Vlan commitVlan(final Long zoneId, final Long podId, final String startIP, final String endIP, final String newVlanGatewayFinal, final String newVlanNetmaskFinal, final String vlanId, final Boolean forVirtualNetwork, final Boolean forSystemVms, final Long networkId, final Long physicalNetworkId, final String startIPv6, final String endIPv6, final String ip6Gateway, final String ip6Cidr, final Domain domain, final Account vlanOwner, final Network network, final Pair> sameSubnet, boolean forNsx) { @@ -5007,28 +5031,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati // same as network's vlan // 2) if vlan is missing, default it to the guest network's vlan if (network.getTrafficType() == TrafficType.Guest) { - String networkVlanId = null; - boolean connectivityWithoutVlan = false; - if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Connectivity)) { - Map connectivityCapabilities = _networkModel.getNetworkServiceCapabilities(network.getId(), Service.Connectivity); - connectivityWithoutVlan = MapUtils.isNotEmpty(connectivityCapabilities) && connectivityCapabilities.containsKey(Capability.NoVlan); - } - - final URI uri = network.getBroadcastUri(); - if (connectivityWithoutVlan) { - networkVlanId = network.getBroadcastDomainType().toUri(network.getUuid()).toString(); - } else if (uri != null) { - // Do not search for the VLAN tag when the network doesn't support VLAN - if (uri.toString().startsWith("vlan")) { - final String[] vlan = uri.toString().split("vlan:\\/\\/"); - networkVlanId = vlan[1]; - // For pvlan - if (network.getBroadcastDomainType() != BroadcastDomainType.Vlan) { - networkVlanId = networkVlanId.split("-")[0]; - } - } - } - + boolean connectivityWithoutVlan = isConnectivityWithoutVlan(network); + String networkVlanId = getNetworkVlanId(network, connectivityWithoutVlan); if (vlanId != null && !connectivityWithoutVlan) { // if vlan is specified, throw an error if it's not equal to // network's vlanId @@ -5160,6 +5164,36 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati return vlan; } + private boolean isConnectivityWithoutVlan(Network network) { + boolean connectivityWithoutVlan = false; + if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Connectivity)) { + Map connectivityCapabilities = _networkModel.getNetworkServiceCapabilities(network.getId(), Service.Connectivity); + connectivityWithoutVlan = MapUtils.isNotEmpty(connectivityCapabilities) && connectivityCapabilities.containsKey(Capability.NoVlan); + } + return connectivityWithoutVlan; + } + + private String getNetworkVlanId(Network network, boolean connectivityWithoutVlan) { + String networkVlanId = null; + if (connectivityWithoutVlan) { + return network.getBroadcastDomainType().toUri(network.getUuid()).toString(); + } + + final URI uri = network.getBroadcastUri(); + if (uri != null) { + // Do not search for the VLAN tag when the network doesn't support VLAN + if (uri.toString().startsWith("vlan")) { + final String[] vlan = uri.toString().split("vlan:\\/\\/"); + networkVlanId = vlan[1]; + // For pvlan + if (network.getBroadcastDomainType() != BroadcastDomainType.Vlan) { + networkVlanId = networkVlanId.split("-")[0]; + } + } + } + return networkVlanId; + } + private void checkZoneVlanIpOverlap(DataCenterVO zone, Network network, String newCidr, String vlanId, String vlanGateway, String vlanNetmask, String startIP, String endIP) { // Throw an exception if this subnet overlaps with subnet on other VLAN, // if this is ip range extension, gateway, network mask should be same and ip range should not overlap diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 4dd07494493..69728812fc2 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -1030,7 +1030,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement } else if (intervalTypeStr != null && volumeId != null) { Type type = SnapshotVO.getSnapshotType(intervalTypeStr); if (type == null) { - throw new InvalidParameterValueException("Unsupported snapstho interval type " + intervalTypeStr); + throw new InvalidParameterValueException("Unsupported snapshot interval type " + intervalTypeStr); } sc.setParameters("snapshotTypeEQ", type.ordinal()); } else { diff --git a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java index d64a42efbec..c46f1f43fd0 100644 --- a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java +++ b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java @@ -26,8 +26,6 @@ import java.util.TimeZone; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.domain.Domain; -import com.cloud.utils.DateUtil; import org.apache.cloudstack.api.command.admin.usage.GenerateUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.RemoveRawUsageRecordsCmd; @@ -42,6 +40,7 @@ import org.jetbrains.annotations.NotNull; import org.springframework.stereotype.Component; import com.cloud.configuration.Config; +import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InvalidParameterValueException; @@ -58,6 +57,8 @@ import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.security.SecurityGroupVO; import com.cloud.network.security.dao.SecurityGroupDao; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.projects.Project; import com.cloud.projects.ProjectManager; import com.cloud.storage.SnapshotVO; @@ -72,6 +73,7 @@ import com.cloud.user.Account; import com.cloud.user.AccountService; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; +import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; import com.cloud.utils.component.Manager; import com.cloud.utils.component.ManagerBase; @@ -121,6 +123,8 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag private IPAddressDao _ipDao; @Inject private HostDao _hostDao; + @Inject + private NetworkOfferingDao _networkOfferingDao; public UsageServiceImpl() { } @@ -245,6 +249,7 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag } Long usageDbId = null; + boolean offeringExistsForNetworkOfferingType = false; switch (usageType.intValue()) { case UsageTypes.NETWORK_BYTES_RECEIVED: @@ -318,13 +323,19 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag usageDbId = ip.getId(); } break; + case UsageTypes.NETWORK_OFFERING: + NetworkOfferingVO networkOffering = _networkOfferingDao.findByUuidIncludingRemoved(usageId); + if (networkOffering != null) { + offeringExistsForNetworkOfferingType = true; + sc.addAnd("offeringId", SearchCriteria.Op.EQ, networkOffering.getId()); + } default: break; } if (usageDbId != null) { sc.addAnd("usageId", SearchCriteria.Op.EQ, usageDbId); - } else { + } else if (!offeringExistsForNetworkOfferingType) { // return an empty list if usageId was not found return new Pair, Integer>(new ArrayList(), new Integer(0)); } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 4eaaabd028a..4e77aa26ea6 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -8444,6 +8444,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir getRootVolumeSizeForVmRestore(newVol, template, userVm, diskOffering, details, true); volumeMgr.saveVolumeDetails(newVol.getDiskOfferingId(), newVol.getId()); + newVol = _volsDao.findById(newVol.getId()); // 1. Save usage event and update resource count for user vm volumes try { @@ -8543,7 +8544,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir Long getRootVolumeSizeForVmRestore(Volume vol, VMTemplateVO template, UserVmVO userVm, DiskOffering diskOffering, Map details, boolean update) { VolumeVO resizedVolume = (VolumeVO) vol; - Long size = null; if (template != null && template.getSize() != null) { UserVmDetailVO vmRootDiskSizeDetail = userVmDetailsDao.findDetail(userVm.getId(), VmDetailConstants.ROOT_DISK_SIZE); diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java index 772b5590411..8873d58de60 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java @@ -55,6 +55,8 @@ import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.Ipv6GuestPrefixSubnetNetworkMapDao; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.offering.DiskOffering; @@ -196,6 +198,8 @@ public class ConfigurationManagerTest { @Mock HostPodDao _podDao; @Mock + NetworkDao _networkDao; + @Mock PhysicalNetworkDao _physicalNetworkDao; @Mock ImageStoreDao _imageStoreDao; @@ -1331,6 +1335,8 @@ public class ConfigurationManagerTest { public void testWrongIpv6CreateVlanAndPublicIpRange() { CreateVlanIpRangeCmd cmd = Mockito.mock(CreateVlanIpRangeCmd.class); Mockito.when(cmd.getIp6Cidr()).thenReturn("fd17:5:8a43:e2a4:c000::/66"); + NetworkVO network = Mockito.mock(NetworkVO.class); + Mockito.when(_networkDao.findById(Mockito.anyLong())).thenReturn(network); try { configurationMgr.createVlanAndPublicIpRange(cmd); } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException e) { diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 04f9c476d47..25d43388b65 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -1089,6 +1089,12 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar try { _itMgr.expunge(ssvm.getUuid()); + ssvm.setPublicIpAddress(null); + ssvm.setPublicMacAddress(null); + ssvm.setPublicNetmask(null); + ssvm.setPrivateMacAddress(null); + ssvm.setPrivateIpAddress(null); + _secStorageVmDao.update(ssvm.getId(), ssvm); _secStorageVmDao.remove(ssvm.getId()); HostVO host = _hostDao.findByTypeNameAndZoneId(ssvm.getDataCenterId(), ssvm.getHostName(), Host.Type.SecondaryStorageVM); if (host != null) { @@ -1373,7 +1379,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar @Override public void finalizeExpunge(VirtualMachine vm) { SecondaryStorageVmVO ssvm = _secStorageVmDao.findByUuid(vm.getUuid()); - + ssvm.setPrivateMacAddress(null); ssvm.setPublicIpAddress(null); ssvm.setPublicMacAddress(null); ssvm.setPublicNetmask(null); diff --git a/test/integration/smoke/test_restore_vm.py b/test/integration/smoke/test_restore_vm.py index aac33460da1..3798bef852a 100644 --- a/test/integration/smoke/test_restore_vm.py +++ b/test/integration/smoke/test_restore_vm.py @@ -148,9 +148,13 @@ class TestRestoreVM(cloudstackTestCase): self.assertEqual(root_vol.state, 'Ready', "Volume should be in Ready state") self.assertEqual(root_vol.size, 16 * 1024 * 1024 * 1024, "Size of volume and custom disk size should match") - old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id)[0] - self.assertEqual(old_root_vol.state, "Destroy", "Old volume should be in Destroy state") - Volume.delete(old_root_vol, self.apiclient) + if self.hypervisor.lower() == "vmware": + old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id) + self.assertEqual(old_root_vol, None, "Old volume should be deleted") + else: + old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id)[0] + self.assertEqual(old_root_vol.state, "Destroy", "Old volume should be in Destroy state") + Volume.delete(old_root_vol, self.apiclient) @attr(tags=["advanced", "basic"], required_hardware="false") def test_04_restore_vm_allocated_root(self): diff --git a/test/integration/smoke/test_ssvm.py b/test/integration/smoke/test_ssvm.py index ad03c3d46e1..0784bc3820c 100644 --- a/test/integration/smoke/test_ssvm.py +++ b/test/integration/smoke/test_ssvm.py @@ -988,6 +988,9 @@ class TestSSVMs(cloudstackTestCase): # Private IP Address of System VMs are allowed to change after reboot - CLOUDSTACK-7745 + # Agent in Up state for a while after reboot, wait for the agent to Disconnect and back Up. + time.sleep(60) + # Wait for the agent to be up self.waitForSystemVMAgent(cpvm_response.name) @@ -1103,6 +1106,9 @@ class TestSSVMs(cloudstackTestCase): "Check whether CPVM is running or not" ) + # Agent in Up state for a while after reboot, wait for the agent to Disconnect and back Up. + time.sleep(60) + # Wait for the agent to be up self.waitForSystemVMAgent(cpvm_response.name) diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index bde48c87616..edacf163db4 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -1080,7 +1080,7 @@ test_data = { "format": "vhd", "hypervisor": "xenserver", "ostype": "Other Linux (64-bit)", - "url": "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64-azure.vhd.tar.gz", + "url": "https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64-azure.vhd.tar.gz", "requireshvm": "True", "ispublic": "True", "isextractable": "True" @@ -1091,10 +1091,10 @@ test_data = { "format": "ova", "hypervisor": "vmware", "ostype": "Other Linux (64-bit)", - "url": "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.ova", + "url": "https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64.ova", "requireshvm": "True", "ispublic": "True", - "deployasis": "True" + "deployasis": "False" }, }, "test_ovf_templates": [ diff --git a/ui/.env.qa b/ui/.env.qa new file mode 100644 index 00000000000..d32df4035d7 --- /dev/null +++ b/ui/.env.qa @@ -0,0 +1 @@ +CS_URL=https://qa.cloudstack.cloud/simulator/pr/10580 diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 0c7c64b3183..c59a292681a 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -2119,6 +2119,7 @@ "label.sharewith": "Share with", "label.showing": "Showing", "label.show.usage.records": "Show usage records", +"label.showing.results.for": "Showing results for \"%x\"", "label.shrinkok": "Shrink OK", "label.shutdown": "Shutdown", "label.shutdown.provider": "Shutdown provider", diff --git a/ui/src/components/header/ProjectMenu.vue b/ui/src/components/header/ProjectMenu.vue index 6fb0c3af350..590a8a2fbd0 100644 --- a/ui/src/components/header/ProjectMenu.vue +++ b/ui/src/components/header/ProjectMenu.vue @@ -17,112 +17,75 @@ diff --git a/ui/src/components/widgets/InfiniteScrollSelect.vue b/ui/src/components/widgets/InfiniteScrollSelect.vue new file mode 100644 index 00000000000..f97faf390f8 --- /dev/null +++ b/ui/src/components/widgets/InfiniteScrollSelect.vue @@ -0,0 +1,298 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/AutogenView.vue b/ui/src/views/AutogenView.vue index 7d7d5ddb6b7..67d955723f9 100644 --- a/ui/src/views/AutogenView.vue +++ b/ui/src/views/AutogenView.vue @@ -919,7 +919,7 @@ export default { if (['listVirtualMachinesMetrics'].includes(this.apiName) && this.dataView) { delete params.details delete params.isvnf - params.details = 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp' + params.details = 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff' } this.loading = true diff --git a/ui/src/views/compute/EditVM.vue b/ui/src/views/compute/EditVM.vue index fad1dcc254e..834e11c3773 100644 --- a/ui/src/views/compute/EditVM.vue +++ b/ui/src/views/compute/EditVM.vue @@ -202,7 +202,7 @@ export default { }, fetchZoneDetails () { api('listZones', { - zoneid: this.resource.zoneid + id: this.resource.zoneid }).then(response => { const zone = response?.listzonesresponse?.zone || [] this.securityGroupsEnabled = zone?.[0]?.securitygroupsenabled || this.$store.getters.showSecurityGroups @@ -336,10 +336,8 @@ export default { params.name = values.name params.displayname = values.displayname params.ostypeid = values.ostypeid - if (this.securityGroupsEnabled) { - if (values.securitygroupids) { - params.securitygroupids = values.securitygroupids - } + if (this.securityGroupsEnabled && Array.isArray(values.securitygroupids) && values.securitygroupids.length > 0) { + params.securitygroupids = values.securitygroupids } if (values.isdynamicallyscalable !== undefined) { params.isdynamicallyscalable = values.isdynamicallyscalable diff --git a/ui/src/views/network/CreateNetworkPermission.vue b/ui/src/views/network/CreateNetworkPermission.vue index 037e91eb9fd..6d73bb07ca3 100644 --- a/ui/src/views/network/CreateNetworkPermission.vue +++ b/ui/src/views/network/CreateNetworkPermission.vue @@ -29,47 +29,27 @@ - - - - - - {{ opt.name || opt.description }} - - - + api="listAccounts" + :apiParams="accountsApiParams" + resourceType="account" + defaultIcon="team-outlined" /> - - - - - - {{ opt.name || opt.description }} - - - + api="listProjects" + :apiParams="projectsApiParams" + resourceType="project" + defaultIcon="project-outlined" />