From ddd2fcaf2d73bb8398a1d4d7ad2c167df48fa8ab Mon Sep 17 00:00:00 2001 From: Daniel Augusto Veronezi Salvador <38945620+GutoVeronezi@users.noreply.github.com> Date: Tue, 23 Nov 2021 07:22:21 -0300 Subject: [PATCH] Refactor few process of VirtualMachineManagerImpl and improve logs (#4966) * Improve logs * Remove unnecessary comments * Use diamond inference * Fix some logs * Remove unnecessary unboxing * Create method to handle job result * Remove unused vars and fix some logics * Extract code to method and few adjusts * Use CollectionUtils * Extract pending work job validation to method * Create new constructors * Extract work job and info creation to a method * Extract submit async job to a method * Extract find vm by id to a method * Change log level from trace to debug * Remove unnused methods and add logs * Undo code remotion * Remove asserts and fix conditionals * Address @GabrielBrascher reviews * Remove double quotes from keys in manual json * Undo code remotion * Add object to log * Remove statement from try/catch * Implement toString with ReflectionToStringBuilderUtils * Fix errors related to merge main Co-authored-by: Daniel Augusto Veronezi Salvador --- .../main/java/com/cloud/vm/NicProfile.java | 13 +- .../src/main/java/com/cloud/vm/VmWork.java | 7 + .../cloud/vm/VirtualMachineManagerImpl.java | 1802 ++++++----------- .../com/cloud/vm/VmWorkAddVmToNetwork.java | 6 + .../main/java/com/cloud/vm/VmWorkMigrate.java | 27 +- .../java/com/cloud/vm/VmWorkMigrateAway.java | 5 + .../com/cloud/vm/VmWorkMigrateForScale.java | 5 + .../cloud/vm/VmWorkMigrateWithStorage.java | 7 + .../main/java/com/cloud/vm/VmWorkReboot.java | 5 + .../java/com/cloud/vm/VmWorkReconfigure.java | 8 + .../com/cloud/vm/VmWorkRemoveNicFromVm.java | 5 + .../cloud/vm/VmWorkRemoveVmFromNetwork.java | 6 + .../main/java/com/cloud/vm/VmWorkRestore.java | 5 + .../main/java/com/cloud/vm/VmWorkStart.java | 4 + .../main/java/com/cloud/vm/VmWorkStop.java | 5 + .../com/cloud/vm/VmWorkStorageMigration.java | 7 +- .../com/cloud/vm/VmWorkUpdateDefaultNic.java | 6 + .../java/com/cloud/storage/VMTemplateVO.java | 10 +- .../com/cloud/agent/manager/Commands.java | 7 + 19 files changed, 745 insertions(+), 1195 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/NicProfile.java b/api/src/main/java/com/cloud/vm/NicProfile.java index ebbe6d88bfe..7781d710d3c 100644 --- a/api/src/main/java/com/cloud/vm/NicProfile.java +++ b/api/src/main/java/com/cloud/vm/NicProfile.java @@ -20,6 +20,7 @@ import java.io.Serializable; import java.net.URI; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import com.cloud.network.Network; import com.cloud.network.Networks.AddressFormat; @@ -430,16 +431,6 @@ public class NicProfile implements InternalIdentity, Serializable { @Override public String toString() { - return new StringBuilder("NicProfile[").append(id) - .append("-") - .append(vmId) - .append("-") - .append(reservationId) - .append("-") - .append(iPv4Address) - .append("-") - .append(broadcastUri) - .append("]") - .toString(); + return String.format("NicProfile %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "vmId", "reservationId", "iPv4Address", "broadcastUri")); } } diff --git a/engine/components-api/src/main/java/com/cloud/vm/VmWork.java b/engine/components-api/src/main/java/com/cloud/vm/VmWork.java index ed9f44cb9bb..a0b5e979524 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/VmWork.java +++ b/engine/components-api/src/main/java/com/cloud/vm/VmWork.java @@ -34,6 +34,13 @@ public class VmWork implements Serializable { this.handlerName = handlerName; } + public VmWork(VmWork vmWork) { + this.userId = vmWork.getUserId(); + this.accountId = vmWork.getAccountId(); + this.vmId = vmWork.getVmId(); + this.handlerName = vmWork.getHandlerName(); + } + public long getUserId() { return userId; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 72d2e3dc1f1..0c8736bbe71 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -39,6 +39,7 @@ import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -59,7 +60,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; import org.apache.cloudstack.framework.jobs.AsyncJobManager; @@ -340,8 +340,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac @Inject private UserVmDetailsDao userVmDetailsDao; @Inject - private ConfigurationDao _configDao; - @Inject private VolumeOrchestrationService volumeMgr; @Inject private DeploymentPlanningManager _dpMgr; @@ -374,7 +372,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this); - Map _vmGurus = new HashMap(); + Map _vmGurus = new HashMap<>(); protected StateMachine2 _stateMachine; static final ConfigKey StartRetry = new ConfigKey("Advanced", Integer.class, "start.retry", "10", @@ -488,13 +486,14 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac s_logger.debug("Allocating disks for " + vmFinal); } + String rootVolumeName = String.format("ROOT-%s", vmFinal.getId()); if (template.getFormat() == ImageFormat.ISO) { - volumeMgr.allocateRawVolume(Type.ROOT, "ROOT-" + vmFinal.getId(), rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), + volumeMgr.allocateRawVolume(Type.ROOT, rootVolumeName, rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), vmFinal, template, owner, null); } else if (template.getFormat() == ImageFormat.BAREMETAL) { - // Do nothing + s_logger.debug(String.format("%s has format [%s]. Skipping ROOT volume [%s] allocation.", template.toString(), ImageFormat.BAREMETAL, rootVolumeName)); } else { - volumeMgr.allocateTemplatedVolumes(Type.ROOT, "ROOT-" + vmFinal.getId(), rootDiskOfferingInfo.getDiskOffering(), rootDiskSizeFinal, + volumeMgr.allocateTemplatedVolumes(Type.ROOT, rootVolumeName, rootDiskOfferingInfo.getDiskOffering(), rootDiskSizeFinal, rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), template, vmFinal, owner); } @@ -526,7 +525,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac @Override public void allocate(final String vmInstanceName, final VirtualMachineTemplate template, final ServiceOffering serviceOffering, final LinkedHashMap> networks, final DeploymentPlan plan, final HypervisorType hyperType) throws InsufficientCapacityException { - allocate(vmInstanceName, template, serviceOffering, new DiskOfferingInfo(serviceOffering), new ArrayList(), networks, plan, hyperType, null, null); + allocate(vmInstanceName, template, serviceOffering, new DiskOfferingInfo(serviceOffering), new ArrayList<>(), networks, plan, hyperType, null, null); } private VirtualMachineGuru getVmGuru(final VirtualMachine vm) { @@ -588,11 +587,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); - s_logger.debug("Cleaning up NICS"); + List vmNics = profile.getNics(); + s_logger.debug(String.format("Cleaning up NICS [%s] of %s.", vmNics.stream().map(nic -> nic.toString()).collect(Collectors.joining(", ")),vm.toString())); final List nicExpungeCommands = hvGuru.finalizeExpungeNics(vm, profile.getNics()); _networkMgr.cleanupNics(profile); - s_logger.debug("Cleaning up hypervisor data structures (ex. SRs in XenServer) for managed storage"); + s_logger.debug(String.format("Cleaning up hypervisor data structures (ex. SRs in XenServer) for managed storage. Data from %s.", vm.toString())); final List volumeExpungeCommands = hvGuru.finalizeExpungeVolumes(vm); @@ -600,7 +600,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac List> targets = getTargets(hostId, vm.getId()); - if (volumeExpungeCommands != null && volumeExpungeCommands.size() > 0 && hostId != null) { + if (CollectionUtils.isNotEmpty(volumeExpungeCommands) && hostId != null) { final Commands cmds = new Commands(Command.OnError.Stop); for (final Command volumeExpungeCommand : volumeExpungeCommands) { @@ -609,23 +609,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } _agentMgr.send(hostId, cmds); - - if (!cmds.isSuccessful()) { - for (final Answer answer : cmds.getAnswers()) { - if (!answer.getResult()) { - s_logger.warn("Failed to expunge vm due to: " + answer.getDetails()); - - throw new CloudRuntimeException("Unable to expunge " + vm + " due to " + answer.getDetails()); - } - } - } + handleUnsuccessfulCommands(cmds, vm); } if (hostId != null) { volumeMgr.revokeAccess(vm.getId(), hostId); } - // Clean up volumes based on the vm's instance id volumeMgr.cleanupVolumes(vm.getId()); if (hostId != null && CollectionUtils.isNotEmpty(targets)) { @@ -634,10 +624,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final VirtualMachineGuru guru = getVmGuru(vm); guru.finalizeExpunge(vm); - //remove the overcommit details from the uservm details + userVmDetailsDao.removeDetails(vm.getId()); - // Remove deploy as-is (if any) userVmDeployAsIsDetailsDao.removeDetails(vm.getId()); // Remove comments (if any) @@ -665,7 +654,35 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (s_logger.isDebugEnabled()) { s_logger.debug("Expunged " + vm); } + } + protected void handleUnsuccessfulCommands(Commands cmds, VMInstanceVO vm) throws CloudRuntimeException { + String cmdsStr = cmds.toString(); + String vmToString = vm.toString(); + + if (cmds.isSuccessful()) { + s_logger.debug(String.format("The commands [%s] to %s were successful.", cmdsStr, vmToString)); + return; + } + + s_logger.info(String.format("The commands [%s] to %s were unsuccessful. Handling answers.", cmdsStr, vmToString)); + + Answer[] answers = cmds.getAnswers(); + if (answers == null) { + s_logger.debug(String.format("There are no answers to commands [%s] to %s.", cmdsStr, vmToString)); + return; + } + + for (Answer answer : answers) { + String details = answer.getDetails(); + if (!answer.getResult()) { + String message = String.format("Unable to expunge %s due to [%s].", vmToString, details); + s_logger.error(message); + throw new CloudRuntimeException(message); + } + + s_logger.debug(String.format("Commands [%s] to %s got answer [%s].", cmdsStr, vmToString, details)); + } } private void addAllExpungeCommandsFromList(List cmdList, Commands cmds, VMInstanceVO vm) { @@ -728,26 +745,22 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac Answer answer = _agentMgr.easySend(hostId, cmd); if (answer == null) { - String msg = "Unable to get an answer to the modify targets command"; - - s_logger.warn(msg); + s_logger.warn(String.format("Unable to get an answer to the modify targets command. Targets [%s].", cmd.getTargets().stream().map(target -> target.toString()).collect(Collectors.joining(", ")))); + return; } - else if (!answer.getResult()) { - String msg = "Unable to modify target on the following host: " + hostId; - s_logger.warn(msg); + if (!answer.getResult()) { + s_logger.warn(String.format("Unable to modify targets [%s] on the host [%s].", cmd.getTargets().stream().map(target -> target.toString()).collect(Collectors.joining(", ")), hostId)); } } @Override public boolean start() { - // TODO, initial delay is hardcoded _executor.scheduleAtFixedRate(new CleanupTask(), 5, VmJobStateReportInterval.value(), TimeUnit.SECONDS); _executor.scheduleAtFixedRate(new TransitionTask(), VmOpCleanupInterval.value(), VmOpCleanupInterval.value(), TimeUnit.SECONDS); cancelWorkItems(_nodeId); volumeMgr.cleanupStorageJobs(); - // cleanup left over place holder works _workJobDao.expungeLeftoverWorkJobs(ManagementServerNode.getManagementServerId()); return true; } @@ -786,15 +799,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac public void start(final String vmUuid, final Map params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) { try { advanceStart(vmUuid, params, planToDeploy, planner); - } catch (final ConcurrentOperationException e) { - throw new CloudRuntimeException("Unable to start a VM due to concurrent operation", e).add(VirtualMachine.class, vmUuid); - } catch (final InsufficientCapacityException e) { - throw new CloudRuntimeException("Unable to start a VM due to insufficient capacity", e).add(VirtualMachine.class, vmUuid); + } catch (ConcurrentOperationException | InsufficientCapacityException e) { + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); } catch (final ResourceUnavailableException e) { if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)){ throw new CloudRuntimeException("Network is unavailable. Please contact administrator", e).add(VirtualMachine.class, vmUuid); } - throw new CloudRuntimeException("Unable to start a VM due to unavailable resources", e).add(VirtualMachine.class, vmUuid); + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); } } @@ -815,7 +826,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac return true; } - // also check DB to get latest VM state to detect vm update from concurrent process before idle waiting to get an early exit final VMInstanceVO instance = _vmDao.findById(vm.getId()); if (instance != null && instance.getState() == State.Running) { if (s_logger.isDebugEnabled()) { @@ -862,10 +872,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (s_logger.isDebugEnabled()) { s_logger.debug("Successfully transitioned to start state for " + vm + " reservation id = " + work.getId()); } - return new Ternary(vm, context, work); + return new Ternary<>(vm, context, work); } - return new Ternary(null, null, work); + return new Ternary<>(null, null, work); } }); @@ -915,7 +925,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } protected boolean changeState(final T vm, final Event event, final Long hostId, final ItWorkVO work, final Step step) throws NoTransitionException { - // FIXME: We should do this better. Step previousStep = null; if (work != null) { previousStep = work.getStep(); @@ -936,10 +945,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final VirtualMachine vm = vmProfile.getVirtualMachine(); final long vmGroupCount = _affinityGroupVMMapDao.countAffinityGroupsForVm(vm.getId()); - if (vmGroupCount > 0) { - return true; - } - return false; + return vmGroupCount > 0; } @Override @@ -954,15 +960,14 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if ( jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - if (s_logger.isTraceEnabled()) { - s_logger.trace(String.format("start parameter value of %s == %s during dispatching", + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("start parameter value of %s == %s during dispatching", VirtualMachineProfile.Param.BootIntoSetup.getName(), (params == null?"":params.get(VirtualMachineProfile.Param.BootIntoSetup)))); } - // avoid re-entrance - VmWorkJobVO placeHolder = null; + final VirtualMachine vm = _vmDao.findByUuid(vmUuid); - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateStart(vmUuid, params, planToDeploy, planner); } finally { @@ -971,35 +976,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } } else { - if (s_logger.isTraceEnabled()) { - s_logger.trace(String.format("start parameter value of %s == %s during processing of queued job", + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("start parameter value of %s == %s during processing of queued job", VirtualMachineProfile.Param.BootIntoSetup.getName(), (params == null?"":params.get(VirtualMachineProfile.Param.BootIntoSetup)))); } final Outcome outcome = startVmThroughJobQueue(vmUuid, params, planToDeploy, planner); - try { - final VirtualMachine vm = outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution exception", e); - } + retrieveVmFromJobOutcome(outcome, vmUuid, "startVm"); - final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobResult != null) { - if (jobResult instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobResult; - } else if (jobResult instanceof ResourceUnavailableException) { - throw (ResourceUnavailableException)jobResult; - } else if (jobResult instanceof InsufficientCapacityException) { - throw (InsufficientCapacityException)jobResult; - } else if (jobResult instanceof RuntimeException) { - throw (RuntimeException)jobResult; - } else if (jobResult instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobResult); - } - } + retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); } } @@ -1083,18 +1069,14 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final DataCenterDeployment originalPlan = plan; int retry = StartRetry.value(); - while (retry-- != 0) { // It's != so that it can match -1. + while (retry-- != 0) { s_logger.debug("VM start attempt #" + (StartRetry.value() - retry)); if (reuseVolume) { - // edit plan if this vm's ROOT volume is in READY state already final List vols = _volsDao.findReadyRootVolumesByInstance(vm.getId()); for (final VolumeVO vol : vols) { - // make sure if the templateId is unchanged. If it is changed, - // let planner - // reassign pool for the volume even if it ready. final Long volTemplateId = vol.getTemplateId(); - if (volTemplateId != null && volTemplateId.longValue() != template.getId()) { + if (volTemplateId != null && volTemplateId != template.getId()) { if (s_logger.isDebugEnabled()) { s_logger.debug(vol + " of " + vm + " is READY, but template ids don't match, let the planner reassign a new pool"); } @@ -1112,9 +1094,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (planToDeploy != null && planToDeploy.getDataCenterId() != 0) { final Long clusterIdSpecified = planToDeploy.getClusterId(); if (clusterIdSpecified != null && rootVolClusterId != null) { - if (rootVolClusterId.longValue() != clusterIdSpecified.longValue()) { - // cannot satisfy the plan passed in to the - // planner + if (!rootVolClusterId.equals(clusterIdSpecified)) { if (s_logger.isDebugEnabled()) { s_logger.debug("Cannot satisfy the deployment plan passed in since the ready Root volume is in different cluster. volume's cluster: " + rootVolClusterId + ", cluster specified: " + clusterIdSpecified); @@ -1153,7 +1133,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (planChangedByVolume) { plan = originalPlan; planChangedByVolume = false; - //do not enter volume reuse for next retry, since we want to look for resources outside the volume's cluster reuseVolume = false; continue; } @@ -1161,11 +1140,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac areAffinityGroupsAssociated(vmProfile)); } - if (dest != null) { - avoids.addHost(dest.getHost().getId()); - if (!template.isDeployAsIs()) { - journal.record("Deployment found - Attempt #" + (StartRetry.value() - retry), vmProfile, dest); - } + avoids.addHost(dest.getHost().getId()); + if (!template.isDeployAsIs()) { + journal.record("Deployment found - Attempt #" + (StartRetry.value() - retry), vmProfile, dest); } long destHostId = dest.getHost().getId(); @@ -1173,7 +1150,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final Long cluster_id = dest.getCluster().getId(); final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.CPU_OVER_COMMIT_RATIO); final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); - //storing the value of overcommit in the user_vm_details table for doing a capacity check in case the cluster overcommit ratio is changed. + if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) == null && (Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f)) { userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); @@ -1202,12 +1179,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac volumeMgr.prepare(vmProfile, dest); } - //since StorageMgr succeeded in volume creation, reuse Volume for further tries until current cluster has capacity if (!reuseVolume) { reuseVolume = true; } - Commands cmds = null; vmGuru.finalizeVirtualMachineProfile(vmProfile, dest, ctx); final VirtualMachineTO vmTO = hvGuru.implement(vmProfile); @@ -1216,13 +1191,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac handlePath(vmTO.getDisks(), vm.getHypervisorType()); - cmds = new Commands(Command.OnError.Stop); + Commands cmds = new Commands(Command.OnError.Stop); cmds.addCommand(new StartCommand(vmTO, dest.getHost(), getExecuteInSequence(vm.getHypervisorType()))); vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx); - // Get VM extraConfig from DB and set to VM TO addExtraConfig(vmTO); work = _workDao.findById(work.getId()); @@ -1257,15 +1231,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new ConcurrentOperationException("Failed to deploy VM"+ vm.getUuid()); } - // Update GPU device capacity final GPUDeviceTO gpuDevice = startAnswer.getVirtualMachine().getGpuDevice(); if (gpuDevice != null) { _resourceMgr.updateGPUDetails(destHostId, gpuDevice.getGroupDetails()); } - // Remove the information on whether it was a deploy vm request.The deployvm=true information - // is set only when the vm is being deployed. When a vm is started from a stop state the - // information isn't set, if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_VM) != null) { userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.DEPLOY_VM); } @@ -1282,7 +1252,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { setupAgentSecurity(vmHost, sshAccessDetails, vm); return; - } catch (final Exception e) { + } catch (final AgentUnavailableException | OperationTimedoutException e) { s_logger.error("Retrying after catching exception while trying to secure agent for systemvm id=" + vm.getId(), e); } } @@ -1306,7 +1276,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (vm.getType() == VirtualMachine.Type.User) { final String platform = stopAns.getPlatform(); if (platform != null) { - final Map vmmetadata = new HashMap(); + final Map vmmetadata = new HashMap<>(); vmmetadata.put(vm.getInstanceName(), platform); syncVMMetaData(vmmetadata); } @@ -1334,7 +1304,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac canRetry = false; throw new AgentUnavailableException("Unable to start " + vm.getHostName(), destHostId, e); } catch (final ResourceUnavailableException e) { - s_logger.info("Unable to contact resource.", e); + s_logger.warn("Unable to contact resource.", e); if (!avoids.add(e)) { if (e.getScope() == Volume.class || e.getScope() == Nic.class) { throw e; @@ -1344,7 +1314,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } } catch (final InsufficientCapacityException e) { - s_logger.info("Insufficient capacity ", e); + s_logger.warn("Insufficient capacity ", e); if (!avoids.add(e)) { if (e.getScope() == Volume.class || e.getScope() == Nic.class) { throw e; @@ -1352,10 +1322,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac s_logger.warn("unexpected InsufficientCapacityException : " + e.getScope().getName(), e); } } - } catch (final ExecutionException e) { - s_logger.error("Failed to start instance " + vm, e); - throw new AgentUnavailableException("Unable to start instance due to " + e.getMessage(), destHostId, e); - } catch (final NoTransitionException e) { + } catch (ExecutionException | NoTransitionException e) { s_logger.error("Failed to start instance " + vm, e); throw new AgentUnavailableException("Unable to start instance due to " + e.getMessage(), destHostId, e); } catch (final StorageAccessException e) { @@ -1364,11 +1331,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (startedVm == null && canRetry) { final Step prevStep = work.getStep(); _workDao.updateStep(work, Step.Release); - // If previous step was started/ing && we got a valid answer - if ((prevStep == Step.Started || prevStep == Step.Starting) && startAnswer != null && startAnswer.getResult()) { //TODO check the response of cleanup and record it in DB for retry + + if ((prevStep == Step.Started || prevStep == Step.Starting) && startAnswer != null && startAnswer.getResult()) { cleanup(vmGuru, vmProfile, work, Event.OperationFailed, false); } else { - //if step is not starting/started, send cleanup command with force=true cleanup(vmGuru, vmProfile, work, Event.OperationFailed, true); } } @@ -1402,7 +1368,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (params == null) { return; } - StringBuffer msgBuf = new StringBuffer("Uefi params "); + + StringBuilder msgBuf = new StringBuilder("Uefi params "); boolean log = false; if (params.get(VirtualMachineProfile.Param.UefiFlag) != null) { msgBuf.append(String.format("UefiFlag: %s ", params.get(VirtualMachineProfile.Param.UefiFlag))); @@ -1445,7 +1412,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - // Add extra config data to the vmTO as a Map private void addExtraConfig(VirtualMachineTO vmTO) { Map details = vmTO.getDetails(); for (String key : details.keySet()) { @@ -1455,7 +1421,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - // for managed storage on KVM, need to make sure the path field of the volume in question is populated with the IQN private void handlePath(final DiskTO[] disks, final HypervisorType hypervisorType) { if (hypervisorType != HypervisorType.KVM) { return; @@ -1486,7 +1451,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - // for managed storage on XenServer and VMware, need to update the DB with a path if the VDI/VMDK file was newly created private void handlePath(final DiskTO[] disks, final Map> iqnToData) { if (disks != null && iqnToData != null) { for (final DiskTO disk : disks) { @@ -1540,9 +1504,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac _volsDao.update(volume.getId(), volume); } - // Use getPath() from VolumeVO to get a fresh copy of what's in the DB. - // Before doing this, in a certain situation, getPath() from VolumeObjectTO - // returned null instead of an actual path (because it was out of date with the DB). if(vol.getPath() != null) { volumeMgr.updateVolumeDiskChain(vol.getId(), vol.getPath(), vol.getChainInfo(), vol.getUpdatedDataStoreUUID()); } else { @@ -1557,9 +1518,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { advanceStop(vmUuid, false); } catch (final OperationTimedoutException e) { - throw new AgentUnavailableException("Unable to stop vm because the operation to stop timed out", e.getAgentId(), e); + throw new AgentUnavailableException(String.format("Unable to stop vm [%s] because the operation to stop timed out", vmUuid), e.getAgentId(), e); } catch (final ConcurrentOperationException e) { - throw new CloudRuntimeException("Unable to stop vm because of a concurrent operation", e); + throw new CloudRuntimeException(String.format("Unable to stop vm because of a concurrent operation", vmUuid), e); } } @@ -1569,22 +1530,30 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { advanceStop(vmUuid, true); } catch (final OperationTimedoutException e) { - throw new AgentUnavailableException("Unable to stop vm because the operation to stop timed out", e.getAgentId(), e); + throw new AgentUnavailableException(String.format("Unable to stop vm [%s] because the operation to stop timed out", vmUuid), e.getAgentId(), e); } catch (final ConcurrentOperationException e) { - throw new CloudRuntimeException("Unable to stop vm because of a concurrent operation", e); + throw new CloudRuntimeException(String.format("Unable to stop vm [%s] because of a concurrent operation", vmUuid), e); } } @Override public boolean getExecuteInSequence(final HypervisorType hypervisorType) { - if (HypervisorType.KVM == hypervisorType || HypervisorType.XenServer == hypervisorType || HypervisorType.Hyperv == hypervisorType || HypervisorType.LXC == hypervisorType) { - return false; - } else if (HypervisorType.VMware == hypervisorType) { - final Boolean fullClone = HypervisorGuru.VmwareFullClone.value(); - return fullClone; - } else { + if (null == hypervisorType) { return ExecuteInSequence.value(); } + + switch (hypervisorType) { + case KVM: + case XenServer: + case Hyperv: + case LXC: + return false; + case VMware: + final Boolean fullClone = HypervisorGuru.VmwareFullClone.value(); + return fullClone; + default: + return ExecuteInSequence.value(); + } } @Override @@ -1610,7 +1579,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); - final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); final VirtualMachineGuru guru = getVmGuru(vm); try { @@ -1660,7 +1628,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac * - If 'unmanage.vm.preserve.nics' = false: then the NICs are removed while unmanaging */ private void unmanageVMNics(VirtualMachineProfile profile, VMInstanceVO vm) { - s_logger.debug("Cleaning up NICs"); + s_logger.debug(String.format("Cleaning up NICs of %s.", vm.toString())); Boolean preserveNics = UnmanagedVMsManager.UnmanageVMPreserveNic.valueIn(vm.getDataCenterId()); if (BooleanUtils.isTrue(preserveNics)) { s_logger.debug("Preserve NICs configuration enabled"); @@ -1747,11 +1715,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac return false; } - } catch (final AgentUnavailableException e) { - if (!force) { - return false; - } - } catch (final OperationTimedoutException e) { + } catch (final AgentUnavailableException | OperationTimedoutException e) { + s_logger.warn(String.format("Unable to stop %s due to [%s].", vm.toString(), e.getMessage()), e); if (!force) { return false; } @@ -1856,7 +1821,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - // avoid re-entrance VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); @@ -1872,27 +1836,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else { final Outcome outcome = stopVmThroughJobQueue(vmUuid, cleanUpEvenIfUnableToStop); - try { - final VirtualMachine vm = outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); - } + retrieveVmFromJobOutcome(outcome, vmUuid, "stopVm"); - final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobResult != null) { - if (jobResult instanceof AgentUnavailableException) { - throw (AgentUnavailableException)jobResult; - } else if (jobResult instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobResult; - } else if (jobResult instanceof OperationTimedoutException) { - throw (OperationTimedoutException)jobResult; - } else if (jobResult instanceof RuntimeException) { - throw (RuntimeException)jobResult; - } else if (jobResult instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobResult); - } + try { + retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + } catch (ResourceUnavailableException | InsufficientCapacityException ex) { + throw new RuntimeException("Unexpected exception", ex); } } } @@ -1977,7 +1926,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } return; } - // grab outstanding work item if any + final ItWorkVO work = _workDao.findByOutstandingWork(vm.getId(), vm.getState()); if (work != null) { if (s_logger.isDebugEnabled()) { @@ -1997,7 +1946,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } catch (final NoTransitionException e) { s_logger.warn(e.getMessage()); } - // mark outstanding work item if any as done + if (work != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Updating work item to Done, id:" + work.getId()); @@ -2019,7 +1968,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { if (!stateTransitTo(vm, Event.StopRequested, vm.getHostId())) { - throw new ConcurrentOperationException("VM is being operated on."); + throw new ConcurrentOperationException(String.format("%s is being operated on.", vm.toString())); } } catch (final NoTransitionException e1) { if (!cleanUpEvenIfUnableToStop) { @@ -2027,7 +1976,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } final boolean doCleanup = true; if (s_logger.isDebugEnabled()) { - s_logger.debug("Unable to transition the state but we're moving on because it's forced stop"); + s_logger.warn("Unable to transition the state but we're moving on because it's forced stop", e1); } if (doCleanup) { @@ -2096,10 +2045,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new CloudRuntimeException("Invalid answer received in response to a StopCommand on " + vm.instanceName); } - } catch (final AgentUnavailableException e) { - s_logger.warn("Unable to stop vm, agent unavailable: " + e.toString()); - } catch (final OperationTimedoutException e) { - s_logger.warn("Unable to stop vm, operation timed out: " + e.toString()); + } catch (AgentUnavailableException | OperationTimedoutException e) { + s_logger.warn(String.format("Unable to stop %s due to [%s].", profile.toString(), e.toString()), e); } finally { if (!stopped) { if (!cleanUpEvenIfUnableToStop) { @@ -2107,7 +2054,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { stateTransitTo(vm, Event.OperationFailed, vm.getHostId()); } catch (final NoTransitionException e) { - s_logger.warn("Unable to transition the state " + vm); + s_logger.warn("Unable to transition the state " + vm, e); } throw new CloudRuntimeException("Unable to stop " + vm); } else { @@ -2143,7 +2090,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac boolean result = stateTransitTo(vm, Event.OperationSucceeded, null); if (result) { if (VirtualMachine.Type.User.equals(vm.type) && ResourceCountRunningVMsonly.value()) { - //update resource count if stop successfully ServiceOfferingVO offering = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()); resourceCountDecrement(vm.getAccountId(),new Long(offering.getCpu()), new Long(offering.getRamSize())); } @@ -2151,8 +2097,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new CloudRuntimeException("unable to stop " + vm); } } catch (final NoTransitionException e) { - s_logger.warn(e.getMessage()); - throw new CloudRuntimeException("Unable to stop " + vm); + String message = String.format("Unable to stop %s due to [%s].", vm.toString(), e.getMessage()); + s_logger.warn(message, e); + throw new CloudRuntimeException(message, e); } } @@ -2161,10 +2108,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } protected boolean stateTransitTo(final VMInstanceVO vm, final VirtualMachine.Event e, final Long hostId, final String reservationId) throws NoTransitionException { - // if there are active vm snapshots task, state change is not allowed - vm.setReservationId(reservationId); - return _stateMachine.transitTo(vm, e, new Pair(vm.getHostId(), hostId), _vmDao); + return _stateMachine.transitTo(vm, e, new Pair<>(vm.getHostId(), hostId), _vmDao); } @Override @@ -2181,7 +2126,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac vm.setLastHostId(vm.getHostId()); } } - return _stateMachine.transitTo(vm, e, new Pair(vm.getHostId(), hostId), _vmDao); + return _stateMachine.transitTo(vm, e, new Pair<>(vm.getHostId(), hostId), _vmDao); } @Override @@ -2219,8 +2164,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } } catch (final NoTransitionException e) { - s_logger.debug(e.getMessage()); - throw new CloudRuntimeException("Unable to destroy " + vm, e); + String message = String.format("Unable to destroy %s due to [%s].", vm.toString(), e.getMessage()); + s_logger.debug(message, e); + throw new CloudRuntimeException(message, e); } } }); @@ -2266,7 +2212,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (command != null) { RestoreVMSnapshotAnswer restoreVMSnapshotAnswer = (RestoreVMSnapshotAnswer) _agentMgr.send(hostId, command); if (restoreVMSnapshotAnswer == null || !restoreVMSnapshotAnswer.getResult()) { - s_logger.warn("Unable to restore the vm snapshot from image file after live migration of vm with vmsnapshots: " + restoreVMSnapshotAnswer.getDetails()); + s_logger.warn("Unable to restore the vm snapshot from image file after live migration of vm with vmsnapshots: " + restoreVMSnapshotAnswer == null ? "null answer" : restoreVMSnapshotAnswer.getDetails()); } } } @@ -2278,10 +2224,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac public void storageMigration(final String vmUuid, final Map volumeToPool) { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - // avoid re-entrance - VmWorkJobVO placeHolder = null; + final VirtualMachine vm = _vmDao.findByUuid(vmUuid); - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateStorageMigration(vmUuid, volumeToPool); } finally { @@ -2292,21 +2237,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else { final Outcome outcome = migrateVmStorageThroughJobQueue(vmUuid, volumeToPool); - try { - final VirtualMachine vm = outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); - } + retrieveVmFromJobOutcome(outcome, vmUuid, "migrateVmStorage"); - final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobResult != null) { - if (jobResult instanceof RuntimeException) { - throw (RuntimeException)jobResult; - } else if (jobResult instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobResult); - } + try { + retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + } catch (ResourceUnavailableException | InsufficientCapacityException ex) { + throw new RuntimeException("Unexpected exception", ex); } } } @@ -2326,17 +2262,17 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac migrateThroughHypervisorOrStorage(vm, volumeToPoolMap); } catch (ConcurrentOperationException - | InsufficientCapacityException // possibly InsufficientVirtualNetworkCapacityException or InsufficientAddressCapacityException + | InsufficientCapacityException | StorageUnavailableException e) { String msg = String.format("Failed to migrate VM: %s", vmUuid); - s_logger.debug(msg); + s_logger.warn(msg, e); throw new CloudRuntimeException(msg, e); } finally { try { stateTransitTo(vm, Event.AgentReportStopped, null); } catch (final NoTransitionException e) { String anotherMEssage = String.format("failed to change vm state of VM: %s", vmUuid); - s_logger.debug(anotherMEssage); + s_logger.warn(anotherMEssage, e); throw new CloudRuntimeException(anotherMEssage, e); } } @@ -2347,18 +2283,14 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac return null; } final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); - // OfflineVmwareMigration: in case of vmware call vcenter to do it for us. - // OfflineVmwareMigration: should we check the proximity of source and destination - // OfflineVmwareMigration: if we are in the same cluster/datacentre/pool or whatever? - // OfflineVmwareMigration: we are checking on success to optionally delete an old vm if we are not + List commandsToSend = hvGuru.finalizeMigrate(vm, volumeToPool); if (CollectionUtils.isNotEmpty(commandsToSend)) { Commands commandsContainer = new Commands(Command.OnError.Stop); commandsContainer.addCommands(commandsToSend); + try { - // OfflineVmwareMigration: change to the call back variety? - // OfflineVmwareMigration: getting a Long seq to be filled with _agentMgr.send(hostId, commandsContainer, this) return _agentMgr.send(hostId, commandsContainer); } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException(String.format("Failed to migrate VM: %s", vm.getUuid()),e); @@ -2373,6 +2305,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac String msg = String.format("Cleaning up after hypervisor pool migration volumes for VM %s(%s)", vm.getInstanceName(), vm.getUuid()); s_logger.debug(msg); } + StoragePool rootVolumePool = null; if (MapUtils.isNotEmpty(volumeToPool)) { for (Map.Entry entry : volumeToPool.entrySet()) { @@ -2391,11 +2324,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } vm.setLastHostId(null); vm.setPodIdToDeployIn(rootVolumePool.getPodId()); - // OfflineVmwareMigration: a consecutive migration will fail probably (no host not pod) - }// else keep last host set for this vm + } + markVolumesInPool(vm, hypervisorMigrationResults); - // OfflineVmwareMigration: deal with answers, if (hypervisorMigrationResults.length > 0) - // OfflineVmwareMigration: iterate over the volumes for data updates } private void markVolumesInPool(VMInstanceVO vm, Answer[] hypervisorMigrationResults) { @@ -2404,8 +2335,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new CloudRuntimeException(String.format("VM ID: %s migration failed. %s", vm.getUuid(), hypervisorMigrationResults[0].getDetails())); } for (Answer answer : hypervisorMigrationResults) { - if (s_logger.isTraceEnabled()) { - s_logger.trace(String.format("Received an %s: %s", answer.getClass().getSimpleName(), answer)); + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Received an %s: %s", answer.getClass().getSimpleName(), answer)); } if (answer instanceof MigrateVmToPoolAnswer) { relevantAnswer = (MigrateVmToPoolAnswer) answer; @@ -2449,7 +2380,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac Answer[] hypervisorMigrationResults = attemptHypervisorMigration(vm, volumeToPool, sourceHostId); boolean migrationResult = false; if (hypervisorMigrationResults == null) { - // OfflineVmwareMigration: if the HypervisorGuru can't do it, let the volume manager take care of it. migrationResult = volumeMgr.storageMigration(profile, volumeToPool); if (migrationResult) { postStorageMigrationCleanup(vm, volumeToPool, _hostDao.findById(sourceHostId), sourceClusterId); @@ -2464,7 +2394,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private Map prepareVmStorageMigration(VMInstanceVO vm, Map volumeToPool) { Map volumeToPoolMap = new HashMap<>(); if (MapUtils.isEmpty(volumeToPool)) { - throw new CloudRuntimeException("Unable to migrate vm: missing volume to pool mapping"); + throw new CloudRuntimeException(String.format("Unable to migrate %s: missing volume to pool mapping.", vm.toString())); } Cluster cluster = null; Long dataCenterId = null; @@ -2496,7 +2426,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac stateTransitTo(vm, Event.StorageMigrationRequested, null); } catch (final NoTransitionException e) { String msg = String.format("Unable to migrate vm: %s", vm.getUuid()); - s_logger.debug(msg); + s_logger.warn(msg, e); throw new CloudRuntimeException(msg, e); } return volumeToPoolMap; @@ -2504,8 +2434,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private void checkDestinationForTags(StoragePool destPool, VMInstanceVO vm) { List vols = _volsDao.findUsableVolumesForInstance(vm.getId()); - // OfflineVmwareMigration: iterate over volumes - // OfflineVmwareMigration: get disk offering + List storageTags = storageMgr.getStoragePoolTagList(destPool.getId()); for(Volume vol : vols) { DiskOfferingVO diskOffering = _diskOfferingDao.findById(vol.getDiskOfferingId()); @@ -2523,11 +2452,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } static boolean matches(List volumeTags, List storagePoolTags) { - // OfflineVmwareMigration: commons collections 4 allows for Collections.containsAll(volumeTags,storagePoolTags); boolean result = true; if (volumeTags != null) { for (String tag : volumeTags) { - // there is a volume tags so if (storagePoolTags == null || !storagePoolTags.contains(tag)) { result = false; break; @@ -2549,22 +2476,17 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } setDestinationPoolAndReallocateNetwork(rootVolumePool, vm); - //when start the vm next time, don;'t look at last_host_id, only choose the host based on volume/storage pool vm.setLastHostId(null); if (rootVolumePool != null) { vm.setPodIdToDeployIn(rootVolumePool.getPodId()); } - // If VM was cold migrated between clusters belonging to two different VMware DCs, - // unregister the VM from the source host and cleanup the associated VM files. if (vm.getHypervisorType().equals(HypervisorType.VMware)) { afterStorageMigrationVmwareVMCleanup(rootVolumePool, vm, srcHost, srcClusterId); } } private void setDestinationPoolAndReallocateNetwork(StoragePool destPool, VMInstanceVO vm) throws InsufficientCapacityException { - //if the vm is migrated to different pod in basic mode, need to reallocate ip - if (destPool != null && destPool.getPodId() != null && !destPool.getPodId().equals(vm.getPodIdToDeployIn())) { if (s_logger.isDebugEnabled()) { String msg = String.format("as the pod for vm %s has changed we are reallocating its network", vm.getInstanceName()); @@ -2577,7 +2499,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } private void afterStorageMigrationVmwareVMCleanup(StoragePool destPool, VMInstanceVO vm, HostVO srcHost, Long srcClusterId) { - // OfflineVmwareMigration: this should only happen on storage migration, else the guru would already have issued the command final Long destClusterId = destPool.getClusterId(); if (srcClusterId != null && destClusterId != null && ! srcClusterId.equals(destClusterId) && srcHost != null) { final String srcDcName = _clusterDetailsDao.getVmwareDcName(srcClusterId); @@ -2588,9 +2509,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - // OfflineVmwareMigration: on port forward refator this to be done in two - // OfflineVmwareMigration: command creation in the guru.migrat method - // OfflineVmwareMigration: sending up in the attemptHypevisorMigration with execute in sequence (responsibility of the guru) private void removeStaleVmFromSource(VMInstanceVO vm, HostVO srcHost) { s_logger.debug("Since VM's storage was successfully migrated across VMware Datacenters, unregistering VM: " + vm.getInstanceName() + " from source host: " + srcHost.getId()); @@ -2598,9 +2516,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac uvc.setCleanupVmFiles(true); try { _agentMgr.send(srcHost.getId(), uvc); - } catch (final Exception e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to unregister VM: " + vm.getInstanceName() + " from source host: " + srcHost.getId() + - " after successfully migrating VM's storage across VMware Datacenters"); + " after successfully migrating VM's storage across VMware Datacenters", e); } } @@ -2610,10 +2528,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - // avoid re-entrance - VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateMigrate(vmUuid, srcHostId, dest); } finally { @@ -2624,26 +2540,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else { final Outcome outcome = migrateVmThroughJobQueue(vmUuid, srcHostId, dest); + retrieveVmFromJobOutcome(outcome, vmUuid, "migrateVm"); + try { - final VirtualMachine vm = outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); - } - - final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobResult != null) { - if (jobResult instanceof ResourceUnavailableException) { - throw (ResourceUnavailableException)jobResult; - } else if (jobResult instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobResult; - } else if (jobResult instanceof RuntimeException) { - throw (RuntimeException)jobResult; - } else if (jobResult instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobResult); - } - + retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + } catch (InsufficientCapacityException ex) { + throw new RuntimeException("Unexpected exception", ex); } } } @@ -2661,7 +2563,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac protected void migrate(final VMInstanceVO vm, final long srcHostId, final DeployDestination dest) throws ResourceUnavailableException, ConcurrentOperationException { s_logger.info("Migrating " + vm + " to " + dest); - final UserVmVO userVm = _userVmDao.findById(vm.getId()); final long dstHostId = dest.getHost().getId(); final Host fromHost = _hostDao.findById(srcHostId); if (fromHost == null) { @@ -2669,7 +2570,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new CloudRuntimeException("Unable to find the host to migrate from: " + srcHostId); } - if (fromHost.getClusterId().longValue() != dest.getCluster().getId() && vm.getHypervisorType() != HypervisorType.VMware) { + if (fromHost.getClusterId() != dest.getCluster().getId() && vm.getHypervisorType() != HypervisorType.VMware) { final List volumes = _volsDao.findCreatedByInstance(vm.getId()); for (final VolumeVO volume : volumes) { if (!_storagePoolDao.findById(volume.getPoolId()).getScope().equals(ScopeType.ZONE)) { @@ -2742,7 +2643,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac vm.setLastHostId(srcHostId); try { - if (vm == null || vm.getHostId() == null || vm.getHostId() != srcHostId || !changeState(vm, Event.MigrationRequested, dstHostId, work, Step.Migrating)) { + if (vm.getHostId() == null || vm.getHostId() != srcHostId || !changeState(vm, Event.MigrationRequested, dstHostId, work, Step.Migrating)) { _networkMgr.rollbackNicForMigration(vmSrc, profile); if (vm != null) { volumeMgr.release(vm.getId(), dstHostId); @@ -2797,7 +2698,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } catch (final OperationTimedoutException e) { if (e.isActive()) { - s_logger.warn("Active migration command so scheduling a restart for " + vm); + s_logger.warn("Active migration command so scheduling a restart for " + vm, e); _haMgr.scheduleRestart(vm, true); } throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId); @@ -2817,13 +2718,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { _agentMgr.send(srcHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null); } catch (final AgentUnavailableException e) { - s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId); + s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId, e); } cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); throw new CloudRuntimeException("Unable to complete migration for " + vm); } } catch (final OperationTimedoutException e) { - s_logger.debug("Error while checking the vm " + vm + " on host " + dstHostId, e); + s_logger.warn("Error while checking the vm " + vm + " on host " + dstHostId, e); } migrated = true; } finally { @@ -2838,7 +2739,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { _agentMgr.send(dstHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null); } catch (final AgentUnavailableException ae) { - s_logger.info("Looks like the destination Host is unavailable for cleanup"); + s_logger.warn("Looks like the destination Host is unavailable for cleanup", ae); } _networkMgr.setHypervisorHostname(profile, dest, false); try { @@ -3067,27 +2968,25 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } private void moveVmToMigratingState(final T vm, final Long hostId, final ItWorkVO work) throws ConcurrentOperationException { - // Put the vm in migrating state. try { if (!changeState(vm, Event.MigrationRequested, hostId, work, Step.Migrating)) { - s_logger.info("Migration cancelled because state has changed: " + vm); + s_logger.error("Migration cancelled because state has changed: " + vm); throw new ConcurrentOperationException("Migration cancelled because state has changed: " + vm); } } catch (final NoTransitionException e) { - s_logger.info("Migration cancelled because " + e.getMessage()); + s_logger.error("Migration cancelled because " + e.getMessage(), e); throw new ConcurrentOperationException("Migration cancelled because " + e.getMessage()); } } private void moveVmOutofMigratingStateOnSuccess(final T vm, final Long hostId, final ItWorkVO work) throws ConcurrentOperationException { - // Put the vm in running state. try { if (!changeState(vm, Event.OperationSucceeded, hostId, work, Step.Started)) { s_logger.error("Unable to change the state for " + vm); throw new ConcurrentOperationException("Unable to change the state for " + vm); } } catch (final NoTransitionException e) { - s_logger.error("Unable to change state due to " + e.getMessage()); + s_logger.error("Unable to change state due to " + e.getMessage(), e); throw new ConcurrentOperationException("Unable to change state due to " + e.getMessage()); } } @@ -3098,11 +2997,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - // avoid re-entrance - - VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateMigrateWithStorage(vmUuid, srcHostId, destHostId, volumeToPool); } finally { @@ -3110,29 +3006,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac _workJobDao.expunge(placeHolder.getId()); } } - } else { final Outcome outcome = migrateVmWithStorageThroughJobQueue(vmUuid, srcHostId, destHostId, volumeToPool); - try { - final VirtualMachine vm = outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); - } + retrieveVmFromJobOutcome(outcome, vmUuid, "migrateVmWithStorage"); - final Object jobException = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobException != null) { - if (jobException instanceof ResourceUnavailableException) { - throw (ResourceUnavailableException)jobException; - } else if (jobException instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobException; - } else if (jobException instanceof RuntimeException) { - throw (RuntimeException)jobException; - } else if (jobException instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobException); - } + try { + retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + } catch (InsufficientCapacityException ex) { + throw new RuntimeException("Unexpected exception", ex); } } } @@ -3160,11 +3042,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()), null, null); profile.setHost(destHost); - // Create a map of which volume should go in which storage pool. final Map volumeToPoolMap = createMappingVolumeAndStoragePool(profile, destHost, volumeToPool); - // If none of the volumes have to be migrated, fail the call. Administrator needs to make a call for migrating - // a vm and not migrating a vm with storage. if (volumeToPoolMap == null || volumeToPoolMap.isEmpty()) { throw new InvalidParameterValueException("Migration of the vm " + vm + "from host " + srcHost + " to destination host " + destHost + " doesn't involve migrating the volumes."); @@ -3188,19 +3067,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac work.setResourceId(destHostId); work = _workDao.persist(work); - // Put the vm in migrating state. vm.setLastHostId(srcHostId); vm.setPodIdToDeployIn(destHost.getPodId()); moveVmToMigratingState(vm, destHostId, work); - List vmData = null; boolean migrated = false; try { - - // config drive: Detach the config drive at source host - // After migration successful attach the config drive in destination host - // On migration failure VM will be stopped, So configIso will be deleted - Nic defaultNic = _networkModel.getDefaultNic(vm.getId()); if (defaultNic != null && VirtualMachine.Type.User.equals(vm.getType())) { @@ -3212,7 +3084,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (_networkModel.isSharedNetworkWithoutServices(network.getId())) { final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(), + List vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(), vm.getUuid(), defaultNic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, VirtualMachineManager.getHypervisorHostname(destination.getHost() != null ? destination.getHost().getName() : "")); String vmName = vm.getInstanceName(); @@ -3223,22 +3095,19 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac profile.setConfigDriveIsoRootFolder(configDriveIsoRootFolder); profile.setConfigDriveIsoFile(isoFile); - // At source host detach the config drive iso. AttachOrDettachConfigDriveCommand dettachCommand = new AttachOrDettachConfigDriveCommand(vm.getInstanceName(), vmData, VmConfigDriveLabel.value(), false); try { _agentMgr.send(srcHost.getId(), dettachCommand); s_logger.debug("Deleted config drive ISO for vm " + vm.getInstanceName() + " In host " + srcHost); } catch (OperationTimedoutException e) { - s_logger.debug("TIme out occured while exeuting command AttachOrDettachConfigDrive " + e.getMessage()); + s_logger.error("TIme out occured while exeuting command AttachOrDettachConfigDrive " + e.getMessage(), e); } } } - // Migrate the vm and its volume. volumeMgr.migrateVolumes(vm, to, srcHost, destHost, volumeToPoolMap); - // Put the vm back to running state. moveVmOutofMigratingStateOnSuccess(vm, destHost.getId(), work); try { @@ -3247,13 +3116,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { _agentMgr.send(srcHostId, new Commands(cleanup(vm.getInstanceName())), null); } catch (final AgentUnavailableException e) { - s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId); + s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId, e); } cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); throw new CloudRuntimeException("VM not found on desintation host. Unable to complete migration for " + vm); } } catch (final OperationTimedoutException e) { - s_logger.warn("Error while checking the vm " + vm + " is on host " + destHost, e); + s_logger.error("Error while checking the vm " + vm + " is on host " + destHost, e); } migrated = true; } finally { @@ -3338,11 +3207,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac public void migrateAway(final String vmUuid, final long srcHostId) throws InsufficientServerCapacityException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - // avoid re-entrance - - VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); try { try { orchestrateMigrateAway(vmUuid, srcHostId, null); @@ -3356,25 +3222,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else { final Outcome outcome = migrateVmAwayThroughJobQueue(vmUuid, srcHostId); - try { - final VirtualMachine vm = outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); - } + retrieveVmFromJobOutcome(outcome, vmUuid, "migrateVmAway"); - final Object jobException = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobException != null) { - if (jobException instanceof InsufficientServerCapacityException) { - throw (InsufficientServerCapacityException)jobException; - } else if (jobException instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobException; - } else if (jobException instanceof RuntimeException) { - throw (RuntimeException)jobException; - } else if (jobException instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobException); - } + try { + retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + } catch (ResourceUnavailableException | InsufficientCapacityException ex) { + throw new RuntimeException("Unexpected exception", ex); } } } @@ -3382,8 +3235,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private void orchestrateMigrateAway(final String vmUuid, final long srcHostId, final DeploymentPlanner planner) throws InsufficientServerCapacityException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); if (vm == null) { - s_logger.debug("Unable to find a VM for " + vmUuid); - throw new CloudRuntimeException("Unable to find " + vmUuid); + String message = String.format("Unable to find VM with uuid [%s].", vmUuid); + s_logger.warn(message); + throw new CloudRuntimeException(message); } ServiceOfferingVO offeringVO = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()); @@ -3391,8 +3245,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final Long hostId = vm.getHostId(); if (hostId == null) { - s_logger.debug("Unable to migrate because the VM doesn't have a host id: " + vm); - throw new CloudRuntimeException("Unable to migrate " + vmUuid); + String message = String.format("Unable to migrate %s due to it does not have a host id.", vm.toString()); + s_logger.warn(message); + throw new CloudRuntimeException(message); } final Host host = _hostDao.findById(hostId); @@ -3416,8 +3271,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac plan.setMigrationPlan(true); dest = _dpMgr.planDeployment(profile, plan, excludes, planner); } catch (final AffinityConflictException e2) { - s_logger.warn("Unable to create deployment, affinity rules associted to the VM conflict", e2); - throw new CloudRuntimeException("Unable to create deployment, affinity rules associted to the VM conflict"); + String message = String.format("Unable to create deployment, affinity rules associted to the %s conflict.", vm.toString()); + s_logger.warn(message, e2); + throw new CloudRuntimeException(message, e2); } if (dest == null) { s_logger.warn("Unable to find destination for migrating the vm " + profile); @@ -3431,23 +3287,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { migrate(vm, srcHostId, dest); return; - } catch (final ResourceUnavailableException e) { - s_logger.debug("Unable to migrate to unavailable " + dest); - } catch (final ConcurrentOperationException e) { - s_logger.debug("Unable to migrate VM due to: " + e.getMessage()); + } catch (ResourceUnavailableException | ConcurrentOperationException e) { + s_logger.warn(String.format("Unable to migrate %s to %s due to [%s]", vm.toString(), dest.getHost().toString(), e.getMessage()), e); } try { advanceStop(vmUuid, true); throw new CloudRuntimeException("Unable to migrate " + vm); - } catch (final ResourceUnavailableException e) { - s_logger.debug("Unable to stop VM due to " + e.getMessage()); - throw new CloudRuntimeException("Unable to migrate " + vm); - } catch (final ConcurrentOperationException e) { - s_logger.debug("Unable to stop VM due to " + e.getMessage()); - throw new CloudRuntimeException("Unable to migrate " + vm); - } catch (final OperationTimedoutException e) { - s_logger.debug("Unable to stop VM due to " + e.getMessage()); + } catch (final ResourceUnavailableException | ConcurrentOperationException | OperationTimedoutException e) { + s_logger.error(String.format("Unable to stop %s due to [%s].", vm.toString(), e.getMessage()), e); throw new CloudRuntimeException("Unable to migrate " + vm); } } @@ -3491,7 +3339,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac protected class CleanupTask extends ManagedContextRunnable { @Override protected void runInContext() { - s_logger.trace("VM Operation Thread Running"); + s_logger.debug("VM Operation Thread Running"); try { _workDao.cleanup(VmOpCleanupWait.value()); final Date cutDate = new Date(DateUtil.currentGMTTime().getTime() - VmOpCleanupInterval.value() * 1000); @@ -3507,9 +3355,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac boolean isMachineUpgradable = true; for (final HostAllocator allocator : hostAllocators) { isMachineUpgradable = allocator.isVirtualMachineUpgradable(vm, offering); - if (isMachineUpgradable) { - continue; - } else { + if (!isMachineUpgradable) { break; } } @@ -3532,13 +3378,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if ( jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - // avoid re-entrance - VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); try { - if (s_logger.isTraceEnabled()) { - s_logger.trace(String.format("reboot parameter value of %s == %s at orchestration", VirtualMachineProfile.Param.BootIntoSetup.getName(), + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("reboot parameter value of %s == %s at orchestration", VirtualMachineProfile.Param.BootIntoSetup.getName(), (params == null? "":params.get(VirtualMachineProfile.Param.BootIntoSetup)))); } orchestrateReboot(vmUuid, params); @@ -3548,41 +3392,21 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } } else { - if (s_logger.isTraceEnabled()) { - s_logger.trace(String.format("reboot parameter value of %s == %s through job-queue", VirtualMachineProfile.Param.BootIntoSetup.getName(), + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("reboot parameter value of %s == %s through job-queue", VirtualMachineProfile.Param.BootIntoSetup.getName(), (params == null? "":params.get(VirtualMachineProfile.Param.BootIntoSetup)))); } final Outcome outcome = rebootVmThroughJobQueue(vmUuid, params); - try { - final VirtualMachine vm = outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); - } + retrieveVmFromJobOutcome(outcome, vmUuid, "rebootVm"); - final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobResult != null) { - if (jobResult instanceof ResourceUnavailableException) { - throw (ResourceUnavailableException)jobResult; - } else if (jobResult instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobResult; - } else if (jobResult instanceof InsufficientCapacityException) { - throw (InsufficientCapacityException)jobResult; - } else if (jobResult instanceof RuntimeException) { - throw (RuntimeException)jobResult; - } else if (jobResult instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobResult); - } - } + retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); } } private void orchestrateReboot(final String vmUuid, final Map params) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); - // if there are active vm snapshots task, state change is not allowed if (_vmSnapshotMgr.hasActiveVMSnapshotTasks(vm.getId())) { s_logger.error("Unable to reboot VM " + vm + " due to: " + vm.getInstanceName() + " has active VM snapshots tasks"); throw new CloudRuntimeException("Unable to reboot VM " + vm + " due to: " + vm.getInstanceName() + " has active VM snapshots tasks"); @@ -3590,7 +3414,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final DataCenter dc = _entityMgr.findById(DataCenter.class, vm.getDataCenterId()); final Host host = _hostDao.findById(vm.getHostId()); if (host == null) { - // Should findById throw an Exception is the host is not found? throw new CloudRuntimeException("Unable to retrieve host with id " + vm.getHostId()); } final Cluster cluster = _entityMgr.findById(Cluster.class, host.getClusterId()); @@ -3610,7 +3433,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (rebootAnswer != null && rebootAnswer.getResult()) { boolean isVmSecurityGroupEnabled = _securityGroupManager.isVmSecurityGroupEnabled(vm.getId()); if (isVmSecurityGroupEnabled && vm.getType() == VirtualMachine.Type.User) { - List affectedVms = new ArrayList(); + List affectedVms = new ArrayList<>(); affectedVms.add(vm.getId()); _securityGroupManager.scheduleRulesetUpdateToHosts(affectedVms, true, null); } @@ -3622,7 +3445,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new CloudRuntimeException(errorMsg); } catch (final OperationTimedoutException e) { s_logger.warn("Unable to send the reboot command to host " + dest.getHost() + " for the vm " + vm + " due to operation timeout", e); - throw new CloudRuntimeException("Failed to reboot the vm on host " + dest.getHost()); + throw new CloudRuntimeException("Failed to reboot the vm on host " + dest.getHost(), e); } } @@ -3631,8 +3454,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (params != null) { enterSetup = (Boolean) params.get(VirtualMachineProfile.Param.BootIntoSetup); } - if (s_logger.isTraceEnabled()) { - s_logger.trace(String.format("orchestrating VM reboot for '%s' %s set to %s", vmTo.getName(), VirtualMachineProfile.Param.BootIntoSetup, enterSetup)); + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("orchestrating VM reboot for '%s' %s set to %s", vmTo.getName(), VirtualMachineProfile.Param.BootIntoSetup, enterSetup)); } vmTo.setEnterHardwareSetup(enterSetup == null ? false : enterSetup); } @@ -3674,14 +3497,21 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } private String getControlNicIpForVM(VirtualMachine vm) { - if (vm.getType() == VirtualMachine.Type.ConsoleProxy || vm.getType() == VirtualMachine.Type.SecondaryStorageVm) { - NicVO nic = _nicsDao.getControlNicForVM(vm.getId()); - return nic.getIPv4Address(); - } else if (vm.getType() == VirtualMachine.Type.DomainRouter) { - return vm.getPrivateIpAddress(); - } else { + if (null == vm.getType()) { return null; } + + switch (vm.getType()) { + case ConsoleProxy: + case SecondaryStorageVm: + NicVO nic = _nicsDao.getControlNicForVM(vm.getId()); + return nic.getIPv4Address(); + case DomainRouter: + return vm.getPrivateIpAddress(); + default: + s_logger.debug(String.format("%s is a [%s], returning null for control Nic IP.", vm.toString(), vm.getType())); + return null; + } } public Command cleanup(final String vmName) { VirtualMachine vm = _vmDao.findVMByInstanceName(vmName); @@ -3695,8 +3525,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac return cmd; } - - // this is XenServer specific public void syncVMMetaData(final Map vmMetadatum) { if (vmMetadatum == null || vmMetadatum.isEmpty()) { return; @@ -3736,7 +3564,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - // this is XenServer specific private void updateVmMetaData(Long vmId, String platform) { UserVmVO userVm = _userVmDao.findById(vmId); _userVmDao.loadDetails(userVm); @@ -3793,8 +3620,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac _syncMgr.processHostVmStatePingReport(agentId, ping.getHostVmStateReport()); } - // take the chance to scan VMs that are stuck in transitional states - // and are missing from the report scanStalledVMInTransitionStateOnUpHost(agentId); processed = true; } @@ -3843,8 +3668,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final Long clusterId = agent.getClusterId(); final long agentId = agent.getId(); - if (agent.getHypervisorType() == HypervisorType.XenServer) { // only for Xen - // initiate the cron job + if (agent.getHypervisorType() == HypervisorType.XenServer) { final ClusterVMMetaDataSyncCommand syncVMMetaDataCmd = new ClusterVMMetaDataSyncCommand(ClusterVMMetaDataSyncInterval.value(), clusterId); try { final long seq_no = _agentMgr.send(agentId, new Commands(syncVMMetaDataCmd), this); @@ -3899,14 +3723,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new InvalidParameterValueException("Invalid parameter, newServiceOffering can't be null"); } - // Check that the VM is stopped / running if (!(vmInstance.getState().equals(State.Stopped) || vmInstance.getState().equals(State.Running))) { s_logger.warn("Unable to upgrade virtual machine " + vmInstance.toString() + " in state " + vmInstance.getState()); throw new InvalidParameterValueException("Unable to upgrade virtual machine " + vmInstance.toString() + " " + " in state " + vmInstance.getState() + "; make sure the virtual machine is stopped/running"); } - // Check if the service offering being upgraded to is what the VM is already running with if (!newServiceOffering.isDynamic() && vmInstance.getServiceOfferingId() == newServiceOffering.getId()) { if (s_logger.isInfoEnabled()) { s_logger.info("Not upgrading vm " + vmInstance.toString() + " since it already has the requested " + "service offering (" + newServiceOffering.getName() + @@ -3921,18 +3743,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac checkIfNewOfferingStorageScopeMatchesStoragePool(vmInstance, newServiceOffering); - // if vm is a system vm, check if it is a system service offering, if yes return with error as it cannot be used for user vms if (currentServiceOffering.isSystemUse() != newServiceOffering.isSystemUse()) { throw new InvalidParameterValueException("isSystem property is different for current service offering and new service offering"); } - // Check that there are enough resources to upgrade the service offering if (!isVirtualMachineUpgradable(vmInstance, newServiceOffering)) { throw new InvalidParameterValueException("Unable to upgrade virtual machine, not enough resources available " + "for an offering of " + newServiceOffering.getCpu() + " cpu(s) at " + newServiceOffering.getSpeed() + " Mhz, and " + newServiceOffering.getRamSize() + " MB of memory"); } - // Check that the service offering being upgraded to has all the tags of the current service offering. final List currentTags = StringUtils.csvTagsToList(currentServiceOffering.getTags()); final List newTags = StringUtils.csvTagsToList(newServiceOffering.getTags()); if (!newTags.containsAll(currentTags)) { @@ -4001,9 +3820,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - // avoid re-entrance - VmWorkJobVO placeHolder = null; - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); + try { return orchestrateAddVmToNetwork(vm, network, requested); } finally { @@ -4014,29 +3832,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else { final Outcome outcome = addVmToNetworkThroughJobQueue(vm, network, requested); - try { - outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution exception", e); - } + retrieveVmFromJobOutcome(outcome, vm.getUuid(), "addVmToNetwork"); - final Object jobException = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobException != null) { - if (jobException instanceof ResourceUnavailableException) { - throw (ResourceUnavailableException)jobException; - } else if (jobException instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobException; - } else if (jobException instanceof InsufficientCapacityException) { - throw (InsufficientCapacityException)jobException; - } else if (jobException instanceof RuntimeException) { - throw (RuntimeException)jobException; - } else if (jobException instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobException); - } else if (jobException instanceof NicProfile) { - return (NicProfile)jobException; - } + Object jobResult = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + + if (jobResult != null && jobResult instanceof NicProfile) { + return (NicProfile) jobResult; } throw new RuntimeException("Unexpected job execution result"); @@ -4057,16 +3858,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final Host host = _hostDao.findById(vm.getHostId()); final DeployDestination dest = new DeployDestination(dc, null, null, host); - //check vm state if (vm.getState() == State.Running) { - //1) allocate and prepare nic final NicProfile nic = _networkMgr.createNicForVm(network, requested, context, vmProfile, true); - //2) Convert vmProfile to vmTO final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vmProfile.getVirtualMachine().getHypervisorType()); final VirtualMachineTO vmTO = hvGuru.implement(vmProfile); - //3) Convert nicProfile to NicTO final NicTO nicTO = toNicTO(nic, vmProfile.getVirtualMachine().getHypervisorType()); //4) plug the nic to the vm @@ -4079,9 +3876,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac _userVmMgr.setupVmForPvlan(true, vm.getHostId(), nic); s_logger.debug("Nic is plugged successfully for vm " + vm + " in network " + network + ". Vm is a part of network now"); final long isDefault = nic.isDefaultNic() ? 1 : 0; - // insert nic's Id into DB as resource_name + if(VirtualMachine.Type.User.equals(vmVO.getType())) { - //Log usage event for user Vms only UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmVO.getAccountId(), vmVO.getDataCenterId(), vmVO.getId(), Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vmVO.getUuid(), vm.isDisplay()); } @@ -4097,7 +3893,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } } else if (vm.getState() == State.Stopped) { - //1) allocate nic return _networkMgr.createNicForVm(network, requested, context, vmProfile, false); } else { s_logger.warn("Unable to add vm " + vm + " to network " + network); @@ -4108,9 +3903,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac @Override public NicTO toNicTO(final NicProfile nic, final HypervisorType hypervisorType) { final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(hypervisorType); - - final NicTO nicTO = hvGuru.toNicTO(nic); - return nicTO; + return hvGuru.toNicTO(nic); } @Override @@ -4119,9 +3912,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - // avoid re-entrance - VmWorkJobVO placeHolder = null; - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); try { return orchestrateRemoveNicFromVm(vm, nic); } finally { @@ -4133,27 +3924,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else { final Outcome outcome = removeNicFromVmThroughJobQueue(vm, nic); - try { - outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); - } + retrieveVmFromJobOutcome(outcome, vm.getUuid(), "removeNicFromVm"); - final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobResult != null) { - if (jobResult instanceof ResourceUnavailableException) { - throw (ResourceUnavailableException)jobResult; - } else if (jobResult instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobResult; - } else if (jobResult instanceof RuntimeException) { - throw (RuntimeException)jobResult; - } else if (jobResult instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobResult); - } else if (jobResult instanceof Boolean) { - return (Boolean)jobResult; + try { + Object jobResult = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + if (jobResult != null && jobResult instanceof Boolean) { + return (Boolean) jobResult; } + } catch (InsufficientCapacityException ex) { + throw new RuntimeException("Unexpected exception", ex); } throw new RuntimeException("Job failed with un-handled exception"); @@ -4178,7 +3957,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), _networkModel.getNetworkRate(network.getId(), vm.getId()), _networkModel.isSecurityGroupSupportedInNetwork(network), _networkModel.getNetworkTag(vmProfile.getVirtualMachine().getHypervisorType(), network)); - //1) Unplug the nic if (vm.getState() == State.Running) { final NicTO nicTO = toNicTO(nicProfile, vmProfile.getVirtualMachine().getHypervisorType()); s_logger.debug("Un-plugging nic " + nic + " for vm " + vm + " from network " + network); @@ -4198,11 +3976,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new ResourceUnavailableException("Unable to remove vm " + vm + " from network, is not in the right state", DataCenter.class, vm.getDataCenterId()); } - //2) Release the nic _networkMgr.releaseNic(vmProfile, nic); s_logger.debug("Successfully released nic " + nic + "for vm " + vm); - //3) Remove the nic _networkMgr.removeNic(vmProfile, nic); _nicsDao.remove(nic.getId()); return true; @@ -4211,7 +3987,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac @Override @DB public boolean removeVmFromNetwork(final VirtualMachine vm, final Network network, final URI broadcastUri) throws ConcurrentOperationException, ResourceUnavailableException { - // TODO will serialize on the VM object later to resolve operation conflicts return orchestrateRemoveVmFromNetwork(vm, network, broadcastUri); } @@ -4241,16 +4016,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac return false; } - // don't delete default NIC on a user VM if (nic.isDefaultNic() && vm.getType() == VirtualMachine.Type.User) { s_logger.warn("Failed to remove nic from " + vm + " in " + network + ", nic is default."); throw new CloudRuntimeException("Failed to remove nic from " + vm + " in " + network + ", nic is default."); } - //Lock on nic is needed here final Nic lock = _nicsDao.acquireInLockTable(nic.getId()); if (lock == null) { - //check if nic is still there. Return if it was released already if (_nicsDao.findById(nic.getId()) == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Not need to remove the vm " + vm + " from network " + network + " as the vm doesn't have nic in this network"); @@ -4269,7 +4041,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), _networkModel.getNetworkRate(network.getId(), vm.getId()), _networkModel.isSecurityGroupSupportedInNetwork(network), _networkModel.getNetworkTag(vmProfile.getVirtualMachine().getHypervisorType(), network)); - //1) Unplug the nic if (vm.getState() == State.Running) { final NicTO nicTO = toNicTO(nicProfile, vmProfile.getVirtualMachine().getHypervisorType()); s_logger.debug("Un-plugging nic for vm " + vm + " from network " + network); @@ -4285,19 +4056,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new ResourceUnavailableException("Unable to remove vm " + vm + " from network, is not in the right state", DataCenter.class, vm.getDataCenterId()); } - //2) Release the nic _networkMgr.releaseNic(vmProfile, nic); s_logger.debug("Successfully released nic " + nic + "for vm " + vm); - //3) Remove the nic _networkMgr.removeNic(vmProfile, nic); return true; } finally { - if (lock != null) { - _nicsDao.releaseFromLockTable(lock.getId()); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Lock is released for nic id " + lock.getId() + " as a part of remove vm " + vm + " from network " + network); - } + _nicsDao.releaseFromLockTable(lock.getId()); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Lock is released for nic id " + lock.getId() + " as a part of remove vm " + vm + " from network " + network); } } } @@ -4325,15 +4092,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final Host host = _hostDao.findById(srcHostId); final DataCenterDeployment plan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), null, null, null); excludes.addHost(vm.getHostId()); - vm.setServiceOfferingId(newSvcOfferingId); // Need to find the destination host based on new svc offering + vm.setServiceOfferingId(newSvcOfferingId); DeployDestination dest = null; try { dest = _dpMgr.planDeployment(profile, plan, excludes, null); } catch (final AffinityConflictException e2) { - s_logger.warn("Unable to create deployment, affinity rules associted to the VM conflict", e2); - throw new CloudRuntimeException("Unable to create deployment, affinity rules associted to the VM conflict"); + String message = String.format("Unable to create deployment, affinity rules associted to the %s conflict.", vm.toString()); + s_logger.warn(message, e2); + throw new CloudRuntimeException(message); } if (dest != null) { @@ -4349,11 +4117,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac excludes.addHost(dest.getHost().getId()); try { migrateForScale(vm.getUuid(), srcHostId, dest, oldSvcOfferingId); - } catch (final ResourceUnavailableException e) { - s_logger.debug("Unable to migrate to unavailable " + dest); - throw e; - } catch (final ConcurrentOperationException e) { - s_logger.debug("Unable to migrate VM due to: " + e.getMessage()); + } catch (ResourceUnavailableException | ConcurrentOperationException e) { + s_logger.warn(String.format("Unable to migrate %s to %s due to [%s]", vm.toString(), dest.getHost().toString(), e.getMessage()), e); throw e; } } @@ -4363,10 +4128,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throws ResourceUnavailableException, ConcurrentOperationException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - // avoid re-entrance - VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateMigrateForScale(vmUuid, srcHostId, dest, oldSvcOfferingId); } finally { @@ -4377,25 +4140,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else { final Outcome outcome = migrateVmForScaleThroughJobQueue(vmUuid, srcHostId, dest, oldSvcOfferingId); - try { - final VirtualMachine vm = outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); - } + retrieveVmFromJobOutcome(outcome, vmUuid, "migrateVmForScale"); - final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobResult != null) { - if (jobResult instanceof ResourceUnavailableException) { - throw (ResourceUnavailableException)jobResult; - } else if (jobResult instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobResult; - } else if (jobResult instanceof RuntimeException) { - throw (RuntimeException)jobResult; - } else if (jobResult instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobResult); - } + try { + retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + } catch (InsufficientCapacityException ex) { + throw new RuntimeException("Unexpected exception", ex); } } } @@ -4427,20 +4177,17 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final VirtualMachineGuru vmGuru = getVmGuru(vm); - final long vmId = vm.getId(); vm = _vmDao.findByUuid(vmUuid); if (vm == null) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Unable to find the vm " + vm); - } - throw new CloudRuntimeException("Unable to find a virtual machine with id " + vmId); + String message = String.format("Unable to find VM {\"uuid\": \"%s\"}.", vmUuid); + s_logger.warn(message); + throw new CloudRuntimeException(message); } if (vm.getState() != State.Running) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("VM is not Running, unable to migrate the vm " + vm); - } - throw new CloudRuntimeException("VM is not Running, unable to migrate the vm currently " + vm + " , current state: " + vm.getState().toString()); + String message = String.format("%s is not in \"Running\" state, unable to migrate it. Current state [%s].", vm.toString(), vm.getState()); + s_logger.warn(message); + throw new CloudRuntimeException(message); } AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM_MIGRATE; @@ -4469,9 +4216,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac pfma = _agentMgr.send(dstHostId, pfmc); if (pfma == null || !pfma.getResult()) { final String details = pfma != null ? pfma.getDetails() : "null answer returned"; - final String msg = "Unable to prepare for migration due to " + details; pfma = null; - throw new AgentUnavailableException(msg, dstHostId); + throw new AgentUnavailableException(String.format("Unable to prepare for migration to destination host [%s] due to [%s].", dstHostId, details), dstHostId); } } catch (final OperationTimedoutException e1) { throw new AgentUnavailableException("Operation timed out", dstHostId); @@ -4484,13 +4230,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac vm.setLastHostId(srcHostId); try { - if (vm == null || vm.getHostId() == null || vm.getHostId() != srcHostId || !changeState(vm, Event.MigrationRequested, dstHostId, work, Step.Migrating)) { - s_logger.info("Migration cancelled because state has changed: " + vm); - throw new ConcurrentOperationException("Migration cancelled because state has changed: " + vm); + if (vm.getHostId() == null || vm.getHostId() != srcHostId || !changeState(vm, Event.MigrationRequested, dstHostId, work, Step.Migrating)) { + String message = String.format("Migration of %s cancelled because state has changed.", vm.toString()); + s_logger.warn(message); + throw new ConcurrentOperationException(message); } } catch (final NoTransitionException e1) { - s_logger.info("Migration cancelled because " + e1.getMessage()); - throw new ConcurrentOperationException("Migration cancelled because " + e1.getMessage()); + String message = String.format("Migration of %s cancelled due to [%s].", vm.toString(), e1.getMessage()); + s_logger.error(message, e1); + throw new ConcurrentOperationException(message); } boolean migrated = false; @@ -4509,22 +4257,21 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { final Answer ma = _agentMgr.send(vm.getLastHostId(), mc); if (ma == null || !ma.getResult()) { - final String details = ma != null ? ma.getDetails() : "null answer returned"; - final String msg = "Unable to migrate due to " + details; + String msg = String.format("Unable to migrate %s due to [%s].", vm.toString(), ma != null ? ma.getDetails() : "null answer returned"); s_logger.error(msg); throw new CloudRuntimeException(msg); } } catch (final OperationTimedoutException e) { if (e.isActive()) { - s_logger.warn("Active migration command so scheduling a restart for " + vm); + s_logger.warn("Active migration command so scheduling a restart for " + vm, e); _haMgr.scheduleRestart(vm, true); } - throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId); + throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId, e); } try { final long newServiceOfferingId = vm.getServiceOfferingId(); - vm.setServiceOfferingId(oldSvcOfferingId); // release capacity for the old service offering only + vm.setServiceOfferingId(oldSvcOfferingId); if (!changeState(vm, VirtualMachine.Event.OperationSucceeded, dstHostId, work, Step.Started)) { throw new ConcurrentOperationException("Unable to change the state for " + vm); } @@ -4539,7 +4286,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { _agentMgr.send(srcHostId, new Commands(cleanup(vm.getInstanceName())), null); } catch (final AgentUnavailableException e) { - s_logger.error(String.format("Unable to cleanup source %s. ", srcHost), e); + s_logger.error(String.format("Unable to cleanup source host [%s] due to [%s].", srcHostId, e.getMessage()), e); } cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); throw new CloudRuntimeException("Unable to complete migration for " + vm); @@ -4566,7 +4313,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { stateTransitTo(vm, Event.OperationFailed, srcHostId); } catch (final NoTransitionException e) { - s_logger.warn(e.getMessage()); + s_logger.warn(e.getMessage(), e); } } else { _networkMgr.setHypervisorHostname(profile, dest, true); @@ -4598,10 +4345,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new AgentUnavailableException("Unable to plug nic for router " + vm.getName() + " in network " + network, host.getId(), e); } } else { - s_logger.warn("Unable to apply ReplugNic, vm " + router + " is not in the right state " + router.getState()); + String message = String.format("Unable to apply ReplugNic, VM [%s] is not in the right state (\"Running\"). VM state [%s].", router.toString(), router.getState()); + s_logger.warn(message); - throw new ResourceUnavailableException("Unable to apply ReplugNic on the backend," + " vm " + vm + " is not in the right state", DataCenter.class, - router.getDataCenterId()); + throw new ResourceUnavailableException(message, DataCenter.class, router.getDataCenterId()); } return result; @@ -4634,9 +4381,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new AgentUnavailableException("Unable to plug nic for router " + vm.getName() + " in network " + network, dest.getHost().getId(), e); } } else { - s_logger.warn("Unable to apply PlugNic, vm " + router + " is not in the right state " + router.getState()); + String message = String.format("Unable to apply PlugNic, VM [%s] is not in the right state (\"Running\"). VM state [%s].", router.toString(), router.getState()); + s_logger.warn(message); - throw new ResourceUnavailableException("Unable to apply PlugNic on the backend," + " vm " + vm + " is not in the right state", DataCenter.class, + throw new ResourceUnavailableException(message, DataCenter.class, router.getDataCenterId()); } @@ -4650,7 +4398,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final VMInstanceVO router = _vmDao.findById(vm.getId()); if (router.getState() == State.Running) { - // collect vm network statistics before unplug a nic UserVmVO userVm = _userVmDao.findById(vm.getId()); if (userVm != null && userVm.getType() == VirtualMachine.Type.User) { _userVmService.collectVmNetworkStatistics(userVm); @@ -4676,10 +4423,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else if (router.getState() == State.Stopped || router.getState() == State.Stopping) { s_logger.debug("Vm " + router.getInstanceName() + " is in " + router.getState() + ", so not sending unplug nic command to the backend"); } else { - s_logger.warn("Unable to apply unplug nic, Vm " + router + " is not in the right state " + router.getState()); + String message = String.format("Unable to apply unplug nic, VM [%s] is not in the right state (\"Running\"). VM state [%s].", router.toString(), router.getState()); + s_logger.warn(message); - throw new ResourceUnavailableException("Unable to apply unplug nic on the backend," + " vm " + router + " is not in the right state", DataCenter.class, - router.getDataCenterId()); + throw new ResourceUnavailableException(message, DataCenter.class, router.getDataCenterId()); } return result; @@ -4692,10 +4439,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - // avoid re-entrance - VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); try { return orchestrateReConfigureVm(vmUuid, oldServiceOffering, newServiceOffering, reconfiguringOnExistingHost); } finally { @@ -4706,29 +4451,17 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else { final Outcome outcome = reconfigureVmThroughJobQueue(vmUuid, oldServiceOffering, newServiceOffering, customParameters, reconfiguringOnExistingHost); - VirtualMachine vm = null; + VirtualMachine vm = retrieveVmFromJobOutcome(outcome, vmUuid, "reconfigureVm"); + + Object result = null; try { - vm = outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); + result = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + } catch (Exception ex) { + throw new RuntimeException("Unhandled exception", ex); } - final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobResult != null) { - if (jobResult instanceof ResourceUnavailableException) { - throw (ResourceUnavailableException)jobResult; - } else if (jobResult instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobResult; - } else if (jobResult instanceof InsufficientServerCapacityException) { - throw (InsufficientServerCapacityException)jobResult; - } else if (jobResult instanceof RuntimeException) { - throw (RuntimeException)jobResult; - } else if (jobResult instanceof Throwable) { - s_logger.error("Unhandled exception", (Throwable)jobResult); - throw new RuntimeException("Unhandled exception", (Throwable)jobResult); - } + if (result != null) { + throw new RuntimeException(String.format("Unexpected job execution result [%s]", result)); } return (VMInstanceVO)vm; @@ -4789,9 +4522,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (reconfiguringOnExistingHost) { vm.setServiceOfferingId(oldServiceOffering.getId()); - _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); //release the old capacity + _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); vm.setServiceOfferingId(newServiceOffering.getId()); - _capacityMgr.allocateVmCapacity(vm, false); // lock the new capacity + _capacityMgr.allocateVmCapacity(vm, false); } } catch (final OperationTimedoutException e) { @@ -4809,7 +4542,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac details.remove(UsageEventVO.DynamicParameters.cpuNumber.name()); details.remove(UsageEventVO.DynamicParameters.cpuSpeed.name()); details.remove(UsageEventVO.DynamicParameters.memory.name()); - List detailList = new ArrayList(); + List detailList = new ArrayList<>(); for(Map.Entry entry: details.entrySet()) { UserVmDetailVO detailVO = new UserVmDetailVO(vmId, entry.getKey(), entry.getValue(), true); detailList.add(detailVO); @@ -4818,12 +4551,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } private void saveCustomOfferingDetails(long vmId, ServiceOffering serviceOffering) { - //save the custom values to the database. Map details = userVmDetailsDao.listDetailsKeyPairs(vmId); details.put(UsageEventVO.DynamicParameters.cpuNumber.name(), serviceOffering.getCpu().toString()); details.put(UsageEventVO.DynamicParameters.cpuSpeed.name(), serviceOffering.getSpeed().toString()); details.put(UsageEventVO.DynamicParameters.memory.name(), serviceOffering.getRamSize().toString()); - List detailList = new ArrayList(); + List detailList = new ArrayList<>(); for (Map.Entry entry: details.entrySet()) { UserVmDetailVO detailVO = new UserVmDetailVO(vmId, entry.getKey(), entry.getValue(), true); detailList.add(detailVO); @@ -4853,9 +4585,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac _storagePoolAllocators = storagePoolAllocators; } - // - // PowerState report handling for out-of-band changes and handling of left-over transitional VM states - // + /** + * PowerState report handling for out-of-band changes and handling of left-over transitional VM states + */ @MessageHandler(topic = Topics.VM_POWER_STATE) protected void HandlePowerStateReport(final String subject, final String senderAddress, final Object args) { @@ -4864,8 +4596,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( VirtualMachine.Type.Instance, vmId); - if (pendingWorkJobs.size() == 0 && !_haMgr.hasPendingHaWork(vmId)) { - // there is no pending operation job + if (CollectionUtils.isEmpty(pendingWorkJobs) && !_haMgr.hasPendingHaWork(vmId)) { final VMInstanceVO vm = _vmDao.findById(vmId); if (vm != null) { switch (vm.getPowerState()) { @@ -4874,13 +4605,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac break; case PowerOff: - case PowerReportMissing: // rigorously set to Migrating? or just do nothing until...? or create a missing state? - // for now handle in line with legacy as power off + case PowerReportMissing: handlePowerOffReportWithNoPendingJobsOnVM(vm); break; - - // PowerUnknown shouldn't be reported, it is a derived - // VM power state from host state (host un-reachable) case PowerUnknown: default: assert false; @@ -4891,22 +4618,14 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } else { s_logger.info("There is pending job or HA tasks working on the VM. vm id: " + vmId + ", postpone power-change report by resetting power-change counters"); - - // reset VM power state tracking so that we won't lost signal when VM has - // been translated to _vmDao.resetVmPowerStateTracking(vmId); } } private void handlePowerOnReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { - // - // 1) handle left-over transitional VM states - // 2) handle out of band VM live migration - // 3) handle out of sync stationary states, marking VM from Stopped to Running with - // alert messages - // Host host = _hostDao.findById(vm.getHostId()); Host poweredHost = _hostDao.findById(vm.getPowerHostId()); + switch (vm.getState()) { case Starting: s_logger.info("VM " + vm.getInstanceName() + " is at " + vm.getState() + " and we received a power-on report while there is no pending jobs on it"); @@ -4919,7 +4638,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac s_logger.info("VM " + vm.getInstanceName() + " is sync-ed to at Running state according to power-on report from hypervisor"); - // we need to alert admin or user about this risky state transition _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(), VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() + ") state is sync-ed (Starting -> Running) from out-of-context transition. VM network environment may need to be reset"); @@ -4927,8 +4645,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac case Running: try { - if (vm.getHostId() != null && vm.getHostId().longValue() != vm.getPowerHostId().longValue()) { - s_logger.info(String.format("Detected out of band VM migration from %s to %s", host, poweredHost)); + if (vm.getHostId() != null && !vm.getHostId().equals(vm.getPowerHostId())) { + s_logger.info("Detected out of band VM migration from host " + vm.getHostId() + " to host " + vm.getPowerHostId()); } stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOnReport, vm.getPowerHostId()); } catch (final NoTransitionException e) { @@ -4978,10 +4696,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } private void handlePowerOffReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { - - // 1) handle left-over transitional VM states - // 2) handle out of sync stationary states, schedule force-stop to release resources - // switch (vm.getState()) { case Starting: case Stopping: @@ -5006,12 +4720,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac return; } - // not when report is missing if (PowerState.PowerOff.equals(vm.getPowerState())) { final VirtualMachineGuru vmGuru = getVmGuru(vm); final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); if (!sendStop(vmGuru, profile, true, true)) { - // In case StopCommand fails, don't proceed further return; } else { // Release resources on StopCommand success @@ -5048,22 +4760,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } private void scanStalledVMInTransitionStateOnUpHost(final long hostId) { - // - // Check VM that is stuck in Starting, Stopping, Migrating states, we won't check - // VMs in expunging state (this need to be handled specially) - // - // checking condition - // 1) no pending VmWork job - // 2) on hostId host and host is UP - // - // When host is UP, soon or later we will get a report from the host about the VM, - // however, if VM is missing from the host report (it may happen in out of band changes - // or from designed behave of XS/KVM), the VM may not get a chance to run the state-sync logic - // - // Therefore, we will scan thoses VMs on UP host based on last update timestamp, if the host is UP - // and a VM stalls for status update, we will consider them to be powered off - // (which is relatively safe to do so) - final long stallThresholdInMs = VmJobStateReportInterval.value() + (VmJobStateReportInterval.value() >> 1); final Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - stallThresholdInMs); final List mostlikelyStoppedVMs = listStalledVMInTransitionStateOnUpHost(hostId, cutTime); @@ -5091,14 +4787,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac for (final Long vmId : stuckAndUncontrollableVMs) { final VMInstanceVO vm = _vmDao.findById(vmId); - // We now only alert administrator about this situation _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(), VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() + ") is stuck in " + vm.getState() + " state and its host is unreachable for too long"); } } - // VMs that in transitional state without recent power state report private List listStalledVMInTransitionStateOnUpHost(final long hostId, final Date cutTime) { final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status = 'UP' " + "AND h.id = ? AND i.power_state_update_time < ? AND i.host_id = h.id " + @@ -5106,35 +4800,27 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac "AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" + "AND i.removed IS NULL"; - final List l = new ArrayList(); - TransactionLegacy txn = null; - try { - txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB); + final List l = new ArrayList<>(); + try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) { + String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime); - PreparedStatement pstmt = null; try { - pstmt = txn.prepareAutoCloseStatement(sql); + PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql); pstmt.setLong(1, hostId); - pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime)); + pstmt.setString(2, cutTimeStr); pstmt.setInt(3, JobInfo.Status.IN_PROGRESS.ordinal()); final ResultSet rs = pstmt.executeQuery(); while (rs.next()) { l.add(rs.getLong(1)); } - } catch (final SQLException e) { - } catch (final Throwable e) { - } - - } finally { - if (txn != null) { - txn.close(); + } catch (SQLException e) { + s_logger.error(String.format("Unable to execute SQL [%s] with params {\"h.id\": %s, \"i.power_state_update_time\": \"%s\"} due to [%s].", sql, hostId, cutTimeStr, e.getMessage()), e); } } return l; } - // VMs that in transitional state and recently have power state update private List listVMInTransitionStateWithRecentReportOnUpHost(final long hostId, final Date cutTime) { final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status = 'UP' " + "AND h.id = ? AND i.power_state_update_time > ? AND i.host_id = h.id " + @@ -5142,29 +4828,25 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac "AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" + "AND i.removed IS NULL"; - final List l = new ArrayList(); - TransactionLegacy txn = null; - try { - txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB); - PreparedStatement pstmt = null; + final List l = new ArrayList<>(); + try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) { + String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime); + int jobStatusInProgress = JobInfo.Status.IN_PROGRESS.ordinal(); + try { - pstmt = txn.prepareAutoCloseStatement(sql); + PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql); pstmt.setLong(1, hostId); - pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime)); - pstmt.setInt(3, JobInfo.Status.IN_PROGRESS.ordinal()); + pstmt.setString(2, cutTimeStr); + pstmt.setInt(3, jobStatusInProgress); final ResultSet rs = pstmt.executeQuery(); while (rs.next()) { l.add(rs.getLong(1)); } } catch (final SQLException e) { - } catch (final Throwable e) { + s_logger.error(String.format("Unable to execute SQL [%s] with params {\"h.id\": %s, \"i.power_state_update_time\": \"%s\", \"j.job_status\": %s} due to [%s].", sql, hostId, cutTimeStr, jobStatusInProgress, e.getMessage()), e); } return l; - } finally { - if (txn != null) { - txn.close(); - } } } @@ -5175,35 +4857,27 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac "AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" + "AND i.removed IS NULL"; - final List l = new ArrayList(); - TransactionLegacy txn = null; - try { - txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB); - PreparedStatement pstmt = null; - try { - pstmt = txn.prepareAutoCloseStatement(sql); + final List l = new ArrayList<>(); + try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) { + String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime); + int jobStatusInProgress = JobInfo.Status.IN_PROGRESS.ordinal(); - pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime)); - pstmt.setInt(2, JobInfo.Status.IN_PROGRESS.ordinal()); + try { + PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql); + + pstmt.setString(1, cutTimeStr); + pstmt.setInt(2, jobStatusInProgress); final ResultSet rs = pstmt.executeQuery(); while (rs.next()) { l.add(rs.getLong(1)); } } catch (final SQLException e) { - } catch (final Throwable e) { + s_logger.error(String.format("Unable to execute SQL [%s] with params {\"i.power_state_update_time\": \"%s\", \"j.job_status\": %s} due to [%s].", sql, cutTimeStr, jobStatusInProgress, e.getMessage()), e); } return l; - } finally { - if (txn != null) { - txn.close(); - } } } - // - // VM operation based on new sync model - // - public class VmStateSyncOutcome extends OutcomeImpl { private long _vmId; @@ -5212,11 +4886,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac @Override public boolean checkCondition() { final AsyncJobVO jobVo = _entityMgr.findById(AsyncJobVO.class, job.getId()); - assert jobVo != null; - if (jobVo == null || jobVo.getStatus() != JobInfo.Status.IN_PROGRESS) { - return true; - } - return false; + return jobVo == null || jobVo.getStatus() != JobInfo.Status.IN_PROGRESS; } }, Topics.VM_POWER_STATE, AsyncJob.Topics.JOB_STATE); _vmId = vmId; @@ -5236,12 +4906,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac @Override public boolean checkCondition() { final AsyncJobVO jobVo = _entityMgr.findById(AsyncJobVO.class, job.getId()); - assert jobVo != null; - if (jobVo == null || jobVo.getStatus() != JobInfo.Status.IN_PROGRESS) { - return true; - } - - return false; + return jobVo == null || jobVo.getStatus() != JobInfo.Status.IN_PROGRESS; } }, AsyncJob.Topics.JOB_STATE); _vmId = vmId; @@ -5253,147 +4918,81 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - // - // TODO build a common pattern to reduce code duplication in following methods - // no time for this at current iteration - // public Outcome startVmThroughJobQueue(final String vmUuid, final Map params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) { + String commandName = VmWorkStart.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName); - final CallContext context = CallContext.current(); - final User callingUser = context.getCallingUser(); - final Account callingAccount = context.getCallingAccount(); + VmWorkJobVO workJob = pendingWorkJob.first(); + Long vmId = pendingWorkJob.second(); - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, VmWorkJobVO.Step.Starting, vmId); - VmWorkJobVO workJob = null; - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, - vm.getId(), VmWorkStart.class.getName()); + workJob = newVmWorkJobAndInfo.first(); + VmWorkStart workInfo = new VmWorkStart(newVmWorkJobAndInfo.second()); - if (pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { - workJob = new VmWorkJobVO(context.getContextId()); - - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkStart.class.getName()); - - workJob.setAccountId(callingAccount.getId()); - workJob.setUserId(callingUser.getId()); - workJob.setStep(VmWorkJobVO.Step.Starting); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkStart workInfo = new VmWorkStart(callingUser.getId(), callingAccount.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER); workInfo.setPlan(planToDeploy); workInfo.setParams(params); if (planner != null) { workInfo.setDeploymentPlanner(planner.getName()); } - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmStateSyncOutcome(workJob, - VirtualMachine.PowerState.PowerOn, vm.getId(), null); + VirtualMachine.PowerState.PowerOn, vmId, null); } public Outcome stopVmThroughJobQueue(final String vmUuid, final boolean cleanup) { - final CallContext context = CallContext.current(); - final Account account = context.getCallingAccount(); - final User user = context.getCallingUser(); + String commandName = VmWorkStop.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(null, vmUuid, null, commandName); - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + VmWorkJobVO workJob = pendingWorkJob.first(); + Long vmId = pendingWorkJob.second(); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - vm.getType(), vm.getId(), - VmWorkStop.class.getName()); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, VmWorkJobVO.Step.Prepare, vmId); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { - workJob = new VmWorkJobVO(context.getContextId()); + workJob = newVmWorkJobAndInfo.first(); + VmWorkStop workInfo = new VmWorkStop(newVmWorkJobAndInfo.second(), cleanup); - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkStop.class.getName()); - - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setStep(VmWorkJobVO.Step.Prepare); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkStop workInfo = new VmWorkStop(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, cleanup); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmStateSyncOutcome(workJob, - VirtualMachine.PowerState.PowerOff, vm.getId(), null); + VirtualMachine.PowerState.PowerOff, vmId, null); } public Outcome rebootVmThroughJobQueue(final String vmUuid, final Map params) { + String commandName = VmWorkReboot.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName); - final CallContext context = CallContext.current(); - final Account account = context.getCallingAccount(); - final User user = context.getCallingUser(); + VmWorkJobVO workJob = pendingWorkJob.first(); + Long vmId = pendingWorkJob.second(); - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, VmWorkJobVO.Step.Prepare, vmId); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vm.getId(), - VmWorkReboot.class.getName()); + workJob = newVmWorkJobAndInfo.first(); + VmWorkReboot workInfo = new VmWorkReboot(newVmWorkJobAndInfo.second(), params); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { - workJob = new VmWorkJobVO(context.getContextId()); - - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkReboot.class.getName()); - - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setStep(VmWorkJobVO.Step.Prepare); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkReboot workInfo = new VmWorkReboot(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, params); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmJobVirtualMachineOutcome(workJob, - vm.getId()); + vmId); } public Outcome migrateVmThroughJobQueue(final String vmUuid, final long srcHostId, final DeployDestination dest) { - final CallContext context = CallContext.current(); - final User user = context.getCallingUser(); - final Account account = context.getCallingAccount(); - Map volumeStorageMap = dest.getStorageForDisks(); if (volumeStorageMap != null) { for (Volume vol : volumeStorageMap.keySet()) { @@ -5401,165 +5000,96 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + Long vmId = vm.getId(); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vm.getId(), - VmWorkMigrate.class.getName()); + String commandName = VmWorkMigrate.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmId, vmUuid, VirtualMachine.Type.Instance, commandName); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { + VmWorkJobVO workJob = pendingWorkJob.first(); - workJob = new VmWorkJobVO(context.getContextId()); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkMigrate.class.getName()); + workJob = newVmWorkJobAndInfo.first(); + VmWorkMigrate workInfo = new VmWorkMigrate(newVmWorkJobAndInfo.second(), srcHostId, dest); - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkMigrate workInfo = new VmWorkMigrate(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, srcHostId, dest); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmStateSyncOutcome(workJob, - VirtualMachine.PowerState.PowerOn, vm.getId(), vm.getPowerHostId()); + VirtualMachine.PowerState.PowerOn, vmId, vm.getPowerHostId()); } public Outcome migrateVmAwayThroughJobQueue(final String vmUuid, final long srcHostId) { - final CallContext context = CallContext.current(); - final User user = context.getCallingUser(); - final Account account = context.getCallingAccount(); + VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + Long vmId = vm.getId(); - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + String commandName = VmWorkMigrateAway.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmId, vmUuid, VirtualMachine.Type.Instance, commandName); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vm.getId(), - VmWorkMigrateAway.class.getName()); + VmWorkJobVO workJob = pendingWorkJob.first(); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { - workJob = new VmWorkJobVO(context.getContextId()); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkMigrateAway.class.getName()); + workJob = newVmWorkJobAndInfo.first(); + VmWorkMigrateAway workInfo = new VmWorkMigrateAway(newVmWorkJobAndInfo.second(), srcHostId); - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkMigrateAway workInfo = new VmWorkMigrateAway(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, srcHostId); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); } - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vmId); AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); - return new VmStateSyncOutcome(workJob, VirtualMachine.PowerState.PowerOn, vm.getId(), vm.getPowerHostId()); + return new VmStateSyncOutcome(workJob, VirtualMachine.PowerState.PowerOn, vmId, vm.getPowerHostId()); } public Outcome migrateVmWithStorageThroughJobQueue( final String vmUuid, final long srcHostId, final long destHostId, final Map volumeToPool) { + String commandName = VmWorkMigrateWithStorage.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName); - final CallContext context = CallContext.current(); - final User user = context.getCallingUser(); - final Account account = context.getCallingAccount(); + VmWorkJobVO workJob = pendingWorkJob.first(); + Long vmId = pendingWorkJob.second(); - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vm.getId(), - VmWorkMigrateWithStorage.class.getName()); + workJob = newVmWorkJobAndInfo.first(); + VmWorkMigrateWithStorage workInfo = new VmWorkMigrateWithStorage(newVmWorkJobAndInfo.second(), srcHostId, destHostId, volumeToPool); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { - - workJob = new VmWorkJobVO(context.getContextId()); - - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkMigrateWithStorage.class.getName()); - - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkMigrateWithStorage workInfo = new VmWorkMigrateWithStorage(user.getId(), account.getId(), vm.getId(), - VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, srcHostId, destHostId, volumeToPool); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmStateSyncOutcome(workJob, - VirtualMachine.PowerState.PowerOn, vm.getId(), destHostId); + VirtualMachine.PowerState.PowerOn, vmId, destHostId); } public Outcome migrateVmForScaleThroughJobQueue( final String vmUuid, final long srcHostId, final DeployDestination dest, final Long newSvcOfferingId) { + String commandName = VmWorkMigrateForScale.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName); - final CallContext context = CallContext.current(); - final User user = context.getCallingUser(); - final Account account = context.getCallingAccount(); + VmWorkJobVO workJob = pendingWorkJob.first(); + Long vmId = pendingWorkJob.second(); - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vm.getId(), - VmWorkMigrateForScale.class.getName()); + workJob = newVmWorkJobAndInfo.first(); + VmWorkMigrateForScale workInfo = new VmWorkMigrateForScale(newVmWorkJobAndInfo.second(), srcHostId, dest, newSvcOfferingId); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { - - workJob = new VmWorkJobVO(context.getContextId()); - - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkMigrateForScale.class.getName()); - - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkMigrateForScale workInfo = new VmWorkMigrateForScale(user.getId(), account.getId(), vm.getId(), - VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, srcHostId, dest, newSvcOfferingId); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); - return new VmJobVirtualMachineOutcome(workJob, vm.getId()); + return new VmJobVirtualMachineOutcome(workJob, vmId); } private void checkConcurrentJobsPerDatastoreThreshhold(final StoragePool destPool) { @@ -5572,13 +5102,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - public Outcome migrateVmStorageThroughJobQueue( - final String vmUuid, final Map volumeToPool) { - - final CallContext context = CallContext.current(); - final User user = context.getCallingUser(); - final Account account = context.getCallingAccount(); - + public Outcome migrateVmStorageThroughJobQueue(final String vmUuid, final Map volumeToPool) { Collection poolIds = volumeToPool.values(); Set uniquePoolIds = new HashSet<>(poolIds); for (Long poolId : uniquePoolIds) { @@ -5586,43 +5110,30 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac checkConcurrentJobsPerDatastoreThreshhold(pool); } - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + String commandName = VmWorkStorageMigration.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vm.getId(), - VmWorkStorageMigration.class.getName()); + VmWorkJobVO workJob = pendingWorkJob.first(); + Long vmId = pendingWorkJob.second(); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); - workJob = new VmWorkJobVO(context.getContextId()); + workJob = newVmWorkJobAndInfo.first(); + VmWorkStorageMigration workInfo = new VmWorkStorageMigration(newVmWorkJobAndInfo.second(), volumeToPool); - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkStorageMigration.class.getName()); - - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkStorageMigration workInfo = new VmWorkStorageMigration(user.getId(), account.getId(), vm.getId(), - VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, volumeToPool); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); - return new VmJobVirtualMachineOutcome(workJob, vm.getId()); + return new VmJobVirtualMachineOutcome(workJob, vmId); } public Outcome addVmToNetworkThroughJobQueue( final VirtualMachine vm, final Network network, final NicProfile requested) { + Long vmId = vm.getId(); + String commandName = VmWorkAddVmToNetwork.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName); final CallContext context = CallContext.current(); final User user = context.getCallingUser(); @@ -5663,302 +5174,198 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac public Outcome removeNicFromVmThroughJobQueue( final VirtualMachine vm, final Nic nic) { + Long vmId = vm.getId(); + String commandName = VmWorkRemoveNicFromVm.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName); - final CallContext context = CallContext.current(); - final User user = context.getCallingUser(); - final Account account = context.getCallingAccount(); + VmWorkJobVO workJob = pendingWorkJob.first(); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vm.getId(), - VmWorkRemoveNicFromVm.class.getName()); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { + workJob = newVmWorkJobAndInfo.first(); + VmWorkRemoveNicFromVm workInfo = new VmWorkRemoveNicFromVm(newVmWorkJobAndInfo.second(), nic.getId()); - workJob = new VmWorkJobVO(context.getContextId()); - - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkRemoveNicFromVm.class.getName()); - - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkRemoveNicFromVm workInfo = new VmWorkRemoveNicFromVm(user.getId(), account.getId(), vm.getId(), - VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, nic.getId()); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); - return new VmJobVirtualMachineOutcome(workJob, vm.getId()); + return new VmJobVirtualMachineOutcome(workJob, vmId); } public Outcome removeVmFromNetworkThroughJobQueue( final VirtualMachine vm, final Network network, final URI broadcastUri) { + Long vmId = vm.getId(); + String commandName = VmWorkRemoveVmFromNetwork.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName); - final CallContext context = CallContext.current(); - final User user = context.getCallingUser(); - final Account account = context.getCallingAccount(); + VmWorkJobVO workJob = pendingWorkJob.first(); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vm.getId(), - VmWorkRemoveVmFromNetwork.class.getName()); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { + workJob = newVmWorkJobAndInfo.first(); + VmWorkRemoveVmFromNetwork workInfo = new VmWorkRemoveVmFromNetwork(newVmWorkJobAndInfo.second(), network, broadcastUri); - workJob = new VmWorkJobVO(context.getContextId()); - - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkRemoveVmFromNetwork.class.getName()); - - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkRemoveVmFromNetwork workInfo = new VmWorkRemoveVmFromNetwork(user.getId(), account.getId(), vm.getId(), - VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, network, broadcastUri); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); - return new VmJobVirtualMachineOutcome(workJob, vm.getId()); + return new VmJobVirtualMachineOutcome(workJob, vmId); } public Outcome reconfigureVmThroughJobQueue( final String vmUuid, final ServiceOffering oldServiceOffering, final ServiceOffering newServiceOffering, Map customParameters, final boolean reconfiguringOnExistingHost) { + String commandName = VmWorkReconfigure.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName); - final CallContext context = CallContext.current(); - final User user = context.getCallingUser(); - final Account account = context.getCallingAccount(); + VmWorkJobVO workJob = pendingWorkJob.first(); + Long vmId = pendingWorkJob.second(); - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vm.getId(), - VmWorkReconfigure.class.getName()); + workJob = newVmWorkJobAndInfo.first(); + VmWorkReconfigure workInfo = new VmWorkReconfigure(newVmWorkJobAndInfo.second(), oldServiceOffering.getId(), newServiceOffering.getId(), customParameters, reconfiguringOnExistingHost); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { - - workJob = new VmWorkJobVO(context.getContextId()); - - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkReconfigure.class.getName()); - - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - // save work context info (there are some duplications) - final VmWorkReconfigure workInfo = new VmWorkReconfigure(user.getId(), account.getId(), vm.getId(), - VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, oldServiceOffering.getId(), newServiceOffering.getId(), customParameters, reconfiguringOnExistingHost); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); - return new VmJobVirtualMachineOutcome(workJob, vm.getId()); + return new VmJobVirtualMachineOutcome(workJob, vmId); } @ReflectionUse private Pair orchestrateStart(final VmWorkStart work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); Boolean enterSetup = (Boolean)work.getParams().get(VirtualMachineProfile.Param.BootIntoSetup); - if (s_logger.isTraceEnabled()) { - s_logger.trace(String.format("orchestrating VM start for '%s' %s set to %s", vm.getInstanceName(), VirtualMachineProfile.Param.BootIntoSetup, enterSetup)); + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("orchestrating VM start for '%s' %s set to %s", vm.getInstanceName(), VirtualMachineProfile.Param.BootIntoSetup, enterSetup)); } try { orchestrateStart(vm.getUuid(), work.getParams(), work.getPlan(), _dpMgr.getDeploymentPlannerByName(work.getDeploymentPlanner())); - } catch (CloudRuntimeException e) { - e.printStackTrace(); - s_logger.info("Caught CloudRuntimeException, returning job failed " + e); - CloudRuntimeException ex = new CloudRuntimeException("Unable to start VM instance"); - return new Pair(JobInfo.Status.FAILED, JobSerializerHelper.toObjectSerializedString(ex)); + } catch (CloudRuntimeException e){ + String message = String.format("Unable to orchestrate start %s due to [%s].", vm.toString(), e.getMessage()); + s_logger.warn(message, e); + CloudRuntimeException ex = new CloudRuntimeException(message); + return new Pair<>(JobInfo.Status.FAILED, JobSerializerHelper.toObjectSerializedString(ex)); } - return new Pair(JobInfo.Status.SUCCEEDED, null); + return new Pair<>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair orchestrateStop(final VmWorkStop work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - throw new CloudRuntimeException("Unable to find VM id=" + work.getVmId()); + String message = String.format("Unable to find VM [%s].", work.getVmId()); + s_logger.warn(message); + throw new CloudRuntimeException(message); } orchestrateStop(vm.getUuid(), work.isCleanup()); - return new Pair(JobInfo.Status.SUCCEEDED, null); + return new Pair<>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair orchestrateMigrate(final VmWorkMigrate work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); orchestrateMigrate(vm.getUuid(), work.getSrcHostId(), work.getDeployDestination()); - return new Pair(JobInfo.Status.SUCCEEDED, null); + return new Pair<>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair orchestrateMigrateAway(final VmWorkMigrateAway work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); try { orchestrateMigrateAway(vm.getUuid(), work.getSrcHostId(), null); } catch (final InsufficientServerCapacityException e) { - s_logger.warn("Failed to deploy vm " + vm.getId() + " with original planner, sending HAPlanner"); + s_logger.warn("Failed to deploy vm " + vm.getId() + " with original planner, sending HAPlanner", e); orchestrateMigrateAway(vm.getUuid(), work.getSrcHostId(), _haMgr.getHAPlanner()); } - return new Pair(JobInfo.Status.SUCCEEDED, null); + return new Pair<>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair orchestrateMigrateWithStorage(final VmWorkMigrateWithStorage work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); orchestrateMigrateWithStorage(vm.getUuid(), work.getSrcHostId(), work.getDestHostId(), work.getVolumeToPool()); - return new Pair(JobInfo.Status.SUCCEEDED, null); + return new Pair<>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair orchestrateMigrateForScale(final VmWorkMigrateForScale work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); orchestrateMigrateForScale(vm.getUuid(), work.getSrcHostId(), work.getDeployDestination(), work.getNewServiceOfferringId()); - return new Pair(JobInfo.Status.SUCCEEDED, null); + return new Pair<>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair orchestrateReboot(final VmWorkReboot work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); orchestrateReboot(vm.getUuid(), work.getParams()); - return new Pair(JobInfo.Status.SUCCEEDED, null); + return new Pair<>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair orchestrateAddVmToNetwork(final VmWorkAddVmToNetwork work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); final Network network = _networkDao.findById(work.getNetworkId()); final NicProfile nic = orchestrateAddVmToNetwork(vm, network, work.getRequestedNicProfile()); - return new Pair(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(nic)); + return new Pair<>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(nic)); } @ReflectionUse private Pair orchestrateRemoveNicFromVm(final VmWorkRemoveNicFromVm work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); final NicVO nic = _entityMgr.findById(NicVO.class, work.getNicId()); final boolean result = orchestrateRemoveNicFromVm(vm, nic); - return new Pair(JobInfo.Status.SUCCEEDED, + return new Pair<>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(result)); } @ReflectionUse private Pair orchestrateRemoveVmFromNetwork(final VmWorkRemoveVmFromNetwork work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); final boolean result = orchestrateRemoveVmFromNetwork(vm, work.getNetwork(), work.getBroadcastUri()); - return new Pair(JobInfo.Status.SUCCEEDED, + return new Pair<>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(result)); } @ReflectionUse private Pair orchestrateReconfigure(final VmWorkReconfigure work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); ServiceOfferingVO oldServiceOffering = _offeringDao.findById(work.getOldServiceOfferingId()); ServiceOfferingVO newServiceOffering = _offeringDao.findById(work.getNewServiceOfferingId()); if (newServiceOffering.isDynamic()) { - // update the service offering object with the custom parameters like cpu, memory newServiceOffering = _offeringDao.getComputeOffering(newServiceOffering, work.getCustomParameters()); } reConfigureVm(vm.getUuid(), oldServiceOffering, newServiceOffering, work.getCustomParameters(), work.isSameHost()); - return new Pair(JobInfo.Status.SUCCEEDED, null); + return new Pair<>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair orchestrateStorageMigration(final VmWorkStorageMigration work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); orchestrateStorageMigration(vm.getUuid(), work.getVolumeToPool()); - - return new Pair(JobInfo.Status.SUCCEEDED, null); + return new Pair<>(JobInfo.Status.SUCCEEDED, null); } @Override @@ -6013,31 +5420,17 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else { final Outcome outcome = restoreVirtualMachineThroughJobQueue(vmId, newTemplateId); - try { - outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution exception", e); + retrieveVmFromJobOutcome(outcome, String.valueOf(vmId), "restoreVirtualMachine"); + + Object jobResult = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + + if (jobResult != null && jobResult instanceof HashMap) { + HashMap passwordMap = (HashMap)jobResult; + UserVmVO userVm = _userVmDao.findById(vmId); + userVm.setPassword(passwordMap.get(vmId)); + return userVm; } - final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobResult != null) { - if (jobResult instanceof ResourceUnavailableException) { - throw (ResourceUnavailableException)jobResult; - } else if (jobResult instanceof ConcurrentOperationException) { - throw (ConcurrentOperationException)jobResult; - } else if (jobResult instanceof RuntimeException) { - throw (RuntimeException)jobResult; - } else if (jobResult instanceof Throwable) { - throw new RuntimeException("Unexpected exception", (Throwable)jobResult); - } else if (jobResult instanceof HashMap) { - HashMap passwordMap = (HashMap)jobResult; - UserVmVO userVm = _userVmDao.findById(vmId); - userVm.setPassword(passwordMap.get(vmId)); - return userVm; - } - } throw new RuntimeException("Unexpected job execution result"); } } @@ -6050,37 +5443,18 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } public Outcome restoreVirtualMachineThroughJobQueue(final long vmId, final Long newTemplateId) { + String commandName = VmWorkRestore.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName); - final CallContext context = CallContext.current(); - final User user = context.getCallingUser(); - final Account account = context.getCallingAccount(); + VmWorkJobVO workJob = pendingWorkJob.first(); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vmId, - VmWorkRestore.class.getName()); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { + workJob = newVmWorkJobAndInfo.first(); + VmWorkRestore workInfo = new VmWorkRestore(newVmWorkJobAndInfo.second(), newTemplateId); - workJob = new VmWorkJobVO(context.getContextId()); - - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkRestore.class.getName()); - - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vmId); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - final VmWorkRestore workInfo = new VmWorkRestore(user.getId(), account.getId(), vmId, - VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, newTemplateId); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vmId); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); @@ -6089,15 +5463,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac @ReflectionUse private Pair orchestrateRestoreVirtualMachine(final VmWorkRestore work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); UserVm uservm = orchestrateRestoreVirtualMachine(vm.getId(), work.getTemplateId()); - HashMap passwordMap = new HashMap(); + HashMap passwordMap = new HashMap<>(); passwordMap.put(uservm.getId(), uservm.getPassword()); - return new Pair(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(passwordMap)); + return new Pair<>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(passwordMap)); } @Override @@ -6105,8 +5475,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { - VmWorkJobVO placeHolder = null; - placeHolder = createPlaceHolderWork(vm.getId()); + VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId()); try { return orchestrateUpdateDefaultNicForVM(vm, nic, defaultNic); } finally { @@ -6117,19 +5486,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else { final Outcome outcome = updateDefaultNicForVMThroughJobQueue(vm, nic, defaultNic); - try { - outcome.get(); - } catch (final InterruptedException e) { - throw new RuntimeException("Operation is interrupted", e); - } catch (final java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution exception", e); - } + retrieveVmFromJobOutcome(outcome, vm.getUuid(), "updateDefaultNicForVM"); - final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); - if (jobResult != null) { - if (jobResult instanceof Boolean) { + try { + Object jobResult = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + + if (jobResult != null && jobResult instanceof Boolean) { return (Boolean)jobResult; } + } catch (ResourceUnavailableException | InsufficientCapacityException ex) { + throw new RuntimeException("Unhandled exception", ex); } throw new RuntimeException("Unexpected job execution result"); @@ -6155,50 +5521,28 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } public Outcome updateDefaultNicForVMThroughJobQueue(final VirtualMachine vm, final Nic nic, final Nic defaultNic) { + Long vmId = vm.getId(); + String commandName = VmWorkUpdateDefaultNic.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName); - final CallContext context = CallContext.current(); - final User user = context.getCallingUser(); - final Account account = context.getCallingAccount(); + VmWorkJobVO workJob = pendingWorkJob.first(); - final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( - VirtualMachine.Type.Instance, vm.getId(), - VmWorkUpdateDefaultNic.class.getName()); + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); - VmWorkJobVO workJob = null; - if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { - assert pendingWorkJobs.size() == 1; - workJob = pendingWorkJobs.get(0); - } else { + workJob = newVmWorkJobAndInfo.first(); + VmWorkUpdateDefaultNic workInfo = new VmWorkUpdateDefaultNic(newVmWorkJobAndInfo.second(), nic.getId(), defaultNic.getId()); - workJob = new VmWorkJobVO(context.getContextId()); - - workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); - workJob.setCmd(VmWorkUpdateDefaultNic.class.getName()); - - workJob.setAccountId(account.getId()); - workJob.setUserId(user.getId()); - workJob.setVmType(VirtualMachine.Type.Instance); - workJob.setVmInstanceId(vm.getId()); - workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); - - final VmWorkUpdateDefaultNic workInfo = new VmWorkUpdateDefaultNic(user.getId(), account.getId(), vm.getId(), - VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, nic.getId(), defaultNic.getId()); - workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); - - _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); - return new VmJobVirtualMachineOutcome(workJob, vm.getId()); + return new VmJobVirtualMachineOutcome(workJob, vmId); } @ReflectionUse private Pair orchestrateUpdateDefaultNic(final VmWorkUpdateDefaultNic work) throws Exception { - final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); - if (vm == null) { - s_logger.info("Unable to find vm " + work.getVmId()); - } - assert vm != null; + VMInstanceVO vm = findVmById(work.getVmId()); final NicVO nic = _entityMgr.findById(NicVO.class, work.getNicId()); if (nic == null) { throw new CloudRuntimeException("Unable to find nic " + work.getNicId()); @@ -6208,7 +5552,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new CloudRuntimeException("Unable to find default nic " + work.getDefaultNicId()); } final boolean result = orchestrateUpdateDefaultNicForVM(vm, nic, defaultNic); - return new Pair(JobInfo.Status.SUCCEEDED, + return new Pair<>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(result)); } @@ -6261,4 +5605,128 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } return findClusterAndHostIdForVm(vm); } + + protected VirtualMachine retrieveVmFromJobOutcome(Outcome jobOutcome, String vmUuid, String jobName) { + try { + return jobOutcome.get(); + } catch (InterruptedException | java.util.concurrent.ExecutionException e) { + throw new RuntimeException(String.format("Unable to retrieve result from job \"%s\" due to [%s]. VM {\"uuid\": \"%s\"}.", jobName, e.getMessage(), vmUuid), e); + } + } + + protected Object retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(Outcome outcome) throws ResourceUnavailableException, InsufficientCapacityException{ + Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); + + if (jobResult == null) { + return null; + } + + if (jobResult instanceof AgentUnavailableException) { + throw (AgentUnavailableException) jobResult; + } + + if (jobResult instanceof InsufficientServerCapacityException) { + throw (InsufficientServerCapacityException) jobResult; + } + + if (jobResult instanceof ResourceUnavailableException) { + throw (ResourceUnavailableException) jobResult; + } + + if (jobResult instanceof InsufficientCapacityException) { + throw (InsufficientCapacityException) jobResult; + } + + if (jobResult instanceof ConcurrentOperationException) { + throw (ConcurrentOperationException) jobResult; + } + + if (jobResult instanceof RuntimeException) { + throw (RuntimeException) jobResult; + } + + if (jobResult instanceof Throwable) { + throw new RuntimeException("Unexpected exception", (Throwable)jobResult); + } + + return jobResult; + } + + protected Pair retrievePendingWorkJob(String vmUuid, String commandName) { + return retrievePendingWorkJob(null, vmUuid, VirtualMachine.Type.Instance, commandName); + } + + protected Pair retrievePendingWorkJob(Long id, String commandName) { + return retrievePendingWorkJob(id, null, VirtualMachine.Type.Instance, commandName); + } + + protected Pair retrievePendingWorkJob(Long vmId, String vmUuid, VirtualMachine.Type vmType, String commandName) { + if (vmId == null) { + VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + + if (vm == null) { + String message = String.format("Could not find a VM with the uuid [%s]. Unable to continue validations with command [%s] through job queue.", vmUuid, commandName); + s_logger.error(message); + throw new RuntimeException(message); + } + + vmId = vm.getId(); + + if (vmType == null) { + vmType = vm.getType(); + } + } + + List pendingWorkJobs = _workJobDao.listPendingWorkJobs(vmType, vmId, commandName); + + if (CollectionUtils.isNotEmpty(pendingWorkJobs)) { + return new Pair<>(pendingWorkJobs.get(0), vmId); + } + + return new Pair<>(null, vmId); + } + + protected Pair createWorkJobAndWorkInfo(String commandName, Long vmId) { + return createWorkJobAndWorkInfo(commandName, null, vmId); + } + + protected Pair createWorkJobAndWorkInfo(String commandName, VmWorkJobVO.Step step, Long vmId) { + CallContext context = CallContext.current(); + long userId = context.getCallingUser().getId(); + long accountId = context.getCallingAccount().getId(); + + VmWorkJobVO workJob = new VmWorkJobVO(context.getContextId()); + workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); + workJob.setCmd(commandName); + workJob.setAccountId(accountId); + workJob.setUserId(userId); + + if (step != null) { + workJob.setStep(step); + } + + workJob.setVmType(VirtualMachine.Type.Instance); + workJob.setVmInstanceId(vmId); + workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); + + VmWork workInfo = new VmWork(userId, accountId, vmId, VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER); + + return new Pair<>(workJob, workInfo); + } + + protected void setCmdInfoAndSubmitAsyncJob(VmWorkJobVO workJob, VmWork workInfo, Long vmId) { + workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); + _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vmId); + } + + protected VMInstanceVO findVmById(Long vmId) { + VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, vmId); + + if (vm == null) { + s_logger.warn(String.format("Could not find VM [%s].", vmId)); + } + + assert vm != null; + return vm; + } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkAddVmToNetwork.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkAddVmToNetwork.java index a56259b93ea..4c2a89a4dbe 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkAddVmToNetwork.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkAddVmToNetwork.java @@ -30,6 +30,12 @@ public class VmWorkAddVmToNetwork extends VmWork { requstedNicProfile = requested; } + public VmWorkAddVmToNetwork(VmWork vmWork, long networkId, NicProfile requested) { + super(vmWork); + this.networkId = networkId; + this.requstedNicProfile = requested; + } + public Long getNetworkId() { return networkId; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrate.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrate.java index 1d7deca2d9e..253b750172c 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrate.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrate.java @@ -41,18 +41,27 @@ public class VmWorkMigrate extends VmWork { public VmWorkMigrate(long userId, long accountId, long vmId, String handlerName, long srcHostId, DeployDestination dst) { super(userId, accountId, vmId, handlerName); + setConstructorParams(srcHostId, dst); + } + + public VmWorkMigrate(VmWork vmWork, long srcHostId, DeployDestination dest) { + super(vmWork); + setConstructorParams(srcHostId, dest); + } + + private void setConstructorParams(long srcHostId, DeployDestination dest) { this.srcHostId = srcHostId; - zoneId = dst.getDataCenter() != null ? dst.getDataCenter().getId() : null; - podId = dst.getPod() != null ? dst.getPod().getId() : null; - clusterId = dst.getCluster() != null ? dst.getCluster().getId() : null; - hostId = dst.getHost() != null ? dst.getHost().getId() : null; - if (dst.getStorageForDisks() != null) { - storage = new HashMap(dst.getStorageForDisks().size()); - for (Map.Entry entry : dst.getStorageForDisks().entrySet()) { - storage.put(entry.getKey().getUuid(), entry.getValue().getUuid()); + this.zoneId = dest.getDataCenter() != null ? dest.getDataCenter().getId() : null; + this.podId = dest.getPod() != null ? dest.getPod().getId() : null; + this.clusterId = dest.getCluster() != null ? dest.getCluster().getId() : null; + this.hostId = dest.getHost() != null ? dest.getHost().getId() : null; + if (dest.getStorageForDisks() != null) { + this.storage = new HashMap<>(dest.getStorageForDisks().size()); + for (Map.Entry entry : dest.getStorageForDisks().entrySet()) { + this.storage.put(entry.getKey().getUuid(), entry.getValue().getUuid()); } } else { - storage = null; + this.storage = null; } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateAway.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateAway.java index 92189edca4d..ef89129d1df 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateAway.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateAway.java @@ -30,6 +30,11 @@ public class VmWorkMigrateAway extends VmWork { this.srcHostId = srcHostId; } + public VmWorkMigrateAway(VmWork vmWork, long srcHostId) { + super(vmWork); + this.srcHostId = srcHostId; + } + public long getSrcHostId() { return srcHostId; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateForScale.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateForScale.java index 2147a54a4ca..35cfbbf758a 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateForScale.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateForScale.java @@ -30,6 +30,11 @@ public class VmWorkMigrateForScale extends VmWorkMigrate { this.newSvcOfferingId = newSvcOfferingId; } + public VmWorkMigrateForScale(VmWork vmWork, long srcHostId, DeployDestination dest, Long newSvcOfferingId) { + super(vmWork, srcHostId, dest); + this.newSvcOfferingId = newSvcOfferingId; + } + public Long getNewServiceOfferringId() { return newSvcOfferingId; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateWithStorage.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateWithStorage.java index 52fe9f28771..3097800667a 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateWithStorage.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrateWithStorage.java @@ -35,6 +35,13 @@ public class VmWorkMigrateWithStorage extends VmWork { this.volumeToPool = volumeToPool; } + public VmWorkMigrateWithStorage(VmWork vmWork, long srcHostId, long destHostId, Map volumeToPool) { + super(vmWork); + this.srcHostId = srcHostId; + this.destHostId = destHostId; + this.volumeToPool = volumeToPool; + } + public long getSrcHostId() { return srcHostId; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkReboot.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkReboot.java index 63c7d004968..6a903b3d781 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkReboot.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkReboot.java @@ -34,6 +34,11 @@ public class VmWorkReboot extends VmWork { setParams(params); } + public VmWorkReboot(VmWork vmWork, Map params) { + super(vmWork); + setParams(params); + } + public Map getParams() { Map map = new HashMap(); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkReconfigure.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkReconfigure.java index f96a587dac0..fdbcb9f6f5e 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkReconfigure.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkReconfigure.java @@ -39,6 +39,14 @@ public class VmWorkReconfigure extends VmWork { this.sameHost = sameHost; } + public VmWorkReconfigure(VmWork vmWork, long oldServiceOfferingId, long newServiceOfferingId, Map customParameters, boolean sameHost) { + super(vmWork); + this.oldServiceOfferingId = oldServiceOfferingId; + this.newServiceOfferingId = newServiceOfferingId; + this.customParameters = customParameters; + this.sameHost = sameHost; + } + public Long getOldServiceOfferingId() { return oldServiceOfferingId; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRemoveNicFromVm.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRemoveNicFromVm.java index 50f158e421d..acbdd22702e 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRemoveNicFromVm.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRemoveNicFromVm.java @@ -27,6 +27,11 @@ public class VmWorkRemoveNicFromVm extends VmWork { this.nicId = nicId; } + public VmWorkRemoveNicFromVm(VmWork vmWork, long nicId) { + super(vmWork); + this.nicId = nicId; + } + public Long getNicId() { return nicId; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRemoveVmFromNetwork.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRemoveVmFromNetwork.java index 535b8d00faa..719c7f9ef1c 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRemoveVmFromNetwork.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRemoveVmFromNetwork.java @@ -33,6 +33,12 @@ public class VmWorkRemoveVmFromNetwork extends VmWork { this.broadcastUri = broadcastUri; } + public VmWorkRemoveVmFromNetwork(VmWork vmWork, Network network, URI broadcastUri) { + super(vmWork); + this.network = network; + this.broadcastUri = broadcastUri; + } + public Network getNetwork() { return network; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRestore.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRestore.java index f8162e48ffa..cb3adae27aa 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRestore.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkRestore.java @@ -27,6 +27,11 @@ public class VmWorkRestore extends VmWork { this.templateId = templateId; } + public VmWorkRestore(VmWork vmWork, Long templateId) { + super(vmWork); + this.templateId = templateId; + } + public Long getTemplateId() { return templateId; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStart.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStart.java index 814e78c2830..1b2a7194d17 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStart.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStart.java @@ -54,6 +54,10 @@ public class VmWorkStart extends VmWork { super(userId, accountId, vmId, handlerName); } + public VmWorkStart(VmWork vmWork){ + super(vmWork); + } + public DeploymentPlan getPlan() { if (podId != null || clusterId != null || hostId != null || poolId != null || physicalNetworkId != null || avoids !=null) { diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStop.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStop.java index 6d4148000c8..00ff7559cba 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStop.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStop.java @@ -26,6 +26,11 @@ public class VmWorkStop extends VmWork { this.cleanup = cleanup; } + public VmWorkStop(VmWork vmWork, boolean cleanup) { + super(vmWork); + this.cleanup = cleanup; + } + public boolean isCleanup() { return cleanup; } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStorageMigration.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStorageMigration.java index 07e8549d224..ce3ef9f121f 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStorageMigration.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkStorageMigration.java @@ -23,7 +23,7 @@ public class VmWorkStorageMigration extends VmWork { Map volumeToPool; - public VmWorkStorageMigration(long userId, long accountId, long vmId, String handlerName, Map volumeToPool) { + public VmWorkStorageMigration(long userId, long accountId, long vmId, String handlerName, Map volumeToPool) { super(userId, accountId, vmId, handlerName); this.volumeToPool = volumeToPool; @@ -32,4 +32,9 @@ public class VmWorkStorageMigration extends VmWork { public Map getVolumeToPool() { return volumeToPool; } + + public VmWorkStorageMigration(VmWork vmWork, Map volumeToPool) { + super(vmWork); + this.volumeToPool = volumeToPool; + } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkUpdateDefaultNic.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkUpdateDefaultNic.java index 14f32399449..b1a936d4c70 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkUpdateDefaultNic.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkUpdateDefaultNic.java @@ -29,6 +29,12 @@ public class VmWorkUpdateDefaultNic extends VmWork { this.defaultNicId = defaultNicId; } + public VmWorkUpdateDefaultNic(VmWork vmWork, long nicId, long defaultNicId) { + super(vmWork); + this.nicId = nicId; + this.defaultNicId = defaultNicId; + } + public Long getNicId() { return nicId; } diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java index 61df40e50d8..8f66da052e9 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java @@ -31,6 +31,8 @@ import javax.persistence.Temporal; import javax.persistence.TemporalType; import javax.persistence.Transient; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; @@ -551,15 +553,9 @@ public class VMTemplateVO implements VirtualMachineTemplate { return uniqueName.hashCode(); } - @Transient - String toString; - @Override public String toString() { - if (toString == null) { - toString = new StringBuilder("Tmpl[").append(id).append("-").append(format).append("-").append(uniqueName).toString(); - } - return toString; + return String.format("Template %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uniqueName", "format")); } public void setRemoved(Date removed) { diff --git a/framework/ipc/src/main/java/com/cloud/agent/manager/Commands.java b/framework/ipc/src/main/java/com/cloud/agent/manager/Commands.java index 50ab85e8b47..208512b5306 100644 --- a/framework/ipc/src/main/java/com/cloud/agent/manager/Commands.java +++ b/framework/ipc/src/main/java/com/cloud/agent/manager/Commands.java @@ -24,6 +24,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.Command.OnError; import com.cloud.utils.exception.CloudRuntimeException; +import java.util.stream.Collectors; public class Commands implements Iterable { OnError _handler; @@ -150,4 +151,10 @@ public class Commands implements Iterable { public Iterator iterator() { return _cmds.iterator(); } + + @Override + public String toString() { + return _cmds.stream().map(cmd -> cmd.toString()).collect(Collectors.joining(", ")); + } + }