From fd74895ad05da991a1ffc809343df8dd427a073b Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 2 May 2025 09:15:03 +0200 Subject: [PATCH] New feature: Reconcile commands (CopyCommand, MigrateCommand, MigrateVolumeCommand) (#10514) --- .../src/main/java/com/cloud/agent/Agent.java | 8 + .../java/com/cloud/agent/api/Command.java | 34 + .../java/com/cloud/agent/api/to/DiskTO.java | 6 +- .../com/cloud/agent/api/to/NetworkTO.java | 6 +- .../java/com/cloud/agent/api/to/NicTO.java | 8 + .../cloud/agent/api/to/VirtualMachineTO.java | 12 +- .../command/ReconcileCommandService.java | 65 + .../com/cloud/agent/api/MigrateCommand.java | 25 +- .../java/com/cloud/agent/api/PingAnswer.java | 15 + .../java/com/cloud/agent/api/PingCommand.java | 11 + .../api/storage/MigrateVolumeCommand.java | 5 + .../com/cloud/resource/ServerResource.java | 2 + .../java/com/cloud/serializer/GsonHelper.java | 2 +- .../cloudstack/command/CommandInfo.java | 124 ++ .../cloudstack/command/ReconcileAnswer.java | 45 + .../cloudstack/command/ReconcileCommand.java | 33 + .../command/ReconcileCommandUtils.java | 192 +++ .../command/ReconcileCopyAnswer.java | 56 + .../command/ReconcileCopyCommand.java | 53 + .../command/ReconcileMigrateAnswer.java | 68 + .../command/ReconcileMigrateCommand.java | 31 + .../command/ReconcileMigrateVolumeAnswer.java | 50 + .../ReconcileMigrateVolumeCommand.java | 48 + .../command/ReconcileVolumeAnswer.java | 46 + .../storage/command/CopyCommand.java | 5 + .../test/CheckGuestOsMappingCommandTest.java | 2 - .../command/ReconcileCommandUtilsTest.java | 69 + .../com/cloud/agent/manager/AgentAttache.java | 74 +- .../cloud/agent/manager/AgentManagerImpl.java | 61 +- .../agent/manager/ClusteredAgentAttache.java | 9 +- .../manager/ClusteredAgentManagerImpl.java | 6 +- .../manager/ClusteredDirectAgentAttache.java | 5 +- .../agent/manager/ConnectedAgentAttache.java | 5 +- .../agent/manager/DirectAgentAttache.java | 5 +- .../com/cloud/agent/manager/DummyAttache.java | 5 +- .../cloud/vm/VirtualMachineManagerImpl.java | 26 +- .../agent/manager/AgentManagerImplTest.java | 3 +- .../manager/ConnectedAgentAttacheTest.java | 17 +- .../agent/manager/DirectAgentAttacheTest.java | 3 +- .../java/com/cloud/storage/dao/VolumeDao.java | 2 + .../com/cloud/storage/dao/VolumeDaoImpl.java | 11 + .../command/ReconcileCommandVO.java | 216 ++++ .../command/dao/ReconcileCommandDao.java | 45 + .../command/dao/ReconcileCommandDaoImpl.java | 134 ++ ...spring-engine-schema-core-daos-context.xml | 1 + .../META-INF/db/schema-42010to42100.sql | 23 + .../StorageSystemDataMotionStrategy.java | 51 +- .../com/cloud/cluster/ClusterManagerImpl.java | 6 + .../jobs/impl/AsyncJobManagerImpl.java | 33 + .../resource/LibvirtComputingResource.java | 156 ++- .../disconnecthook/DisconnectHook.java | 59 + .../disconnecthook/MigrationCancelHook.java | 50 + .../VolumeMigrationCancelHook.java | 53 + ...virtCheckVirtualMachineCommandWrapper.java | 7 + .../LibvirtCopyVolumeCommandWrapper.java | 10 + .../wrapper/LibvirtMigrateCommandWrapper.java | 20 + .../LibvirtMigrateVolumeCommandWrapper.java | 32 +- .../LibvirtReconcileCommandWrapper.java | 258 ++++ .../wrapper/LibvirtRequestWrapper.java | 9 +- .../kvm/storage/KVMStorageProcessor.java | 20 +- .../kvm/resource/DisconnectHooksTest.java | 191 +++ .../LibvirtComputingResourceTest.java | 10 +- .../vmware/resource/VmwareResource.java | 6 +- .../resource/CitrixResourceBase.java | 4 +- .../driver/ScaleIOPrimaryDataStoreDriver.java | 51 +- .../cloud/ha/HighAvailabilityManagerImpl.java | 8 +- .../java/com/cloud/hypervisor/KVMGuru.java | 2 +- .../command/ReconcileCommandServiceImpl.java | 1144 +++++++++++++++++ .../spring-server-core-managers-context.xml | 3 + .../com/cloud/hypervisor/KVMGuruTest.java | 4 +- 70 files changed, 3769 insertions(+), 90 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/command/ReconcileCommandService.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/CommandInfo.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/ReconcileAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/ReconcileCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/ReconcileCommandUtils.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/ReconcileCopyAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/ReconcileCopyCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/command/ReconcileVolumeAnswer.java create mode 100644 core/src/test/java/org/apache/cloudstack/command/ReconcileCommandUtilsTest.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/command/ReconcileCommandVO.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/command/dao/ReconcileCommandDao.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/command/dao/ReconcileCommandDaoImpl.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/DisconnectHook.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/MigrationCancelHook.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/VolumeMigrationCancelHook.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReconcileCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/DisconnectHooksTest.java create mode 100644 server/src/main/java/org/apache/cloudstack/command/ReconcileCommandServiceImpl.java diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index ad480fef4e5..fcd4234a136 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -800,6 +800,9 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater } commandsInProgress.incrementAndGet(); try { + if (cmd.isReconcile()) { + cmd.setRequestSequence(request.getSequence()); + } answer = serverResource.executeRequest(cmd); } finally { commandsInProgress.decrementAndGet(); @@ -1021,6 +1024,8 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater if ((answer.isSendStartup()) && reconnectAllowed) { logger.info("Management server requested startup command to reinitialize the agent"); sendStartup(link); + } else { + serverResource.processPingAnswer((PingAnswer) answer); } shell.setAvoidHosts(answer.getAvoidMsList()); } @@ -1087,6 +1092,9 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater Answer answer = null; commandsInProgress.incrementAndGet(); try { + if (command.isReconcile()) { + command.setRequestSequence(req.getSequence()); + } answer = serverResource.executeRequest(command); } finally { commandsInProgress.decrementAndGet(); diff --git a/api/src/main/java/com/cloud/agent/api/Command.java b/api/src/main/java/com/cloud/agent/api/Command.java index eb979c0060b..4766c30ead2 100644 --- a/api/src/main/java/com/cloud/agent/api/Command.java +++ b/api/src/main/java/com/cloud/agent/api/Command.java @@ -35,6 +35,23 @@ public abstract class Command { Continue, Stop } + public enum State { + CREATED, // Command is created by management server + STARTED, // Command is started by agent + PROCESSING, // Processing by agent + PROCESSING_IN_BACKEND, // Processing in backend by agent + COMPLETED, // Operation succeeds by agent or management server + FAILED, // Operation fails by agent + RECONCILE_RETRY, // Ready for retry of reconciliation + RECONCILING, // Being reconciled by management server + RECONCILED, // Reconciled by management server + RECONCILE_SKIPPED, // Skip the reconciliation as the resource state is inconsistent with the command + RECONCILE_FAILED, // Fail to reconcile by management server + TIMED_OUT, // Timed out on management server or agent + INTERRUPTED, // Interrupted by management server or agent (for example agent is restarted), + DANGLED_IN_BACKEND // Backend process which cannot be processed normally (for example agent is restarted) + } + public static final String HYPERVISOR_TYPE = "hypervisorType"; // allow command to carry over hypervisor or other environment related context info @@ -42,6 +59,7 @@ public abstract class Command { protected Map contextMap = new HashMap(); private int wait; //in second private boolean bypassHostMaintenance = false; + private transient long requestSequence = 0L; protected Command() { this.wait = 0; @@ -82,6 +100,10 @@ public abstract class Command { return contextMap.get(name); } + public Map getContextMap() { + return contextMap; + } + public boolean allowCaching() { return true; } @@ -94,6 +116,18 @@ public abstract class Command { this.bypassHostMaintenance = bypassHostMaintenance; } + public boolean isReconcile() { + return false; + } + + public long getRequestSequence() { + return requestSequence; + } + + public void setRequestSequence(long requestSequence) { + this.requestSequence = requestSequence; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/api/src/main/java/com/cloud/agent/api/to/DiskTO.java b/api/src/main/java/com/cloud/agent/api/to/DiskTO.java index d22df2df172..5664de79091 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DiskTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DiskTO.java @@ -46,7 +46,7 @@ public class DiskTO { private Long diskSeq; private String path; private Volume.Type type; - private Map _details; + private Map details; public DiskTO() { @@ -92,10 +92,10 @@ public class DiskTO { } public void setDetails(Map details) { - _details = details; + this.details = details; } public Map getDetails() { - return _details; + return details; } } diff --git a/api/src/main/java/com/cloud/agent/api/to/NetworkTO.java b/api/src/main/java/com/cloud/agent/api/to/NetworkTO.java index bd08ce81101..d65ec0e3daa 100644 --- a/api/src/main/java/com/cloud/agent/api/to/NetworkTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/NetworkTO.java @@ -36,7 +36,7 @@ public class NetworkTO { protected TrafficType type; protected URI broadcastUri; protected URI isolationUri; - protected boolean isSecurityGroupEnabled; + protected boolean securityGroupEnabled; protected String name; protected String ip6address; protected String ip6gateway; @@ -112,7 +112,7 @@ public class NetworkTO { } public void setSecurityGroupEnabled(boolean enabled) { - this.isSecurityGroupEnabled = enabled; + this.securityGroupEnabled = enabled; } /** @@ -221,7 +221,7 @@ public class NetworkTO { } public boolean isSecurityGroupEnabled() { - return this.isSecurityGroupEnabled; + return this.securityGroupEnabled; } public void setIp6Dns1(String ip6Dns1) { diff --git a/api/src/main/java/com/cloud/agent/api/to/NicTO.java b/api/src/main/java/com/cloud/agent/api/to/NicTO.java index 573363c04fb..ca95fcfd679 100644 --- a/api/src/main/java/com/cloud/agent/api/to/NicTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/NicTO.java @@ -86,6 +86,14 @@ public class NicTO extends NetworkTO { this.nicUuid = uuid; } + public String getNicUuid() { + return nicUuid; + } + + public void setNicUuid(String nicUuid) { + this.nicUuid = nicUuid; + } + @Override public String toString() { return new StringBuilder("[Nic:").append(type).append("-").append(ip).append("-").append(broadcastUri).append("]").toString(); diff --git a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java index 6f24b1cd6ca..6c415ec1df1 100644 --- a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java @@ -61,7 +61,7 @@ public class VirtualMachineTO { @LogLevel(LogLevel.Log4jLevel.Off) String vncPassword; String vncAddr; - Map params; + Map details; String uuid; String bootType; String bootMode; @@ -191,7 +191,11 @@ public class VirtualMachineTO { return maxSpeed; } - public boolean getLimitCpuUse() { + public boolean isEnableHA() { + return enableHA; + } + + public boolean isLimitCpuUse() { return limitCpuUse; } @@ -289,11 +293,11 @@ public class VirtualMachineTO { } public Map getDetails() { - return params; + return details; } public void setDetails(Map params) { - this.params = params; + this.details = params; } public String getUuid() { diff --git a/api/src/main/java/org/apache/cloudstack/command/ReconcileCommandService.java b/api/src/main/java/org/apache/cloudstack/command/ReconcileCommandService.java new file mode 100644 index 00000000000..89ab97990df --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/command/ReconcileCommandService.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.command; + + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.hypervisor.Hypervisor; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.framework.config.ConfigKey; + +import java.util.Arrays; +import java.util.List; + +public interface ReconcileCommandService { + + ConfigKey ReconcileCommandsEnabled = new ConfigKey<>("Advanced", Boolean.class, + "reconcile.commands.enabled", "false", + "Indicates whether the background task to reconcile the commands is enabled or not", + false); + + ConfigKey ReconcileCommandsInterval = new ConfigKey<>("Advanced", Integer.class, + "reconcile.commands.interval", "60", + "Interval (in seconds) for the background task to reconcile the commands", + false); + ConfigKey ReconcileCommandsMaxAttempts = new ConfigKey<>("Advanced", Integer.class, + "reconcile.commands.max.attempts", "30", + "The maximum number of attempts to reconcile the commands", + true); + + ConfigKey ReconcileCommandsWorkers = new ConfigKey<>("Advanced", Integer.class, + "reconcile.commands.workers", "100", + "The Number of worker threads to reconcile the commands", + false); + + List SupportedHypervisorTypes = Arrays.asList(Hypervisor.HypervisorType.KVM); + + void persistReconcileCommands(Long hostId, Long requestSequence, Command[] cmd); + + boolean updateReconcileCommand(long requestSeq, Command command, Answer answer, Command.State newStateByManagement, Command.State newStateByAgent); + + void processCommand(Command pingCommand, Answer pingAnswer); + + void processAnswers(long requestSeq, Command[] commands, Answer[] answers); + + void updateReconcileCommandToInterruptedByManagementServerId(long managementServerId); + + void updateReconcileCommandToInterruptedByHostId(long hostId); + + boolean isReconcileResourceNeeded(long resourceId, ApiCommandResourceType resourceType); +} diff --git a/core/src/main/java/com/cloud/agent/api/MigrateCommand.java b/core/src/main/java/com/cloud/agent/api/MigrateCommand.java index 3acdb9c351b..5ac4e9ae445 100644 --- a/core/src/main/java/com/cloud/agent/api/MigrateCommand.java +++ b/core/src/main/java/com/cloud/agent/api/MigrateCommand.java @@ -29,14 +29,14 @@ import com.cloud.agent.api.to.VirtualMachineTO; public class MigrateCommand extends Command { private String vmName; - private String destIp; + private String destinationIp; private Map migrateStorage; private boolean migrateStorageManaged; private boolean migrateNonSharedInc; private boolean autoConvergence; private String hostGuid; - private boolean isWindows; - private VirtualMachineTO vmTO; + private boolean windows; + private VirtualMachineTO virtualMachine; private boolean executeInSequence = false; private List migrateDiskInfoList = new ArrayList<>(); private Map dpdkInterfaceMapping = new HashMap<>(); @@ -64,11 +64,11 @@ public class MigrateCommand extends Command { protected MigrateCommand() { } - public MigrateCommand(String vmName, String destIp, boolean isWindows, VirtualMachineTO vmTO, boolean executeInSequence) { + public MigrateCommand(String vmName, String destinationIp, boolean windows, VirtualMachineTO virtualMachine, boolean executeInSequence) { this.vmName = vmName; - this.destIp = destIp; - this.isWindows = isWindows; - this.vmTO = vmTO; + this.destinationIp = destinationIp; + this.windows = windows; + this.virtualMachine = virtualMachine; this.executeInSequence = executeInSequence; } @@ -105,15 +105,15 @@ public class MigrateCommand extends Command { } public boolean isWindows() { - return isWindows; + return windows; } public VirtualMachineTO getVirtualMachine() { - return vmTO; + return virtualMachine; } public String getDestinationIp() { - return destIp; + return destinationIp; } public String getVmName() { @@ -233,4 +233,9 @@ public class MigrateCommand extends Command { this.isSourceDiskOnStorageFileSystem = isDiskOnFileSystemStorage; } } + + @Override + public boolean isReconcile() { + return true; + } } diff --git a/core/src/main/java/com/cloud/agent/api/PingAnswer.java b/core/src/main/java/com/cloud/agent/api/PingAnswer.java index 3a40ad3925f..22eb4443f8a 100644 --- a/core/src/main/java/com/cloud/agent/api/PingAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/PingAnswer.java @@ -19,6 +19,7 @@ package com.cloud.agent.api; +import java.util.ArrayList; import java.util.List; public class PingAnswer extends Answer { @@ -27,6 +28,8 @@ public class PingAnswer extends Answer { private boolean sendStartup = false; private List avoidMsList; + private List reconcileCommands = new ArrayList<>(); + protected PingAnswer() { } @@ -49,6 +52,18 @@ public class PingAnswer extends Answer { this.sendStartup = sendStartup; } + public List getReconcileCommands() { + return reconcileCommands; + } + + public void setReconcileCommands(List reconcileCommands) { + this.reconcileCommands = reconcileCommands; + } + + public void addReconcileCommand(String reconcileCommand) { + this.reconcileCommands.add(reconcileCommand); + } + public List getAvoidMsList() { return avoidMsList; } diff --git a/core/src/main/java/com/cloud/agent/api/PingCommand.java b/core/src/main/java/com/cloud/agent/api/PingCommand.java index 4192fc2e747..4ae64da89c3 100644 --- a/core/src/main/java/com/cloud/agent/api/PingCommand.java +++ b/core/src/main/java/com/cloud/agent/api/PingCommand.java @@ -20,11 +20,14 @@ package com.cloud.agent.api; import com.cloud.host.Host; +import org.apache.cloudstack.command.CommandInfo; public class PingCommand extends Command { Host.Type hostType; long hostId; boolean outOfBand; + @LogLevel(LogLevel.Log4jLevel.Trace) + private CommandInfo[] commandInfos = new CommandInfo[] {}; protected PingCommand() { } @@ -78,4 +81,12 @@ public class PingCommand extends Command { result = 31 * result + (int) (hostId ^ (hostId >>> 32)); return result; } + + public CommandInfo[] getCommandInfos() { + return commandInfos; + } + + public void setCommandInfos(CommandInfo[] commandInfos) { + this.commandInfos = commandInfos; + } } diff --git a/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java b/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java index 9fed0f913e1..70375c30a1b 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java +++ b/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java @@ -145,4 +145,9 @@ public class MigrateVolumeCommand extends Command { } public String getChainInfo() { return chainInfo; } + + @Override + public boolean isReconcile() { + return true; + } } diff --git a/core/src/main/java/com/cloud/resource/ServerResource.java b/core/src/main/java/com/cloud/resource/ServerResource.java index 845ac8a48fa..23d200942a2 100644 --- a/core/src/main/java/com/cloud/resource/ServerResource.java +++ b/core/src/main/java/com/cloud/resource/ServerResource.java @@ -22,6 +22,7 @@ package com.cloud.resource; import com.cloud.agent.IAgentControl; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; +import com.cloud.agent.api.PingAnswer; import com.cloud.agent.api.PingCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.host.Host; @@ -90,4 +91,5 @@ public interface ServerResource extends Manager { return false; } + default void processPingAnswer(PingAnswer answer) {}; } diff --git a/core/src/main/java/com/cloud/serializer/GsonHelper.java b/core/src/main/java/com/cloud/serializer/GsonHelper.java index 2d2cecf2618..7de98c08b7e 100644 --- a/core/src/main/java/com/cloud/serializer/GsonHelper.java +++ b/core/src/main/java/com/cloud/serializer/GsonHelper.java @@ -62,7 +62,7 @@ public class GsonHelper { LOGGER.info("Default Builder inited."); } - static Gson setDefaultGsonConfig(GsonBuilder builder) { + public static Gson setDefaultGsonConfig(GsonBuilder builder) { builder.setVersion(1.5); InterfaceTypeAdaptor dsAdaptor = new InterfaceTypeAdaptor(); builder.registerTypeAdapter(DataStoreTO.class, dsAdaptor); diff --git a/core/src/main/java/org/apache/cloudstack/command/CommandInfo.java b/core/src/main/java/org/apache/cloudstack/command/CommandInfo.java new file mode 100644 index 00000000000..b9bb702e345 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/CommandInfo.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.command; + +import com.cloud.agent.api.Command; +import com.cloud.serializer.GsonHelper; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import java.util.Date; + +public class CommandInfo { + public static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSSZ"; + public static final Gson GSON = GsonHelper.setDefaultGsonConfig(new GsonBuilder().setDateFormat(DATE_FORMAT)); + + long requestSeq; + Command.State state; + Date startTime; + Date updateTime; + String commandName; + String command; + int timeout; + String answerName; + String answer; + + public CommandInfo() { + } + + public CommandInfo(long requestSeq, Command command, Command.State state) { + this.requestSeq = requestSeq; + this.state = state; + this.startTime = this.updateTime = new Date(); + this.commandName = command.getClass().getName(); + this.command = GSON.toJson(command); + this.timeout = command.getWait(); + } + + public long getRequestSeq() { + return requestSeq; + } + + public void setRequestSeq(long requestSeq) { + this.requestSeq = requestSeq; + } + + public Command.State getState() { + return state; + } + + public void setState(Command.State state) { + this.state = state; + this.updateTime = new Date(); + } + + public Date getStartTime() { + return startTime; + } + + public void setStartTime(Date startTime) { + this.startTime = startTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getCommandName() { + return commandName; + } + + public void setCommandName(String commandName) { + this.commandName = commandName; + } + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + + public int getTimeout() { + return timeout; + } + + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + public String getAnswerName() { + return answerName; + } + + public void setAnswerName(String answerName) { + this.answerName = answerName; + } + + public String getAnswer() { + return answer; + } + + public void setAnswer(String answer) { + this.answer = answer; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileAnswer.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileAnswer.java new file mode 100644 index 00000000000..e8d27e1fa71 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileAnswer.java @@ -0,0 +1,45 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.command; + +import com.cloud.agent.api.Answer; +import org.apache.cloudstack.api.ApiCommandResourceType; + +public class ReconcileAnswer extends Answer { + + ApiCommandResourceType resourceType; + Long resourceId; + + public ApiCommandResourceType getResourceType() { + return resourceType; + } + + public void setResourceType(ApiCommandResourceType resourceType) { + this.resourceType = resourceType; + } + + public Long getResourceId() { + return resourceId; + } + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileCommand.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileCommand.java new file mode 100644 index 00000000000..262aefb30ac --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileCommand.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.command; + +import com.cloud.agent.api.Command; + +public class ReconcileCommand extends Command { + + @Override + public boolean executeInSequence() { + return false; + } + + @Override + public int getWait() { + return 30; // timeout is 30 seconds + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileCommandUtils.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileCommandUtils.java new file mode 100644 index 00000000000..8acc02a730f --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileCommandUtils.java @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.command; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.JsonSyntaxException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.text.SimpleDateFormat; + +public class ReconcileCommandUtils { + + protected static final Logger LOGGER = LogManager.getLogger(ReconcileCommandUtils.class.getName()); + + public static void createLogFileForCommand(final String logPath, final Command cmd) { + updateLogFileForCommand(logPath, cmd, Command.State.CREATED); + } + + public static void updateLogFileForCommand(final String logPath, final Command cmd, final Command.State state) { + if (cmd.isReconcile()) { + String logFileName = getLogFileNameForCommand(logPath, cmd); + LOGGER.debug(String.format("Updating log file %s with %s state", logFileName, state)); + File logFile = new File(logFileName); + CommandInfo commandInfo = null; + if (logFile.exists()) { + commandInfo = readLogFileForCommand(logFileName); + logFile.delete(); + } + if (commandInfo == null) { + commandInfo = new CommandInfo(cmd.getRequestSequence(), cmd, state); + } else { + commandInfo.setState(state); + } + try { + BufferedWriter writer = new BufferedWriter(new FileWriter(logFile)); + writer.write(CommandInfo.GSON.toJson(commandInfo)); + writer.close(); + } catch (IOException e) { + LOGGER.error(String.format("Failed to write log file %s", logFile)); + } + } + } + + public static void updateLogFileForCommand(final String logFullPath, final Command.State state) { + File logFile = new File(logFullPath); + LOGGER.debug(String.format("Updating log file %s with %s state", logFile.getName(), state)); + if (!logFile.exists()) { + return; + } + CommandInfo commandInfo = readLogFileForCommand(logFullPath); + if (commandInfo != null) { + commandInfo.setState(state); + } + logFile.delete(); + try { + BufferedWriter writer = new BufferedWriter(new FileWriter(logFile)); + writer.write(CommandInfo.GSON.toJson(commandInfo)); + writer.close(); + } catch (IOException e) { + LOGGER.error(String.format("Failed to write log file %s", logFile)); + } + } + + public static void deleteLogFileForCommand(final String logPath, final Command cmd) { + if (cmd.isReconcile()) { + File logFile = new File(getLogFileNameForCommand(logPath, cmd)); + LOGGER.debug(String.format("Removing log file %s", logFile.getName())); + if (logFile.exists()) { + logFile.delete(); + } + } + } + + public static void deleteLogFile(final String logFullPath) { + File logFile = new File(logFullPath); + LOGGER.debug(String.format("Removing log file %s ", logFile.getName())); + if (logFile.exists()) { + logFile.delete(); + } + } + + public static String getLogFileNameForCommand(final String logPath, final Command cmd) { + return String.format("%s/%s-%s.json", logPath, cmd.getRequestSequence(), cmd); + } + + public static CommandInfo readLogFileForCommand(final String logFullPath) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + SimpleDateFormat df = new SimpleDateFormat(CommandInfo.DATE_FORMAT); + objectMapper.setDateFormat(df); + return objectMapper.readValue(new File(logFullPath), CommandInfo.class); + } catch (IOException e) { + LOGGER.error(String.format("Failed to read log file %s: %s", logFullPath, e.getMessage())); + return null; + } + } + + public static Command parseCommandInfo(final CommandInfo commandInfo) { + if (commandInfo.getCommandName() == null || commandInfo.getCommand() == null) { + return null; + } + return parseCommandInfo(commandInfo.getCommandName(), commandInfo.getCommand()); + } + + public static Command parseCommandInfo(final String commandName, final String commandInfo) { + Object parsedObject = null; + try { + Class commandClazz = Class.forName(commandName); + parsedObject = CommandInfo.GSON.fromJson(commandInfo, commandClazz); + } catch (ClassNotFoundException | JsonSyntaxException e) { + LOGGER.error(String.format("Failed to parse command from CommandInfo %s due to %s", commandInfo, e.getMessage())); + } + if (parsedObject != null) { + return (Command) parsedObject; + } + return null; + } + + public static Answer parseAnswerFromCommandInfo(final CommandInfo commandInfo) { + if (commandInfo.getAnswerName() == null || commandInfo.getAnswer() == null) { + return null; + } + return parseAnswerFromAnswerInfo(commandInfo.getAnswerName(), commandInfo.getAnswer()); + } + + public static Answer parseAnswerFromAnswerInfo(final String answerName, final String answerInfo) { + Object parsedObject = null; + try { + Class commandClazz = Class.forName(answerName); + parsedObject = CommandInfo.GSON.fromJson(answerInfo, commandClazz); + } catch (ClassNotFoundException | JsonSyntaxException e) { + LOGGER.error(String.format("Failed to parse answer from answerInfo %s due to %s", answerInfo, e.getMessage())); + } + if (parsedObject != null) { + return (Answer) parsedObject; + } + return null; + } + + public static void updateLogFileWithAnswerForCommand(final String logPath, final Command cmd, final Answer answer) { + if (cmd.isReconcile()) { + String logFileName = getLogFileNameForCommand(logPath, cmd); + LOGGER.debug(String.format("Updating log file %s with answer %s", logFileName, answer)); + File logFile = new File(logFileName); + if (!logFile.exists()) { + return; + } + CommandInfo commandInfo = readLogFileForCommand(logFile.getAbsolutePath()); + if (commandInfo == null) { + return; + } + if (Command.State.STARTED.equals(commandInfo.getState())) { + if (answer.getResult()) { + commandInfo.setState(Command.State.COMPLETED); + } else { + commandInfo.setState(Command.State.FAILED); + } + } + commandInfo.setAnswerName(answer.toString()); + commandInfo.setAnswer(CommandInfo.GSON.toJson(answer)); + try { + BufferedWriter writer = new BufferedWriter(new FileWriter(logFile)); + writer.write(CommandInfo.GSON.toJson(commandInfo)); + writer.close(); + } catch (IOException e) { + LOGGER.error(String.format("Failed to write log file %s", logFile)); + } + } + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyAnswer.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyAnswer.java new file mode 100644 index 00000000000..82a24fa7fa5 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyAnswer.java @@ -0,0 +1,56 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.command; + +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +public class ReconcileCopyAnswer extends ReconcileVolumeAnswer { + + boolean isSkipped = false; + String reason; + + public ReconcileCopyAnswer(boolean isSkipped, String reason) { + super(); + this.isSkipped = isSkipped; + this.reason = reason; + } + + public ReconcileCopyAnswer(boolean isSkipped, boolean result, String reason) { + super(); + this.isSkipped = isSkipped; + this.result = result; + this.reason = reason; + } + + public ReconcileCopyAnswer(VolumeOnStorageTO volumeOnSource, VolumeOnStorageTO volumeOnDestination) { + this.isSkipped = false; + this.result = true; + this.volumeOnSource = volumeOnSource; + this.volumeOnDestination = volumeOnDestination; + } + + public boolean isSkipped() { + return isSkipped; + } + + public String getReason() { + return reason; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyCommand.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyCommand.java new file mode 100644 index 00000000000..36a678833d0 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyCommand.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.command; + +import com.cloud.agent.api.to.DataTO; + +import java.util.Map; + +public class ReconcileCopyCommand extends ReconcileCommand { + + DataTO srcData; + DataTO destData; + Map option; // details of source volume + Map option2; // details of destination volume + + public ReconcileCopyCommand(DataTO srcData, DataTO destData, Map option, Map option2) { + this.srcData = srcData; + this.destData = destData; + this.option = option; + this.option2 = option2; + } + + public DataTO getSrcData() { + return srcData; + } + + public DataTO getDestData() { + return destData; + } + + public Map getOption() { + return option; + } + + public Map getOption2() { + return option2; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateAnswer.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateAnswer.java new file mode 100644 index 00000000000..6313267c7e4 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateAnswer.java @@ -0,0 +1,68 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.command; + +import com.cloud.vm.VirtualMachine; + +import java.util.List; + +public class ReconcileMigrateAnswer extends ReconcileAnswer { + + Long hostId; + String vmName; + VirtualMachine.State vmState; + List vmDisks; + + public ReconcileMigrateAnswer() { + } + + public ReconcileMigrateAnswer(String vmName, VirtualMachine.State vmState) { + this.vmName = vmName; + this.vmState = vmState; + } + + public Long getHostId() { + return hostId; + } + + public void setHostId(Long hostId) { + this.hostId = hostId; + } + + public String getVmName() { + return vmName; + } + + public VirtualMachine.State getVmState() { + return vmState; + } + + public void setVmState(VirtualMachine.State vmState) { + this.vmState = vmState; + } + + public List getVmDisks() { + return vmDisks; + } + + public void setVmDisks(List vmDisks) { + this.vmDisks = vmDisks; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateCommand.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateCommand.java new file mode 100644 index 00000000000..50e1c68a65f --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateCommand.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.command; + +public class ReconcileMigrateCommand extends ReconcileCommand { + + String vmName; + + public ReconcileMigrateCommand(String vmName) { + this.vmName = vmName; + } + + public String getVmName() { + return vmName; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeAnswer.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeAnswer.java new file mode 100644 index 00000000000..ebbd913f971 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeAnswer.java @@ -0,0 +1,50 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.command; + +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +import java.util.List; + +public class ReconcileMigrateVolumeAnswer extends ReconcileVolumeAnswer { + + String vmName; + List vmDiskPaths; + + public ReconcileMigrateVolumeAnswer(VolumeOnStorageTO volumeOnSource, VolumeOnStorageTO volumeOnDestination) { + super(volumeOnSource, volumeOnDestination); + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public List getVmDiskPaths() { + return vmDiskPaths; + } + + public void setVmDiskPaths(List vmDiskPaths) { + this.vmDiskPaths = vmDiskPaths; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeCommand.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeCommand.java new file mode 100644 index 00000000000..e3e75249406 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeCommand.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.command; + +import com.cloud.agent.api.to.DataTO; + +public class ReconcileMigrateVolumeCommand extends ReconcileCommand { + + DataTO srcData; + DataTO destData; + String vmName; + + public ReconcileMigrateVolumeCommand(DataTO srcData, DataTO destData) { + this.srcData = srcData; + this.destData = destData; + } + + public DataTO getSrcData() { + return srcData; + } + + public DataTO getDestData() { + return destData; + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileVolumeAnswer.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileVolumeAnswer.java new file mode 100644 index 00000000000..da10bf23aee --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileVolumeAnswer.java @@ -0,0 +1,46 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.command; + +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +public class ReconcileVolumeAnswer extends ReconcileAnswer { + + // (1) null object: volume is not available. For example the source is secondary storage + // (2) otherwise, if volumeOnSource.getPath() is null, the volume cannot be found on primary storage pool + VolumeOnStorageTO volumeOnSource; + VolumeOnStorageTO volumeOnDestination; + + public ReconcileVolumeAnswer() { + } + + public ReconcileVolumeAnswer(VolumeOnStorageTO volumeOnSource, VolumeOnStorageTO volumeOnDestination) { + this.volumeOnSource = volumeOnSource; + this.volumeOnDestination = volumeOnDestination; + } + + public VolumeOnStorageTO getVolumeOnSource() { + return volumeOnSource; + } + + public VolumeOnStorageTO getVolumeOnDestination() { + return volumeOnDestination; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/CopyCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/CopyCommand.java index aac082a0133..36550420909 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/CopyCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/CopyCommand.java @@ -93,4 +93,9 @@ public class CopyCommand extends StorageSubSystemCommand { public void setExecuteInSequence(final boolean inSeq) { executeInSequence = inSeq; } + + @Override + public boolean isReconcile() { + return true; + } } diff --git a/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckGuestOsMappingCommandTest.java b/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckGuestOsMappingCommandTest.java index 3f68422d502..f021009cec0 100644 --- a/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckGuestOsMappingCommandTest.java +++ b/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckGuestOsMappingCommandTest.java @@ -25,8 +25,6 @@ import static org.junit.Assert.assertFalse; import com.cloud.agent.api.CheckGuestOsMappingCommand; import org.junit.Test; -import com.cloud.agent.api.AgentControlCommand; - public class CheckGuestOsMappingCommandTest { @Test diff --git a/core/src/test/java/org/apache/cloudstack/command/ReconcileCommandUtilsTest.java b/core/src/test/java/org/apache/cloudstack/command/ReconcileCommandUtilsTest.java new file mode 100644 index 00000000000..f69ada58e1a --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/command/ReconcileCommandUtilsTest.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.command; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.MigrateCommand; +import com.cloud.agent.api.to.VirtualMachineTO; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; + +public class ReconcileCommandUtilsTest { + + final static String COMMANDS_LOG_PATH = "/tmp"; + + @Test + public void createAndReadAndDeleteLogFilesForCommand() { + final String vmName = "TestVM"; + final String destIp = "DestinationHost"; + final VirtualMachineTO vmTO = Mockito.mock(VirtualMachineTO.class); + final MigrateCommand command = new MigrateCommand(vmName, destIp, false, vmTO, false); + long requestSequence = 10000L; + command.setRequestSequence(requestSequence); + + String logFile = ReconcileCommandUtils.getLogFileNameForCommand(COMMANDS_LOG_PATH, command); + + ReconcileCommandUtils.createLogFileForCommand(COMMANDS_LOG_PATH, command); + Assert.assertTrue((new File(logFile).exists())); + + CommandInfo commandInfo = ReconcileCommandUtils.readLogFileForCommand(logFile); + Assert.assertNotNull(commandInfo); + Assert.assertEquals(command.getClass().getName(), commandInfo.getCommandName()); + Assert.assertEquals(Command.State.CREATED, commandInfo.getState()); + + Command parseCommand = ReconcileCommandUtils.parseCommandInfo(commandInfo); + System.out.println("command state is " + commandInfo); + Assert.assertNotNull(parseCommand); + Assert.assertTrue(parseCommand instanceof MigrateCommand); + Assert.assertEquals(vmName,((MigrateCommand) parseCommand).getVmName()); + Assert.assertEquals(destIp,((MigrateCommand) parseCommand).getDestinationIp()); + + ReconcileCommandUtils.updateLogFileForCommand(COMMANDS_LOG_PATH, command, Command.State.PROCESSING); + CommandInfo newCommandInfo = ReconcileCommandUtils.readLogFileForCommand(logFile); + System.out.println("new command state is " + newCommandInfo); + Assert.assertNotNull(newCommandInfo); + Assert.assertEquals(command.getClass().getName(), newCommandInfo.getCommandName()); + Assert.assertEquals(Command.State.PROCESSING, newCommandInfo.getState()); + + ReconcileCommandUtils.deleteLogFileForCommand(COMMANDS_LOG_PATH, command); + Assert.assertFalse((new File(logFile).exists())); + } +} diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 30a58d405c9..0a6bbc87654 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -32,7 +32,11 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.agent.lb.SetupMSListCommand; +import org.apache.cloudstack.command.ReconcileAnswer; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.logging.log4j.Logger; @@ -114,6 +118,7 @@ public abstract class AgentAttache { protected final long _id; protected String _uuid; protected String _name = null; + protected HypervisorType _hypervisorType; protected final ConcurrentHashMap _waitForList; protected final LinkedList _requests; protected Long _currentSequence; @@ -135,10 +140,11 @@ public abstract class AgentAttache { Arrays.sort(s_commandsNotAllowedInConnectingMode); } - protected AgentAttache(final AgentManagerImpl agentMgr, final long id, final String uuid, final String name, final boolean maintenance) { + protected AgentAttache(final AgentManagerImpl agentMgr, final long id, final String uuid, final String name, final HypervisorType hypervisorType, final boolean maintenance) { _id = id; _uuid = uuid; _name = name; + _hypervisorType = hypervisorType; _waitForList = new ConcurrentHashMap(); _currentSequence = null; _maintenance = maintenance; @@ -261,6 +267,10 @@ public abstract class AgentAttache { return _name; } + public HypervisorType get_hypervisorType() { + return _hypervisorType; + } + public int getQueueSize() { return _requests.size(); } @@ -406,10 +416,17 @@ public abstract class AgentAttache { try { for (int i = 0; i < 2; i++) { Answer[] answers = null; - try { - answers = sl.waitFor(wait); - } catch (final InterruptedException e) { - logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted"); + Command[] cmds = req.getCommands(); + if (cmds != null && cmds.length == 1 && (cmds[0] != null) && cmds[0].isReconcile() + && !sl.isDisconnected() && _agentMgr.isReconcileCommandsEnabled(_hypervisorType)) { + // only available if (1) the only command is a Reconcile command (2) agent is connected; (3) reconciliation is enabled; (4) hypervisor is KVM; + answers = waitForAnswerOfReconcileCommand(sl, seq, cmds[0], wait); + } else { + try { + answers = sl.waitFor(wait); + } catch (final InterruptedException e) { + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted"); + } } if (answers != null) { new Response(req, answers).logD("Received: ", false); @@ -428,11 +445,13 @@ public abstract class AgentAttache { if (current != null && seq != current) { logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Waited too long."); + _agentMgr.updateReconcileCommandsIfNeeded(req.getSequence(), req.getCommands(), Command.State.TIMED_OUT); throw new OperationTimedoutException(req.getCommands(), _id, seq, wait, false); } logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Waiting some more time because this is the current command"); } + _agentMgr.updateReconcileCommandsIfNeeded(req.getSequence(), req.getCommands(), Command.State.TIMED_OUT); throw new OperationTimedoutException(req.getCommands(), _id, seq, wait * 2, true); } catch (OperationTimedoutException e) { logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Timed out on " + req.toString()); @@ -449,12 +468,57 @@ public abstract class AgentAttache { if (req.executeInSequence() && (current != null && current == seq)) { sendNext(seq); } + _agentMgr.updateReconcileCommandsIfNeeded(req.getSequence(), req.getCommands(), Command.State.TIMED_OUT); throw new OperationTimedoutException(req.getCommands(), _id, seq, wait, false); } finally { unregisterListener(seq); } } + private Answer[] waitForAnswerOfReconcileCommand(SynchronousListener sl, final long seq, final Command command, final int wait) { + Answer[] answers = null; + int waitTimeLeft = wait; + while (waitTimeLeft > 0) { + int waitTime = Math.min(waitTimeLeft, _agentMgr.getReconcileInterval()); + logger.debug(String.format("Waiting %s seconds for the answer of reconcile command %s-%s", waitTime, seq, command)); + if (sl.isDisconnected()) { + logger.debug(String.format("Disconnected while waiting for the answer of reconcile command %s-%s", seq, command)); + break; + } + try { + answers = sl.waitFor(waitTime); + } catch (final InterruptedException e) { + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted"); + } + if (answers != null) { + break; + } + + logger.debug(String.format("Getting the answer of reconcile command from cloudstack database for %s-%s", seq, command)); + Pair commandInfo = _agentMgr.getStateAndAnswerOfReconcileCommand(seq, command); + if (commandInfo == null) { + logger.debug(String.format("Cannot get the answer of reconcile command from cloudstack database for %s-%s", seq, command)); + continue; + } + Command.State state = commandInfo.first(); + if (Command.State.INTERRUPTED.equals(state)) { + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted by agent, will reconcile it"); + throw new CloudRuntimeException("Interrupted by agent"); + } else if (Command.State.DANGLED_IN_BACKEND.equals(state)) { + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Dangling in backend, it seems the agent was restarted, will reconcile it"); + throw new CloudRuntimeException("It is not being processed by agent"); + } + Answer answer = commandInfo.second(); + logger.debug(String.format("Got the answer of reconcile command from cloudstack database for %s-%s: %s", seq, command, answer)); + if (answer != null && !(answer instanceof ReconcileAnswer)) { + answers = new Answer[] { answer }; + break; + } + waitTimeLeft -= waitTime; + } + return answers; + } + protected synchronized void sendNext(final long seq) { _currentSequence = null; if (_requests.isEmpty()) { diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 9b410ce82a5..53e25b6b009 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -43,6 +43,10 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.agent.lb.IndirectAgentLB; import org.apache.cloudstack.ca.CAManager; +import org.apache.cloudstack.command.ReconcileCommandService; +import org.apache.cloudstack.command.ReconcileCommandUtils; +import org.apache.cloudstack.command.ReconcileCommandVO; +import org.apache.cloudstack.command.dao.ReconcileCommandDao; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; @@ -177,6 +181,10 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected HighAvailabilityManager _haMgr = null; @Inject protected AlertManager _alertMgr = null; + @Inject + protected ReconcileCommandService reconcileCommandService; + @Inject + ReconcileCommandDao reconcileCommandDao; @Inject protected HypervisorGuruManager _hvGuruMgr; @@ -207,6 +215,9 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl private final ConcurrentHashMap newAgentConnections = new ConcurrentHashMap<>(); protected ScheduledExecutorService newAgentConnectionsMonitor; + private boolean _reconcileCommandsEnabled = false; + private Integer _reconcileCommandInterval; + @Inject ResourceManager _resourceMgr; @Inject @@ -275,6 +286,9 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl initializeCommandTimeouts(); + _reconcileCommandsEnabled = ReconcileCommandService.ReconcileCommandsEnabled.value(); + _reconcileCommandInterval = ReconcileCommandService.ReconcileCommandsInterval.value(); + return true; } @@ -643,7 +657,13 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl final Request req = new Request(hostId, agent.getName(), _nodeId, cmds, commands.stopOnError(), true); req.setSequence(agent.getNextSequence()); + + reconcileCommandService.persistReconcileCommands(hostId, req.getSequence(), cmds); + final Answer[] answers = agent.send(req, wait); + + reconcileCommandService.processAnswers(req.getSequence(), cmds, answers); + notifyAnswersToMonitors(hostId, req.getSequence(), answers); commands.setAnswers(answers); return answers; @@ -940,7 +960,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected AgentAttache createAttacheForDirectConnect(final Host host, final ServerResource resource) { logger.debug("create DirectAgentAttache for {}", host); - final DirectAgentAttache attache = new DirectAgentAttache(this, host.getId(), host.getUuid(), host.getName(), resource, host.isInMaintenanceStates()); + final DirectAgentAttache attache = new DirectAgentAttache(this, host.getId(), host.getUuid(), host.getName(), host.getHypervisorType(), resource, host.isInMaintenanceStates()); AgentAttache old; synchronized (_agents) { @@ -1123,6 +1143,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl host = _hostDao.findById(hostId); // Maybe the host magically reappeared? if (host != null && host.getStatus() == Status.Down) { _haMgr.scheduleRestartForVmsOnHost(host, true, HighAvailabilityManager.ReasonType.HostDown); + reconcileCommandService.updateReconcileCommandToInterruptedByHostId(hostId); } return true; } @@ -1285,7 +1306,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected AgentAttache createAttacheForConnect(final HostVO host, final Link link) { logger.debug("create ConnectedAgentAttache for {}", host); - final AgentAttache attache = new ConnectedAgentAttache(this, host.getId(), host.getUuid(), host.getName(), link, host.isInMaintenanceStates()); + final AgentAttache attache = new ConnectedAgentAttache(this, host.getId(), host.getUuid(), host.getName(), host.getHypervisorType(), link, host.isInMaintenanceStates()); link.attach(attache); AgentAttache old; @@ -1629,6 +1650,9 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl } final List avoidMsList = _mshostDao.listNonUpStateMsIPs(); answer = new PingAnswer((PingCommand)cmd, avoidMsList, requestStartupCommand); + + // Add or update reconcile tasks + reconcileCommandService.processCommand(cmd, answer); } else if (cmd instanceof ReadyAnswer) { final HostVO host = _hostDao.findById(attache.getId()); if (host == null) { @@ -2082,6 +2106,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString())); params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString())); params.put(NetworkOrchestrationService.TUNGSTEN_ENABLED.key(), String.valueOf(NetworkOrchestrationService.TUNGSTEN_ENABLED.valueIn(host.getDataCenterId()))); + params.put(ReconcileCommandService.ReconcileCommandsEnabled.key(), String.valueOf(_reconcileCommandsEnabled)); try { SetHostParamsCommand cmds = new SetHostParamsCommand(params); @@ -2168,4 +2193,36 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl private GlobalLock getHostJoinLock(Long hostId) { return GlobalLock.getInternLock(String.format("%s-%s", "Host-Join", hostId)); } + + public boolean isReconcileCommandsEnabled(HypervisorType hypervisorType) { + return _reconcileCommandsEnabled && ReconcileCommandService.SupportedHypervisorTypes.contains(hypervisorType); + } + + public void updateReconcileCommandsIfNeeded(long requestSeq, Command[] commands, Command.State state) { + if (!_reconcileCommandsEnabled) { + return; + } + for (Command command: commands) { + if (command.isReconcile()) { + reconcileCommandService.updateReconcileCommand(requestSeq, command, null, state, null); + } + } + } + + public Pair getStateAndAnswerOfReconcileCommand(long requestSeq, Command command) { + ReconcileCommandVO reconcileCommandVO = reconcileCommandDao.findCommand(requestSeq, command.toString()); + if (reconcileCommandVO == null) { + return null; + } + Command.State state = reconcileCommandVO.getStateByAgent(); + if (reconcileCommandVO.getAnswerName() == null || reconcileCommandVO.getAnswerInfo() == null) { + return new Pair<>(state, null); + } + Answer answer = ReconcileCommandUtils.parseAnswerFromAnswerInfo(reconcileCommandVO.getAnswerName(), reconcileCommandVO.getAnswerInfo()); + return new Pair<>(state, answer); + } + + public Integer getReconcileInterval() { + return _reconcileCommandInterval; + } } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentAttache.java index e36b145c8bc..5e4ccfa67c6 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentAttache.java @@ -31,6 +31,7 @@ import com.cloud.agent.api.Command; import com.cloud.agent.transport.Request; import com.cloud.exception.AgentUnavailableException; import com.cloud.host.Status; +import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.nio.Link; public class ClusteredAgentAttache extends ConnectedAgentAttache implements Routable { @@ -44,14 +45,14 @@ public class ClusteredAgentAttache extends ConnectedAgentAttache implements Rout s_clusteredAgentMgr = agentMgr; } - public ClusteredAgentAttache(final AgentManagerImpl agentMgr, final long id, final String uuid, final String name) { - super(agentMgr, id, uuid, name, null, false); + public ClusteredAgentAttache(final AgentManagerImpl agentMgr, final long id, final String uuid, final String name, final Hypervisor.HypervisorType hypervisorType) { + super(agentMgr, id, uuid, name, hypervisorType, null, false); _forward = true; _transferRequests = new LinkedList(); } - public ClusteredAgentAttache(final AgentManagerImpl agentMgr, final long id, final String uuid, final String name, final Link link, final boolean maintenance) { - super(agentMgr, id, uuid, name, link, maintenance); + public ClusteredAgentAttache(final AgentManagerImpl agentMgr, final long id, final String uuid, final String name, final Hypervisor.HypervisorType hypervisorType, final Link link, final boolean maintenance) { + super(agentMgr, id, uuid, name, hypervisorType, link, maintenance); _forward = link == null; _transferRequests = new LinkedList(); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index dad7d401b94..40899c7d8f4 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -264,7 +264,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust protected AgentAttache createAttache(final HostVO host) { logger.debug("create forwarding ClusteredAgentAttache for {}", host); long id = host.getId(); - final AgentAttache attache = new ClusteredAgentAttache(this, id, host.getUuid(), host.getName()); + final AgentAttache attache = new ClusteredAgentAttache(this, id, host.getUuid(), host.getName(), host.getHypervisorType()); AgentAttache old; synchronized (_agents) { old = _agents.get(host.getId()); @@ -280,7 +280,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust @Override protected AgentAttache createAttacheForConnect(final HostVO host, final Link link) { logger.debug("create ClusteredAgentAttache for {}", host); - final AgentAttache attache = new ClusteredAgentAttache(this, host.getId(), host.getUuid(), host.getName(), link, host.isInMaintenanceStates()); + final AgentAttache attache = new ClusteredAgentAttache(this, host.getId(), host.getUuid(), host.getName(), host.getHypervisorType(), link, host.isInMaintenanceStates()); link.attach(attache); AgentAttache old; synchronized (_agents) { @@ -296,7 +296,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust @Override protected AgentAttache createAttacheForDirectConnect(final Host host, final ServerResource resource) { logger.debug("Create ClusteredDirectAgentAttache for {}.", host); - final DirectAgentAttache attache = new ClusteredDirectAgentAttache(this, host.getId(), host.getUuid(), host.getName(), _nodeId, resource, host.isInMaintenanceStates()); + final DirectAgentAttache attache = new ClusteredDirectAgentAttache(this, host.getId(), host.getUuid(), host.getName(), host.getHypervisorType(), _nodeId, resource, host.isInMaintenanceStates()); AgentAttache old; synchronized (_agents) { old = _agents.get(host.getId()); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredDirectAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredDirectAgentAttache.java index e36ea6cedc1..5e796d8e0d8 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredDirectAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredDirectAgentAttache.java @@ -20,14 +20,15 @@ import com.cloud.agent.transport.Request; import com.cloud.agent.transport.Response; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.UnsupportedVersionException; +import com.cloud.hypervisor.Hypervisor; import com.cloud.resource.ServerResource; import com.cloud.utils.exception.CloudRuntimeException; public class ClusteredDirectAgentAttache extends DirectAgentAttache implements Routable { private final long _nodeId; - public ClusteredDirectAgentAttache(ClusteredAgentManagerImpl agentMgr, long id, String uuid, String name, long mgmtId, ServerResource resource, boolean maintenance) { - super(agentMgr, id, uuid, name, resource, maintenance); + public ClusteredDirectAgentAttache(ClusteredAgentManagerImpl agentMgr, long id, String uuid, String name, final Hypervisor.HypervisorType hypervisorType, long mgmtId, ServerResource resource, boolean maintenance) { + super(agentMgr, id, uuid, name, hypervisorType, resource, maintenance); _nodeId = mgmtId; } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java index 523f98fd010..f208a81b422 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java @@ -22,6 +22,7 @@ import java.nio.channels.ClosedChannelException; import com.cloud.agent.transport.Request; import com.cloud.exception.AgentUnavailableException; import com.cloud.host.Status; +import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.nio.Link; /** @@ -31,8 +32,8 @@ public class ConnectedAgentAttache extends AgentAttache { protected Link _link; - public ConnectedAgentAttache(final AgentManagerImpl agentMgr, final long id, final String uuid, final String name, final Link link, final boolean maintenance) { - super(agentMgr, id, uuid, name, maintenance); + public ConnectedAgentAttache(final AgentManagerImpl agentMgr, final long id, final String uuid, final String name, final Hypervisor.HypervisorType hypervisorType, final Link link, final boolean maintenance) { + super(agentMgr, id, uuid, name, hypervisorType, maintenance); _link = link; } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java index 07d5bf80393..81148c5db30 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java @@ -35,6 +35,7 @@ import com.cloud.agent.transport.Request; import com.cloud.agent.transport.Response; import com.cloud.exception.AgentUnavailableException; import com.cloud.host.Status; +import com.cloud.hypervisor.Hypervisor; import com.cloud.resource.ServerResource; import org.apache.logging.log4j.ThreadContext; @@ -51,8 +52,8 @@ public class DirectAgentAttache extends AgentAttache { AtomicInteger _outstandingTaskCount; AtomicInteger _outstandingCronTaskCount; - public DirectAgentAttache(AgentManagerImpl agentMgr, long id, String uuid,String name, ServerResource resource, boolean maintenance) { - super(agentMgr, id, uuid, name, maintenance); + public DirectAgentAttache(AgentManagerImpl agentMgr, long id, String uuid, String name, final Hypervisor.HypervisorType hypervisorType, ServerResource resource, boolean maintenance) { + super(agentMgr, id, uuid, name, hypervisorType, maintenance); _resource = resource; _outstandingTaskCount = new AtomicInteger(0); _outstandingCronTaskCount = new AtomicInteger(0); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/DummyAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/DummyAttache.java index 2f15e7af43c..f2b253fffcb 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/DummyAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/DummyAttache.java @@ -19,11 +19,12 @@ package com.cloud.agent.manager; import com.cloud.agent.transport.Request; import com.cloud.exception.AgentUnavailableException; import com.cloud.host.Status; +import com.cloud.hypervisor.Hypervisor; public class DummyAttache extends AgentAttache { - public DummyAttache(AgentManagerImpl agentMgr, long id, String uuid, String name, boolean maintenance) { - super(agentMgr, id, uuid, name, maintenance); + public DummyAttache(AgentManagerImpl agentMgr, long id, String uuid, String name, final Hypervisor.HypervisorType hypervisorType, boolean maintenance) { + super(agentMgr, id, uuid, name, hypervisorType, maintenance); } @Override diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 15e3110f51b..e3307caa54b 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -2843,11 +2843,29 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new CloudRuntimeException(details); } } catch (final OperationTimedoutException e) { - if (e.isActive()) { - logger.warn("Active migration command so scheduling a restart for {}", vm, e); - _haMgr.scheduleRestart(vm, true); + boolean success = false; + if (HypervisorType.KVM.equals(vm.getHypervisorType())) { + try { + final Answer answer = _agentMgr.send(vm.getHostId(), new CheckVirtualMachineCommand(vm.getInstanceName())); + if (answer != null && answer.getResult() && answer instanceof CheckVirtualMachineAnswer) { + final CheckVirtualMachineAnswer vmAnswer = (CheckVirtualMachineAnswer) answer; + if (VirtualMachine.PowerState.PowerOn.equals(vmAnswer.getState())) { + logger.info(String.format("Vm %s is found on destination host %s. Migration is successful", vm, vm.getHostId())); + success = true; + } + } + } catch (Exception ex) { + logger.error(String.format("Failed to get state of VM %s on destination host %s: %s", vm, vm.getHostId(), ex.getMessage())); + } + } + if (!success) { + if (e.isActive()) { + logger.warn("Active migration command so scheduling a restart for {}", vm, e); + _haMgr.scheduleRestart(vm, true); + + throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId); + } } - throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId); } try { diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentManagerImplTest.java index 52b7ed77533..fb42f247788 100644 --- a/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentManagerImplTest.java @@ -25,6 +25,7 @@ import com.cloud.exception.ConnectionException; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.Pair; import org.junit.Assert; import org.junit.Before; @@ -47,7 +48,7 @@ public class AgentManagerImplTest { host = new HostVO("some-Uuid"); host.setDataCenterId(1L); cmds = new StartupCommand[]{new StartupRoutingCommand()}; - attache = new ConnectedAgentAttache(null, 1L, "uuid", "kvm-attache", null, false); + attache = new ConnectedAgentAttache(null, 1L, "uuid", "kvm-attache", Hypervisor.HypervisorType.KVM, null, false); hostDao = Mockito.mock(HostDao.class); storagePoolMonitor = Mockito.mock(Listener.class); diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java index 0b42b505668..66e6bbae5e2 100644 --- a/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock; import org.junit.Test; +import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.nio.Link; public class ConnectedAgentAttacheTest { @@ -31,8 +32,8 @@ public class ConnectedAgentAttacheTest { Link link = mock(Link.class); - ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 0, "uuid", null, link, false); - ConnectedAgentAttache agentAttache2 = new ConnectedAgentAttache(null, 0, "uuid", null, link, false); + ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 0, "uuid", null, Hypervisor.HypervisorType.KVM, link, false); + ConnectedAgentAttache agentAttache2 = new ConnectedAgentAttache(null, 0, "uuid", null, Hypervisor.HypervisorType.KVM,link, false); assertTrue(agentAttache1.equals(agentAttache2)); } @@ -42,7 +43,7 @@ public class ConnectedAgentAttacheTest { Link link = mock(Link.class); - ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 0, "uuid", null, link, false); + ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 0, "uuid", null, Hypervisor.HypervisorType.KVM, link, false); assertFalse(agentAttache1.equals(null)); } @@ -53,8 +54,8 @@ public class ConnectedAgentAttacheTest { Link link1 = mock(Link.class); Link link2 = mock(Link.class); - ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 0, "uuid", null, link1, false); - ConnectedAgentAttache agentAttache2 = new ConnectedAgentAttache(null, 0, "uuid", null, link2, false); + ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 0, "uuid", null, Hypervisor.HypervisorType.KVM, link1, false); + ConnectedAgentAttache agentAttache2 = new ConnectedAgentAttache(null, 0, "uuid", null, Hypervisor.HypervisorType.KVM, link2, false); assertFalse(agentAttache1.equals(agentAttache2)); } @@ -64,8 +65,8 @@ public class ConnectedAgentAttacheTest { Link link1 = mock(Link.class); - ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 1, "uuid", null, link1, false); - ConnectedAgentAttache agentAttache2 = new ConnectedAgentAttache(null, 2, "uuid", null, link1, false); + ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 1, "uuid", null, Hypervisor.HypervisorType.KVM, link1, false); + ConnectedAgentAttache agentAttache2 = new ConnectedAgentAttache(null, 2, "uuid", null, Hypervisor.HypervisorType.KVM, link1, false); assertFalse(agentAttache1.equals(agentAttache2)); } @@ -75,7 +76,7 @@ public class ConnectedAgentAttacheTest { Link link1 = mock(Link.class); - ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 1, "uuid", null, link1, false); + ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 1, "uuid", null, Hypervisor.HypervisorType.KVM, link1, false); assertFalse(agentAttache1.equals("abc")); } diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java index 65e31c271a4..4ba276460e3 100644 --- a/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java @@ -24,6 +24,7 @@ import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.hypervisor.Hypervisor; import com.cloud.resource.ServerResource; import java.util.UUID; @@ -42,7 +43,7 @@ public class DirectAgentAttacheTest { @Before public void setup() { - directAgentAttache = new DirectAgentAttache(_agentMgr, _id, _uuid, "myDirectAgentAttache", _resource, false); + directAgentAttache = new DirectAgentAttache(_agentMgr, _id, _uuid, "myDirectAgentAttache", Hypervisor.HypervisorType.KVM, _resource, false); MockitoAnnotations.initMocks(directAgentAttache); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index e6ffca06f9e..0f47c9e7155 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -162,4 +162,6 @@ public interface VolumeDao extends GenericDao, StateDao searchRemovedByVms(List vmIds, Long batchSize); VolumeVO findOneByIScsiName(String iScsiName); + + VolumeVO findByLastIdAndState(long lastVolumeId, Volume.State...states); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index 750dbf2bee0..f8b6bb3ed68 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -49,6 +49,7 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Func; @@ -397,6 +398,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol AllFieldsSearch.and("name", AllFieldsSearch.entity().getName(), Op.EQ); AllFieldsSearch.and("passphraseId", AllFieldsSearch.entity().getPassphraseId(), Op.EQ); AllFieldsSearch.and("iScsiName", AllFieldsSearch.entity().get_iScsiName(), Op.EQ); + AllFieldsSearch.and("path", AllFieldsSearch.entity().getPath(), Op.EQ); AllFieldsSearch.done(); RootDiskStateSearch = createSearchBuilder(); @@ -904,9 +906,18 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol return searchIncludingRemoved(sc, filter, null, false); } + @Override public VolumeVO findOneByIScsiName(String iScsiName) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("iScsiName", iScsiName); return findOneIncludingRemovedBy(sc); } + + @Override + public VolumeVO findByLastIdAndState(long lastVolumeId, State ...states) { + QueryBuilder sc = QueryBuilder.create(VolumeVO.class); + sc.and(sc.entity().getLastId(), SearchCriteria.Op.EQ, lastVolumeId); + sc.and(sc.entity().getState(), SearchCriteria.Op.IN, (Object[]) states); + return sc.find(); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/command/ReconcileCommandVO.java b/engine/schema/src/main/java/org/apache/cloudstack/command/ReconcileCommandVO.java new file mode 100644 index 00000000000..150c9662ada --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/command/ReconcileCommandVO.java @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.command; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import com.cloud.agent.api.Command; +import com.cloud.utils.db.GenericDao; + +import org.apache.cloudstack.acl.InfrastructureEntity; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.InternalIdentity; + +@Entity +@Table(name = "reconcile_commands") +public class ReconcileCommandVO implements InfrastructureEntity, InternalIdentity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "management_server_id") + private long managementServerId; + + @Column(name = "host_id") + private long hostId; + + @Column(name = "request_sequence") + private long requestSequence; + + @Column(name = "resource_id") + private long resourceId; + + @Column(name = "resource_type") + private ApiCommandResourceType resourceType; + + @Column(name = "state_by_management") + private Command.State stateByManagement; + + @Column(name = "state_by_agent") + private Command.State stateByAgent; + + @Column(name = "command_name") + private String commandName; + + @Column(name = "command_info", length = 65535) + private String commandInfo; + + @Column(name = "answer_name") + private String answerName; + + @Column(name = "answer_info", length = 65535) + private String answerInfo; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name= GenericDao.REMOVED_COLUMN) + private Date removed; + + @Temporal(TemporalType.TIMESTAMP) + @Column(name= "updated") + private Date updated; + + @Column(name= "retry_count") + private Long retryCount = 0L; + + @Override + public long getId() { + return id; + } + + public long getManagementServerId() { + return managementServerId; + } + + public void setManagementServerId(long managementServerId) { + this.managementServerId = managementServerId; + } + + public long getHostId() { + return hostId; + } + + public void setHostId(long hostId) { + this.hostId = hostId; + } + + public long getRequestSequence() { + return requestSequence; + } + + public void setRequestSequence(long requestSequence) { + this.requestSequence = requestSequence; + } + + public long getResourceId() { + return resourceId; + } + + public void setResourceId(long resourceId) { + this.resourceId = resourceId; + } + + public ApiCommandResourceType getResourceType() { + return resourceType; + } + + public void setResourceType(ApiCommandResourceType resourceType) { + this.resourceType = resourceType; + } + + public Command.State getStateByManagement() { + return stateByManagement; + } + + public void setStateByManagement(Command.State stateByManagement) { + this.stateByManagement = stateByManagement; + } + + public Command.State getStateByAgent() { + return stateByAgent; + } + + public void setStateByAgent(Command.State stateByAgent) { + this.stateByAgent = stateByAgent; + } + + public String getCommandName() { + return commandName; + } + + public void setCommandName(String commandName) { + this.commandName = commandName; + } + + public String getCommandInfo() { + return commandInfo; + } + + public void setCommandInfo(String commandInfo) { + this.commandInfo = commandInfo; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public String getAnswerName() { + return answerName; + } + + public void setAnswerName(String answerName) { + this.answerName = answerName; + } + + public String getAnswerInfo() { + return answerInfo; + } + + public void setAnswerInfo(String answerInfo) { + this.answerInfo = answerInfo; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public Date getUpdated() { + return updated; + } + + public void setUpdated(Date updated) { + this.updated = updated; + } + + public Long getRetryCount() { + return retryCount; + } + + public void setRetryCount(Long retryCount) { + this.retryCount = retryCount; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/command/dao/ReconcileCommandDao.java b/engine/schema/src/main/java/org/apache/cloudstack/command/dao/ReconcileCommandDao.java new file mode 100644 index 00000000000..59c4605d548 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/command/dao/ReconcileCommandDao.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.command.dao; + +import com.cloud.agent.api.Command; +import com.cloud.utils.db.GenericDao; + +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.command.ReconcileCommandVO; + +import java.util.List; + +public interface ReconcileCommandDao extends GenericDao { + + List listByManagementServerId(long managementServerId); + + List listByHostId(long hostId); + + List listByState(Command.State... states); + + void removeCommand(long commandId, String commandName, Command.State state); + + ReconcileCommandVO findCommand(long reqSequence, String commandName); + + void updateCommandsToInterruptedByManagementServerId(long managementServerId); + + void updateCommandsToInterruptedByHostId(long hostId); + + List listByResourceIdAndTypeAndStates(long resourceId, ApiCommandResourceType resourceType, Command.State... states); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/command/dao/ReconcileCommandDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/command/dao/ReconcileCommandDaoImpl.java new file mode 100644 index 00000000000..593f464f9ad --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/command/dao/ReconcileCommandDaoImpl.java @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.command.dao; + +import java.util.Date; +import java.util.List; + +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.command.ReconcileCommandVO; +import org.springframework.stereotype.Component; + +import com.cloud.agent.api.Command; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.QueryBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +@DB +public class ReconcileCommandDaoImpl extends GenericDaoBase implements ReconcileCommandDao { + + final SearchBuilder updateCommandSearch; + final SearchBuilder resourceSearch; + + public ReconcileCommandDaoImpl() { + + updateCommandSearch = createSearchBuilder(); + updateCommandSearch.and("managementServerId", updateCommandSearch.entity().getManagementServerId(), SearchCriteria.Op.EQ); + updateCommandSearch.and("stateByManagement", updateCommandSearch.entity().getStateByManagement(), SearchCriteria.Op.IN); + updateCommandSearch.and("hostId", updateCommandSearch.entity().getHostId(), SearchCriteria.Op.EQ); + updateCommandSearch.and("stateByAgent", updateCommandSearch.entity().getStateByAgent(), SearchCriteria.Op.IN); + updateCommandSearch.and("reqSequence", updateCommandSearch.entity().getRequestSequence(), SearchCriteria.Op.EQ); + updateCommandSearch.and("commandName", updateCommandSearch.entity().getCommandName(), SearchCriteria.Op.EQ); + updateCommandSearch.done(); + + resourceSearch = createSearchBuilder(); + resourceSearch.and("resourceId", resourceSearch.entity().getResourceId(), SearchCriteria.Op.EQ); + resourceSearch.and("resourceType", resourceSearch.entity().getResourceType(), SearchCriteria.Op.EQ); + resourceSearch.and("stateByManagement", resourceSearch.entity().getStateByManagement(), SearchCriteria.Op.IN); + resourceSearch.done(); + } + + @Override + public List listByManagementServerId(long managementServerId) { + QueryBuilder sc = QueryBuilder.create(ReconcileCommandVO.class); + sc.and(sc.entity().getManagementServerId(), SearchCriteria.Op.EQ, managementServerId); + return sc.list(); + } + + @Override + public List listByHostId(long hostId) { + QueryBuilder sc = QueryBuilder.create(ReconcileCommandVO.class); + sc.and(sc.entity().getHostId(), SearchCriteria.Op.EQ, hostId); + return sc.list(); + } + + @Override + public List listByState(Command.State... states) { + QueryBuilder sc = QueryBuilder.create(ReconcileCommandVO.class); + sc.and(sc.entity().getStateByManagement(), SearchCriteria.Op.IN, (Object[]) states); + return sc.list(); + } + + @Override + public void removeCommand(long reqSequence, String commandName, Command.State state) { + SearchCriteria sc = updateCommandSearch.create(); + sc.setParameters("reqSequence", reqSequence); + sc.setParameters("commandName", commandName); + + ReconcileCommandVO vo = createForUpdate(); + if (state != null) { + vo.setStateByManagement(state); + } + vo.setRemoved(new Date()); + update(vo, sc); + } + + @Override + public ReconcileCommandVO findCommand(long reqSequence, String commandName) { + SearchCriteria sc = updateCommandSearch.create(); + sc.setParameters("reqSequence", reqSequence); + sc.setParameters("commandName", commandName); + return findOneBy(sc); + } + + @Override + public void updateCommandsToInterruptedByManagementServerId(long managementServerId) { + SearchCriteria sc = updateCommandSearch.create(); + sc.setParameters("managementServerId", managementServerId); + sc.setParameters("stateByManagement", Command.State.CREATED, Command.State.RECONCILING); + + ReconcileCommandVO vo = createForUpdate(); + vo.setStateByManagement(Command.State.INTERRUPTED); + + update(vo, sc); + } + + @Override + public void updateCommandsToInterruptedByHostId(long hostId) { + SearchCriteria sc = updateCommandSearch.create(); + sc.setParameters("hostId", hostId); + sc.setParameters("stateByAgent", Command.State.STARTED, Command.State.PROCESSING, Command.State.PROCESSING_IN_BACKEND); + + ReconcileCommandVO vo = createForUpdate(); + vo.setStateByAgent(Command.State.INTERRUPTED); + + update(vo, sc); + } + + @Override + public List listByResourceIdAndTypeAndStates(long resourceId, ApiCommandResourceType resourceType, Command.State... states) { + QueryBuilder sc = QueryBuilder.create(ReconcileCommandVO.class); + sc.and(sc.entity().getResourceId(), SearchCriteria.Op.EQ, resourceId); + sc.and(sc.entity().getResourceType(), SearchCriteria.Op.EQ, resourceType); + sc.and(sc.entity().getStateByManagement(), SearchCriteria.Op.IN, (Object[]) states); + return sc.list(); + } +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 4f22234d7bf..d05635f4614 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -300,4 +300,5 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index b01243ad989..f4c58b7bec6 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -37,3 +37,26 @@ WHERE rp.rule = 'quotaStatement' AND NOT EXISTS(SELECT 1 FROM cloud.role_permissions rp_ WHERE rp.role_id = rp_.role_id AND rp_.rule = 'quotaCreditsList'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.host', 'last_mgmt_server_id', 'bigint unsigned DEFAULT NULL COMMENT "last management server this host is connected to" AFTER `mgmt_server_id`'); + +-- Add table for reconcile commands +CREATE TABLE IF NOT EXISTS `cloud`.`reconcile_commands` ( + `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT, + `management_server_id` bigint unsigned NOT NULL COMMENT 'node id of the management server', + `host_id` bigint unsigned NOT NULL COMMENT 'id of the host', + `request_sequence` bigint unsigned NOT NULL COMMENT 'sequence of the request', + `resource_id` bigint unsigned DEFAULT NULL COMMENT 'id of the resource', + `resource_type` varchar(255) COMMENT 'type if the resource', + `state_by_management` varchar(255) COMMENT 'state of the command updated by management server', + `state_by_agent` varchar(255) COMMENT 'state of the command updated by cloudstack agent', + `command_name` varchar(255) COMMENT 'name of the command', + `command_info` MEDIUMTEXT COMMENT 'info of the command', + `answer_name` varchar(255) COMMENT 'name of the answer', + `answer_info` MEDIUMTEXT COMMENT 'info of the answer', + `created` datetime COMMENT 'date the reconcile command was created', + `removed` datetime COMMENT 'date the reconcile command was removed', + `updated` datetime COMMENT 'date the reconcile command was updated', + `retry_count` bigint unsigned DEFAULT 0 COMMENT 'The retry count of reconciliation', + PRIMARY KEY(`id`), + INDEX `i_reconcile_command__host_id`(`host_id`), + CONSTRAINT `fk_reconcile_command__host_id` FOREIGN KEY (`host_id`) REFERENCES `host`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index cdf823eaf5b..2e13080494f 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -32,6 +32,8 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; +import com.cloud.agent.api.CheckVirtualMachineAnswer; +import com.cloud.agent.api.CheckVirtualMachineCommand; import com.cloud.agent.api.PrepareForMigrationAnswer; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; @@ -2011,6 +2013,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { @Override public void copyAsync(Map volumeDataStoreMap, VirtualMachineTO vmTO, Host srcHost, Host destHost, AsyncCompletionCallback callback) { String errMsg = null; + boolean success = false; + Map srcVolumeInfoToDestVolumeInfo = new HashMap<>(); try { if (srcHost.getHypervisorType() != HypervisorType.KVM) { @@ -2024,7 +2028,6 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { List migrateDiskInfoList = new ArrayList(); Map migrateStorage = new HashMap<>(); - Map srcVolumeInfoToDestVolumeInfo = new HashMap<>(); boolean managedStorageDestination = false; boolean migrateNonSharedInc = false; @@ -2140,20 +2143,38 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); migrateCommand.setAutoConvergence(kvmAutoConvergence); - MigrateAnswer migrateAnswer = (MigrateAnswer)agentManager.send(srcHost.getId(), migrateCommand); - - boolean success = migrateAnswer != null && migrateAnswer.getResult(); + MigrateAnswer migrateAnswer = null; + try { + migrateAnswer = (MigrateAnswer)agentManager.send(srcHost.getId(), migrateCommand); + success = migrateAnswer != null && migrateAnswer.getResult(); + } catch (OperationTimedoutException ex) { + if (HypervisorType.KVM.equals(vm.getHypervisorType())) { + final Answer answer = agentManager.send(destHost.getId(), new CheckVirtualMachineCommand(vm.getInstanceName())); + if (answer != null && answer.getResult() && answer instanceof CheckVirtualMachineAnswer) { + final CheckVirtualMachineAnswer vmAnswer = (CheckVirtualMachineAnswer)answer; + if (VirtualMachine.PowerState.PowerOn.equals(vmAnswer.getState())) { + logger.info(String.format("Vm %s is found on destination host %s. Migration is successful", vm, destHost)); + success = true; + } + } + } + if (!success) { + throw ex; + } + } handlePostMigration(success, srcVolumeInfoToDestVolumeInfo, vmTO, destHost); - if (migrateAnswer == null) { - throw new CloudRuntimeException("Unable to get an answer to the migrate command"); - } + if (!success) { + if (migrateAnswer == null) { + throw new CloudRuntimeException("Unable to get an answer to the migrate command"); + } - if (!migrateAnswer.getResult()) { - errMsg = migrateAnswer.getDetails(); + if (!migrateAnswer.getResult()) { + errMsg = migrateAnswer.getDetails(); - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(errMsg); + } } } catch (AgentUnavailableException | OperationTimedoutException | CloudRuntimeException ex) { String volumesAndStorages = volumeDataStoreMap.entrySet().stream().map(entry -> formatEntryOfVolumesAndStoragesAsJsonToDisplayOnLog(entry)).collect(Collectors.joining(",")); @@ -2163,6 +2184,15 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { throw new CloudRuntimeException(errMsg); } finally { + if (!success && !srcVolumeInfoToDestVolumeInfo.isEmpty()) { + for (VolumeInfo destVolumeInfo : srcVolumeInfoToDestVolumeInfo.values()) { + logger.info(String.format("Expunging dest volume [id: %s, state: %s] as part of failed VM migration with volumes command for VM [%s].", destVolumeInfo.getId(), destVolumeInfo.getState(), vmTO.getId())); + destVolumeInfo.processEvent(Event.OperationFailed); + destVolumeInfo.processEvent(Event.DestroyRequested); + _volumeService.expungeVolumeAsync(destVolumeInfo); + } + } + CopyCmdAnswer copyCmdAnswer = new CopyCmdAnswer(errMsg); CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); @@ -2372,6 +2402,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { newVol.setPodId(storagePoolVO.getPodId()); newVol.setPoolId(storagePoolVO.getId()); newVol.setLastPoolId(lastPoolId); + newVol.setLastId(volume.getId()); if (volume.getPassphraseId() != null) { newVol.setPassphraseId(volume.getPassphraseId()); diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java index 309d66e9d30..78924a10b32 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java @@ -40,6 +40,7 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.command.ReconcileCommandService; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; @@ -104,6 +105,9 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C private StatusAdministrator statusAdministrator; + @Inject + protected ReconcileCommandService reconcileCommandService; + // // pay attention to _mshostId and _msid // _mshostId is the primary key of management host table @@ -1013,6 +1017,8 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C logger.warn("Management node " + host.getId() + " is detected inactive by timestamp and did not send node status to all other nodes"); host.setState(ManagementServerHost.State.Down); _mshostDao.update(host.getId(), host); + + reconcileCommandService.updateReconcileCommandToInterruptedByManagementServerId(host.getMsid()); } } } else { diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 41af291bd69..ea7de4f79c6 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -37,6 +37,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.command.ReconcileCommandService; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; @@ -71,6 +72,7 @@ import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.storage.Snapshot; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; @@ -167,6 +169,8 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, private NetworkDao networkDao; @Inject private NetworkOrchestrationService networkOrchestrationService; + @Inject + private ReconcileCommandService reconcileCommandService; private volatile long _executionRunNumber = 1; @@ -1197,6 +1201,23 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, return true; } if (vol.getState().isTransitional()) { + if (Volume.State.Migrating.equals(vol.getState())) { + if (ReconcileCommandService.ReconcileCommandsEnabled.value()) { + if (reconcileCommandService.isReconcileResourceNeeded(volumeId, ApiCommandResourceType.Volume)) { + logger.debug(String.format("Skipping cleaning up Migrating volume: %s, it will be reconciled", vol)); + return true; + } + if (vol.getInstanceId() != null && reconcileCommandService.isReconcileResourceNeeded(vol.getInstanceId(), ApiCommandResourceType.VirtualMachine)) { + logger.debug(String.format("Skipping cleaning up Migrating volume: %s, the vm %s will be reconciled", vol, _vmInstanceDao.findById(vol.getInstanceId()))); + return true; + } + } + VolumeVO destVolume = _volsDao.findByLastIdAndState(vol.getId(), Volume.State.Migrating, Volume.State.Creating); + if (destVolume != null) { + logger.debug(String.format("Found destination volume of Migrating volume %s: %s", vol, destVolume)); + cleanupVolume(destVolume.getId()); + } + } logger.debug("Cleaning up volume with Id: " + volumeId); boolean status = vol.stateTransit(Volume.Event.OperationFailed); cleanupFailedVolumesCreatedFromSnapshots(volumeId); @@ -1213,6 +1234,18 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, return true; } if (vmInstanceVO.getState().isTransitional()) { + if (VirtualMachine.State.Migrating.equals(vmInstanceVO.getState())) { + if (ReconcileCommandService.ReconcileCommandsEnabled.value() + && reconcileCommandService.isReconcileResourceNeeded(vmId, ApiCommandResourceType.VirtualMachine)) { + logger.debug(String.format("Skipping cleaning up Instance %s, it will be reconciled", vmInstanceVO)); + return true; + } + logger.debug("Cleaning up volumes with instance Id: " + vmId); + List volumes = _volsDao.findByInstance(vmInstanceVO.getId()); + for (VolumeVO volume : volumes) { + cleanupVolume(volume.getId()); + } + } logger.debug("Cleaning up Instance with Id: " + vmId); return virtualMachineManager.stateTransitTo(vmInstanceVO, VirtualMachine.Event.OperationFailed, vmInstanceVO.getHostId()); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index e3c141f69c1..98c71857673 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -43,6 +43,7 @@ import java.util.Properties; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -53,12 +54,16 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; +import org.apache.cloudstack.command.CommandInfo; +import org.apache.cloudstack.command.ReconcileCommandService; +import org.apache.cloudstack.command.ReconcileCommandUtils; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; import org.apache.cloudstack.storage.configdrive.ConfigDrive; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.cryptsetup.CryptSetup; import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; @@ -110,6 +115,7 @@ import org.xml.sax.SAXException; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.HostVmStateReportEntry; +import com.cloud.agent.api.PingAnswer; import com.cloud.agent.api.PingCommand; import com.cloud.agent.api.PingRoutingCommand; import com.cloud.agent.api.PingRoutingWithNwGroupsCommand; @@ -144,6 +150,7 @@ import com.cloud.exception.InternalErrorException; import com.cloud.host.Host.Type; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.kvm.dpdk.DpdkHelper; +import com.cloud.hypervisor.kvm.resource.disconnecthook.DisconnectHook; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ClockDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ConsoleDef; @@ -213,6 +220,7 @@ import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.PowerState; import com.cloud.vm.VmDetailConstants; + import com.google.gson.Gson; /** @@ -342,6 +350,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final int LIBVIRT_CGROUPV2_WEIGHT_MIN = 2; public static final int LIBVIRT_CGROUPV2_WEIGHT_MAX = 10000; + public static final String COMMANDS_LOG_PATH = "/usr/share/cloudstack-agent/tmp/commands"; + private String modifyVlanPath; private String versionStringPath; private String patchScriptPath; @@ -512,6 +522,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private boolean isTungstenEnabled = false; + private boolean isReconcileCommandsEnabled = false; + private static Gson gson = new Gson(); /** @@ -555,6 +567,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } + protected List _disconnectHooks = new CopyOnWriteArrayList<>(); + @Override public ExecutionResult executeInVR(final String routerIp, final String script, final String args) { return executeInVR(routerIp, script, args, timeout); @@ -774,6 +788,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public StorageSubsystemCommandHandler getStorageHandler() { return storageHandler; } + private static final class KeyValueInterpreter extends OutputInterpreter { private final Map map = new HashMap(); @@ -1517,6 +1532,18 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv isTungstenEnabled = Boolean.parseBoolean(params.get(NetworkOrchestrationService.TUNGSTEN_ENABLED.key())); } + if (params.get(ReconcileCommandService.ReconcileCommandsEnabled.key()) != null) { + isReconcileCommandsEnabled = Boolean.parseBoolean(params.get(ReconcileCommandService.ReconcileCommandsEnabled.key())); + } + if (isReconcileCommandsEnabled) { + File commandsLogPath = new File(COMMANDS_LOG_PATH); + if (!commandsLogPath.exists()) { + commandsLogPath.mkdirs(); + } + // Update state of reconcile commands + getCommandInfosFromLogFiles(true); + } + return true; } @@ -1963,6 +1990,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv */ @Override public Answer executeRequest(final Command cmd) { + if (isReconcileCommandsEnabled) { + ReconcileCommandUtils.updateLogFileForCommand(COMMANDS_LOG_PATH, cmd, Command.State.STARTED); + } final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); try { @@ -1972,6 +2002,67 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } + public CommandInfo[] getCommandInfosFromLogFiles(boolean update) { + File commandsLogPath = new File(COMMANDS_LOG_PATH); + File[] files = commandsLogPath.listFiles(); + if (files != null) { + CommandInfo[] commandInfos = new CommandInfo[files.length]; + int i = 0; + for (File file : files) { + CommandInfo commandInfo = ReconcileCommandUtils.readLogFileForCommand(file.getAbsolutePath()); + if (commandInfo == null) { + continue; + } + if (update) { + if (Command.State.PROCESSING.equals(commandInfo.getState())) { + ReconcileCommandUtils.updateLogFileForCommand(file.getAbsolutePath(), Command.State.INTERRUPTED); + } else if (Command.State.PROCESSING_IN_BACKEND.equals(commandInfo.getState())) { + ReconcileCommandUtils.updateLogFileForCommand(file.getAbsolutePath(), Command.State.DANGLED_IN_BACKEND); + } + } + logger.debug(String.format("Adding reconcile command with seq: %s, command: %s, answer: %s", commandInfo.getRequestSeq(), commandInfo.getCommandName(), commandInfo.getAnswer())); + commandInfos[i++] = commandInfo; + } + return commandInfos; + } + return new CommandInfo[0]; + } + + public void createOrUpdateLogFileForCommand(Command command, Command.State state) { + if (isReconcileCommandsEnabled) { + ReconcileCommandUtils.updateLogFileForCommand(COMMANDS_LOG_PATH, command, state); + } + } + + public void createOrUpdateLogFileForCommand(Command command, Answer answer) { + if (isReconcileCommandsEnabled) { + ReconcileCommandUtils.updateLogFileWithAnswerForCommand(LibvirtComputingResource.COMMANDS_LOG_PATH, command, answer); + } + } + + @Override + public void processPingAnswer(PingAnswer answer) { + PingCommand pingCommand = answer.getCommand(); + List reconcileCommands = answer.getReconcileCommands(); + CommandInfo[] commandInfos = pingCommand.getCommandInfos(); + for (CommandInfo commandInfo : commandInfos) { + String commandKey = getCommandKey(commandInfo.getRequestSeq(), commandInfo.getCommandName()); + if (Arrays.asList(Command.State.COMPLETED, Command.State.FAILED, Command.State.INTERRUPTED, Command.State.TIMED_OUT).contains(commandInfo.getState())) { + logger.debug(String.format("Removing command %s in %s state as it has been received by the management server", commandKey, commandInfo.getState())); + String fileName = String.format("%s/%s-%s.json", COMMANDS_LOG_PATH, commandInfo.getRequestSeq(), commandInfo.getCommandName()); + ReconcileCommandUtils.deleteLogFile(fileName); + } else if (!reconcileCommands.contains(commandKey)) { + logger.debug(String.format("Removing command %s in %s state as it cannot be found by the management server", commandKey, commandInfo.getState())); + String fileName = String.format("%s/%s-%s.json", COMMANDS_LOG_PATH, commandInfo.getRequestSeq(), commandInfo.getCommandName()); + ReconcileCommandUtils.deleteLogFile(fileName); + } + } + } + + private String getCommandKey(long requestSeq, String commandName) { + return requestSeq + "-" + commandName; + } + public synchronized boolean destroyTunnelNetwork(final String bridge) { findOrCreateTunnelNetwork(bridge); @@ -2534,7 +2625,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv * Set quota and period tags on 'ctd' when CPU limit use is set */ protected void setQuotaAndPeriod(VirtualMachineTO vmTO, CpuTuneDef ctd) { - if (vmTO.getLimitCpuUse() && vmTO.getCpuQuotaPercentage() != null) { + if (vmTO.isLimitCpuUse() && vmTO.getCpuQuotaPercentage() != null) { Double cpuQuotaPercentage = vmTO.getCpuQuotaPercentage(); int period = CpuTuneDef.DEFAULT_PERIOD; int quota = (int) (period * cpuQuotaPercentage); @@ -3712,6 +3803,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv if (healthCheckResult != HealthCheckResult.IGNORE) { pingRoutingCommand.setHostHealthCheckResult(healthCheckResult == HealthCheckResult.SUCCESS); } + if (isReconcileCommandsEnabled) { + pingRoutingCommand.setCommandInfos(getCommandInfosFromLogFiles(false)); + } return pingRoutingCommand; } @@ -5656,4 +5750,64 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv throw new RuntimeException(e); } } + + @Override + public void disconnected() { + LOGGER.info("Detected agent disconnect event, running through " + _disconnectHooks.size() + " disconnect hooks"); + for (DisconnectHook hook : _disconnectHooks) { + hook.start(); + } + long start = System.currentTimeMillis(); + for (DisconnectHook hook : _disconnectHooks) { + try { + long elapsed = System.currentTimeMillis() - start; + long remaining = hook.getTimeoutMs() - elapsed; + long joinWait = remaining > 0 ? remaining : 1; + hook.join(joinWait); + hook.interrupt(); + } catch (InterruptedException ex) { + LOGGER.warn("Interrupted disconnect hook: " + ex.getMessage()); + } + } + _disconnectHooks.clear(); + } + + public void addDisconnectHook(DisconnectHook hook) { + LOGGER.debug("Adding disconnect hook " + hook); + _disconnectHooks.add(hook); + } + + public void removeDisconnectHook(DisconnectHook hook) { + LOGGER.debug("Removing disconnect hook " + hook); + if (_disconnectHooks.contains(hook)) { + LOGGER.debug("Removing disconnect hook " + hook); + _disconnectHooks.remove(hook); + } else { + LOGGER.debug("Requested removal of disconnect hook, but hook not found: " + hook); + } + } + + public VolumeOnStorageTO getVolumeOnStorage(PrimaryDataStoreTO primaryStore, String volumePath) { + try { + if (primaryStore.isManaged()) { + if (!storagePoolManager.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volumePath, primaryStore.getDetails())) { + logger.warn(String.format("Failed to connect src volume %s, in storage pool %s", volumePath, primaryStore)); + } + } + final KVMPhysicalDisk srcVolume = storagePoolManager.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volumePath); + if (srcVolume == null) { + logger.debug("Failed to get physical disk for volume: " + volumePath); + throw new CloudRuntimeException("Failed to get physical disk for volume at path: " + volumePath); + } + return new VolumeOnStorageTO(HypervisorType.KVM, srcVolume.getName(), srcVolume.getName(), srcVolume.getPath(), + srcVolume.getFormat().toString(), srcVolume.getSize(), srcVolume.getVirtualSize()); + } catch (final CloudRuntimeException e) { + logger.debug(String.format("Failed to get volume %s on storage %s: %s", volumePath, primaryStore, e)); + return new VolumeOnStorageTO(); + } finally { + if (primaryStore.isManaged()) { + storagePoolManager.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volumePath); + } + } + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/DisconnectHook.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/DisconnectHook.java new file mode 100644 index 00000000000..9cc0bb00eaa --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/DisconnectHook.java @@ -0,0 +1,59 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.disconnecthook; + +/** +DisconnectHooks can be used to cleanup/cancel long running commands when +connection to the management server is interrupted (which results in job +failure). Agent CommandWrappers can register a hook with the +libvirtComputingResource at the beginning of processing, and +libvirtComputingResource will call it upon disconnect. The CommandWrapper can +also remove the hook upon completion of the command. + +DisconnectHooks should implement a run() method that is safe to call and will +fail cleanly if there is no cleanup to do. Otherwise the CommandWrapper +registering/deregistering the hook should account for any race conditions +introduced by the ordering of when the command is processed and when the hook +is registered/deregistered. + +If a timeout is set, the hook's run() will be interrupted. It will be up to +run() to determine what to do with the InterruptedException, but the hook +processing will not wait any longer for the hook to complete. + +Avoid doing anything time intensive as DisconnectHooks will delay agent +shutdown. +*/ + +public abstract class DisconnectHook extends Thread { + // Default timeout is 10 seconds + long timeoutMs = 10000; + + public DisconnectHook(String name) { + super(); + this.setName(this.getClass().getName() + "-" + name); + } + + public DisconnectHook(String name, long timeout) { + this(name); + this.timeoutMs = timeout; + } + + public long getTimeoutMs(){ return timeoutMs; } + +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/MigrationCancelHook.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/MigrationCancelHook.java new file mode 100644 index 00000000000..04e9e04c645 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/MigrationCancelHook.java @@ -0,0 +1,50 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.disconnecthook; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; + +import org.libvirt.Domain; +import org.libvirt.LibvirtException; + +public class MigrationCancelHook extends DisconnectHook { + private static final Logger LOGGER = LogManager.getLogger(MigrationCancelHook.class); + + Domain _migratingDomain; + String _vmName; + + public MigrationCancelHook(Domain migratingDomain) throws LibvirtException { + super(migratingDomain.getName()); + _migratingDomain = migratingDomain; + _vmName = migratingDomain.getName(); + } + + @Override + public void run() { + LOGGER.info("Interrupted migration of " + _vmName); + try { + if (_migratingDomain.abortJob() == 0) { + LOGGER.warn("Aborted migration job for " + _vmName); + } + } catch (LibvirtException ex) { + LOGGER.warn("Failed to abort migration job for " + _vmName, ex); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/VolumeMigrationCancelHook.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/VolumeMigrationCancelHook.java new file mode 100644 index 00000000000..f73720f7908 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/disconnecthook/VolumeMigrationCancelHook.java @@ -0,0 +1,53 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.disconnecthook; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; + +import org.libvirt.Domain; +import org.libvirt.LibvirtException; + +public class VolumeMigrationCancelHook extends DisconnectHook { + private static final Logger LOGGER = LogManager.getLogger(VolumeMigrationCancelHook.class); + + Domain _migratingDomain; + String _vmName; + String _destDiskLabel; + + public VolumeMigrationCancelHook(Domain migratingDomain, String destDiskLabel) throws LibvirtException { + super(migratingDomain.getName()); + _migratingDomain = migratingDomain; + _vmName = migratingDomain.getName(); + _destDiskLabel = destDiskLabel; + } + + @Override + public void run() { + LOGGER.info("Interrupted volume migration of " + _vmName); + if (_migratingDomain != null && _destDiskLabel != null) { + try { + _migratingDomain.blockJobAbort(_destDiskLabel, Domain.BlockJobAbortFlags.ASYNC); + LOGGER.warn(String.format("Aborted block job for vm %s and volume: %s", _vmName, _destDiskLabel)); + } catch (LibvirtException ex) { + LOGGER.error(String.format("Failed to abort block job for vm %s and volume: %s due to %s", _vmName, _destDiskLabel, ex.getMessage())); + } + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java index de99f4841ca..99005755cbb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java @@ -20,6 +20,8 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainInfo; import org.libvirt.LibvirtException; import com.cloud.agent.api.Answer; @@ -45,6 +47,11 @@ public final class LibvirtCheckVirtualMachineCommandWrapper extends CommandWrapp vncPort = libvirtComputingResource.getVncPort(conn, command.getVmName()); } + Domain vm = conn.domainLookupByName(command.getVmName()); + if (state == PowerState.PowerOn && DomainInfo.DomainState.VIR_DOMAIN_PAUSED.equals(vm.getInfo().state)) { + return new CheckVirtualMachineAnswer(command, PowerState.PowerUnknown, vncPort); + } + return new CheckVirtualMachineAnswer(command, state, vncPort); } catch (final LibvirtException e) { return new CheckVirtualMachineAnswer(command, e.getMessage()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyVolumeCommandWrapper.java index 4e42af6899a..e8c6f40acb4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyVolumeCommandWrapper.java @@ -23,6 +23,7 @@ import java.io.File; import java.util.Map; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; import com.cloud.agent.api.storage.CopyVolumeAnswer; import com.cloud.agent.api.storage.CopyVolumeCommand; import com.cloud.agent.api.to.DiskTO; @@ -92,7 +93,10 @@ public final class LibvirtCopyVolumeCommandWrapper extends CommandWrapper ifaces = null; List disks; @@ -121,6 +124,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper> vmsnapshots = null; + MigrationCancelHook cancelHook = null; try { final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); @@ -237,6 +241,12 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc, migrateStorage, migrateNonSharedInc, command.isAutoConvergence(), vmName, command.getDestinationIp(), migrateDiskLabels); @@ -278,6 +288,8 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(final ReconcileCommand command, final LibvirtComputingResource libvirtComputingResource) { + + if (command instanceof ReconcileMigrateCommand) { + return handle((ReconcileMigrateCommand) command, libvirtComputingResource); + } else if (command instanceof ReconcileCopyCommand) { + return handle((ReconcileCopyCommand) command, libvirtComputingResource); + } else if (command instanceof ReconcileMigrateVolumeCommand) { + return handle((ReconcileMigrateVolumeCommand) command, libvirtComputingResource); + } + return new ReconcileAnswer(); + } + + private ReconcileAnswer handle(final ReconcileMigrateCommand reconcileCommand, final LibvirtComputingResource libvirtComputingResource) { + String vmName = reconcileCommand.getVmName(); + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + + ReconcileMigrateAnswer answer; + try { + Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName); + Domain vm = conn.domainLookupByName(vmName); + DomainState domainState = vm.getInfo().state; + logger.debug(String.format("Found VM %s with domain state %s", vmName, domainState)); + VirtualMachine.State state = getState(domainState); + List disks = null; + if (VirtualMachine.State.Running.equals(state)) { + disks = getVmDiskPaths(libvirtComputingResource.getDisks(conn, vmName)); + } + answer = new ReconcileMigrateAnswer(vmName, state); + answer.setVmDisks(disks); + } catch (LibvirtException e) { + logger.debug(String.format("Failed to get state of VM %s, assume it is Stopped", vmName)); + VirtualMachine.State state = VirtualMachine.State.Stopped; + answer = new ReconcileMigrateAnswer(vmName, state); + } + return answer; + } + + static VirtualMachine.State getState(DomainState domainState) { + VirtualMachine.State state; + if (domainState == DomainState.VIR_DOMAIN_RUNNING) { + state = VirtualMachine.State.Running; + } else if (Arrays.asList(DomainState.VIR_DOMAIN_SHUTDOWN, DomainState.VIR_DOMAIN_SHUTOFF, DomainState.VIR_DOMAIN_CRASHED).contains(domainState)) { + state = VirtualMachine.State.Stopped; + } else if (domainState == DomainState.VIR_DOMAIN_PAUSED) { + state = VirtualMachine.State.Unknown; + } else { + state = VirtualMachine.State.Unknown; + } + return state; + } + + private List getVmDiskPaths(List diskDefs) { + List diskPaths = new ArrayList(); + for (LibvirtVMDef.DiskDef diskDef : diskDefs) { + if (diskDef.getDiskPath() != null) { + diskPaths.add(diskDef.getDiskPath()); + } + } + return diskPaths; + } + + private ReconcileAnswer handle(final ReconcileCopyCommand reconcileCommand, final LibvirtComputingResource libvirtComputingResource) { + DataTO srcData = reconcileCommand.getSrcData(); + DataTO destData = reconcileCommand.getDestData(); + DataStoreTO srcDataStore = srcData.getDataStore(); + DataStoreTO destDataStore = destData.getDataStore(); + + // consistent with StorageSubsystemCommandHandlerBase.execute(CopyCommand cmd) + if (srcData.getObjectType() == DataObjectType.TEMPLATE && + (srcData.getDataStore().getRole() == DataStoreRole.Image || srcData.getDataStore().getRole() == DataStoreRole.ImageCache) && + destData.getDataStore().getRole() == DataStoreRole.Primary) { + String reason = "copy template to primary storage"; + return new ReconcileCopyAnswer(true, reason); + } else if (srcData.getObjectType() == DataObjectType.TEMPLATE && srcDataStore.getRole() == DataStoreRole.Primary && + destDataStore.getRole() == DataStoreRole.Primary) { + String reason = "clone template to a volume"; + return new ReconcileCopyAnswer(true, reason); + } else if (srcData.getObjectType() == DataObjectType.VOLUME && + (srcData.getDataStore().getRole() == DataStoreRole.ImageCache || srcDataStore.getRole() == DataStoreRole.Image)) { + logger.debug("Reconciling: copy volume from image cache to primary"); + return reconcileCopyVolumeFromImageCacheToPrimary(srcData, destData, reconcileCommand.getOption2(), libvirtComputingResource); + } else if (srcData.getObjectType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Primary) { + if (destData.getObjectType() == DataObjectType.VOLUME) { + if ((srcData instanceof VolumeObjectTO && ((VolumeObjectTO)srcData).isDirectDownload()) || + destData.getDataStore().getRole() == DataStoreRole.Primary) { + logger.debug("Reconciling: copy volume from primary to primary"); + return reconcileCopyVolumeFromPrimaryToPrimary(srcData, destData, libvirtComputingResource); + } else { + logger.debug("Reconciling: copy volume from primary to secondary"); + return reconcileCopyVolumeFromPrimaryToSecondary(srcData, destData, reconcileCommand.getOption(), libvirtComputingResource); + } + } else if (destData.getObjectType() == DataObjectType.TEMPLATE) { + String reason = "create volume from template"; + return new ReconcileCopyAnswer(true, reason); + } + } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.SNAPSHOT && + srcData.getDataStore().getRole() == DataStoreRole.Primary) { + String reason = "backup snapshot from primary"; + return new ReconcileCopyAnswer(true, reason); + } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.VOLUME) { + String reason = "create volume from snapshot"; + return new ReconcileCopyAnswer(true, reason); + } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.TEMPLATE) { + String reason = "create template from snapshot"; + return new ReconcileCopyAnswer(true, reason); + } + + return new ReconcileCopyAnswer(true, "not implemented yet"); + } + + private ReconcileCopyAnswer reconcileCopyVolumeFromImageCacheToPrimary(DataTO srcData, DataTO destData, Map details, LibvirtComputingResource libvirtComputingResource) { + // consistent with KVMStorageProcessor.copyVolumeFromImageCacheToPrimary + final DataStoreTO srcStore = srcData.getDataStore(); + if (!(srcStore instanceof NfsTO)) { + return new ReconcileCopyAnswer(true, "can only handle nfs storage as source"); + } + final DataStoreTO destStore = destData.getDataStore(); + final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)destStore; + String path = destData.getPath(); + if (path == null) { + path = details != null ? details.get(DiskTO.PATH) : null; + } + if (path == null) { + path = details != null ? details.get(DiskTO.IQN) : null; + } + if (path == null) { + return new ReconcileCopyAnswer(true, "path and iqn on destination storage are null"); + } + try { + VolumeOnStorageTO volumeOnDestination = libvirtComputingResource.getVolumeOnStorage(primaryStore, path); + return new ReconcileCopyAnswer(null, volumeOnDestination); + } catch (final CloudRuntimeException e) { + logger.debug("Failed to reconcile CopyVolumeFromImageCacheToPrimary: ", e); + return new ReconcileCopyAnswer(false, false, e.toString()); + } + } + private ReconcileCopyAnswer reconcileCopyVolumeFromPrimaryToPrimary(DataTO srcData, DataTO destData, LibvirtComputingResource libvirtComputingResource) { + // consistent with KVMStorageProcessor.copyVolumeFromPrimaryToPrimary + final String srcVolumePath = srcData.getPath(); + final String destVolumePath = destData.getPath(); + final DataStoreTO srcStore = srcData.getDataStore(); + final DataStoreTO destStore = destData.getDataStore(); + final PrimaryDataStoreTO srcPrimaryStore = (PrimaryDataStoreTO)srcStore; + final PrimaryDataStoreTO destPrimaryStore = (PrimaryDataStoreTO)destStore; + + VolumeOnStorageTO volumeOnSource = null; + VolumeOnStorageTO volumeOnDestination = null; + try { + volumeOnSource = libvirtComputingResource.getVolumeOnStorage(srcPrimaryStore, srcVolumePath); + if (destPrimaryStore.isManaged() || destVolumePath != null) { + volumeOnDestination = libvirtComputingResource.getVolumeOnStorage(destPrimaryStore, destVolumePath); + } + return new ReconcileCopyAnswer(volumeOnSource, volumeOnDestination); + } catch (final CloudRuntimeException e) { + logger.debug("Failed to reconcile CopyVolumeFromPrimaryToPrimary: ", e); + return new ReconcileCopyAnswer(false, false, e.toString()); + } + } + + private ReconcileCopyAnswer reconcileCopyVolumeFromPrimaryToSecondary(DataTO srcData, DataTO destData, Map details, LibvirtComputingResource libvirtComputingResource) { + // consistent with KVMStorageProcessor.copyVolumeFromPrimaryToSecondary + final String srcVolumePath = srcData.getPath(); + final DataStoreTO srcStore = srcData.getDataStore(); + final DataStoreTO destStore = destData.getDataStore(); + final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)srcStore; + if (!(destStore instanceof NfsTO)) { + return new ReconcileCopyAnswer(true, "can only handle nfs storage as destination"); + } + VolumeOnStorageTO volumeOnSource = libvirtComputingResource.getVolumeOnStorage(primaryStore, srcVolumePath); + return new ReconcileCopyAnswer(volumeOnSource, null); + } + + private ReconcileAnswer handle(final ReconcileMigrateVolumeCommand reconcileCommand, final LibvirtComputingResource libvirtComputingResource) { + // consistent with LibvirtMigrateVolumeCommandWrapper.execute + DataTO srcData = reconcileCommand.getSrcData(); + DataTO destData = reconcileCommand.getDestData(); + PrimaryDataStoreTO srcDataStore = (PrimaryDataStoreTO) srcData.getDataStore(); + PrimaryDataStoreTO destDataStore = (PrimaryDataStoreTO) destData.getDataStore(); + + VolumeOnStorageTO volumeOnSource = libvirtComputingResource.getVolumeOnStorage(srcDataStore, srcData.getPath()); + VolumeOnStorageTO volumeOnDestination = libvirtComputingResource.getVolumeOnStorage(destDataStore, destData.getPath()); + + ReconcileMigrateVolumeAnswer answer = new ReconcileMigrateVolumeAnswer(volumeOnSource, volumeOnDestination); + String vmName = reconcileCommand.getVmName(); + if (vmName != null) { + try { + LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName); + List disks = getVmDiskPaths(libvirtComputingResource.getDisks(conn, vmName)); + answer.setVmName(vmName); + answer.setVmDiskPaths(disks); + } catch (LibvirtException e) { + logger.error(String.format("Unable to get disks for %s due to %s", vmName, e.getMessage())); + } + } + + return answer; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRequestWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRequestWrapper.java index 73694f224a4..4dd36ef3771 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRequestWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRequestWrapper.java @@ -75,6 +75,13 @@ public class LibvirtRequestWrapper extends RequestWrapper { if (commandWrapper == null) { throw new CommandNotSupported("No way to handle " + command.getClass()); } - return commandWrapper.execute(command, serverResource); + Answer answer = commandWrapper.execute(command, serverResource); + + if (answer != null && command.isReconcile() && serverResource instanceof LibvirtComputingResource) { + LibvirtComputingResource libvirtComputingResource = (LibvirtComputingResource) serverResource; + libvirtComputingResource.createOrUpdateLogFileForCommand(command, answer); + } + + return answer; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index c35c7e9a62b..36d9cad1796 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -43,6 +43,7 @@ import java.util.stream.Collectors; import javax.naming.ConfigurationException; +import com.cloud.agent.api.Command; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.direct.download.DirectDownloadHelper; @@ -341,8 +342,8 @@ public class KVMStorageProcessor implements StorageProcessor { } } - private String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map details) { - String path; + public static String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map details) { + String path = null; if (primaryStore.getPoolType() == StoragePoolType.FiberChannel) { path = destData.getPath(); } else { @@ -527,6 +528,11 @@ public class KVMStorageProcessor implements StorageProcessor { final String volumeName = UUID.randomUUID().toString(); + // Update path in the command for reconciliation + if (destData.getPath() == null) { + ((VolumeObjectTO) destData).setPath(volumeName); + } + final int index = srcVolumePath.lastIndexOf(File.separator); final String volumeDir = srcVolumePath.substring(0, index); String srcVolumeName = srcVolumePath.substring(index + 1); @@ -543,7 +549,9 @@ public class KVMStorageProcessor implements StorageProcessor { volume.setDispName(srcVol.getName()); volume.setVmName(srcVol.getVmName()); + resource.createOrUpdateLogFileForCommand(cmd, Command.State.PROCESSING_IN_BACKEND); final KVMPhysicalDisk newDisk = storagePoolMgr.copyPhysicalDisk(volume, path != null ? path : volumeName, primaryPool, cmd.getWaitInMillSeconds()); + resource.createOrUpdateLogFileForCommand(cmd, Command.State.COMPLETED); storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path); @@ -2556,22 +2564,30 @@ public class KVMStorageProcessor implements StorageProcessor { } else { final String volumeName = UUID.randomUUID().toString(); destVolumeName = volumeName + "." + destFormat.getFileExtension(); + + // Update path in the command for reconciliation + if (destData.getPath() == null) { + ((VolumeObjectTO) destData).setPath(destVolumeName); + } } destPool = storagePoolMgr.getStoragePool(destPrimaryStore.getPoolType(), destPrimaryStore.getUuid()); try { Volume.Type volumeType = srcVol.getVolumeType(); + resource.createOrUpdateLogFileForCommand(cmd, Command.State.PROCESSING_IN_BACKEND); if (srcVol.getPassphrase() != null && (Volume.Type.ROOT.equals(volumeType) || Volume.Type.DATADISK.equals(volumeType))) { volume.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds(), srcVol.getPassphrase(), destVol.getPassphrase(), srcVol.getProvisioningType()); } else { storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds()); } + resource.createOrUpdateLogFileForCommand(cmd, Command.State.COMPLETED); } catch (Exception e) { // Any exceptions while copying the disk, should send failed answer with the error message String errMsg = String.format("Failed to copy volume [uuid: %s, name: %s] to dest storage [id: %s, name: %s], due to %s", srcVol.getUuid(), srcVol.getName(), destPrimaryStore.getUuid(), destPrimaryStore.getName(), e.toString()); logger.debug(errMsg, e); + resource.createOrUpdateLogFileForCommand(cmd, Command.State.FAILED); throw new CloudRuntimeException(errMsg); } finally { if (srcPrimaryStore.isManaged()) { diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/DisconnectHooksTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/DisconnectHooksTest.java new file mode 100644 index 00000000000..9193d9a2e1f --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/DisconnectHooksTest.java @@ -0,0 +1,191 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource; + +import com.cloud.hypervisor.kvm.resource.disconnecthook.DisconnectHook; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class DisconnectHooksTest { + class TestHook extends DisconnectHook { + private boolean _started = false; + private boolean _executed = false; + private boolean _withTimeout = false; + + private long _runtime; + + public TestHook() { super("foo"); } + + public TestHook(long timeout, long runtime) { + super("foo", timeout); + _withTimeout = true; + _runtime = runtime; + } + + @Override + public void run() { + _started = true; + if (this._withTimeout) { + try { + Thread.sleep(this._runtime); + } catch (InterruptedException e) { + throw new RuntimeException("TestHook interrupted while sleeping"); + } + } + this._executed = true; + } + + protected boolean hasRun() { + return _executed; + } + + protected boolean hasStarted() { + return _started; + } + } + + LibvirtComputingResource libvirtComputingResource; + + @Before + public void setup() { + libvirtComputingResource = new LibvirtComputingResource(); + } + + @Test + public void addHookWithoutRun() { + TestHook hook = new TestHook(); + libvirtComputingResource = new LibvirtComputingResource(); + libvirtComputingResource.addDisconnectHook(hook); + + // test that we added hook but did not run it + Assert.assertEquals(1, libvirtComputingResource._disconnectHooks.size()); + Assert.assertFalse(hook.hasRun()); + } + + @Test + public void addHookWithRun() { + TestHook hook = new TestHook(); + libvirtComputingResource = new LibvirtComputingResource(); + libvirtComputingResource.addDisconnectHook(hook); + + // test that we added hook but did not run it + Assert.assertEquals(1, libvirtComputingResource._disconnectHooks.size()); + Assert.assertFalse(hook.hasRun()); + + // test that we run and remove hook on disconnect + libvirtComputingResource.disconnected(); + + Assert.assertTrue(hook.hasRun()); + Assert.assertEquals(0, libvirtComputingResource._disconnectHooks.size()); + } + + @Test + public void addAndRemoveHooksWithAndWithoutRun() { + TestHook hook1 = new TestHook(); + TestHook hook2 = new TestHook(); + libvirtComputingResource.addDisconnectHook(hook1); + libvirtComputingResource.addDisconnectHook(hook2); + + Assert.assertEquals(2, libvirtComputingResource._disconnectHooks.size()); + Assert.assertFalse(hook1.hasRun()); + Assert.assertFalse(hook2.hasRun()); + + // remove first hook but leave second hook + libvirtComputingResource.removeDisconnectHook(hook1); + libvirtComputingResource.disconnected(); + + // ensure removed hook did not run + Assert.assertFalse(hook1.hasRun()); + + // ensure remaining hook did run + Assert.assertTrue(hook2.hasRun()); + Assert.assertEquals(0, libvirtComputingResource._disconnectHooks.size()); + } + + @Test + public void addAndRunHooksOneWithTimeout() { + // test that hook stops running when we exceed timeout + long timeout = 500; + TestHook hook1 = new TestHook(timeout, timeout + 100); + TestHook hook2 = new TestHook(); + libvirtComputingResource.addDisconnectHook(hook1); + libvirtComputingResource.addDisconnectHook(hook2); + libvirtComputingResource.disconnected(); + Assert.assertTrue(hook2.hasRun()); + + try { + Thread.sleep(timeout); + } catch (Exception ignored){} + + Assert.assertTrue(hook1.hasStarted()); + Assert.assertFalse(hook1.isAlive()); + Assert.assertFalse(hook1.hasRun()); + + Assert.assertEquals(0, libvirtComputingResource._disconnectHooks.size()); + } + + @Test + public void addAndRunTwoHooksWithTimeout() { + // test that hooks stop running when we exceed timeout + // test for parallel timeout rather than additive + long timeout = 500; + TestHook hook1 = new TestHook(timeout, timeout + 100); + TestHook hook2 = new TestHook(timeout, timeout + 100); + libvirtComputingResource.addDisconnectHook(hook1); + libvirtComputingResource.addDisconnectHook(hook2); + libvirtComputingResource.disconnected(); + + // if the timeouts were additive (e.g. if we were sequentially looping through join(timeout)), the second Hook + // would get enough time to complete (500 for first Hook and 500 for itself) and not be interrupted. + try { + Thread.sleep(timeout*2); + } catch (Exception ignored){} + + Assert.assertTrue(hook1.hasStarted()); + Assert.assertFalse(hook1.isAlive()); + Assert.assertFalse(hook1.hasRun()); + Assert.assertTrue(hook2.hasStarted()); + Assert.assertFalse(hook2.isAlive()); + Assert.assertFalse(hook2.hasRun()); + + Assert.assertEquals(0, libvirtComputingResource._disconnectHooks.size()); + } + + @Test + public void addAndRunTimeoutHooksToCompletion() { + // test we can run to completion if we don't take as long as timeout, and they run parallel + long timeout = 500; + TestHook hook1 = new TestHook(timeout, timeout - 100); + TestHook hook2 = new TestHook(timeout, timeout - 100); + libvirtComputingResource.addDisconnectHook(hook1); + libvirtComputingResource.addDisconnectHook(hook2); + libvirtComputingResource.disconnected(); + + try { + Thread.sleep(timeout); + } catch (Exception ignored){} + + Assert.assertTrue(hook1.hasStarted()); + Assert.assertTrue(hook1.hasRun()); + Assert.assertTrue(hook2.hasStarted()); + Assert.assertTrue(hook2.hasRun()); + Assert.assertEquals(0, libvirtComputingResource._disconnectHooks.size()); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 598f1dff2fc..d586f01d4eb 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -1760,6 +1760,8 @@ public class LibvirtComputingResourceTest { @Test public void testCheckVirtualMachineCommand() { final Connect conn = Mockito.mock(Connect.class); + final Domain vm = Mockito.mock(Domain.class); + final DomainInfo domainInfo = Mockito.mock(DomainInfo.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); final String vmName = "Test"; @@ -1768,6 +1770,8 @@ public class LibvirtComputingResourceTest { when(libvirtComputingResourceMock.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); try { when(libvirtUtilitiesHelper.getConnectionByVmName(vmName)).thenReturn(conn); + when(conn.domainLookupByName(vmName)).thenReturn(vm); + when(vm.getInfo()).thenReturn(domainInfo); } catch (final LibvirtException e) { fail(e.getMessage()); } @@ -5521,7 +5525,7 @@ public class LibvirtComputingResourceTest { @Test public void testSetQuotaAndPeriod() { double pct = 0.33d; - Mockito.when(vmTO.getLimitCpuUse()).thenReturn(true); + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(true); Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(pct); CpuTuneDef cpuTuneDef = new CpuTuneDef(); final LibvirtComputingResource lcr = new LibvirtComputingResource(); @@ -5532,7 +5536,7 @@ public class LibvirtComputingResourceTest { @Test public void testSetQuotaAndPeriodNoCpuLimitUse() { - Mockito.when(vmTO.getLimitCpuUse()).thenReturn(false); + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(false); CpuTuneDef cpuTuneDef = new CpuTuneDef(); final LibvirtComputingResource lcr = new LibvirtComputingResource(); lcr.setQuotaAndPeriod(vmTO, cpuTuneDef); @@ -5543,7 +5547,7 @@ public class LibvirtComputingResourceTest { @Test public void testSetQuotaAndPeriodMinQuota() { double pct = 0.01d; - Mockito.when(vmTO.getLimitCpuUse()).thenReturn(true); + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(true); Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(pct); CpuTuneDef cpuTuneDef = new CpuTuneDef(); final LibvirtComputingResource lcr = new LibvirtComputingResource(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 25565ceeb33..b279a79a328 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1961,7 +1961,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes // Check if license supports the feature VmwareHelper.isFeatureLicensed(hyperHost, FeatureKeyConstants.HOTPLUG); VmwareHelper.setVmScaleUpConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), getReservedCpuMHZ(vmSpec), (int) requestedMaxMemoryInMb, ramMb, - vmSpec.getLimitCpuUse()); + vmSpec.isLimitCpuUse()); if (!vmMo.configureVm(vmConfigSpec)) { throw new Exception("Unable to execute ScaleVmCommand"); @@ -2188,7 +2188,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes } tearDownVm(vmMo); } else if (!hyperHost.createBlankVm(vmNameOnVcenter, vmInternalCSName, vmSpec.getCpus(), vmSpec.getMaxSpeed().intValue(), getReservedCpuMHZ(vmSpec), - vmSpec.getLimitCpuUse(), (int) (vmSpec.getMaxRam() / ResourceType.bytesToMiB), getReservedMemoryMb(vmSpec), guestOsId, rootDiskDataStoreDetails.first(), false, + vmSpec.isLimitCpuUse(), (int) (vmSpec.getMaxRam() / ResourceType.bytesToMiB), getReservedMemoryMb(vmSpec), guestOsId, rootDiskDataStoreDetails.first(), false, controllerInfo, systemVm)) { throw new Exception("Failed to create VM. vmName: " + vmInternalCSName); } @@ -2232,7 +2232,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes VirtualDeviceConfigSpec[] deviceConfigSpecArray = new VirtualDeviceConfigSpec[totalChangeDevices]; DiskTO[] sortedDisks = sortVolumesByDeviceId(disks); VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), getReservedCpuMHZ(vmSpec), (int) (vmSpec.getMaxRam() / (1024 * 1024)), - getReservedMemoryMb(vmSpec), guestOsId, vmSpec.getLimitCpuUse(), deployAsIs); + getReservedMemoryMb(vmSpec), guestOsId, vmSpec.isLimitCpuUse(), deployAsIs); // Check for multi-cores per socket settings int numCoresPerSocket = 1; diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index c7e56f3421b..5286df9ac2f 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -1387,7 +1387,7 @@ public abstract class CitrixResourceBase extends ServerResourceBase implements S cpuWeight = _maxWeight; } - if (vmSpec.getLimitCpuUse()) { + if (vmSpec.isLimitCpuUse()) { // CPU cap is per VM, so need to assign cap based on the number // of vcpus utilization = (int)(vmSpec.getMaxSpeed() * 0.99 * vmSpec.getCpus() / _host.getSpeed() * 100); @@ -4709,7 +4709,7 @@ public abstract class CitrixResourceBase extends ServerResourceBase implements S cpuWeight = _maxWeight; } - if (vmSpec.getLimitCpuUse()) { + if (vmSpec.isLimitCpuUse()) { long utilization; // max CPU cap, default is unlimited utilization = (int)(vmSpec.getMaxSpeed() * 0.99 * vmSpec.getCpus() / _host.getSpeed() * 100); // vm.addToVCPUsParamsLive(conn, "cap", diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index 192ae4636e9..7015d1f782a 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -62,6 +62,7 @@ import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.storage.volume.VolumeObject; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; @@ -79,6 +80,7 @@ import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.alert.AlertManager; import com.cloud.configuration.Config; +import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; @@ -893,9 +895,7 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { boolean migrateStatus = answer.getResult(); if (migrateStatus) { - updateVolumeAfterCopyVolume(srcData, destData); - updateSnapshotsAfterCopyVolume(srcData, destData); - deleteSourceVolumeAfterSuccessfulBlockCopy(srcData, host); + updateAfterSuccessfulVolumeMigration(srcData, destData, host); logger.debug("Successfully migrated migrate PowerFlex volume {} to storage pool {}", srcData, destStore); answer = new Answer(null, true, null); } else { @@ -906,6 +906,17 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { } catch (Exception e) { logger.error("Failed to migrate PowerFlex volume: {} due to: {}", srcData, e.getMessage()); answer = new Answer(null, false, e.getMessage()); + if (e.getMessage().contains(OperationTimedoutException.class.getName())) { + logger.error(String.format("The PowerFlex volume %s might have been migrated because the exception is %s, checking the volume on destination pool", srcData, OperationTimedoutException.class.getName())); + Boolean volumeOnDestination = getVolumeStateOnPool(destStore, destVolumePath); + if (volumeOnDestination) { + logger.error(String.format("The PowerFlex volume %s has been migrated to destination pool %s", srcData, destStore.getName())); + updateAfterSuccessfulVolumeMigration(srcData, destData, host); + answer = new Answer(null, true, null); + } else { + logger.error(String.format("The PowerFlex volume %s has not been migrated completely to destination pool %s", srcData, destStore.getName())); + } + } } if (destVolumePath != null && !answer.getResult()) { @@ -915,6 +926,40 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { return answer; } + private void updateAfterSuccessfulVolumeMigration(DataObject srcData, DataObject destData, Host host) { + try { + updateVolumeAfterCopyVolume(srcData, destData); + updateSnapshotsAfterCopyVolume(srcData, destData); + deleteSourceVolumeAfterSuccessfulBlockCopy(srcData, host); + } catch (Exception ex) { + logger.error(String.format("Error while update PowerFlex volume: %s after successfully migration due to: %s", srcData, ex.getMessage())); + } + } + + public Boolean getVolumeStateOnPool(DataStore srcStore, String srcVolumePath) { + try { + // check the state of volume on pool via ScaleIO gateway + final ScaleIOGatewayClient client = getScaleIOClient(srcStore); + final String sourceScaleIOVolumeId = ScaleIOUtil.getVolumePath(srcVolumePath); + final org.apache.cloudstack.storage.datastore.api.Volume sourceScaleIOVolume = client.getVolume(sourceScaleIOVolumeId); + logger.debug(String.format("The PowerFlex volume %s on pool %s is: %s", srcVolumePath, srcStore.getName(), + ReflectionToStringBuilderUtils.reflectOnlySelectedFields(sourceScaleIOVolume, "id", "name", "vtreeId", "sizeInGB", "volumeSizeInGb"))); + if (sourceScaleIOVolume == null || StringUtils.isEmpty(sourceScaleIOVolume.getVtreeId())) { + return false; + } + Pair volumeStats = getVolumeStats(storagePoolDao.findById(srcStore.getId()), srcVolumePath); + if (volumeStats == null) { + logger.debug(String.format("Unable to find volume stats for %s on pool %s", srcVolumePath, srcStore.getName())); + return false; + } + logger.debug(String.format("Found volume stats for %s: provisionedSizeInBytes = %s, allocatedSizeInBytes = %s on pool %s", srcVolumePath, volumeStats.first(), volumeStats.second(), srcStore.getName())); + return volumeStats.first().equals(volumeStats.second()); + } catch (Exception ex) { + logger.error(String.format("Failed to check if PowerFlex volume %s exists on source pool %s", srcVolumePath, srcStore.getName())); + } + return null; + } + protected void updateVolumeAfterCopyVolume(DataObject srcData, DataObject destData) { // destination volume is already created and volume path is set in database by this time at "CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId());" final long srcVolumeId = srcData.getId(); diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index 1522402ae32..7c0d1bf4afc 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -379,7 +379,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur protected void wakeupWorkers() { logger.debug("Wakeup workers HA"); for (WorkerThread worker : _workers) { - worker.wakup(); + worker.wakeup(); } } @@ -589,6 +589,10 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur vm.getUpdated() + " previous updated = " + work.getUpdateTime()); return null; } + if (vm.getHostId() != null && !vm.getHostId().equals(work.getHostId())) { + logger.info("VM " + vm + " has been changed. Current host id = " + vm.getHostId() + " Previous host id = " + work.getHostId()); + return null; + } AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM; if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) { @@ -1209,7 +1213,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur } } - public synchronized void wakup() { + public synchronized void wakeup() { notifyAll(); } } diff --git a/server/src/main/java/com/cloud/hypervisor/KVMGuru.java b/server/src/main/java/com/cloud/hypervisor/KVMGuru.java index 9edaa5e6d64..64881df4c82 100644 --- a/server/src/main/java/com/cloud/hypervisor/KVMGuru.java +++ b/server/src/main/java/com/cloud/hypervisor/KVMGuru.java @@ -128,7 +128,7 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { * @param vmProfile vm profile */ protected void setVmQuotaPercentage(VirtualMachineTO to, VirtualMachineProfile vmProfile) { - if (to.getLimitCpuUse()) { + if (to.isLimitCpuUse()) { VirtualMachine vm = vmProfile.getVirtualMachine(); HostVO host = hostDao.findById(vm.getHostId()); if (host == null) { diff --git a/server/src/main/java/org/apache/cloudstack/command/ReconcileCommandServiceImpl.java b/server/src/main/java/org/apache/cloudstack/command/ReconcileCommandServiceImpl.java new file mode 100644 index 00000000000..d0dcc1f86de --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/command/ReconcileCommandServiceImpl.java @@ -0,0 +1,1144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.command; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.Command.State; +import com.cloud.agent.api.MigrateCommand; +import com.cloud.agent.api.PingAnswer; +import com.cloud.agent.api.PingCommand; +import com.cloud.agent.api.storage.MigrateVolumeCommand; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.DataTO; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; + +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.command.dao.ReconcileCommandDao; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +@Component +public class ReconcileCommandServiceImpl extends ManagerBase implements ReconcileCommandService, Configurable { + + final static long ManagementServerId = ManagementServerNode.getManagementServerId(); + final static int GracePeriod = 10 * 60; // 10 minutes + private boolean _reconcileCommandsEnabled = false; + + private ScheduledExecutorService reconcileCommandsExecutor; + private ExecutorService reconcileCommandTaskExecutor; + CompletionService completionService; + + @Inject + ReconcileCommandDao reconcileCommandDao; + @Inject + ManagementServerHostDao managementServerHostDao; + @Inject + HostDao hostDao; + @Inject + AgentManager agentManager; + @Inject + VMInstanceDao vmInstanceDao; + @Inject + VolumeDao volumeDao; + @Inject + EndPointSelector endPointSelector; + @Inject + DataStoreManager dataStoreManager; + @Inject + VolumeDataStoreDao volumeDataStoreDao; + @Inject + VolumeOrchestrationService volumeManager; + @Inject + private VolumeApiService volumeApiService; + @Inject + private AccountManager accountManager; + + @Override + public boolean configure(final String name, final Map params) throws ConfigurationException { + _reconcileCommandsEnabled = ReconcileCommandsEnabled.value(); + if (_reconcileCommandsEnabled) { + // create thread pool and blocking queue + final int workersCount = ReconcileCommandsWorkers.value(); + reconcileCommandTaskExecutor = Executors.newFixedThreadPool(workersCount, new NamedThreadFactory("Reconcile-Command-Task-Executor")); + final BlockingQueue> queue = new LinkedBlockingQueue<>(workersCount); + completionService = new ExecutorCompletionService<>(reconcileCommandTaskExecutor, queue); + + reconcileCommandsExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("Reconcile-Commands-Worker")); + reconcileCommandsExecutor.scheduleWithFixedDelay(new ReconcileCommandsWorker(), + ReconcileCommandsInterval.value(), ReconcileCommandsInterval.value(), TimeUnit.SECONDS); + } + + return true; + } + + @Override + public boolean stop() { + if (reconcileCommandsExecutor != null) { + reconcileCommandsExecutor.shutdownNow(); + } + if (reconcileCommandTaskExecutor != null) { + reconcileCommandTaskExecutor.shutdownNow(); + } + if (_reconcileCommandsEnabled) { + reconcileCommandDao.updateCommandsToInterruptedByManagementServerId(ManagementServerId); + } + + return true; + } + + @Override + public String getConfigComponentName() { + return ReconcileCommandService.class.getName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ ReconcileCommandsEnabled, ReconcileCommandsInterval, ReconcileCommandsWorkers, ReconcileCommandsMaxAttempts }; + } + + @Override + public void persistReconcileCommands(Long hostId, Long requestSequence, Command[] commands) { + if (!_reconcileCommandsEnabled) { + return; + } + HostVO host = hostDao.findById(hostId); + if (host == null || !SupportedHypervisorTypes.contains(host.getHypervisorType())) { + return; + } + for (Command cmd : commands) { + if (cmd.isReconcile()) { + persistReconcileCommand(hostId, requestSequence, cmd); + } + } + } + + private void persistReconcileCommand(Long hostId, Long requestSequence, Command cmd) { + ReconcileCommandVO reconcileCommandVO = new ReconcileCommandVO(); + reconcileCommandVO.setManagementServerId(ManagementServerId); + reconcileCommandVO.setCommandInfo(CommandInfo.GSON.toJson(cmd)); + reconcileCommandVO.setCommandName(cmd.toString()); + reconcileCommandVO.setCreated(new Date()); + reconcileCommandVO.setUpdated(new Date()); + reconcileCommandVO.setStateByManagement(State.CREATED); + reconcileCommandVO.setHostId(hostId); + reconcileCommandVO.setRequestSequence(requestSequence); + if (cmd instanceof CopyCommand) { + CopyCommand copyCmd = (CopyCommand)cmd; + DataTO srcData = copyCmd.getSrcTO(); + if (srcData != null && srcData.getDataStore() instanceof PrimaryDataStoreTO) { + reconcileCommandVO.setResourceType(ApiCommandResourceType.Volume); + reconcileCommandVO.setResourceId(srcData.getId()); + } + } else if (cmd instanceof MigrateCommand) { + MigrateCommand migrateCommand = (MigrateCommand)cmd; + reconcileCommandVO.setResourceType(ApiCommandResourceType.VirtualMachine); + reconcileCommandVO.setResourceId(migrateCommand.getVirtualMachine().getId()); + } else if (cmd instanceof MigrateVolumeCommand) { + MigrateVolumeCommand migrateVolumeCommand = (MigrateVolumeCommand)cmd; + reconcileCommandVO.setResourceType(ApiCommandResourceType.Volume); + reconcileCommandVO.setResourceId(migrateVolumeCommand.getVolumeId()); + } + reconcileCommandDao.persist(reconcileCommandVO); + } + + @Override + public boolean updateReconcileCommand(long requestSeq, Command command, Answer answer, State newStateByManagement, State newStateByAgent) { + String commandKey = getCommandKey(requestSeq, command); + logger.debug(String.format("Updating reconcile command %s with answer %s and new states %s-%s", commandKey, answer, newStateByManagement, newStateByAgent)); + ReconcileCommandVO reconcileCommandVO = reconcileCommandDao.findCommand(requestSeq, command.toString()); + if (reconcileCommandVO == null) { + logger.debug(String.format("Skipped updating reconcile command %s due to no record is found in DB", commandKey)); + return false; + } + boolean updated = false; + if (newStateByManagement != null) { + if (State.RECONCILE_RETRY.equals(newStateByManagement)) { + if (State.RECONCILING.equals(reconcileCommandVO.getStateByManagement())) { + reconcileCommandVO.setStateByManagement(newStateByManagement); + updated = true; + } else { + logger.debug(String.format("Skipping the update of state by management of command %s from %s to %s", commandKey, reconcileCommandVO.getStateByManagement(), newStateByManagement)); + } + } else if (!newStateByManagement.equals(reconcileCommandVO.getStateByManagement())) { + reconcileCommandVO.setStateByManagement(newStateByManagement); + updated = true; + } + if (State.RECONCILE_FAILED.equals(newStateByManagement)) { + reconcileCommandVO.setRetryCount(reconcileCommandVO.getRetryCount() + 1); + updated = true; + } + if (ManagementServerId != ManagementServerNode.getManagementServerId()) { + reconcileCommandVO.setManagementServerId(ManagementServerId); + updated = true; + } + } + if (newStateByAgent != null) { + if (!newStateByAgent.equals(reconcileCommandVO.getStateByAgent())) { + reconcileCommandVO.setStateByAgent(newStateByAgent); + updated = true; + } + } + String commandInfo = CommandInfo.GSON.toJson(command); + if (!commandInfo.equals(reconcileCommandVO.getCommandInfo())) { + reconcileCommandVO.setCommandInfo(commandInfo); + updated = true; + } + if (answer != null && (reconcileCommandVO.getAnswerName() == null || answer instanceof ReconcileAnswer + || reconcileCommandVO.getAnswerName().equals(answer.toString()))) { + reconcileCommandVO.setAnswerName(answer.toString()); + reconcileCommandVO.setAnswerInfo(CommandInfo.GSON.toJson(answer)); + updated = true; + } + if (updated) { + reconcileCommandVO.setUpdated(new Date()); + reconcileCommandDao.update(reconcileCommandVO.getId(), reconcileCommandVO); + } + return true; + } + + private String getCommandKey(long requestSeq, Command command) { + return requestSeq + "-" + command; + } + + private class ReconcileCommandsWorker extends ManagedContextRunnable { + @Override + protected void runInContext() { + GlobalLock gcLock = GlobalLock.getInternLock("Reconcile.Commands.Lock"); + try { + if (gcLock.lock(3)) { + try { + reallyRun(); + } finally { + gcLock.unlock(); + } + } + } finally { + gcLock.releaseRef(); + } + } + + private List getReconcileCommands() { + ManagementServerHostVO msHost = managementServerHostDao.findOneInUpState(new Filter(ManagementServerHostVO.class, "id", true, 0L, 1L)); + if (msHost == null || msHost.getMsid() != ManagementServerId) { + return new ArrayList<>(); + } + return reconcileCommandDao.listByState(State.INTERRUPTED, State.TIMED_OUT, State.RECONCILE_RETRY, State.RECONCILING, State.RECONCILE_FAILED, State.CREATED); + } + + public void reallyRun() { + List reconcileCommands = getReconcileCommands(); + logger.debug(String.format("Reconciling %s command(s) ...", reconcileCommands.size())); + for (ReconcileCommandVO reconcileCommand : reconcileCommands) { + ReconcileCommandTask task = new ReconcileCommandTask(reconcileCommand); + completionService.submit(task); + } + for (int i = 0; i < reconcileCommands.size(); i++) { + try { + Future future = completionService.take(); + ReconcileCommandResult result = future.get(); + long requestSequence = result.getRequestSequence(); + Command command = result.getCommand(); + ReconcileAnswer answer = result.getAnswer(); + String commandKey = getCommandKey(requestSequence, command); + if (result.isFailed()) { + throw new CloudRuntimeException(String.format("Failed to reconcile command %s due to: %s", commandKey, result.getResult())); + } + if (answer != null && answer.getResult()) { + logger.debug(String.format("Command %s has been reconciled with answer %s", commandKey, answer)); + if (result.isReconciled()) { + if (processReconcileAnswer(requestSequence, command, answer)) { + updateReconcileCommand(requestSequence, result.getCommand(), answer, State.RECONCILED, null); + reconcileCommandDao.removeCommand(requestSequence, result.getCommand().toString(), null); + } else { + updateReconcileCommand(requestSequence, result.getCommand(), answer, State.RECONCILE_RETRY, null); + } + } else { + updateReconcileCommand(requestSequence, result.getCommand(), answer, State.RECONCILE_FAILED, null); + } + } else if (result.isReconciled()) { + logger.info(String.format("Command %s is reconciled but answer is null, skipping the reconciliation", commandKey)); + updateReconcileCommand(requestSequence, result.getCommand(), answer, State.RECONCILE_SKIPPED, null); + reconcileCommandDao.removeCommand(requestSequence, result.getCommand().toString(), null); + } else { + logger.info(String.format("Command %s is not reconciled, will retry", commandKey)); + updateReconcileCommand(requestSequence, result.getCommand(), answer, State.RECONCILE_RETRY, null); + } + } catch (InterruptedException | ExecutionException e) { + logger.error(String.format("Failed to reconcile command due to: %s", e.getMessage()), e); + throw new CloudRuntimeException("Failed to reconcile command"); + } + } + } + } + + + public static class ReconcileCommandResult extends CommandResult { + long requestSequence; + Command command; + ReconcileAnswer answer; + boolean isReconciled; + + public ReconcileCommandResult(long requestSequence, Command command, ReconcileAnswer answer, boolean isReconciled) { + super(); + this.requestSequence = requestSequence; + this.command = command; + this.answer = answer; + this.isReconciled = isReconciled; + } + + public long getRequestSequence() { + return requestSequence; + } + + public Command getCommand() { + return command; + } + + public ReconcileAnswer getAnswer() { + return answer; + } + + public boolean isReconciled() { + return isReconciled; + } + } + + protected class ReconcileCommandTask implements Callable { + long requestSequence; + Command command; + State stateByManagement; + State stateByAgent; + Long hostId; + Long retryCount; + ReconcileCommandVO reconcileCommand; + + public ReconcileCommandTask(ReconcileCommandVO reconcileCommand) { + this.requestSequence = reconcileCommand.getRequestSequence(); + this.stateByManagement = reconcileCommand.getStateByManagement(); + this.stateByAgent = reconcileCommand.getStateByAgent(); + this.hostId = reconcileCommand.getHostId(); + this.retryCount = reconcileCommand.getRetryCount(); + this.reconcileCommand = reconcileCommand; + this.command = ReconcileCommandUtils.parseCommandInfo(reconcileCommand.getCommandName(), reconcileCommand.getCommandInfo()); + } + + @Override + public ReconcileCommandResult call() { + String commandKey = getCommandKey(requestSequence, command); + HostVO host = hostDao.findByIdIncludingRemoved(hostId); + assert host != null; + if (!SupportedHypervisorTypes.contains(host.getHypervisorType())) { + return new ReconcileCommandResult(requestSequence, command, null, false); + } + + logger.debug(String.format("Reconciling command %s with state %s-%s", commandKey, stateByManagement, stateByAgent)); + + if (State.TIMED_OUT.equals(stateByManagement)) { + logger.debug(String.format("The command %s timed out on management server. Reconciling ...", commandKey)); + return reconcile(reconcileCommand); + } else if (Arrays.asList(State.INTERRUPTED, State.RECONCILE_RETRY).contains(stateByManagement)) { + logger.debug(String.format("The command %s is %s on management server. Reconciling ...", commandKey, stateByManagement)); + return reconcile(reconcileCommand); + } else if (State.RECONCILING.equals(stateByManagement)) { + Date now = new Date(); + if (reconcileCommand.getUpdated() != null && reconcileCommand.getUpdated().getTime() > now.getTime() - GracePeriod * 1000) { + logger.debug(String.format("The command %s is being reconciled, skipping and wait for next run", commandKey)); + } else { + logger.debug(String.format("The command %s is %s, the state seems out of date, updating to RECONCILE_READY", commandKey, stateByManagement)); + reconcileCommand = reconcileCommandDao.findById(reconcileCommand.getId()); + reconcileCommand.setStateByManagement(State.RECONCILE_RETRY); + reconcileCommandDao.update(reconcileCommand.getId(), reconcileCommand); + } + } else if (State.RECONCILE_FAILED.equals(stateByManagement)) { + if (retryCount != null && retryCount <= ReconcileCommandsMaxAttempts.value()) { + logger.debug(String.format("The command %s has been reconciled %s times, retrying", commandKey, retryCount)); + return reconcile(reconcileCommand); + } else { + logger.debug(String.format("The command %s has been reconciled %s times, skipping", commandKey, retryCount)); + return new ReconcileCommandResult(requestSequence, command, null, true); + } + } else if (State.RECONCILED.equals(stateByManagement)) { + logger.debug(String.format("The command %s has been reconciled, skipping", commandKey)); + } else if (stateByAgent == null) { // state by management is CREATED + logger.debug(String.format("Skipping the reconciliation of command %s, because the state by agent is null", commandKey)); + } else if (Arrays.asList(State.STARTED, State.PROCESSING, State.PROCESSING_IN_BACKEND).contains(stateByAgent)) { + if (host.getRemoved() == null && Status.Up.equals(host.getStatus())) { + logger.debug(String.format("Skipping the reconciliation of command %s, because the host %s is Up, the command may be still in processing", commandKey, host)); + } else if (host.getRemoved() != null) { + logger.debug(String.format("The host %s has been removed on %s, Reconciling command %s ...", host, host.getRemoved(), commandKey)); + return reconcile(reconcileCommand); + } else { + logger.debug(String.format("The host %s is in %s state, Reconciling command %s ...", host, host.getStatus(), commandKey)); + return reconcile(reconcileCommand); + } + } else if (Arrays.asList(State.COMPLETED, State.FAILED).contains(stateByAgent)) { + Date now = new Date(); + if (reconcileCommand.getUpdated() != null && reconcileCommand.getUpdated().getTime() > now.getTime() - GracePeriod * 1000) { + logger.debug(String.format("The command %s is %s on host %s, it seems the answer is not processed by any management server. Skipping ...", commandKey, stateByAgent, host)); + } else { + logger.debug(String.format("The command %s is %s on host %s, it seems the answer is not processed by any management server. Reconciling ...", commandKey, stateByAgent, host)); + return reconcile(reconcileCommand); + } + } else if (Arrays.asList(State.INTERRUPTED, State.DANGLED_IN_BACKEND).contains(stateByAgent)) { + logger.debug(String.format("The command %s is %s on host %s, the cloudstack agent might has been restarted. Reconciling ...", commandKey, stateByAgent, host)); + return reconcile(reconcileCommand); + } + + return new ReconcileCommandResult(requestSequence, command, null, false); + } + } + + protected ReconcileCommandResult reconcile(ReconcileCommandVO reconcileCommandVO) { + Command command = ReconcileCommandUtils.parseCommandInfo(reconcileCommandVO.getCommandName(), reconcileCommandVO.getCommandInfo()); + + if (!preReconcileCheck(command)) { + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, true); + } + + if (command instanceof MigrateCommand) { + updateReconcileCommand(reconcileCommandVO.getRequestSequence(), command, null, State.RECONCILING, null); + return reconcile(reconcileCommandVO, (MigrateCommand) command); + } else if (command instanceof CopyCommand) { + updateReconcileCommand(reconcileCommandVO.getRequestSequence(), command, null, State.RECONCILING, null); + return reconcile(reconcileCommandVO, (CopyCommand) command); + } else if (command instanceof MigrateVolumeCommand) { + updateReconcileCommand(reconcileCommandVO.getRequestSequence(), command, null, State.RECONCILING, null); + return reconcile(reconcileCommandVO, (MigrateVolumeCommand) command); + } else { + logger.error(String.format("Unsupported reconcile command %s ", command)); + } + + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, true); + } + + boolean preReconcileCheck(Command command) { + if (command instanceof MigrateCommand) { + MigrateCommand migrateCommand = (MigrateCommand) command; + Long vmId = migrateCommand.getVirtualMachine().getId(); + VMInstanceVO vm = vmInstanceDao.findById(vmId); + if (vm == null) { + logger.debug(String.format("Skipping reconciliation of command %s as vm %s has been removed", command, vm)); + return false; + } + if (MapUtils.isEmpty(migrateCommand.getMigrateStorage())) { + logger.debug(String.format("Skipping reconciliation of command %s as the migration does not migrate volumes of vm %s", command, vm)); + return false; + } + List volumes = volumeDao.findByInstance(vmId); + for (VolumeVO volume : volumes) { + if (Volume.State.Migrating.equals(volume.getState())) { + return true; + } + } + logger.debug(String.format("Skipping reconciliation of command %s as the vm %s does not have volume in Migrating state", command, vm)); + return false; + } else if (command instanceof CopyCommand) { + DataTO srcData = ((CopyCommand) command).getSrcTO(); + DataTO destData = ((CopyCommand) command).getDestTO(); + if (srcData != null && srcData.getDataStore() instanceof PrimaryDataStoreTO) { + VolumeVO volumeVO = volumeDao.findById(srcData.getId()); + if (volumeVO == null || !Arrays.asList(Volume.State.Migrating, Volume.State.Ready).contains(volumeVO.getState())) { + logger.debug(String.format("Skipping reconciliation of command %s as source volume %s is removed or not Migrating or Ready", command, volumeVO)); + return false; + } + } + if (destData != null && destData.getDataStore() instanceof PrimaryDataStoreTO) { + VolumeVO volumeVO = volumeDao.findById(destData.getId()); + if (volumeVO == null || !Arrays.asList(Volume.State.Migrating, Volume.State.Ready, Volume.State.Creating).contains(volumeVO.getState())) { + logger.debug(String.format("Skipping reconciliation of command %s as destination volume %s is removed or not Migrating or Ready or Creating", command, volumeVO)); + return false; + } + } + } else if (command instanceof MigrateVolumeCommand) { + DataTO srcData = ((MigrateVolumeCommand) command).getSrcData(); + DataTO destData = ((MigrateVolumeCommand) command).getDestData(); + if (srcData == null || destData == null) { + logger.debug(String.format("Skipping reconciliation of command %s as the source volume (%s) or destination volume (%s) is NULL", command, srcData, destData)); + return false; + } + if (srcData.getId() != destData.getId()) { + logger.debug(String.format("Skipping reconciliation of command %s as the source volume (id: %s) and destination volume (id: %s) have different ID", command, srcData.getId(), destData.getId())); + return false; + } + VolumeVO volumeVO = volumeDao.findById(srcData.getId()); + if (volumeVO == null) { + logger.debug(String.format("Skipping reconciliation of command %s as the volume (id: %s) has been removed", command, srcData.getId())); + return false; + } else if (!Volume.State.Migrating.equals(volumeVO.getState())) { + logger.debug(String.format("Skipping reconciliation of command %s as the volume %s (state: %s) is not Migrating state", command, volumeVO, volumeVO.getState())); + return false; + } + if (!volumeVO.getPath().equals(destData.getPath())) { + logger.debug(String.format("Skipping reconciliation of command %s as the volume path (%s) is not same as destination volume path (%s)", command, volumeVO.getPath(), destData.getPath())); + return false; + } + } + return true; + } + + private ReconcileCommandResult reconcile(ReconcileCommandVO reconcileCommandVO, MigrateCommand command) { + Long vmId = command.getVirtualMachine().getId(); + VMInstanceVO vm = vmInstanceDao.findById(vmId); + if (vm == null) { + logger.debug(String.format("VM (id: %s) has been removed", vmId)); + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, true); + } + if (!VirtualMachine.State.Running.equals(vm.getState())) { + logger.debug(String.format("VM %s (state: %s) is not Running state, wait for next run", vm, vm.getState())); + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, false); + } + + ReconcileMigrateAnswer reconcileMigrateAnswer = new ReconcileMigrateAnswer(); + reconcileMigrateAnswer.setResourceType(ApiCommandResourceType.VirtualMachine); + reconcileMigrateAnswer.setResourceId(vmId); + + Long hostId = vm.getHostId(); + HostVO sourceHost = hostDao.findById(hostId); + if (sourceHost != null && sourceHost.getStatus() == Status.Up) { + ReconcileMigrateCommand reconcileMigrateCommand = new ReconcileMigrateCommand(command.getVmName()); + Answer reconcileAnswer = agentManager.easySend(sourceHost.getId(), reconcileMigrateCommand); + reconcileMigrateAnswer.setHostId(sourceHost.getId()); + if (reconcileAnswer instanceof ReconcileMigrateAnswer) { + reconcileMigrateAnswer.setVmState(((ReconcileMigrateAnswer) reconcileAnswer).getVmState()); + reconcileMigrateAnswer.setVmDisks(((ReconcileMigrateAnswer) reconcileAnswer).getVmDisks()); + } + } + + boolean isReconciled = (reconcileMigrateAnswer.getVmState() != null); + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, reconcileMigrateAnswer, isReconciled); + } + + private EndPoint getEndpoint(DataTO srcData, DataTO destData) { + EndPoint endPoint = null; + if (srcData.getDataStore() instanceof PrimaryDataStoreTO) { + PrimaryDataStoreTO srcDataStore = (PrimaryDataStoreTO) srcData.getDataStore(); + if (srcDataStore.isManaged()) { + return null; + } + DataStore store = dataStoreManager.getPrimaryDataStore(srcDataStore.getId()); + endPoint = endPointSelector.select(store); + } else if (destData != null && destData.getDataStore() instanceof PrimaryDataStoreTO) { + PrimaryDataStoreTO destDataStore = (PrimaryDataStoreTO) destData.getDataStore(); + if (destDataStore.isManaged()) { + return null; + } + DataStore store = dataStoreManager.getPrimaryDataStore(destDataStore.getId()); + endPoint = endPointSelector.select(store); + } + return endPoint; + } + + private ReconcileCommandResult reconcile(ReconcileCommandVO reconcileCommandVO, CopyCommand command) { + DataTO srcData = command.getSrcTO(); + DataTO destData = command.getDestTO(); + if (srcData == null || destData == null) { + logger.debug(String.format("Unable to reconcile command %s with srcData %s and destData %s", command, srcData, destData)); + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, false); + } + Long hostId = reconcileCommandVO.getHostId(); + HostVO host = hostDao.findById(hostId); + if (host == null || !Status.Up.equals(host.getStatus())) { + EndPoint endPoint = getEndpoint(srcData, destData); + if (endPoint == null) { + logger.debug(String.format("Unable to reconcile command %s with srcData %s and destData %s as endpoint is null", command, srcData, destData)); + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, false); + } + host = hostDao.findById(endPoint.getId()); + } + + // Send reconcileCommand to the host + logger.info(String.format("Reconciling command %s via host %s", command, host)); + ReconcileCopyCommand reconcileCommand = new ReconcileCopyCommand(srcData, destData, command.getOptions(), command.getOptions2()); + Answer reconcileAnswer = agentManager.easySend(host.getId(), reconcileCommand); + if (!(reconcileAnswer instanceof ReconcileAnswer)) { + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, true); + } + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, (ReconcileAnswer) reconcileAnswer, true); + } + + private ReconcileCommandResult reconcile(ReconcileCommandVO reconcileCommandVO, MigrateVolumeCommand command) { + DataTO srcData = command.getSrcData(); + DataTO destData = command.getDestData(); + if (srcData == null || destData == null) { + logger.debug(String.format("Unable to reconcile command %s with srcData %s and destData %s", command, srcData, destData)); + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, true); + } + VolumeVO volume = volumeDao.findById(srcData.getId()); + if (volume == null) { + logger.debug(String.format("Unable to reconcile command %s with removed volume (id: %s)", command, srcData.getId())); + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, true); + } + + Long hostId = reconcileCommandVO.getHostId(); + HostVO host = hostDao.findById(hostId); + if (host == null || !Status.Up.equals(host.getStatus())) { + EndPoint endPoint = getEndpoint(srcData, destData); + if (endPoint == null) { + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, false); + } + host = hostDao.findById(endPoint.getId()); + } + + // Send reconcileCommand to the host + logger.info(String.format("Reconciling command %s via host %s", command, host)); + ReconcileMigrateVolumeCommand reconcileCommand = new ReconcileMigrateVolumeCommand(srcData, destData); + if (volume.getInstanceId() != null) { + VMInstanceVO vmInstance = vmInstanceDao.findById(volume.getInstanceId()); + if (vmInstance != null) { + reconcileCommand.setVmName(vmInstance.getInstanceName()); + } + } + Answer reconcileAnswer = agentManager.easySend(host.getId(), reconcileCommand); + if (!(reconcileAnswer instanceof ReconcileAnswer)) { + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, null, true); + } + return new ReconcileCommandResult(reconcileCommandVO.getRequestSequence(), command, (ReconcileAnswer) reconcileAnswer, true); + } + + @Override + public void processCommand(Command pingCommand, Answer pingAnswer) { + if (pingCommand instanceof PingCommand && pingAnswer instanceof PingAnswer) { + CommandInfo[] commandInfos = ((PingCommand) pingCommand).getCommandInfos(); + for (CommandInfo commandInfo : commandInfos) { + processCommandInfo(commandInfo, (PingAnswer) pingAnswer); + } + } + } + + private void processCommandInfo(CommandInfo commandInfo, PingAnswer pingAnswer) { + Command parsedCommand = ReconcileCommandUtils.parseCommandInfo(commandInfo); + Answer parsedAnswer = ReconcileCommandUtils.parseAnswerFromCommandInfo(commandInfo); + if (parsedCommand != null && parsedCommand.isReconcile()) { + if (updateReconcileCommand(commandInfo.getRequestSeq(), parsedCommand, parsedAnswer, null, commandInfo.getState())) { + pingAnswer.addReconcileCommand(getCommandKey(commandInfo.getRequestSeq(), parsedCommand)); + } + } + } + + @Override + public void processAnswers(long requestSeq, Command[] commands, Answer[] answers) { + if (commands.length != answers.length) { + logger.error(String.format("Incorrect number of commands (%s) and answers (%s)", commands.length, answers.length)); + } + for (int i = 0; i < answers.length; i++) { + Command command = commands[i]; + Answer answer = answers[i]; + if (command.isReconcile() && answer.getResult()) { + reconcileCommandDao.removeCommand(requestSeq, command.toString(), State.COMPLETED); + } + } + } + + @Override + public void updateReconcileCommandToInterruptedByManagementServerId(long managementServerId) { + logger.debug("Updating reconcile command to interrupted by management server id " + managementServerId); + reconcileCommandDao.updateCommandsToInterruptedByManagementServerId(managementServerId); + } + + @Override + public void updateReconcileCommandToInterruptedByHostId(long hostId) { + logger.debug("Updating reconcile command to interrupted by host id " + hostId); + reconcileCommandDao.updateCommandsToInterruptedByHostId(hostId); + } + + private boolean processReconcileAnswer(long requestSequence, Command cmd, ReconcileAnswer reconcileAnswer) { + if (cmd instanceof MigrateCommand && reconcileAnswer instanceof ReconcileMigrateAnswer) { + MigrateCommand command = (MigrateCommand) cmd; + ReconcileMigrateAnswer answer = (ReconcileMigrateAnswer) reconcileAnswer; + return processReconcileMigrateAnswer(command, answer); + } else if (cmd instanceof CopyCommand && reconcileAnswer instanceof ReconcileCopyAnswer) { + CopyCommand command = (CopyCommand) cmd; + ReconcileCopyAnswer answer = (ReconcileCopyAnswer) reconcileAnswer; + return processReconcileCopyAnswer(requestSequence, command, answer); + } else if (cmd instanceof MigrateVolumeCommand && reconcileAnswer instanceof ReconcileMigrateVolumeAnswer) { + MigrateVolumeCommand command = (MigrateVolumeCommand) cmd; + ReconcileMigrateVolumeAnswer answer = (ReconcileMigrateVolumeAnswer) reconcileAnswer; + return processReconcileMigrateVolumeAnswer(requestSequence, command, answer); + } + return true; + } + + private boolean processReconcileMigrateAnswer(MigrateCommand command, ReconcileMigrateAnswer reconcileAnswer) { + Long vmId = command.getVirtualMachine().getId(); + VMInstanceVO vm = vmInstanceDao.findById(vmId); + if (vm == null) { + logger.debug(String.format("VM (id: %s) has been removed", vmId)); + return true; + } + if (!VirtualMachine.State.Running.equals(vm.getState())) { + logger.debug(String.format("VM %s (state: %s) is not Running state", vm, vm.getState())); + return true; + } + Map migrateDiskInfoMap = command.getMigrateStorage(); + if (MapUtils.isEmpty(migrateDiskInfoMap)) { + return true; + } + if (!VirtualMachine.State.Running.equals(reconcileAnswer.getVmState())) { + logger.debug(String.format("VM %s is %s state on the host, will retry", vm, vm.getState())); + return false; + } + List diskPaths = reconcileAnswer.getVmDisks(); + logger.debug(String.format("The disks attached to the VM %s after live vm migration are: %s", vm, diskPaths)); + + List volumes = volumeDao.findByInstance(vmId); + for (VolumeVO volumeVO : volumes) { + if (Volume.State.Migrating.equals(volumeVO.getState())) { + logger.debug(String.format("Reconciling vm %s with volume %s in Migrating state", vm, volumeVO)); + logger.debug(String.format("Searching for volumes with last_id = %s", volumeVO.getId())); + VolumeVO destVolume = volumeDao.findByLastIdAndState(volumeVO.getId(), Volume.State.Migrating); + if (destVolume != null) { + logger.debug(String.format("Found destination volume with last_id = %s: %s", volumeVO.getId(), destVolume)); + Boolean isVolumeMigrated = isVolumeMigrated(diskPaths, volumeVO.getPath(), destVolume.getPath()); + if (isVolumeMigrated == null) { + logger.debug(String.format("Unable to determine if source volume %s has been migrated to destination volume %s", volumeVO, destVolume)); + continue; + } + if (isVolumeMigrated) { + logger.debug(String.format("Adding destination volume %s to vm %s as part of reconciliation of command %s and answer %s", destVolume, vm, command, reconcileAnswer)); + destVolume.setState(Volume.State.Ready); + destVolume.setInstanceId(vmId); + volumeDao.update(destVolume.getId(), destVolume); + + logger.debug(String.format("Removing volume %s from vm %s as part of reconciliation of command %s and answer %s", volumeVO, vm, command, reconcileAnswer)); + volumeVO.setState(Volume.State.Destroy); + volumeVO.setVolumeType(Volume.Type.DATADISK); + volumeVO.setInstanceId(null); + volumeDao.update(volumeVO.getId(), volumeVO); + } else { + logger.debug(String.format("Removing destination volume %s from vm %s as part of reconciliation of command %s and answer %s", destVolume, vm, command, reconcileAnswer)); + destVolume.setState(Volume.State.Destroy); + destVolume.setVolumeType(Volume.Type.DATADISK); + destVolume.setInstanceId(null); + volumeDao.update(destVolume.getId(), destVolume); + + logger.debug(String.format("Adding volume %s to vm %s as part of reconciliation of command %s and answer %s", volumeVO, vm, command, reconcileAnswer)); + volumeVO.setState(Volume.State.Ready); + volumeVO.setInstanceId(vmId); + volumeDao.update(volumeVO.getId(), volumeVO); + } + } + } + } + volumes = volumeDao.findByInstance(vmId); + logger.debug(String.format("The disks of volumes attached to the VM %s after reconciliation of successful vm migration are: %s", vm, volumes.stream().map(VolumeVO::getPath).collect(Collectors.toList()))); + + return true; + } + + private Boolean isVolumeMigrated(List diskPaths, String sourceVolumePath, String destVolumePath) { + if (StringUtils.isAnyBlank(sourceVolumePath, destVolumePath)) { + return null; + } + for (String diskPath : diskPaths) { + if (diskPath.contains(sourceVolumePath) && !diskPath.contains(destVolumePath)) { + return false; // Not migrated + } else if (!diskPath.contains(sourceVolumePath) && diskPath.contains(destVolumePath)) { + return true; // Migrated + } + } + return null; // Unable to determine + } + + private boolean processReconcileCopyAnswer(long requestSequence, CopyCommand command, ReconcileCopyAnswer reconcileAnswer) { + DataTO srcData = command.getSrcTO(); + DataTO destData = command.getDestTO(); + + final Long srcStoreId = srcData.getDataStore() instanceof PrimaryDataStoreTO ? ((PrimaryDataStoreTO) srcData.getDataStore()).getId() : null; + final Long destStoreId = destData.getDataStore() instanceof PrimaryDataStoreTO ? ((PrimaryDataStoreTO) destData.getDataStore()).getId() : null; + + VolumeVO sourceVolume = srcData.getObjectType().equals(DataObjectType.VOLUME) && srcData.getDataStore() instanceof PrimaryDataStoreTO ? volumeDao.findByIdIncludingRemoved(srcData.getId()) : null; + VolumeVO destVolume = destData.getObjectType().equals(DataObjectType.VOLUME) && destData.getDataStore() instanceof PrimaryDataStoreTO ? volumeDao.findByIdIncludingRemoved(destData.getId()) : null; + + if (reconcileAnswer.isSkipped()) { + logger.debug(String.format("The reconcile command for source volume (id: %s) to destination volume (id: %s) is ignored because it is skipped, due to reason: %s", srcData.getId(), destData.getId(), reconcileAnswer.getReason())); + processVolumesIfReconcileCopyCommandIsSkipped(srcStoreId, sourceVolume, destStoreId, destVolume, command, reconcileAnswer); + return true; + } + if (!reconcileAnswer.getResult()) { + logger.debug(String.format("The reconcile command for source volume (id: %s) to destination volume (id: %s) is ignored because the result is false, due to %s", srcData.getId(), destData.getId(), reconcileAnswer.getDetails())); + return false; + } + + ReconcileCommandVO reconcileCommandVO = reconcileCommandDao.findCommand(requestSequence, command.toString()); + if (reconcileCommandVO == null) { + logger.debug(String.format("The reconcile command for source volume (id: %s) to destination volume (id: %s) is not found in database, ignoring", srcData.getId(), destData.getId())); + return true; + } + + ReconcileCopyAnswer previousReconcileAnswer = null; + if (destVolume != null) { + if (reconcileCommandVO.getAnswerName() == null) { + logger.debug(String.format("The reconcile command for source volume (id: %s) to destination volume (id: %s) does not have previous answer in database, ignoring this time", srcData.getId(), destData.getId())); + return false; + } + Answer previousAnswer = ReconcileCommandUtils.parseAnswerFromAnswerInfo(reconcileCommandVO.getAnswerName(), reconcileCommandVO.getAnswerInfo()); + if (!(previousAnswer instanceof ReconcileCopyAnswer)) { + logger.debug(String.format("The reconcile command for source volume (id: %s) to destination volume (id: %s) does not have previous reconcileAnswer in database, ignoring this time", srcData.getId(), destData.getId())); + return false; + } + previousReconcileAnswer = (ReconcileCopyAnswer) previousAnswer; + } + + VolumeOnStorageTO volumeOnSource = reconcileAnswer.getVolumeOnSource(); + VolumeOnStorageTO volumeOnDestination = reconcileAnswer.getVolumeOnDestination(); + Pair statePair = getVolumeStateOnSourceAndDestination(srcData, destData, volumeOnSource, volumeOnDestination, previousReconcileAnswer); + Volume.State sourceVolumeState = statePair.first(); + Volume.State destVolumeState = statePair.second(); + logger.debug(String.format("Processing volume (id: %s, state: %s) on source store and volume (id: %s, state: %s) on destination store", srcData.getId(), sourceVolumeState, destData.getId(), destVolumeState)); + + if (sourceVolume != null && destVolume != null) { + // copy from primary to primary (offline volume migration) + return processReconcileCopyAnswerFromPrimaryToPrimary(srcStoreId, sourceVolume, sourceVolumeState, destStoreId, destVolume, destVolumeState, volumeOnDestination); + } else if (sourceVolume == null && destVolume != null) { + // copy from secondary to primary + return processReconcileCopyAnswerFromImageCacheToPrimary(destStoreId, destVolume, destVolumeState, volumeOnDestination, command, reconcileAnswer); + } else if (sourceVolume != null && sourceVolumeState != null) { + // copy from primary to secondary + return processReconcileCopyAnswerFromPrimaryToSecondary(srcStoreId, sourceVolume, sourceVolumeState, command, reconcileAnswer); + } + return false; + } + + private boolean processVolumesIfReconcileCopyCommandIsSkipped(Long srcStoreId, VolumeVO sourceVolume, Long destStoreId, VolumeVO destVolume, + CopyCommand command, ReconcileCopyAnswer reconcileAnswer) { + if (sourceVolume == null && destVolume != null) { + // copy from secondary to primary + return processReconcileCopyAnswerFromImageCacheToPrimary(destStoreId, destVolume, Volume.State.Destroy, null, command, reconcileAnswer); + } else if (sourceVolume != null && destVolume == null) { + // copy from primary to secondary + return processReconcileCopyAnswerFromPrimaryToSecondary(srcStoreId, sourceVolume, Volume.State.Ready, command, reconcileAnswer); + } + return false; + } + + private boolean processReconcileCopyAnswerFromPrimaryToPrimary(Long srcStoreId, VolumeVO sourceVolume, Volume.State sourceVolumeState, + Long destStoreId, VolumeVO destVolume, Volume.State destVolumeState, VolumeOnStorageTO volumeOnDestination) { + boolean isSourceMigrating = sourceVolume != null && sourceVolume.getRemoved() == null && sourceVolume.getState().equals(Volume.State.Migrating); + if (Volume.State.Ready.equals(sourceVolumeState)) { + if (isSourceMigrating && srcStoreId != null && srcStoreId.equals(sourceVolume.getPoolId()) && destVolumeState != null) { + logger.debug(String.format("Updating source volume %s to %s state", sourceVolume, Volume.State.Ready)); + sourceVolume.setState(Volume.State.Ready); + sourceVolume.setUpdated(new Date()); + volumeDao.update(sourceVolume.getId(), sourceVolume); + } + if (Volume.State.Creating.equals(destVolume.getState()) && destStoreId != null && destStoreId.equals(destVolume.getPoolId()) && destVolumeState != null) { + logger.debug(String.format("Updating destination volume %s to %s state", destVolume, destVolumeState)); + destVolume.setState(destVolumeState); + destVolume.setUpdated(new Date()); + volumeDao.update(destVolume.getId(), destVolume); + } + return true; + } else if (Volume.State.Ready.equals(destVolumeState)) { + if (Volume.State.Creating.equals(destVolume.getState()) && destStoreId != null && destStoreId.equals(destVolume.getPoolId())) { + logger.debug(String.format("Updating destination volume %s to %s state", destVolume, Volume.State.Ready)); + destVolume.setState(Volume.State.Ready); + destVolume.setUpdated(new Date()); + destVolume.setInstanceId(sourceVolume.getInstanceId()); + destVolume.setPath(volumeOnDestination.getPath()); // Update path of destination volume + destVolume.set_iScsiName(volumeOnDestination.getPath()); + volumeDao.update(destVolume.getId(), destVolume); + } + if (isSourceMigrating && srcStoreId != null && srcStoreId.equals(sourceVolume.getPoolId()) && sourceVolumeState != null) { + logger.debug(String.format("Updating source volume %s to %s state", sourceVolume, sourceVolumeState)); + sourceVolume.setState(sourceVolumeState); + sourceVolume.setUpdated(new Date()); + sourceVolume.setInstanceId(null); + volumeDao.update(sourceVolume.getId(), sourceVolume); + } + return true; + } + return false; + } + + private boolean processReconcileCopyAnswerFromImageCacheToPrimary(Long destStoreId, VolumeVO destVolume, Volume.State destVolumeState, VolumeOnStorageTO volumeOnDestination, + CopyCommand command, ReconcileCopyAnswer reconcileAnswer) { + if (destVolume.getRemoved() == null && Volume.State.Creating.equals(destVolume.getState()) && destStoreId != null && destStoreId.equals(destVolume.getPoolId())) { + Long lastVolumeId = destVolume.getLastId(); + logger.debug(String.format("Searching for last volume with id = %s", destVolume.getLastId())); + VolumeVO lastVolume = volumeDao.findById(lastVolumeId); + if (lastVolume != null && Arrays.asList(Volume.State.Migrating, Volume.State.Ready).contains(lastVolume.getState()) + && Volume.State.Destroy.equals(destVolumeState)) { + destVolume.setState(destVolumeState); + if (volumeOnDestination != null) { + destVolume.setPath(volumeOnDestination.getPath()); // Update path of destination volume + } + volumeDao.update(destVolume.getId(), destVolume); + if (Volume.State.Migrating.equals(lastVolume.getState())) { + lastVolume.setState(Volume.State.Ready); // Update last volume to Ready + volumeDao.update(lastVolume.getId(), lastVolume); + } + // remove record from volume_store_ref with Copying state + VolumeDataStoreVO volumeDataStoreVO = volumeDataStoreDao.findByVolume(lastVolume.getId()); + if (volumeDataStoreVO != null && volumeDataStoreVO.getState().equals(ObjectInDataStoreStateMachine.State.Copying)) { + logger.debug(String.format("Removing record (id: %s) for volume %s from volume_store_ref as part of reconciliation of command %s and answer %s", volumeDataStoreVO.getId(), lastVolume, command, reconcileAnswer)); + volumeDataStoreDao.remove(volumeDataStoreVO.getId()); + } + } + return true; + } + return false; + } + + private boolean processReconcileCopyAnswerFromPrimaryToSecondary(Long srcStoreId, VolumeVO sourceVolume, Volume.State sourceVolumeState, + CopyCommand command, ReconcileCopyAnswer reconcileAnswer) { + boolean isSourceMigrating = sourceVolume != null && sourceVolume.getRemoved() == null && sourceVolume.getState().equals(Volume.State.Migrating); + if (isSourceMigrating && srcStoreId != null && srcStoreId.equals(sourceVolume.getPoolId())) { + logger.debug(String.format("Updating source volume %s to %s state", sourceVolume, sourceVolumeState)); + sourceVolume.setState(sourceVolumeState); // Update source volume state + volumeDao.update(sourceVolume.getId(), sourceVolume); + + // remove record from volume_store_ref with Creating state + VolumeDataStoreVO volumeDataStoreVO = volumeDataStoreDao.findByVolume(sourceVolume.getId()); + if (volumeDataStoreVO != null && volumeDataStoreVO.getState().equals(ObjectInDataStoreStateMachine.State.Creating)) { + logger.debug(String.format("Removing record (id: %s) for volume %s from volume_store_ref as part of reconciliation of command %s and answer %s", volumeDataStoreVO.getId(), sourceVolume, command, reconcileAnswer)); + volumeDataStoreDao.remove(volumeDataStoreVO.getId()); + } + } + logger.debug(String.format("Searching for volumes with last_id = %s", sourceVolume.getId())); + VolumeVO newVolume = volumeDao.findByLastIdAndState(sourceVolume.getId(), Volume.State.Creating); + if (newVolume != null) { + logger.debug(String.format("Removing volume %s as part of reconciliation of command %s and answer %s", newVolume, command, reconcileAnswer)); + newVolume.setState(Volume.State.Destroy); + newVolume.setVolumeType(Volume.Type.DATADISK); + newVolume.setInstanceId(null); + newVolume.setRemoved(new Date()); + volumeDao.update(newVolume.getId(), newVolume); + } + return true; + } + + private boolean processReconcileMigrateVolumeAnswer(long requestSequence, MigrateVolumeCommand command, ReconcileMigrateVolumeAnswer reconcileAnswer) { + DataTO srcData = command.getSrcData(); + DataTO destData = command.getDestData(); + if (srcData == null || destData == null) { + logger.debug(String.format("The source (%s) and destination (%s) of MigrateCommand must be non-empty", srcData, destData)); + return true; + } + if (srcData.getId() != destData.getId()) { + logger.debug(String.format("The source volume (id: %s) and destination volume (id: %s) of MigrateCommand must be same ID", srcData.getId(), destData.getId())); + return true; + } + VolumeVO sourceVolume = volumeDao.findByIdIncludingRemoved(srcData.getId()); + if (sourceVolume == null || sourceVolume.getRemoved() != null) { + logger.debug(String.format("Volume (id: %s) has been removed in CloudStack", srcData.getId())); + return true; + } + if (!sourceVolume.getState().equals(Volume.State.Migrating)) { + logger.debug(String.format("Volume %s (state: %s) is not in Migrating state", sourceVolume, sourceVolume.getState())); + return true; + } + + if (!(srcData.getDataStore() instanceof PrimaryDataStoreTO) && (destData.getDataStore() instanceof PrimaryDataStoreTO)) { + logger.debug(String.format("The source (role: %s) and destination (role: %s) of MigrateCommand must be Primary", srcData.getDataStore().getRole(), destData.getDataStore().getRole())); + return true; + } + + ReconcileCommandVO reconcileCommandVO = reconcileCommandDao.findCommand(requestSequence, command.toString()); + if (reconcileCommandVO == null) { + logger.debug(String.format("The reconcile command for migrating volume %s is not found in database, ignoring", sourceVolume)); + return true; + } + if (reconcileCommandVO.getAnswerName() == null) { + logger.debug(String.format("The reconcile command for migrating volume %s does not have previous answer in database, ignoring this time", sourceVolume)); + return false; + } + Answer previousAnswer = ReconcileCommandUtils.parseAnswerFromAnswerInfo(reconcileCommandVO.getAnswerName(), reconcileCommandVO.getAnswerInfo()); + if (!(previousAnswer instanceof ReconcileMigrateVolumeAnswer)) { + logger.debug(String.format("The reconcile command for for migrating volume %s does not have previous reconcileAnswer in database, ignoring this time", sourceVolume)); + return false; + } + List diskPaths = reconcileAnswer.getVmDiskPaths(); + logger.debug(String.format("The disks attached to the VM %s after live volume migration are: %s", reconcileAnswer.getVmName(), diskPaths)); + + PrimaryDataStoreTO srcDataStore = (PrimaryDataStoreTO) srcData.getDataStore(); + PrimaryDataStoreTO destDataStore = (PrimaryDataStoreTO) destData.getDataStore(); + + VolumeOnStorageTO volumeOnSource = reconcileAnswer.getVolumeOnSource(); + VolumeOnStorageTO volumeOnDestination = reconcileAnswer.getVolumeOnDestination(); + ReconcileMigrateVolumeAnswer previousReconcileAnswer = (ReconcileMigrateVolumeAnswer) previousAnswer; + Pair statePair = getVolumeStateOnSourceAndDestination(srcData, destData, volumeOnSource, volumeOnDestination, previousReconcileAnswer); + Volume.State sourceVolumeState = statePair.first(); + Volume.State destVolumeState = statePair.second(); + logger.debug(String.format("Processing volume (id: %s, state: %s) on source pool and volume (id: %s, state: %s) on destination pool", srcData.getId(), sourceVolumeState, destData.getId(), destVolumeState)); + if (Volume.State.Ready.equals(sourceVolumeState)) { + updateVolumeAndDestroyOldVolume(sourceVolume, srcData, srcDataStore, destData, destDataStore, volumeOnDestination); + return true; + } else if (Volume.State.Ready.equals(destVolumeState)) { + updateVolumeAndDestroyOldVolume(sourceVolume, destData, destDataStore, srcData, srcDataStore, volumeOnSource); + return true; + } else if (CollectionUtils.isNotEmpty(diskPaths)) { + // VM is Running + if (!Volume.State.Migrating.equals(destVolumeState) && volumeOnDestination != null) { + for (String diskPath : diskPaths) { + if (diskPath.equals(volumeOnDestination.getFullPath())) { + logger.debug(String.format("The VM %s is running with the volume %s on destination pool %s, the volume has been migrated", reconcileAnswer.getVmName(), volumeOnDestination.getFullPath(), destData.getDataStore())); + updateVolumeAndDestroyOldVolume(sourceVolume, destData, destDataStore, srcData, srcDataStore, volumeOnSource); + return true; + } else if (diskPath.equals(volumeOnSource.getFullPath())) { + logger.debug(String.format("The VM %s is running with the volume %s on source pool %s, the volume has not been migrated", reconcileAnswer.getVmName(), volumeOnSource.getFullPath(), srcData.getDataStore())); + updateVolumeAndDestroyOldVolume(sourceVolume, srcData, srcDataStore, destData, destDataStore, volumeOnDestination); + return true; + } + } + } + } + return false; + } + + private void updateVolumeAndDestroyOldVolume(VolumeVO sourceVolume, DataTO srcData, PrimaryDataStoreTO srcDataStore, DataTO destData, PrimaryDataStoreTO destDataStore, VolumeOnStorageTO volumeToBeDeleted) { + + logger.debug(String.format("Updating volume %s to %s state", sourceVolume, Volume.State.Ready)); + sourceVolume.setState(Volume.State.Ready); + sourceVolume.setPoolId(srcDataStore.getId()); // restore pool_id and update path + sourceVolume.setPath(srcData.getPath()); + sourceVolume.set_iScsiName(srcData.getPath()); + sourceVolume.setUpdated(new Date()); + volumeDao.update(sourceVolume.getId(), sourceVolume); + + if (volumeToBeDeleted != null) { + logger.debug(String.format("Creating a dummy volume from %s on pool %s", sourceVolume, destDataStore.getId())); + Volume newVol = volumeManager.allocateDuplicateVolume(sourceVolume, null, null); + VolumeVO newVolume = (VolumeVO) newVol; + newVolume.setInstanceId(null); + newVolume.setPoolId(destDataStore.getId()); + newVolume.setState(Volume.State.Creating); + newVolume.setPath(destData.getPath()); + newVolume.set_iScsiName(destData.getPath()); + volumeDao.update(newVolume.getId(), newVolume); + + logger.debug(String.format("Deleting the dummy volume %s on pool %s", newVolume, destDataStore.getId())); + volumeApiService.destroyVolume(newVolume.getId(), accountManager.getAccount(Account.ACCOUNT_ID_SYSTEM), true, true); + } + } + + private Pair getVolumeStateOnSourceAndDestination(DataTO srcData, DataTO destData, VolumeOnStorageTO volumeOnSource, VolumeOnStorageTO volumeOnDestination, ReconcileVolumeAnswer previousReconcileAnswer) { + final Long srcStoreId = srcData.getDataStore() instanceof PrimaryDataStoreTO ? ((PrimaryDataStoreTO) srcData.getDataStore()).getId() : null; + final Long destStoreId = destData.getDataStore() instanceof PrimaryDataStoreTO ? ((PrimaryDataStoreTO) destData.getDataStore()).getId() : null; + + VolumeOnStorageTO previousVolumeOnDestination = previousReconcileAnswer != null ? previousReconcileAnswer.getVolumeOnDestination() : null; + + if (volumeOnSource != null) { + if (volumeOnDestination == null) { + if (volumeOnSource.getPath() != null) { + logger.debug(String.format("Volume (id :%s) exist on source (id: %s) and volume (id: %s) does not exist on destination (id: %s), updating state to Ready on source pool", srcData.getId(), srcStoreId, destData.getId(), destStoreId)); + return new Pair<>(Volume.State.Ready, null); + } else { + logger.debug(String.format("Volume (id :%s) cannot be found on source (id: %s) and volume (id: %s) does not exist on destination (id: %s), updating state to Ready on source pool", srcData.getId(), srcStoreId, destData.getId(), destStoreId)); + return new Pair<>(Volume.State.Destroy, null); + } + } + if (volumeOnDestination.getPath() == null) { + logger.debug(String.format("Volume (id :%s) exist on source (id: %s) and volume (id: %s) cannot be found on destination (id: %s), updating state to Ready on source pool", srcData.getId(), srcStoreId, destData.getId(), destStoreId)); + return new Pair<>(Volume.State.Ready, Volume.State.Destroy); + } + boolean isDestinationVolumeChanged = (previousVolumeOnDestination != null && volumeOnDestination.getSize() != previousVolumeOnDestination.getSize()); + if (isDestinationVolumeChanged) { + logger.debug(String.format("Volume (id :%s) on destination (id: %s) is still being updated, skipping", destData.getId(), destStoreId)); + return new Pair<>(Volume.State.Migrating, Volume.State.Migrating); + } else if (destData.getId() == srcData.getId()) { + logger.debug(String.format("Volume (id :%s) on destination (id: %s) is not updated, cannot determine the state on source and destination pool", destData.getId(), destStoreId)); + return new Pair<>(null, null); + } else { + logger.debug(String.format("Volume (id :%s) on destination (id: %s) is not updated, updating state to Ready on source pool and Destroy on destination pool", destData.getId(), destStoreId)); + return new Pair<>(Volume.State.Ready, Volume.State.Destroy); + } + } else if (volumeOnDestination != null) { + boolean isDestinationVolumeChanged = (previousVolumeOnDestination != null && volumeOnDestination.getSize() != previousVolumeOnDestination.getSize()); + if (isDestinationVolumeChanged) { + logger.debug(String.format("Volume (id :%s) on destination (id: %s) is still being updated, skipping", destData.getId(), destStoreId)); + return new Pair<>(srcStoreId != null ? Volume.State.Migrating : null, Volume.State.Migrating); + } else if (srcStoreId != null) { + // from primary to primary + logger.debug(String.format("Volume (id: %s) does not exist on source (id: %s) but volume (id: %s) exist on destination (id: %s), updating state to Ready on destination pool", srcData.getId(), srcStoreId, destData.getId(), destStoreId)); + return new Pair<>(Volume.State.Destroy, Volume.State.Ready); + } else { + // from secondary to primary + logger.debug(String.format("Volume (id: %s) exist on destination (id: %s), however it is copied from secondary, updating state to Destroy on destination pool", destData.getId(), destStoreId)); + return new Pair<>(null, Volume.State.Destroy); + } + } + + return new Pair<>(null, null); + } + + @Override + public boolean isReconcileResourceNeeded(long resourceId, ApiCommandResourceType resourceType) { + return !reconcileCommandDao.listByResourceIdAndTypeAndStates(resourceId, resourceType, + State.INTERRUPTED, State.TIMED_OUT, State.RECONCILE_RETRY, State.RECONCILING, State.RECONCILE_FAILED, State.CREATED) + .isEmpty(); + } +} diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 60c2095d5f4..6edf206709c 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -382,4 +382,7 @@ + + + diff --git a/server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java b/server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java index eea8bb9de68..17b4eed33d3 100644 --- a/server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java +++ b/server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java @@ -114,7 +114,7 @@ public class KVMGuruTest { @Before public void setup() throws UnsupportedEncodingException { - Mockito.when(vmTO.getLimitCpuUse()).thenReturn(true); + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(true); Mockito.when(vmProfile.getVirtualMachine()).thenReturn(vm); Mockito.when(vm.getHostId()).thenReturn(hostId); Mockito.when(hostDao.findById(hostId)).thenReturn(host); @@ -156,7 +156,7 @@ public class KVMGuruTest { @Test public void testSetVmQuotaPercentageNotCPULimit() { - Mockito.when(vmTO.getLimitCpuUse()).thenReturn(false); + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(false); guru.setVmQuotaPercentage(vmTO, vmProfile); Mockito.verify(vmProfile, Mockito.never()).getVirtualMachine(); Mockito.verify(vmTO, Mockito.never()).setCpuQuotaPercentage(Mockito.anyDouble());