CLOUDSTACK-4939 - Failed to create snapshot (KVM, Multiple hosts, Sharedstorage)

Conflicts:

	engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
	engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
	plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java

Conflicts:

	engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java
	plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java

engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java

@@ -23,10 +23,14 @@ import java.util.List;
 public interface EndPointSelector {
     EndPoint select(DataObject srcData, DataObject destData);
 
+    EndPoint select(DataObject srcData, DataObject destData, StorageAction action);
+
     EndPoint select(DataObject object);
 
     EndPoint select(DataStore store);
 
+    EndPoint select(DataObject object, StorageAction action);
+
     List<EndPoint> selectAll(DataStore store);
 
     EndPoint select(Scope scope, Long storeId);

engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/StorageAction.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.engine.subsystem.api.storage;
+
+public enum StorageAction {
+    TAKESNAPSHOT,
+    BACKUPSNAPSHOT,
+    DELETESNAPSHOT
+}

engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java

@@ -22,6 +22,7 @@ import com.cloud.agent.api.Answer;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.offering.DiskOffering.DiskCacheMode;
 import com.cloud.storage.Volume;
+import com.cloud.vm.VirtualMachine;
 
 public interface VolumeInfo extends DataObject, Volume {
     boolean isAttachedVM();
@@ -35,6 +36,7 @@ public interface VolumeInfo extends DataObject, Volume {
     Long getLastPoolId();
 
     String getAttachedVmName();
+    VirtualMachine getAttachedVM();
 
     void processEventOnly(ObjectInDataStoreStateMachine.Event event);
 

engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java

@@ -23,15 +23,7 @@ import java.util.Map;
 
 import javax.inject.Inject;
 
-import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
+import org.apache.cloudstack.engine.subsystem.api.storage.*;
-import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
-import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
-import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
 import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
 import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
@@ -498,7 +490,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
             } else {
                 CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait, VirtualMachineManager.ExecuteInSequence.value());
                 cmd.setOptions(options);
-                EndPoint ep = selector.select(srcData, destData);
+                EndPoint ep = selector.select(srcData, destData, StorageAction.BACKUPSNAPSHOT);
                 if (ep == null) {
                     String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
                     s_logger.error(errMsg);
@@ -506,6 +498,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
                 } else {
                     answer = ep.sendMessage(cmd);
                 }
+
             }
             // clean up cache entry
             if (cacheData != null) {

engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java

@@ -27,23 +27,25 @@ import java.util.List;
 
 import javax.inject.Inject;
 
-import com.cloud.utils.db.Transaction;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
 import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.StorageAction;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.storage.LocalHostEndpoint;
 import org.apache.cloudstack.storage.RemoteHostEndPoint;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
 
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
 import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage.TemplateType;
@@ -52,6 +54,7 @@ import com.cloud.utils.db.QueryBuilder;
 import com.cloud.utils.db.SearchCriteria.Op;
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.VirtualMachine;
 
 @Component
 public class DefaultEndPointSelector implements EndPointSelector {
@@ -206,6 +209,21 @@ public class DefaultEndPointSelector implements EndPointSelector {
         return null;
     }
 
+    @Override
+    public EndPoint select(DataObject srcData, DataObject destData, StorageAction action) {
+        if (action == StorageAction.BACKUPSNAPSHOT) {
+            SnapshotInfo srcSnapshot = (SnapshotInfo)srcData;
+            if (srcSnapshot.getHypervisorType() == Hypervisor.HypervisorType.KVM) {
+                VolumeInfo volumeInfo = srcSnapshot.getBaseVolume();
+                VirtualMachine vm = volumeInfo.getAttachedVM();
+                if (vm != null && vm.getState() == VirtualMachine.State.Running) {
+                    return getEndPointFromHostId(vm.getHostId());
+                }
+            }
+        }
+        return select(srcData, destData);
+    }
+
     protected EndPoint findEndpointForPrimaryStorage(DataStore store) {
         return findEndPointInScope(store.getScope(), findOneHostOnPrimaryStorage, store.getId());
     }
@@ -268,6 +286,27 @@ public class DefaultEndPointSelector implements EndPointSelector {
         }
     }
 
+    private EndPoint getEndPointFromHostId(Long hostId) {
+        HostVO host = hostDao.findById(hostId);
+        return RemoteHostEndPoint.getHypervisorHostEndPoint(host);
+    }
+
+    @Override
+    public EndPoint select(DataObject object, StorageAction action) {
+        if (action == StorageAction.TAKESNAPSHOT) {
+            SnapshotInfo snapshotInfo = (SnapshotInfo)object;
+            if (snapshotInfo.getHypervisorType() == Hypervisor.HypervisorType.KVM) {
+                VolumeInfo volumeInfo = snapshotInfo.getBaseVolume();
+                VirtualMachine vm = volumeInfo.getAttachedVM();
+                if ((vm != null) && (vm.getState() == VirtualMachine.State.Running)) {
+                    Long hostId = vm.getHostId();
+                    return getEndPointFromHostId(hostId);
+                }
+            }
+        }
+        return select(object);
+    }
+
     @Override
     public EndPoint select(Scope scope, Long storeId) {
         return findEndPointInScope(scope, findOneHostOnPrimaryStorage, storeId);

engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java

@@ -22,6 +22,9 @@ import javax.inject.Inject;
 
 import org.apache.log4j.Logger;
 
+import com.cloud.storage.DiskOfferingVO;
+import com.cloud.storage.dao.DiskOfferingDao;
+import com.cloud.vm.VirtualMachine;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
@@ -40,11 +43,9 @@ import com.cloud.agent.api.to.DataTO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.offering.DiskOffering.DiskCacheMode;
 import com.cloud.storage.DataStoreRole;
-import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Volume;
 import com.cloud.storage.VolumeVO;
-import com.cloud.storage.dao.DiskOfferingDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.utils.component.ComponentContext;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -100,6 +101,15 @@ public class VolumeObject implements VolumeInfo {
         return null;
     }
 
+    @Override
+    public VirtualMachine getAttachedVM() {
+        Long vmId = this.volumeVO.getInstanceId();
+        if (vmId != null) {
+            VMInstanceVO vm = vmInstanceDao.findById(vmId);
+            return vm;
+        }
+        return null;
+    }
     @Override
     public String getUuid() {
         return volumeVO.getUuid();

plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java

@@ -24,8 +24,6 @@ import java.util.UUID;
 
 import javax.inject.Inject;
 
-import org.apache.log4j.Logger;
-
 import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
@@ -37,6 +35,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.StorageAction;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
@@ -51,6 +50,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.volume.VolumeObject;
+import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.ResizeVolumeAnswer;
@@ -269,10 +269,11 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri
             }
 
             CreateObjectCommand cmd = new CreateObjectCommand(snapshotTO);
-            EndPoint ep = epSelector.select(snapshot);
+            EndPoint ep = this.epSelector.select(snapshot, StorageAction.TAKESNAPSHOT);
             Answer answer = null;
-            if (ep == null) {
+
-                String errMsg = "No remote endpoint to send DeleteCommand, check if host or ssvm is down?";
+            if ( ep == null ){
+                String errMsg = "No remote endpoint to send createObjectCommand, check if host or ssvm is down?";
                 s_logger.error(errMsg);
                 answer = new Answer(cmd, false, errMsg);
             } else {