Support of snapshot copy to primary storage in different zones. (#9478)

* Support of snapshot copy to different StorPool primary storage between zones

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -113,10 +113,10 @@ public interface VolumeApiService {
 
     Volume detachVolumeFromVM(DetachVolumeCmd cmd);
 
-    Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map<String, String> tags, List<Long> zoneIds)
+    Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map<String, String> tags, List<Long> zoneIds, List<Long> poolIds, Boolean useStorageReplication)
             throws ResourceAllocationException;
 
-    Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, List<Long> zoneIds) throws ResourceAllocationException;
+    Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, List<Long> zoneIds, List<Long> storagePoolIds, Boolean useStorageReplication) throws ResourceAllocationException;
 
     Volume updateVolume(long volumeId, String path, String state, Long storageId,
                         Boolean displayVolume, Boolean deleteProtection,

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -539,6 +539,9 @@ public class ApiConstants {
     public static final String SNAPSHOT_POLICY_ID = "snapshotpolicyid";
     public static final String SNAPSHOT_TYPE = "snapshottype";
     public static final String SNAPSHOT_QUIESCEVM = "quiescevm";
+
+    public static final String USE_STORAGE_REPLICATION = "usestoragereplication";
+
     public static final String SOURCE_CIDR_LIST = "sourcecidrlist";
     public static final String SOURCE_ZONE_ID = "sourcezoneid";
     public static final String SSL_VERIFICATION = "sslverification";
@@ -1159,6 +1162,7 @@ public class ApiConstants {
 
     public static final String ZONE_ID_LIST = "zoneids";
     public static final String DESTINATION_ZONE_ID_LIST = "destzoneids";
+    public static final String STORAGE_ID_LIST = "storageids";
     public static final String ADMIN = "admin";
     public static final String CHECKSUM_PARAMETER_PREFIX_DESCRIPTION = "The parameter containing the checksum will be considered a MD5sum if it is not prefixed\n"
             + " and just a plain ascii/utf8 representation of a hexadecimal string. If it is required to\n"

api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java

@@ -17,9 +17,13 @@
 
 package org.apache.cloudstack.api.command.user.snapshot;
 
-import java.util.ArrayList;
+import com.cloud.dc.DataCenter;
-import java.util.List;
+import com.cloud.event.EventTypes;
-
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.exception.StorageUnavailableException;
+import com.cloud.storage.Snapshot;
+import com.cloud.user.Account;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
@@ -31,26 +35,24 @@ import org.apache.cloudstack.api.ResponseObject;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.command.user.UserCmd;
 import org.apache.cloudstack.api.response.SnapshotResponse;
+import org.apache.cloudstack.api.response.StoragePoolResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.commons.collections.CollectionUtils;
-
+import org.apache.commons.lang3.BooleanUtils;
-import com.cloud.dc.DataCenter;
-import com.cloud.event.EventTypes;
-import com.cloud.exception.ResourceAllocationException;
-import com.cloud.exception.ResourceUnavailableException;
-import com.cloud.exception.StorageUnavailableException;
-import com.cloud.storage.Snapshot;
-import com.cloud.user.Account;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.util.ArrayList;
+import java.util.List;
+
 @APICommand(name = "copySnapshot", description = "Copies a snapshot from one zone to another.",
         responseObject = SnapshotResponse.class, responseView = ResponseObject.ResponseView.Restricted,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0",
         authorized = {RoleType.Admin,  RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
 public class CopySnapshotCmd extends BaseAsyncCmd implements UserCmd {
     public static final Logger logger = LogManager.getLogger(CopySnapshotCmd.class.getName());
+    private Snapshot snapshot;
 
     /////////////////////////////////////////////////////
     //////////////// API parameters /////////////////////
@@ -84,6 +86,20 @@ public class CopySnapshotCmd extends BaseAsyncCmd implements UserCmd {
                     "Do not specify destzoneid and destzoneids together, however one of them is required.")
     protected List<Long> destZoneIds;
 
+    @Parameter(name = ApiConstants.STORAGE_ID_LIST,
+            type=CommandType.LIST,
+            collectionType = CommandType.UUID,
+            entityType = StoragePoolResponse.class,
+            required = false,
+            authorized = RoleType.Admin,
+            since = "4.21.0",
+            description = "A comma-separated list of IDs of the storage pools in other zones in which the snapshot will be made available. " +
+                    "The snapshot will always be made available in the zone in which the volume is present. Currently supported for StorPool only")
+    protected List<Long> storagePoolIds;
+
+    @Parameter (name = ApiConstants.USE_STORAGE_REPLICATION, type=CommandType.BOOLEAN, required = false, since = "4.21.0", description = "This parameter enables the option the snapshot to be copied to supported primary storage")
+    protected Boolean useStorageReplication;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -106,7 +122,15 @@ public class CopySnapshotCmd extends BaseAsyncCmd implements UserCmd {
             destIds.add(destZoneId);
             return destIds;
         }
-        return null;
+        return new ArrayList<>();
+    }
+
+    public List<Long> getStoragePoolIds() {
+        return storagePoolIds;
+    }
+
+    public Boolean useStorageReplication() {
+        return BooleanUtils.toBoolean(useStorageReplication);
     }
 
     @Override
@@ -152,7 +176,7 @@ public class CopySnapshotCmd extends BaseAsyncCmd implements UserCmd {
     @Override
     public void execute() throws ResourceUnavailableException {
         try {
-            if (destZoneId == null && CollectionUtils.isEmpty(destZoneIds))
+            if (destZoneId == null && CollectionUtils.isEmpty(destZoneIds) && useStorageReplication())
                 throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
                         "Either destzoneid or destzoneids parameters have to be specified.");
 
@@ -161,7 +185,7 @@ public class CopySnapshotCmd extends BaseAsyncCmd implements UserCmd {
                         "Both destzoneid and destzoneids cannot be specified at the same time.");
 
             CallContext.current().setEventDetails(getEventDescription());
-            Snapshot snapshot = _snapshotService.copySnapshot(this);
+            snapshot = _snapshotService.copySnapshot(this);
 
             if (snapshot != null) {
                 SnapshotResponse response = _queryService.listSnapshot(this);
@@ -177,6 +201,13 @@ public class CopySnapshotCmd extends BaseAsyncCmd implements UserCmd {
             logger.warn("Exception: ", ex);
             throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage());
         }
+    }
 
+    public Snapshot getSnapshot() {
+        return snapshot;
+    }
+
+    public void setSnapshot(Snapshot snapshot) {
+        this.snapshot = snapshot;
     }
 }

api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java

@@ -16,11 +16,13 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.snapshot;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.api.ApiConstants;
@@ -32,6 +34,7 @@ import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.response.DomainResponse;
 import org.apache.cloudstack.api.response.SnapshotPolicyResponse;
 import org.apache.cloudstack.api.response.SnapshotResponse;
+import org.apache.cloudstack.api.response.StoragePoolResponse;
 import org.apache.cloudstack.api.response.VolumeResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.commons.collections.MapUtils;
@@ -99,6 +102,19 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd {
             since = "4.19.0")
     protected List<Long> zoneIds;
 
+    @Parameter(name = ApiConstants.STORAGE_ID_LIST,
+            type=CommandType.LIST,
+            collectionType = CommandType.UUID,
+            entityType = StoragePoolResponse.class,
+            authorized = RoleType.Admin,
+            description = "A comma-separated list of IDs of the storage pools in other zones in which the snapshot will be made available. " +
+                    "The snapshot will always be made available in the zone in which the volume is present.",
+            since = "4.21.0")
+    protected List<Long> storagePoolIds;
+
+    @Parameter (name = ApiConstants.USE_STORAGE_REPLICATION, type=CommandType.BOOLEAN, required = false, description = "This parameter enables the option the snapshot to be copied to supported primary storage")
+    protected Boolean useStorageReplication;
+
     private String syncObjectType = BaseAsyncCmd.snapshotHostSyncObject;
 
     // ///////////////////////////////////////////////////
@@ -161,6 +177,17 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd {
         return zoneIds;
     }
 
+    public List<Long> getStoragePoolIds() {
+        return storagePoolIds == null ? new ArrayList<>() : storagePoolIds;
+    }
+
+    public Boolean useStorageReplication() {
+        if (useStorageReplication == null) {
+            return false;
+        }
+        return useStorageReplication;
+    }
+
     // ///////////////////////////////////////////////////
     // ///////////// API Implementation///////////////////
     // ///////////////////////////////////////////////////
@@ -209,7 +236,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd {
 
     @Override
     public void create() throws ResourceAllocationException {
-        Snapshot snapshot = _volumeService.allocSnapshot(getVolumeId(), getPolicyId(), getSnapshotName(), getLocationType(), getZoneIds());
+        Snapshot snapshot = _volumeService.allocSnapshot(getVolumeId(), getPolicyId(), getSnapshotName(), getLocationType(), getZoneIds(), getStoragePoolIds(), useStorageReplication());
         if (snapshot != null) {
             setEntityId(snapshot.getId());
             setEntityUuid(snapshot.getUuid());
@@ -223,7 +250,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd {
         Snapshot snapshot;
         try {
             snapshot =
-                _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags(), getZoneIds());
+                _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags(), getZoneIds(), getStoragePoolIds(), useStorageReplication());
 
             if (snapshot != null) {
                 SnapshotResponse response = _responseGenerator.createSnapshotResponse(snapshot);
@@ -243,7 +270,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd {
         }
     }
 
-    private Snapshot.LocationType getLocationType() {
+    public Snapshot.LocationType getLocationType() {
 
         if (Snapshot.LocationType.values() == null || Snapshot.LocationType.values().length == 0 || locationType == null) {
             return null;

api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java

@@ -16,11 +16,13 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.snapshot;
 
-import java.util.Collection;
+import com.cloud.exception.InvalidParameterValueException;
-import java.util.HashMap;
+import com.cloud.exception.PermissionDeniedException;
-import java.util.List;
+import com.cloud.projects.Project;
-import java.util.Map;
+import com.cloud.storage.Volume;
-
+import com.cloud.storage.snapshot.SnapshotPolicy;
+import com.cloud.user.Account;
+import java.util.ArrayList;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
@@ -30,16 +32,16 @@ import org.apache.cloudstack.api.BaseCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.response.SnapshotPolicyResponse;
+import org.apache.cloudstack.api.response.StoragePoolResponse;
 import org.apache.cloudstack.api.response.VolumeResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.commons.collections.MapUtils;
 
-import com.cloud.exception.InvalidParameterValueException;
+import java.util.Collection;
-import com.cloud.exception.PermissionDeniedException;
+import java.util.HashMap;
-import com.cloud.projects.Project;
+import java.util.List;
-import com.cloud.storage.Volume;
+import java.util.Map;
-import com.cloud.storage.snapshot.SnapshotPolicy;
+import org.apache.commons.lang3.BooleanUtils;
-import com.cloud.user.Account;
 
 @APICommand(name = "createSnapshotPolicy", description = "Creates a snapshot policy for the account.", responseObject = SnapshotPolicyResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@@ -83,6 +85,17 @@ public class CreateSnapshotPolicyCmd extends BaseCmd {
                     "The snapshots will always be made available in the zone in which the volume is present.")
     protected List<Long> zoneIds;
 
+    @Parameter(name = ApiConstants.STORAGE_ID_LIST,
+            type=CommandType.LIST,
+            collectionType = CommandType.UUID,
+            entityType = StoragePoolResponse.class,
+            description = "A comma-separated list of IDs of the storage pools in other zones in which the snapshot will be made available. " +
+                    "The snapshot will always be made available in the zone in which the volume is present.",
+            since = "4.21.0")
+    protected List<Long> storagePoolIds;
+
+    @Parameter (name = ApiConstants.USE_STORAGE_REPLICATION, type=CommandType.BOOLEAN, required = false, since = "4.21.0", description = "This parameter enables the option the snapshot to be copied to supported primary storage")
+    protected Boolean useStorageReplication;
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -119,6 +132,14 @@ public class CreateSnapshotPolicyCmd extends BaseCmd {
         return zoneIds;
     }
 
+    public  List<Long> getStoragePoolIds() {
+        return storagePoolIds == null ? new ArrayList<>() : storagePoolIds;
+    }
+
+    public Boolean useStorageReplication() {
+        return BooleanUtils.toBoolean(useStorageReplication);
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java

@@ -16,17 +16,16 @@
 // under the License.
 package org.apache.cloudstack.api.response;
 
-import java.util.LinkedHashSet;
+import com.cloud.serializer.Param;
-import java.util.Set;
+import com.cloud.storage.snapshot.SnapshotPolicy;
-
+import com.google.gson.annotations.SerializedName;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponseWithTagInformation;
 import org.apache.cloudstack.api.EntityReference;
 
-import com.cloud.serializer.Param;
+import java.util.LinkedHashSet;
-import com.cloud.storage.snapshot.SnapshotPolicy;
+import java.util.Set;
-import com.google.gson.annotations.SerializedName;
 
 @EntityReference(value = SnapshotPolicy.class)
 public class SnapshotPolicyResponse extends BaseResponseWithTagInformation {
@@ -62,9 +61,14 @@ public class SnapshotPolicyResponse extends BaseResponseWithTagInformation {
     @Param(description = "The list of zones in which snapshot backup is scheduled", responseObject = ZoneResponse.class, since = "4.19.0")
     protected Set<ZoneResponse> zones;
 
+    @SerializedName(ApiConstants.STORAGE)
+    @Param(description = "The list of pools in which snapshot backup is scheduled", responseObject = StoragePoolResponse.class, since = "4.21.0")
+    protected Set<StoragePoolResponse> storagePools;
+
     public SnapshotPolicyResponse() {
         tags = new LinkedHashSet<ResourceTagResponse>();
         zones = new LinkedHashSet<>();
+        storagePools = new LinkedHashSet<>();
     }
 
     public String getId() {
@@ -130,4 +134,6 @@ public class SnapshotPolicyResponse extends BaseResponseWithTagInformation {
     public void setZones(Set<ZoneResponse> zones) {
         this.zones = zones;
     }
+
+    public void  setStoragePools(Set<StoragePoolResponse> pools) { this.storagePools = pools; }
 }

api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java

@@ -93,7 +93,7 @@ public class CreateSnapshotCmdTest extends TestCase {
         Snapshot snapshot = Mockito.mock(Snapshot.class);
         try {
             Mockito.when(volumeApiService.takeSnapshot(nullable(Long.class), nullable(Long.class), isNull(),
-                    nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), nullable(Map.class), nullable(List.class))).thenReturn(snapshot);
+                    nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), nullable(Map.class), nullable(List.class), nullable(List.class), Mockito.anyBoolean())).thenReturn(snapshot);
 
         } catch (Exception e) {
             Assert.fail("Received exception when success expected " + e.getMessage());
@@ -126,7 +126,7 @@ public class CreateSnapshotCmdTest extends TestCase {
 
         try {
                 Mockito.when(volumeApiService.takeSnapshot(nullable(Long.class), nullable(Long.class), nullable(Long.class),
-                        nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), any(), Mockito.anyList())).thenReturn(null);
+                        nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), any(), Mockito.anyList(), Mockito.anyList(), Mockito.anyBoolean())).thenReturn(null);
         } catch (Exception e) {
             Assert.fail("Received exception when success expected " + e.getMessage());
         }

api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmdTest.java

@@ -87,7 +87,12 @@ public class CopySnapshotCmdTest {
 
     @Test (expected = ServerApiException.class)
     public void testExecuteWrongNoParams() {
+        UUIDManager uuidManager = Mockito.mock(UUIDManager.class);
+        SnapshotApiService snapshotApiService = Mockito.mock(SnapshotApiService.class);
         final CopySnapshotCmd cmd = new CopySnapshotCmd();
+        cmd._uuidMgr = uuidManager;
+        cmd._snapshotService = snapshotApiService;
+
         try {
             cmd.execute();
         } catch (ResourceUnavailableException e) {

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreCapabilities.java

@@ -40,5 +40,13 @@ public enum DataStoreCapabilities {
     /**
      * indicates that this driver supports reverting a volume to a snapshot state
      */
-    CAN_REVERT_VOLUME_TO_SNAPSHOT
+    CAN_REVERT_VOLUME_TO_SNAPSHOT,
+    /**
+    * indicates that the driver supports copying snapshot between zones on pools of the same type
+    */
+    CAN_COPY_SNAPSHOT_BETWEEN_ZONES_AND_SAME_POOL_TYPE,
+    /**
+     * indicates that this driver supports the option to create a template from the back-end snapshot
+     */
+    CAN_CREATE_TEMPLATE_FROM_SNAPSHOT
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java

@@ -46,4 +46,6 @@ public interface SnapshotService {
     AsyncCallFuture<SnapshotResult> copySnapshot(SnapshotInfo snapshot, String copyUrl, DataStore dataStore) throws ResourceUnavailableException;
 
     AsyncCallFuture<CreateCmdResult> queryCopySnapshot(SnapshotInfo snapshot) throws ResourceUnavailableException;
+
+    AsyncCallFuture<SnapshotResult> copySnapshot(SnapshotInfo sourceSnapshot, SnapshotInfo destSnapshot, SnapshotStrategy strategy);
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotStrategy.java

@@ -16,12 +16,14 @@
 // under the License.
 package org.apache.cloudstack.engine.subsystem.api.storage;
 
+
 import com.cloud.storage.Snapshot;
+import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
 
 public interface SnapshotStrategy {
 
     enum SnapshotOperation {
-        TAKE, BACKUP, DELETE, REVERT
+        TAKE, BACKUP, DELETE, REVERT, COPY
     }
 
     SnapshotInfo takeSnapshot(SnapshotInfo snapshot);
@@ -35,4 +37,7 @@ public interface SnapshotStrategy {
     StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op);
 
     void postSnapshotCreation(SnapshotInfo snapshot);
+
+    default void copySnapshot(DataObject snapshotSource, DataObject snapshotDest, AsyncCompletionCallback<CreateCmdResult> caller) {
+    }
 }

engine/components-api/src/main/java/com/cloud/vm/VmWorkTakeVolumeSnapshot.java

@@ -30,12 +30,12 @@ public class VmWorkTakeVolumeSnapshot extends VmWork {
     private boolean quiesceVm;
     private Snapshot.LocationType locationType;
     private boolean asyncBackup;
-
+    private List<Long> poolIds;
     private List<Long> zoneIds;
 
     public VmWorkTakeVolumeSnapshot(long userId, long accountId, long vmId, String handlerName,
             Long volumeId, Long policyId, Long snapshotId, boolean quiesceVm, Snapshot.LocationType locationType,
-            boolean asyncBackup, List<Long> zoneIds) {
+            boolean asyncBackup, List<Long> zoneIds, List<Long> poolIds) {
         super(userId, accountId, vmId, handlerName);
         this.volumeId = volumeId;
         this.policyId = policyId;
@@ -44,6 +44,7 @@ public class VmWorkTakeVolumeSnapshot extends VmWork {
         this.locationType = locationType;
         this.asyncBackup = asyncBackup;
         this.zoneIds = zoneIds;
+        this.poolIds = poolIds;
     }
 
     public Long getVolumeId() {
@@ -71,4 +72,8 @@ public class VmWorkTakeVolumeSnapshot extends VmWork {
     public List<Long> getZoneIds() {
         return zoneIds;
     }
+
+    public List<Long> getPoolIds() {
+        return poolIds;
+    }
 }

engine/components-api/src/test/java/com/cloud/vm/VmWorkTakeVolumeSnapshotTest.java

@@ -26,8 +26,9 @@ public class VmWorkTakeVolumeSnapshotTest {
     @Test
     public void testVmWorkTakeVolumeSnapshotZoneIds() {
         List<Long> zoneIds = List.of(10L, 20L);
+        List<Long> poolIds = List.of(10L, 20L);
         VmWorkTakeVolumeSnapshot work = new VmWorkTakeVolumeSnapshot(1L, 1L, 1L, "handler",
-                1L, 1L, 1L, false, null, false, zoneIds);
+                1L, 1L, 1L, false, null, false, zoneIds, poolIds);
         Assert.assertNotNull(work.getZoneIds());
         Assert.assertEquals(zoneIds.size(), work.getZoneIds().size());
         Assert.assertEquals(zoneIds.get(0), work.getZoneIds().get(0));

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -577,14 +577,18 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
         }
 
         VolumeInfo vol = volFactory.getVolume(volume.getId());
+        long zoneId = volume.getDataCenterId();
         DataStore store = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary);
-        DataStoreRole dataStoreRole = snapshotHelper.getDataStoreRole(snapshot);
+        DataStoreRole dataStoreRole = snapshotHelper.getDataStoreRole(snapshot, zoneId);
-        SnapshotInfo snapInfo = snapshotFactory.getSnapshotWithRoleAndZone(snapshot.getId(), dataStoreRole, volume.getDataCenterId());
+        SnapshotInfo snapInfo = snapshotFactory.getSnapshotWithRoleAndZone(snapshot.getId(), dataStoreRole, zoneId);
 
-        boolean kvmSnapshotOnlyInPrimaryStorage = snapshotHelper.isKvmSnapshotOnlyInPrimaryStorage(snapshot, dataStoreRole);
+        boolean kvmSnapshotOnlyInPrimaryStorage = snapshotHelper.isKvmSnapshotOnlyInPrimaryStorage(snapshot, dataStoreRole, volume.getDataCenterId());
+        boolean storageSupportSnapshotToTemplateEnabled = snapshotHelper.isStorageSupportSnapshotToTemplate(snapInfo);
 
         try {
+            if (!storageSupportSnapshotToTemplateEnabled) {
                 snapInfo = snapshotHelper.backupSnapshotToSecondaryStorageIfNotExists(snapInfo, dataStoreRole, snapshot, kvmSnapshotOnlyInPrimaryStorage);
+            }
         } catch (CloudRuntimeException e) {
             snapshotHelper.expungeTemporarySnapshot(kvmSnapshotOnlyInPrimaryStorage, snapInfo);
             throw e;
@@ -596,7 +600,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
         }
 
         // don't try to perform a sync if the DataStoreRole of the snapshot is equal to DataStoreRole.Primary
-        if (!DataStoreRole.Primary.equals(dataStoreRole) || kvmSnapshotOnlyInPrimaryStorage) {
+        if (!DataStoreRole.Primary.equals(dataStoreRole) || !storageSupportSnapshotToTemplateEnabled) {
             try {
                 // sync snapshot to region store if necessary
                 DataStore snapStore = snapInfo.getDataStore();

engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java

@@ -33,6 +33,13 @@ public interface ResourceDetailsDao<R extends ResourceDetail> extends GenericDao
      */
     R findDetail(long resourceId, String name);
 
+    /**
+     * Find details by key
+     * @param key
+     * @return
+     */
+    List<R> findDetails(String key);
+
     /**
      * Find details by resourceId and key
      * @param resourceId

engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java

@@ -65,6 +65,12 @@ public abstract class ResourceDetailsDaoBase<R extends ResourceDetail> extends G
         return findOneBy(sc);
     }
 
+    public List<R> findDetails(String key) {
+        SearchCriteria<R> sc = AllFieldsSearch.create();
+        sc.setParameters("name", key);
+        return listBy(sc);
+    }
+
     public List<R> findDetails(long resourceId, String key) {
         SearchCriteria<R> sc = AllFieldsSearch.create();
         sc.setParameters("resourceId", resourceId);

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java

@@ -168,4 +168,7 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
     List<StoragePoolVO> listByIds(List<Long> ids);
 
     List<StoragePoolVO> findStoragePoolsByEmptyStorageAccessGroups(Long dcId, Long podId, Long clusterId, ScopeType scope, HypervisorType hypervisorType);
+
+    List<StoragePoolVO> findPoolsByStorageTypeAndZone(Storage.StoragePoolType storageType, Long zoneId);
+
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java

@@ -916,6 +916,14 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
         return listBy(sc);
     }
 
+    @Override
+    public List<StoragePoolVO> findPoolsByStorageTypeAndZone(Storage.StoragePoolType storageType, Long zoneId) {
+        SearchCriteria<StoragePoolVO> sc = AllFieldSearch.create();
+        sc.setParameters("poolType", storageType);
+        sc.addAnd("dataCenterId", Op.EQ, zoneId);
+        return listBy(sc);
+    }
+
     private SearchCriteria<StoragePoolVO> createStoragePoolSearchCriteria(Long storagePoolId, String storagePoolName,
                                                                           Long zoneId, String path, Long podId, Long clusterId, Long hostId, String address, ScopeType scopeType,
                                                                           StoragePoolStatus status, String keyword, String storageAccessGroup) {

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java

@@ -61,8 +61,11 @@ StateDao<ObjectInDataStoreStateMachine.State, ObjectInDataStoreStateMachine.Even
 
     List<SnapshotDataStoreVO> listExtractedSnapshotsBeforeDate(Date beforeDate);
 
+    List<SnapshotDataStoreVO> listSnapshotsBySnapshotId(long snapshotId);
+
     List<SnapshotDataStoreVO> listReadyBySnapshot(long snapshotId, DataStoreRole role);
 
+    List<SnapshotDataStoreVO> listReadyBySnapshotId(long snapshotId);
     SnapshotDataStoreVO findBySourceSnapshot(long snapshotId, DataStoreRole role);
 
     List<SnapshotDataStoreVO> findBySnapshotIdAndNotInDestroyedHiddenState(long snapshotId);

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java

@@ -16,24 +16,6 @@
 // under the License.
 package org.apache.cloudstack.storage.datastore.db;
 
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-
-import javax.inject.Inject;
-import javax.naming.ConfigurationException;
-
-import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
-import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
-import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
-import org.apache.commons.collections.CollectionUtils;
-import org.springframework.stereotype.Component;
-
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.SnapshotVO;
@@ -47,6 +29,25 @@ import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.UpdateBuilder;
 
+import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
+
+import org.apache.commons.collections.CollectionUtils;
+
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
 @Component
 public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO, Long> implements SnapshotDataStoreDao {
     private static final String STORE_ID = "store_id";
@@ -76,6 +77,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
     private SearchBuilder<SnapshotDataStoreVO> searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore;
     private SearchBuilder<SnapshotDataStoreVO> searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq;
 
+    private SearchBuilder<SnapshotDataStoreVO> searchBySnapshotId;
 
     protected static final List<Hypervisor.HypervisorType> HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING = List.of(Hypervisor.HypervisorType.XenServer);
 
@@ -187,6 +189,11 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
         searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.and(STORE_ROLE, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.entity().getRole(), SearchCriteria.Op.EQ);
         searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.and(STORE_ID, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.entity().getDataStoreId(), SearchCriteria.Op.IN);
 
+        searchBySnapshotId = createSearchBuilder();
+        searchBySnapshotId.and(SNAPSHOT_ID, searchBySnapshotId.entity().getSnapshotId(), SearchCriteria.Op.EQ);
+        searchBySnapshotId.and(STATE, searchBySnapshotId.entity().getState(), SearchCriteria.Op.EQ);
+        searchBySnapshotId.done();
+
         return true;
     }
 
@@ -403,6 +410,13 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
         return listBy(sc);
     }
 
+    @Override
+    public List<SnapshotDataStoreVO> listSnapshotsBySnapshotId(long snapshotId) {
+        SearchCriteria<SnapshotDataStoreVO> sc = searchBySnapshotId.create();
+        sc.setParameters(SNAPSHOT_ID, snapshotId);
+        return listBy(sc);
+    }
+
     @Override
     public List<SnapshotDataStoreVO> listReadyBySnapshot(long snapshotId, DataStoreRole role) {
         SearchCriteria<SnapshotDataStoreVO> sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role);
@@ -410,6 +424,14 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
         return listBy(sc);
     }
 
+    @Override
+    public List<SnapshotDataStoreVO> listReadyBySnapshotId(long snapshotId) {
+        SearchCriteria<SnapshotDataStoreVO> sc = searchBySnapshotId.create();
+        sc.setParameters(SNAPSHOT_ID, snapshotId);
+        sc.setParameters(STATE, State.Ready);
+        return listBy(sc);
+    }
+
     @Override
     public SnapshotDataStoreVO findBySourceSnapshot(long snapshotId, DataStoreRole role) {
         SearchCriteria<SnapshotDataStoreVO> sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role);

engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/CephSnapshotStrategy.java

@@ -23,6 +23,7 @@ import javax.inject.Inject;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
@@ -46,6 +47,9 @@ public class CephSnapshotStrategy extends StorageSystemSnapshotStrategy {
 
     @Override
     public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) {
+        if (SnapshotOperation.COPY.equals(op)) {
+            return StrategyPriority.CANT_HANDLE;
+        }
         long volumeId = snapshot.getVolumeId();
         VolumeVO volumeVO = volumeDao.findByIdIncludingRemoved(volumeId);
         boolean baseVolumeExists = volumeVO.getRemoved() == null;

engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java

@@ -627,9 +627,14 @@ public class DefaultSnapshotStrategy extends SnapshotStrategyBase {
 
     @Override
     public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) {
+        if (SnapshotOperation.COPY.equals(op)) {
+            return StrategyPriority.CANT_HANDLE;
+        }
+
         if (SnapshotOperation.TAKE.equals(op)) {
             return validateVmSnapshot(snapshot);
         }
+
         if (SnapshotOperation.REVERT.equals(op)) {
             long volumeId = snapshot.getVolumeId();
             VolumeVO volumeVO = volumeDao.findById(volumeId);

engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/ScaleIOSnapshotStrategy.java

@@ -22,6 +22,7 @@ import javax.inject.Inject;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
@@ -44,6 +45,9 @@ public class ScaleIOSnapshotStrategy extends StorageSystemSnapshotStrategy {
 
     @Override
     public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) {
+        if (SnapshotOperation.COPY.equals(op)) {
+            return StrategyPriority.CANT_HANDLE;
+        }
         long volumeId = snapshot.getVolumeId();
         VolumeVO volumeVO = volumeDao.findByIdIncludingRemoved(volumeId);
         boolean baseVolumeExists = volumeVO.getRemoved() == null;

engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java

@@ -46,6 +46,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy;
 import org.apache.cloudstack.engine.subsystem.api.storage.StorageAction;
 import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
@@ -899,4 +900,35 @@ public class SnapshotServiceImpl implements SnapshotService {
         ep.sendMessageAsync(cmd, caller);
         return future;
     }
+
+    public AsyncCallFuture<SnapshotResult> copySnapshot(SnapshotInfo sourceSnapshot, SnapshotInfo destSnapshot, SnapshotStrategy strategy) {
+        try {
+            if (destSnapshot.getStatus() == ObjectInDataStoreStateMachine.State.Allocated) {
+                destSnapshot.processEvent(Event.CreateOnlyRequested);
+            } else if (sourceSnapshot.getStatus() == ObjectInDataStoreStateMachine.State.Ready) {
+                destSnapshot.processEvent(Event.CopyRequested);
+            } else {
+                logger.info(String.format("Cannot copy snapshot to another storage in different zone. It's not in the right state %s", sourceSnapshot.getStatus()));
+                sourceSnapshot.processEvent(Event.OperationFailed);
+                throw new CloudRuntimeException(String.format("Cannot copy snapshot to another storage in different zone. It's not in the right state %s", sourceSnapshot.getStatus()));
+            }
+        } catch (Exception e) {
+            logger.debug("Failed to change snapshot state: " + e.toString());
+            sourceSnapshot.processEvent(Event.OperationFailed);
+            throw new CloudRuntimeException(e);
+        }
+
+        AsyncCallFuture<SnapshotResult> future = new AsyncCallFuture<SnapshotResult>();
+        try {
+            CopySnapshotContext<CommandResult> context = new CopySnapshotContext<>(null, sourceSnapshot, destSnapshot, future);
+            AsyncCallbackDispatcher<SnapshotServiceImpl, CreateCmdResult> caller = AsyncCallbackDispatcher.create(this);
+            caller.setCallback(caller.getTarget().copySnapshotZoneAsyncCallback(null, null)).setContext(context);
+            strategy.copySnapshot(sourceSnapshot, destSnapshot, caller);
+        } catch (Exception e) {
+            logger.debug("Failed to take snapshot: " + destSnapshot.getId(), e);
+            destSnapshot.processEvent(Event.OperationFailed);
+            throw new CloudRuntimeException("Failed to copy snapshot" + destSnapshot.getId());
+        }
+        return future;
+    }
 }

engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java

@@ -38,6 +38,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation;
 import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer;
 import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
@@ -912,7 +913,9 @@ public class StorageSystemSnapshotStrategy extends SnapshotStrategyBase {
     @Override
     public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) {
         Snapshot.LocationType locationType = snapshot.getLocationType();
-
+        if (SnapshotOperation.COPY.equals(op)) {
+            return StrategyPriority.CANT_HANDLE;
+        }
         // If the snapshot exists on Secondary Storage, we can't delete it.
         if (SnapshotOperation.DELETE.equals(op)) {
             if (Snapshot.LocationType.SECONDARY.equals(locationType)) {

plugins/storage/volume/storpool/src/main/java/com/cloud/agent/api/storage/StorPoolModifyStoragePoolAnswer.java

@@ -36,14 +36,16 @@ public class StorPoolModifyStoragePoolAnswer extends Answer{
     private List<ModifyStoragePoolAnswer> datastoreClusterChildren = new ArrayList<>();
     private String clusterId;
     private String clientNodeId;
+    private String clusterLocation;
 
-    public StorPoolModifyStoragePoolAnswer(StorPoolModifyStoragePoolCommand cmd, long capacityBytes, long availableBytes, Map<String, TemplateProp> tInfo, String clusterId, String clientNodeId) {
+    public StorPoolModifyStoragePoolAnswer(StorPoolModifyStoragePoolCommand cmd, long capacityBytes, long availableBytes, Map<String, TemplateProp> tInfo, String clusterId, String clientNodeId, String clusterLocation) {
         super(cmd);
         result = true;
         poolInfo = new StoragePoolInfo(null, cmd.getPool().getHost(), cmd.getPool().getPath(), cmd.getLocalPath(), cmd.getPool().getType(), capacityBytes, availableBytes);
         templateInfo = tInfo;
         this.clusterId = clusterId;
         this.clientNodeId = clientNodeId;
+        this.clusterLocation = clusterLocation;
     }
 
     public StorPoolModifyStoragePoolAnswer(String errMsg) {
@@ -101,4 +103,12 @@ public class StorPoolModifyStoragePoolAnswer extends Answer{
     public void setClientNodeId(String clientNodeId) {
         this.clientNodeId = clientNodeId;
     }
+
+    public String getClusterLocation() {
+        return clusterLocation;
+    }
+
+    public void setClusterLocation(String clusterLocation) {
+        this.clusterLocation = clusterLocation;
+    }
 }

plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/StorPoolModifyStorageCommandWrapper.java

@@ -24,6 +24,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 
+import org.apache.commons.lang3.StringUtils;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.StorPoolModifyStoragePoolAnswer;
@@ -38,7 +39,9 @@ import com.cloud.resource.ResourceWrapper;
 import com.cloud.storage.template.TemplateProp;
 import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
+import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 
 @ResourceWrapper(handles =  StorPoolModifyStoragePoolCommand.class)
@@ -51,6 +54,7 @@ public final class StorPoolModifyStorageCommandWrapper extends CommandWrapper<St
             logger.debug(String.format("Could not get StorPool cluster id for a command [%s]", command.getClass()));
             return new Answer(command, false, "spNotFound");
         }
+        String clusterLocation = getStorPoolClusterLocation(clusterId);
         try {
             String result = attachOrDetachVolume("attach", "volume", command.getVolumeName());
             if (result != null) {
@@ -66,7 +70,7 @@ public final class StorPoolModifyStorageCommandWrapper extends CommandWrapper<St
             }
 
             final Map<String, TemplateProp> tInfo = new HashMap<>();
-            return new StorPoolModifyStoragePoolAnswer(command, storagepool.getCapacity(), storagepool.getAvailable(), tInfo, clusterId, storagepool.getStorageNodeId());
+            return new StorPoolModifyStoragePoolAnswer(command, storagepool.getCapacity(), storagepool.getAvailable(), tInfo, clusterId, storagepool.getStorageNodeId(), clusterLocation);
         } catch (Exception e) {
             logger.debug(String.format("Could not modify storage due to %s", e.getMessage()));
             return new Answer(command, e);
@@ -118,4 +122,28 @@ public final class StorPoolModifyStorageCommandWrapper extends CommandWrapper<St
         }
         return res;
     }
+
+    private String getStorPoolClusterLocation(String clusterId) {
+        Script sc = new Script("storpool", 300000, logger);
+        sc.add("-j");
+        sc.add("location");
+        sc.add("list");
+
+        OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
+
+        String res = sc.execute(parser);
+        if (res == null) {
+            JsonObject jsonObj = new JsonParser().parse(parser.getLines()).getAsJsonObject();
+            if (jsonObj.getAsJsonObject("data") != null) {
+                JsonArray arr = jsonObj.getAsJsonObject("data").getAsJsonArray("locations");
+                for (JsonElement jsonElement : arr) {
+                    JsonObject obj = jsonElement.getAsJsonObject();
+                    if (StringUtils.contains(clusterId, obj.get("id").getAsString())) {
+                        return obj.get("name").getAsString();
+                    }
+                }
+            }
+        }
+        return null;
+    }
 }

plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java

@@ -25,10 +25,12 @@ import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
+
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 import com.google.gson.JsonPrimitive;
+
 import org.apache.cloudstack.storage.datastore.util.StorPoolUtil;
 import org.apache.cloudstack.utils.qemu.QemuImg;
 import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
@@ -49,6 +51,7 @@ import java.util.Calendar;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+
 import java.util.UUID;
 
 public class StorPoolStorageAdaptor implements StorageAdaptor {

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/collector/StorPoolAbandonObjectsCollector.java

@@ -19,27 +19,10 @@
 
 package org.apache.cloudstack.storage.collector;
 
-import java.sql.PreparedStatement;
+import com.cloud.dc.dao.ClusterDao;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
 
-import javax.inject.Inject;
+import com.cloud.storage.dao.SnapshotDetailsDao;
-
+import com.cloud.storage.dao.SnapshotDetailsVO;
-import org.apache.cloudstack.framework.config.ConfigKey;
-import org.apache.cloudstack.framework.config.Configurable;
-import org.apache.cloudstack.managed.context.ManagedContextRunnable;
-import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
-import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
-//import org.apache.cloudstack.storage.datastore.util.StorPoolHelper;
-import org.apache.cloudstack.storage.datastore.util.StorPoolUtil;
-import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
@@ -48,16 +31,47 @@ import com.cloud.utils.db.Transaction;
 import com.cloud.utils.db.TransactionCallbackNoReturn;
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.TransactionStatus;
+
 import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
 
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.datastore.util.StorPoolHelper;
+import org.apache.cloudstack.storage.datastore.util.StorPoolUtil;
+import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse;
+import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpConnectionDesc;
+
+import org.apache.commons.collections.CollectionUtils;
+
+import javax.inject.Inject;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
 public class StorPoolAbandonObjectsCollector extends ManagerBase implements Configurable {
     @Inject
     private PrimaryDataStoreDao storagePoolDao;
     @Inject
     private StoragePoolDetailsDao storagePoolDetailsDao;
+    @Inject
+    private SnapshotDetailsDao snapshotDetailsDao;
+    @Inject
+    private ClusterDao clusterDao;
 
     private ScheduledExecutorService _volumeTagsUpdateExecutor;
+    private ScheduledExecutorService snapshotRecoveryCheckExecutor;
     private static final String ABANDON_LOGGER = "/var/log/cloudstack/management/storpool-abandoned-objects";
 
 
@@ -69,6 +83,9 @@ public class StorPoolAbandonObjectsCollector extends ManagerBase implements Conf
             "storpool.snapshot.tags.checkup", "86400",
             "Minimal interval (in seconds) to check and report if StorPool snapshot exists in CloudStack snapshots database",
             false);
+    static final ConfigKey<Integer> snapshotRecoveryFromRemoteCheck = new ConfigKey<Integer>("Advanced", Integer.class,
+            "storpool.snapshot.recovery.from.remote.check", "300",
+            "Minimal interval (in seconds) to check and recover StorPool snapshot from remote", false);
 
     @Override
     public String getConfigComponentName() {
@@ -77,7 +94,7 @@ public class StorPoolAbandonObjectsCollector extends ManagerBase implements Conf
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] { volumeCheckupTagsInterval, snapshotCheckupTagsInterval };
+        return new ConfigKey<?>[] { volumeCheckupTagsInterval, snapshotCheckupTagsInterval, snapshotRecoveryFromRemoteCheck };
     }
 
     @Override
@@ -93,6 +110,8 @@ public class StorPoolAbandonObjectsCollector extends ManagerBase implements Conf
         }
         _volumeTagsUpdateExecutor = Executors.newScheduledThreadPool(2,
                 new NamedThreadFactory("StorPoolAbandonObjectsCollector"));
+        snapshotRecoveryCheckExecutor = Executors.newScheduledThreadPool(1,
+                new NamedThreadFactory("StorPoolSnapshotRecoveryCheck"));
 
         if (volumeCheckupTagsInterval.value() > 0) {
             _volumeTagsUpdateExecutor.scheduleAtFixedRate(new StorPoolVolumesTagsUpdate(),
@@ -102,6 +121,10 @@ public class StorPoolAbandonObjectsCollector extends ManagerBase implements Conf
             _volumeTagsUpdateExecutor.scheduleAtFixedRate(new StorPoolSnapshotsTagsUpdate(),
                     snapshotCheckupTagsInterval.value(), snapshotCheckupTagsInterval.value(), TimeUnit.SECONDS);
         }
+        if (snapshotRecoveryFromRemoteCheck.value() > 0) {
+            snapshotRecoveryCheckExecutor.scheduleAtFixedRate(new StorPoolSnapshotRecoveryCheck(),
+                    snapshotRecoveryFromRemoteCheck.value(), snapshotRecoveryFromRemoteCheck.value(), TimeUnit.SECONDS);
+        }
     }
 
     class StorPoolVolumesTagsUpdate extends ManagedContextRunnable {
@@ -322,4 +345,84 @@ public class StorPoolAbandonObjectsCollector extends ManagerBase implements Conf
         }
         return map;
     }
+
+    class StorPoolSnapshotRecoveryCheck extends ManagedContextRunnable {
+
+        @Override
+        protected void runInContext() {
+            List<StoragePoolVO> spPools = storagePoolDao.findPoolsByProvider(StorPoolUtil.SP_PROVIDER_NAME);
+            if (CollectionUtils.isEmpty(spPools)) {
+                return;
+            }
+            List<SnapshotDetailsVO> snapshotDetails = snapshotDetailsDao.findDetails(StorPoolUtil.SP_RECOVERED_SNAPSHOT);
+            if (CollectionUtils.isEmpty(snapshotDetails)) {
+                return;
+            }
+            Map<Long, StoragePoolVO> onePoolforZone = new HashMap<>();
+            for (StoragePoolVO storagePoolVO : spPools) {
+                onePoolforZone.put(storagePoolVO.getDataCenterId(), storagePoolVO);
+            }
+            List<Long> recoveredSnapshots = new ArrayList<>();
+            for (StoragePoolVO storagePool : onePoolforZone.values()) {
+                collectRecoveredSnapshotAfterExport(snapshotDetails, recoveredSnapshots, storagePool);
+            }
+            for (Long recoveredSnapshot : recoveredSnapshots) {
+                snapshotDetailsDao.remove(recoveredSnapshot);
+            }
+        }
+
+        private void collectRecoveredSnapshotAfterExport(List<SnapshotDetailsVO> snapshotDetails, List<Long> recoveredSnapshots, StoragePoolVO storagePool) {
+            try {
+                logger.debug(String.format("Checking StorPool recovered snapshots for zone [%s]",
+                        storagePool.getDataCenterId()));
+                SpConnectionDesc conn = StorPoolUtil.getSpConnection(storagePool.getUuid(),
+                        storagePool.getId(), storagePoolDetailsDao, storagePoolDao);
+                JsonArray arr = StorPoolUtil.snapshotsList(conn);
+                List<String> snapshots = snapshotsForRecovery(arr);
+                if (snapshots.isEmpty()) {
+                    return;
+                }
+                for (SnapshotDetailsVO snapshot : snapshotDetails) {
+                    String[] snapshotOnRemote = snapshot.getValue().split(";");
+                    if (snapshotOnRemote.length != 2) {
+                        continue;
+                    }
+                    String name = snapshot.getValue().split(";")[0];
+                    String location = snapshot.getValue().split(";")[1];
+                    if (name == null || location == null) {
+                        StorPoolUtil.spLog("Could not find name or location for the snapshot %s", snapshot.getValue());
+                        continue;
+                    }
+                    if (snapshots.contains(name)) {
+                        findRecoveredSnapshots(recoveredSnapshots, conn, snapshot, name, location);
+                    }
+                }
+            } catch (Exception e) {
+                logger.debug(String.format("Could not collect StorPool recovered snapshots %s", e.getMessage()));
+            }
+        }
+
+        private void findRecoveredSnapshots(List<Long> recoveredSnapshots, SpConnectionDesc conn, SnapshotDetailsVO snapshot, String name, String location) {
+            Long clusterId = StorPoolHelper.findClusterIdByGlobalId(StorPoolUtil.getSnapshotClusterId(name, conn), clusterDao);
+            conn = StorPoolHelper.getSpConnectionDesc(conn, clusterId);
+            SpApiResponse resp = StorPoolUtil.snapshotUnexport(name, location, conn);
+            if (resp.getError() == null) {
+                StorPoolUtil.spLog("Unexport of snapshot %s was successful", name);
+                recoveredSnapshots.add(snapshot.getId());
+            } else {
+                StorPoolUtil.spLog("Could not recover StorPool snapshot %s", resp.getError());
+            }
+        }
+    }
+
+    private static List<String> snapshotsForRecovery(JsonArray arr) {
+        List<String> snapshots = new ArrayList<>();
+        for (int i = 0; i < arr.size(); i++) {
+            boolean recoveringFromRemote = arr.get(i).getAsJsonObject().get("recoveringFromRemote").getAsBoolean();
+            if (!recoveringFromRemote) {
+               snapshots.add(arr.get(i).getAsJsonObject().get("name").getAsString());
+            }
+        }
+        return snapshots;
+    }
 }

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java

@@ -18,18 +18,32 @@
  */
 package org.apache.cloudstack.storage.datastore.driver;
 
+import com.cloud.storage.dao.SnapshotDetailsVO;
+
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import javax.inject.Inject;
 
-import com.cloud.storage.dao.SnapshotDetailsVO;
+import com.cloud.storage.dao.StoragePoolHostDao;
+import com.cloud.storage.dao.VMTemplateDetailsDao;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.storage.dao.VolumeDetailsDao;
+import com.cloud.tags.dao.ResourceTagDao;
+import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.VMInstanceVO;
+import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.VirtualMachineManager;
+import com.cloud.vm.dao.VMInstanceDao;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
@@ -68,6 +82,7 @@ import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
 import org.apache.cloudstack.storage.volume.VolumeObject;
+
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 
@@ -112,17 +127,7 @@ import com.cloud.storage.VolumeDetailVO;
 import com.cloud.storage.VolumeVO;
 import com.cloud.storage.dao.SnapshotDao;
 import com.cloud.storage.dao.SnapshotDetailsDao;
-import com.cloud.storage.dao.StoragePoolHostDao;
+
-import com.cloud.storage.dao.VMTemplateDetailsDao;
-import com.cloud.storage.dao.VolumeDao;
-import com.cloud.storage.dao.VolumeDetailsDao;
-import com.cloud.tags.dao.ResourceTagDao;
-import com.cloud.utils.Pair;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.vm.VMInstanceVO;
-import com.cloud.vm.VirtualMachine.State;
-import com.cloud.vm.VirtualMachineManager;
-import com.cloud.vm.dao.VMInstanceDao;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -187,7 +192,10 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
 
     @Override
     public Map<String, String> getCapabilities() {
-        return null;
+        Map<String, String> mapCapabilities = new HashMap<>();
+        mapCapabilities.put(DataStoreCapabilities.CAN_COPY_SNAPSHOT_BETWEEN_ZONES_AND_SAME_POOL_TYPE.toString(), Boolean.TRUE.toString());
+        mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_TEMPLATE_FROM_SNAPSHOT.toString(), Boolean.TRUE.toString());
+        return mapCapabilities;
     }
 
     @Override
@@ -520,6 +528,8 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
             } catch (Exception e) {
                 err = String.format("Could not delete volume due to %s", e.getMessage());
             }
+        } else if (data.getType() == DataObjectType.SNAPSHOT) {
+            err = deleteSnapshot((SnapshotInfo) data, err);
         } else {
             err = String.format("Invalid DataObjectType \"%s\" passed to deleteAsync", data.getType());
         }
@@ -534,6 +544,18 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         callback.complete(res);
     }
 
+    private String deleteSnapshot(SnapshotInfo data, String err) {
+        SnapshotInfo snapshot = data;
+        SpConnectionDesc conn = StorPoolUtil.getSpConnection(snapshot.getDataStore().getUuid(), snapshot.getDataStore().getId(), storagePoolDetailsDao, primaryStoreDao);
+        String name = StorPoolStorageAdaptor.getVolumeNameFromPath(snapshot.getPath(), true);
+        SpApiResponse resp = StorPoolUtil.snapshotDelete(name, conn);
+        if (resp.getError() != null) {
+            err = String.format("Failed to clean-up Storpool snapshot %s. Error: %s", name, resp.getError());
+            StorPoolUtil.spLog(err);
+        }
+        return err;
+    }
+
     private void tryToSnapshotVolumeBeforeDelete(VolumeInfo vinfo, DataStore dataStore, String name, SpConnectionDesc conn) {
         Integer deleteAfter = StorPoolConfigurationManager.DeleteAfterInterval.valueIn(dataStore.getId());
         if (deleteAfter != null && deleteAfter > 0 && vinfo.getPassphraseId() == null) {
@@ -606,7 +628,22 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
 
     @Override
     public boolean canCopy(DataObject srcData, DataObject dstData) {
+        DataObjectType srcType = srcData.getType();
+        DataObjectType dstType = dstData.getType();
+        if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.VOLUME) {
             return true;
+        } else if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.SNAPSHOT) {
+            return true;
+        } else if (srcType == DataObjectType.VOLUME && dstType == DataObjectType.TEMPLATE) {
+            return true;
+        } else if (srcType == DataObjectType.TEMPLATE && dstType == DataObjectType.TEMPLATE) {
+            return true;
+        } else if (srcType == DataObjectType.TEMPLATE && dstType == DataObjectType.VOLUME) {
+            return true;
+        } else if (srcType == DataObjectType.VOLUME && dstType == DataObjectType.VOLUME) {
+            return true;
+        }
+        return false;
     }
 
     @Override
@@ -624,13 +661,12 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         try {
             if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.VOLUME) {
                 SnapshotInfo sinfo = (SnapshotInfo)srcData;
-                final String snapshotName = StorPoolHelper.getSnapshotName(srcData.getId(), srcData.getUuid(), snapshotDataStoreDao, snapshotDetailsDao);
-
                 VolumeInfo vinfo = (VolumeInfo)dstData;
                 final String volumeName = vinfo.getUuid();
                 final Long size = vinfo.getSize();
 
                 SpConnectionDesc conn = StorPoolUtil.getSpConnection(vinfo.getDataStore().getUuid(), vinfo.getDataStore().getId(), storagePoolDetailsDao, primaryStoreDao);
+                String snapshotName =  StorPoolStorageAdaptor.getVolumeNameFromPath(((SnapshotInfo) srcData).getPath(), true);
 
                 StorPoolVolumeDef spVolume = createVolumeWithTags(sinfo, snapshotName, vinfo, volumeName, size, conn);
                 SpApiResponse resp = StorPoolUtil.volumeCreate(spVolume, conn);
@@ -640,9 +676,10 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                     VolumeObjectTO to = (VolumeObjectTO)dstData.getTO();
                     to.setPath(StorPoolUtil.devPath(StorPoolUtil.getNameFromResponse(resp, false)));
                     to.setSize(size);
+                    updateVolumePoolType(vinfo);
 
                     answer = new CopyCmdAnswer(to);
-                    StorPoolUtil.spLog("Created volume=%s with uuid=%s from snapshot=%s with uuid=%s", StorPoolUtil.getNameFromResponse(resp, false), to.getUuid(), snapshotName, sinfo.getUuid());
+                    StorPoolUtil.spLog("Created volume=%s with uuid=%s from snapshot=%s with uuid=%s", StorPoolUtil.getNameFromResponse(resp, false), volumeName, snapshotName, sinfo.getUuid());
                 } else if (resp.getError().getName().equals("objectDoesNotExist")) {
                     //check if snapshot is on secondary storage
                     StorPoolUtil.spLog("Snapshot %s does not exists on StorPool, will try to create a volume from a snapshot on secondary storage", snapshotName);
@@ -658,8 +695,24 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                         } else {
                             answer = new Answer(cmd, false, String.format("Could not create Storpool volume %s from snapshot %s. Error: %s", volumeName, snapshotName, emptyVolumeCreateResp.getError()));
                         }
+                        VolumeObjectTO to = (VolumeObjectTO) dstData.getTO();
+                        to.setPath(StorPoolUtil.devPath(StorPoolUtil.getNameFromResponse(resp, false)));
+                        to.setSize(size);
+
+                        answer = new CopyCmdAnswer(to);
+                        StorPoolUtil.spLog("Created volume=%s with uuid=%s from snapshot=%s with uuid=%s", StorPoolUtil.getNameFromResponse(resp, false), to.getUuid(), snapshotName, sinfo.getUuid());
                     } else {
-                        answer = new Answer(cmd, false, String.format("The snapshot %s does not exists neither on primary, neither on secondary storage. Cannot create volume from snapshot", snapshotName));
+                        err = String.format("Could not create volume from a snapshot due to {}", resp.getError());
+                    }
+                } else if (sinfo.getDataStore().getRole().equals(DataStoreRole.Image)) {
+                    //check if snapshot is on secondary storage
+                    StorPoolUtil.spLog("Snapshot %s does not exists on StorPool, will try to create a volume from a snapshot on secondary storage", sinfo.getName());
+                    SnapshotDataStoreVO snap = getSnapshotImageStoreRef(sinfo.getId(), vinfo.getDataCenterId());
+                    SpApiResponse emptyVolumeCreateResp = StorPoolUtil.volumeCreate(volumeName, null, size, null, null, "volume", null, conn);
+                    if (emptyVolumeCreateResp.getError() == null) {
+                        answer = createVolumeFromSnapshot(srcData, dstData, size, emptyVolumeCreateResp);
+                    } else {
+                        answer = new Answer(cmd, false, String.format("Could not create Storpool volume %s from snapshot %s. Error: %s", volumeName, snapshotName, emptyVolumeCreateResp.getError()));
                     }
                 } else {
                     answer = new Answer(cmd, false, String.format("Could not create Storpool volume %s from snapshot %s. Error: %s", volumeName, snapshotName, resp.getError()));
@@ -668,7 +721,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                 SnapshotInfo sinfo = (SnapshotInfo)srcData;
                 SnapshotDetailsVO snapshotDetail = snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE);
                 // bypass secondary storage
-                if (StorPoolConfigurationManager.BypassSecondaryStorage.value() || snapshotDetail != null) {
+                if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value())) {
                     SnapshotObjectTO snapshot = (SnapshotObjectTO) srcData.getTO();
                     answer = new CopyCmdAnswer(snapshot);
                 } else {
@@ -678,9 +731,9 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                     final String snapName =  StorPoolStorageAdaptor.getVolumeNameFromPath(((SnapshotInfo) srcData).getPath(), true);
                     SpConnectionDesc conn = StorPoolUtil.getSpConnection(srcData.getDataStore().getUuid(), srcData.getDataStore().getId(), storagePoolDetailsDao, primaryStoreDao);
                     try {
-                        Long clusterId = StorPoolHelper.findClusterIdByGlobalId(snapName, clusterDao);
+                        Long clusterId = StorPoolHelper.findClusterIdByGlobalId(StorPoolUtil.getSnapshotClusterId(snapName, conn), clusterDao);
-                        EndPoint ep = clusterId != null ? RemoteHostEndPoint.getHypervisorHostEndPoint(StorPoolHelper.findHostByCluster(clusterId, hostDao)) : selector.select(srcData, dstData);
+                        HostVO host = clusterId != null ? StorPoolHelper.findHostByCluster(clusterId, hostDao) : null;
-                        if (ep == null) {
+                        EndPoint ep = host != null ? RemoteHostEndPoint.getHypervisorHostEndPoint(host) : selector.select(srcData, dstData);                        if (ep == null) {
                             err = "No remote endpoint to send command, check if host or ssvm is down?";
                         } else {
                             answer = ep.sendMessage(cmd);
@@ -712,8 +765,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                         StorPoolHelper.getTimeout(StorPoolHelper.PrimaryStorageDownloadWait, configDao), VirtualMachineManager.ExecuteInSequence.value());
 
                 try {
-                    Long clusterId = StorPoolHelper.findClusterIdByGlobalId(volumeName, clusterDao);
+                    EndPoint ep2 = selector.select(srcData, dstData);
-                    EndPoint ep2 = clusterId != null ? RemoteHostEndPoint.getHypervisorHostEndPoint(StorPoolHelper.findHostByCluster(clusterId, hostDao)) : selector.select(srcData, dstData);
                     if (ep2 == null) {
                         err = "No remote endpoint to send command, check if host or ssvm is down?";
                     } else {
@@ -937,8 +989,9 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                             StorPoolUtil.spLog("StorpoolPrimaryDataStoreDriverImpl.copyAsnc command=%s ", cmd);
 
                             try {
-                                Long clusterId = StorPoolHelper.findClusterIdByGlobalId(snapshotName, clusterDao);
+                                Long clusterId = StorPoolHelper.findClusterIdByGlobalId(StorPoolUtil.getSnapshotClusterId(snapshotName, conn), clusterDao);
-                                EndPoint ep = clusterId != null ? RemoteHostEndPoint.getHypervisorHostEndPoint(StorPoolHelper.findHostByCluster(clusterId, hostDao)) : selector.select(srcData, dstData);
+                                HostVO host = clusterId != null ? StorPoolHelper.findHostByCluster(clusterId, hostDao) : null;
+                                EndPoint ep = host != null ? RemoteHostEndPoint.getHypervisorHostEndPoint(host) : selector.select(srcData, dstData);
                                 StorPoolUtil.spLog("selector.select(srcData, dstData) ", ep);
                                 if (ep == null) {
                                     ep = selector.select(dstData);

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/provider/StorPoolHostListener.java

@@ -170,6 +170,7 @@ public class StorPoolHostListener implements HypervisorHostListener {
         }
 
         StorPoolHelper.setSpClusterIdIfNeeded(hostId, mspAnswer.getClusterId(), clusterDao, hostDao, clusterDetailsDao);
+        StorPoolHelper.setLocationIfNeeded(pool, storagePoolDetailsDao, mspAnswer.getClusterLocation());
 
         StorPoolUtil.spLog("Connection established between storage pool [%s] and host [%s]", poolVO, host);
         return true;

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolHelper.java

@@ -19,26 +19,6 @@
 
 package org.apache.cloudstack.storage.datastore.util;
 
-import java.sql.PreparedStatement;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
-import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
-import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
-import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
-import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
-import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse;
-import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager;
-import org.apache.cloudstack.storage.to.VolumeObjectTO;
-import org.apache.commons.collections4.CollectionUtils;
-
 import com.cloud.dc.ClusterDetailsDao;
 import com.cloud.dc.ClusterDetailsVO;
 import com.cloud.dc.ClusterVO;
@@ -49,6 +29,7 @@ import com.cloud.hypervisor.kvm.storage.StorPoolStorageAdaptor;
 import com.cloud.server.ResourceTag;
 import com.cloud.server.ResourceTag.ResourceObjectType;
 import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.StoragePool;
 import com.cloud.storage.VMTemplateStoragePoolVO;
 import com.cloud.storage.VolumeVO;
 import com.cloud.storage.dao.SnapshotDetailsDao;
@@ -65,6 +46,28 @@ import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.dao.VMInstanceDao;
 
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
+import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse;
+import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
+
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.sql.PreparedStatement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+
 public class StorPoolHelper {
 
     private static final String UPDATE_SNAPSHOT_DETAILS_VALUE = "UPDATE `cloud`.`snapshot_details` SET value=? WHERE id=?";
@@ -218,6 +221,22 @@ public class StorPoolHelper {
         }
     }
 
+    public static void setLocationIfNeeded(StoragePool storagePool, StoragePoolDetailsDao storagePoolDetails,
+            String location) {
+        if (location == null) {
+            return;
+        }
+        StoragePoolDetailVO storagePoolDetailVO = storagePoolDetails.findDetail(storagePool.getId(),
+                StorPoolConfigurationManager.StorPoolClusterLocation.key());
+        if (storagePoolDetailVO == null) {
+            storagePoolDetails.persist(new StoragePoolDetailVO(storagePool.getId(),
+                    StorPoolConfigurationManager.StorPoolClusterLocation.key(), location, true));
+        } else if (storagePoolDetailVO.getValue() == null || !storagePoolDetailVO.getValue().equals(location)) {
+            storagePoolDetailVO.setValue(location);
+            storagePoolDetails.update(storagePoolDetailVO.getId(), storagePoolDetailVO);
+        }
+    }
+
     public static Long findClusterIdByGlobalId(String globalId, ClusterDao clusterDao) {
         List<Long> clusterIds = clusterDao.listAllIds();
         if (clusterIds.size() == 1) {
@@ -238,7 +257,7 @@ public class StorPoolHelper {
 
     public static HostVO findHostByCluster(Long clusterId, HostDao hostDao) {
         List<HostVO> host = hostDao.findByClusterId(clusterId);
-        return host != null ? host.get(0) : null;
+        return CollectionUtils.isNotEmpty(host) ? host.get(0) : null;
     }
 
     public static int getTimeout(String cfg, ConfigurationDao configDao) {
@@ -289,4 +308,15 @@ public class StorPoolHelper {
         }
         return true;
     }
+
+    public static StorPoolUtil.SpConnectionDesc getSpConnectionDesc(StorPoolUtil.SpConnectionDesc connectionLocal, Long clusterId) {
+
+        String subClusterEndPoint = StorPoolConfigurationManager.StorPoolSubclusterEndpoint.valueIn(clusterId);
+        if (StringUtils.isNotEmpty(subClusterEndPoint)) {
+            String host = subClusterEndPoint.split(";")[0].split("=")[1];
+            String token = subClusterEndPoint.split(";")[1].split("=")[1];
+            connectionLocal = new StorPoolUtil.SpConnectionDesc(host, token, connectionLocal.getTemplateName());
+        }
+        return connectionLocal;
+    }
 }

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java

@@ -137,6 +137,9 @@ public class StorPoolUtil {
     public static final String DELAY_DELETE = "delayDelete";
 
     public static final String SP_TIER = "SP_QOSCLASS";
+    public static final String SP_RECOVERED_SNAPSHOT = "SP_RECOVERED_SNAPSHOT";
+
+    public static final String SP_REMOTE_LOCATION = "SP_REMOTE_LOCATION";
 
     public static final String OBJECT_DOES_NOT_EXIST = "objectDoesNotExist";
 
@@ -429,6 +432,14 @@ public class StorPoolUtil {
         return resp.getError() == null ? true : objectExists(resp.getError());
     }
 
+    public static boolean snapshotRecovered(final String name, SpConnectionDesc conn) {
+        SpApiResponse resp = GET("Snapshot/" + name, conn);
+        JsonObject obj = resp.fullJson.getAsJsonObject();
+        JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject();
+        boolean recoveringFromRemote = data.getAsJsonPrimitive("recoveringFromRemote").getAsBoolean();
+        return recoveringFromRemote;
+    }
+
     public static JsonArray snapshotsList(SpConnectionDesc conn) {
         SpApiResponse resp = GET("MultiCluster/SnapshotsList", conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
@@ -675,6 +686,42 @@ public class StorPoolUtil {
         return resp.getError() == null ? POST("MultiCluster/SnapshotDelete/" + name, null, conn) : resp;
     }
 
+    public static SpApiResponse snapshotExport(String name, String location, SpConnectionDesc conn) {
+        Map<String, Object> json = new HashMap<>();
+        json.put("snapshot", name);
+        json.put("location", location);
+        return POST("SnapshotExport", json, conn);
+    }
+
+    public static SpApiResponse snapshotUnexport(String name, String location, SpConnectionDesc conn) {
+        Map<String, Object> json = new HashMap<>();
+        json.put("snapshot", name);
+        json.put("force", true);
+        json.put("all", true);
+        return POST("SnapshotUnexport", json, conn);
+    }
+
+    public static String getSnapshotClusterId(String snapshotName, SpConnectionDesc conn) {
+        SpApiResponse resp = POST("MultiCluster/SnapshotUpdate/" + snapshotName, new HashMap<>(), conn);
+        JsonObject json = resp.fullJson.getAsJsonObject();
+        return json.get("clusterId").getAsString();
+    }
+
+    public static SpApiResponse snapshotFromRemote(String name, String remoteLocation, String template, Map<String, String> tags,
+            SpConnectionDesc conn) {
+        Map<String, Object> json = new HashMap<>();
+        json.put("remoteId", name);
+        json.put("remoteLocation", remoteLocation);
+        json.put("template", template);
+        json.put("name", "");
+        json.put("tags", tags);
+        return POST("SnapshotFromRemote", json, conn);
+    }
+
+    public static SpApiResponse snapshotReconcile(String name, SpConnectionDesc conn) {
+        return POST("SnapshotReconcile/" + name, null, conn);
+    }
+
     public static SpApiResponse detachAllForced(final String name, final boolean snapshot, SpConnectionDesc conn) {
         final String type = snapshot ? "snapshot" : "volume";
         List<Map<String, Object>> json = new ArrayList<>();

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java

@@ -19,12 +19,42 @@
 
 package org.apache.cloudstack.storage.motion;
 
-import java.util.ArrayList;
+import com.cloud.agent.AgentManager;
-import java.util.HashMap;
+import com.cloud.agent.api.Answer;
-import java.util.List;
+import com.cloud.agent.api.Command;
-import java.util.Map;
+import com.cloud.agent.api.MigrateAnswer;
-
+import com.cloud.agent.api.MigrateCommand;
-import javax.inject.Inject;
+import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo;
+import com.cloud.agent.api.ModifyTargetsAnswer;
+import com.cloud.agent.api.ModifyTargetsCommand;
+import com.cloud.agent.api.PrepareForMigrationCommand;
+import com.cloud.agent.api.storage.StorPoolBackupTemplateFromSnapshotCommand;
+import com.cloud.agent.api.to.DataObjectType;
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.dc.dao.ClusterDao;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.StorageManager;
+import com.cloud.storage.VMTemplateDetailVO;
+import com.cloud.storage.Volume;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.GuestOSCategoryDao;
+import com.cloud.storage.dao.GuestOSDao;
+import com.cloud.storage.dao.SnapshotDao;
+import com.cloud.storage.dao.SnapshotDetailsDao;
+import com.cloud.storage.dao.SnapshotDetailsVO;
+import com.cloud.storage.dao.VMTemplateDetailsDao;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.VMInstanceVO;
+import com.cloud.vm.VirtualMachineManager;
+import com.cloud.vm.dao.VMInstanceDao;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy;
@@ -55,48 +85,21 @@ import org.apache.cloudstack.storage.datastore.util.StorPoolHelper;
 import org.apache.cloudstack.storage.datastore.util.StorPoolUtil;
 import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse;
 import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpConnectionDesc;
-import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager;
 import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
+
 import org.apache.commons.collections.MapUtils;
-import org.apache.logging.log4j.Logger;
+
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 import org.springframework.stereotype.Component;
 
-import com.cloud.agent.AgentManager;
+import javax.inject.Inject;
-import com.cloud.agent.api.Answer;
+import java.util.ArrayList;
-import com.cloud.agent.api.Command;
+import java.util.HashMap;
-import com.cloud.agent.api.MigrateAnswer;
+import java.util.List;
-import com.cloud.agent.api.MigrateCommand;
+import java.util.Map;
-import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo;
-import com.cloud.agent.api.ModifyTargetsAnswer;
-import com.cloud.agent.api.ModifyTargetsCommand;
-import com.cloud.agent.api.PrepareForMigrationCommand;
-import com.cloud.agent.api.storage.StorPoolBackupTemplateFromSnapshotCommand;
-import com.cloud.agent.api.to.DataObjectType;
-import com.cloud.agent.api.to.VirtualMachineTO;
-import com.cloud.dc.dao.ClusterDao;
-import com.cloud.exception.AgentUnavailableException;
-import com.cloud.exception.OperationTimedoutException;
-import com.cloud.host.Host;
-import com.cloud.host.dao.HostDao;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.storage.Storage.ImageFormat;
-import com.cloud.storage.StorageManager;
-import com.cloud.storage.VMTemplateDetailVO;
-import com.cloud.storage.Volume;
-import com.cloud.storage.VolumeVO;
-import com.cloud.storage.dao.GuestOSCategoryDao;
-import com.cloud.storage.dao.GuestOSDao;
-import com.cloud.storage.dao.SnapshotDao;
-import com.cloud.storage.dao.SnapshotDetailsDao;
-import com.cloud.storage.dao.SnapshotDetailsVO;
-import com.cloud.storage.dao.VMTemplateDetailsDao;
-import com.cloud.storage.dao.VolumeDao;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.vm.VMInstanceVO;
-import com.cloud.vm.VirtualMachineManager;
-import com.cloud.vm.dao.VMInstanceDao;
 
 @Component
 public class StorPoolDataMotionStrategy implements DataMotionStrategy {
@@ -149,10 +152,13 @@ public class StorPoolDataMotionStrategy implements DataMotionStrategy {
     public StrategyPriority canHandle(DataObject srcData, DataObject destData) {
         DataObjectType srcType = srcData.getType();
         DataObjectType dstType = destData.getType();
+
         if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.TEMPLATE) {
             SnapshotInfo sinfo = (SnapshotInfo) srcData;
-            VolumeInfo volume = sinfo.getBaseVolume();
+            if (!sinfo.getDataStore().getRole().equals(DataStoreRole.Primary)) {
-            StoragePoolVO storagePool = _storagePool.findById(volume.getPoolId());
+                return StrategyPriority.CANT_HANDLE;
+            }
+            StoragePoolVO storagePool = _storagePool.findById(sinfo.getDataStore().getId());
             if (!storagePool.getStorageProviderName().equals(StorPoolUtil.SP_PROVIDER_NAME)) {
                 return StrategyPriority.CANT_HANDLE;
             }
@@ -163,7 +169,7 @@ public class StorPoolDataMotionStrategy implements DataMotionStrategy {
             String snapshotName = StorPoolHelper.getSnapshotName(sinfo.getId(), sinfo.getUuid(), _snapshotStoreDao,
                     _snapshotDetailsDao);
             StorPoolUtil.spLog("StorPoolDataMotionStrategy.canHandle snapshot name=%s", snapshotName);
-            if (snapshotName != null && StorPoolConfigurationManager.BypassSecondaryStorage.value()) {
+            if (snapshotName != null) {
                 return StrategyPriority.HIGHEST;
             }
         }
@@ -175,13 +181,12 @@ public class StorPoolDataMotionStrategy implements DataMotionStrategy {
             AsyncCompletionCallback<CopyCommandResult> callback) {
         SnapshotObjectTO snapshot = (SnapshotObjectTO) srcData.getTO();
         TemplateObjectTO template = (TemplateObjectTO) destData.getTO();
-        DataStore store = _dataStore.getDataStore(snapshot.getVolume().getDataStore().getUuid(),
+        DataStore store = _dataStore.getDataStore(snapshot.getDataStore().getUuid(),
-                snapshot.getVolume().getDataStore().getRole());
+                snapshot.getDataStore().getRole());
         SnapshotInfo sInfo = _snapshotDataFactory.getSnapshot(snapshot.getId(), store);
 
-        VolumeInfo vInfo = sInfo.getBaseVolume();
+        SpConnectionDesc conn = StorPoolUtil.getSpConnection(sInfo.getDataStore().getUuid(),
-        SpConnectionDesc conn = StorPoolUtil.getSpConnection(vInfo.getDataStore().getUuid(),
+                sInfo.getDataStore().getId(), _storagePoolDetails, _storagePool);
-                vInfo.getDataStore().getId(), _storagePoolDetails, _storagePool);
         String name = template.getUuid();
         String volumeName = "";
 
@@ -209,11 +214,9 @@ public class StorPoolDataMotionStrategy implements DataMotionStrategy {
                 // final String snapName =
                 // StorpoolStorageAdaptor.getVolumeNameFromPath(((SnapshotInfo)
                 // srcData).getPath(), true);
-                Long clusterId = StorPoolHelper.findClusterIdByGlobalId(parentName, _clusterDao);
+                Long clusterId = StorPoolHelper.findClusterIdByGlobalId(StorPoolUtil.getSnapshotClusterId(parentName, conn), _clusterDao);
-                EndPoint ep2 = clusterId != null
+                HostVO host = clusterId != null ? StorPoolHelper.findHostByCluster(clusterId, _hostDao) : null;
-                        ? RemoteHostEndPoint
+                EndPoint ep2 = host != null ? RemoteHostEndPoint.getHypervisorHostEndPoint(host) : _selector.select(srcData, destData);
-                                .getHypervisorHostEndPoint(StorPoolHelper.findHostByCluster(clusterId, _hostDao))
-                        : _selector.select(sInfo, destData);
                 if (ep2 == null) {
                     err = "No remote endpoint to send command, check if host or ssvm is down?";
                 } else {
@@ -238,7 +241,7 @@ public class StorPoolDataMotionStrategy implements DataMotionStrategy {
             StorPoolUtil.volumeDelete(volumeName, conn);
         }
         _vmTemplateDetailsDao.persist(new VMTemplateDetailVO(template.getId(), StorPoolUtil.SP_STORAGE_POOL_ID,
-                String.valueOf(vInfo.getDataStore().getId()), false));
+                String.valueOf(sInfo.getDataStore().getId()), false));
         StorPoolUtil.spLog("StorPoolDataMotionStrategy.copyAsync Creating snapshot=%s for StorPool template=%s",
                 volumeName, conn.getTemplateName());
         final CopyCommandResult cmd = new CopyCommandResult(null, answer);

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolConfigurationManager.java

@@ -53,6 +53,10 @@ public class StorPoolConfigurationManager implements Configurable {
             "storpool.list.snapshots.delete.after.interval", "360",
             "The interval (in seconds) to fetch the StorPool snapshots with deleteAfter flag",
             false);
+    public static final ConfigKey<String> StorPoolClusterLocation = new ConfigKey<String>(String.class, "sp.cluster.location", "Advanced", null,
+            "StorPool cluster location", true, ConfigKey.Scope.StoragePool, null);
+    public static final ConfigKey<String> StorPoolSubclusterEndpoint = new ConfigKey<>(String.class, "sp.cluster.endpoint", "Advanced", null,
+            "StorPool sub-cluster endpoint", true, ConfigKey.Scope.Cluster, null);
 
     @Override
     public String getConfigComponentName() {
@@ -61,6 +65,6 @@ public class StorPoolConfigurationManager implements Configurable {
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] { BypassSecondaryStorage, StorPoolClusterId, AlternativeEndPointEnabled, AlternativeEndpoint, VolumesStatsInterval, StorageStatsInterval, DeleteAfterInterval, ListSnapshotsWithDeleteAfterInterval };
+        return new ConfigKey<?>[] { BypassSecondaryStorage, StorPoolClusterId, AlternativeEndPointEnabled, AlternativeEndpoint, VolumesStatsInterval, StorageStatsInterval, DeleteAfterInterval, ListSnapshotsWithDeleteAfterInterval, StorPoolClusterLocation, StorPoolSubclusterEndpoint };
     }
 }

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java

@@ -16,12 +16,15 @@
 // under the License.
 package org.apache.cloudstack.storage.snapshot;
 
+import com.cloud.api.query.dao.SnapshotJoinDao;
+import com.cloud.api.query.vo.SnapshotJoinVO;
+import com.cloud.dc.dao.ClusterDao;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.hypervisor.kvm.storage.StorPoolStorageAdaptor;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.SnapshotVO;
-import com.cloud.storage.VolumeVO;
+import com.cloud.storage.Storage;
 import com.cloud.storage.dao.SnapshotDao;
 import com.cloud.storage.dao.SnapshotDetailsDao;
 import com.cloud.storage.dao.SnapshotDetailsVO;
@@ -30,8 +33,13 @@ import com.cloud.storage.dao.VolumeDao;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.fsm.NoTransitionException;
 
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
 import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
 import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
@@ -39,18 +47,25 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy;
 import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority;
+import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.datastore.util.StorPoolHelper;
 import org.apache.cloudstack.storage.datastore.util.StorPoolUtil;
 import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse;
 import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpConnectionDesc;
+import org.apache.cloudstack.storage.to.SnapshotObjectTO;
+
 import org.apache.commons.collections.CollectionUtils;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+
+import org.jetbrains.annotations.NotNull;
 import org.springframework.stereotype.Component;
 
 import javax.inject.Inject;
@@ -82,6 +97,10 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
     DataStoreManager dataStoreMgr;
     @Inject
     SnapshotZoneDao snapshotZoneDao;
+    @Inject
+    SnapshotJoinDao snapshotJoinDao;
+    @Inject
+    private ClusterDao clusterDao;
 
     @Override
     public SnapshotInfo backupSnapshot(SnapshotInfo snapshotInfo) {
@@ -104,48 +123,86 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
     public boolean deleteSnapshot(Long snapshotId, Long zoneId) {
 
         final SnapshotVO snapshotVO = _snapshotDao.findById(snapshotId);
-        VolumeVO volume = _volumeDao.findByIdIncludingRemoved(snapshotVO.getVolumeId());
         String name = StorPoolHelper.getSnapshotName(snapshotId, snapshotVO.getUuid(), _snapshotStoreDao, _snapshotDetailsDao);
         boolean res = false;
         // clean-up snapshot from Storpool storage pools
-        StoragePoolVO storage = _primaryDataStoreDao.findById(volume.getPoolId());
+        List<SnapshotDataStoreVO> snapshotDataStoreVOS;
-        if (storage.getStorageProviderName().equals(StorPoolUtil.SP_PROVIDER_NAME)) {
+        List<SnapshotJoinVO> snapshotJoinVOList = snapshotJoinDao.listBySnapshotIdAndZoneId(zoneId, snapshotId);
         try {
-                SpConnectionDesc conn = StorPoolUtil.getSpConnection(storage.getUuid(), storage.getId(), storagePoolDetailsDao, _primaryDataStoreDao);
+            for (SnapshotJoinVO snapshot: snapshotJoinVOList) {
-                SpApiResponse resp = StorPoolUtil.snapshotDelete(name, conn);
+                if (State.Destroyed.equals(snapshot.getStatus())) {
-                if (resp.getError() != null) {
+                    continue;
-                    final String err = String.format("Failed to clean-up Storpool snapshot %s. Error: %s", name, resp.getError());
+                }
-                    StorPoolUtil.spLog(err);
+                if (snapshot.getStoreRole().isImageStore()) {
-                    markSnapshotAsDestroyedIfAlreadyRemoved(snapshotId, resp.getError().getName().equals(StorPoolUtil.OBJECT_DOES_NOT_EXIST));
+                    continue;
-                    throw new CloudRuntimeException(err);
+                }
-                } else {
+                StoragePoolVO storage = _primaryDataStoreDao.findById(snapshot.getStoreId());
-                    res = deleteSnapshotFromDbIfNeeded(snapshotVO, zoneId);
+                if (zoneId != null) {
-                    markSnapshotAsDestroyedIfAlreadyRemoved(snapshotId,true);
+                    if (!zoneId.equals(snapshot.getDataCenterId())) {
-                    StorPoolUtil.spLog("StorpoolSnapshotStrategy.deleteSnapshot: executed successfully=%s, snapshot %s, name=%s", res, snapshotVO, name);
+                        continue;
+                    }
+                    res = deleteSnapshot(snapshotId, zoneId, snapshotVO, name, storage);
+                    break;
+                }
+                res = deleteSnapshot(snapshotId, zoneId, snapshotVO, name, storage);
             }
         } catch (Exception e) {
             String errMsg = String.format("Cannot delete snapshot due to %s", e.getMessage());
             throw new CloudRuntimeException(errMsg);
         }
-        }
+        snapshotDataStoreVOS = _snapshotStoreDao.listSnapshotsBySnapshotId(snapshotId);
-
+        boolean areAllSnapshotsDestroyed = snapshotDataStoreVOS.stream().allMatch(v -> v.getState().equals(State.Destroyed) || v.getState().equals(State.Destroying));
-        List<SnapshotDataStoreVO> snapshots = _snapshotStoreDao.listBySnapshotIdAndState(snapshotId, State.Ready);
+        if (areAllSnapshotsDestroyed) {
-        if (res || CollectionUtils.isEmpty(snapshots)) {
             updateSnapshotToDestroyed(snapshotVO);
             return true;
         }
         return res;
     }
 
-    private void markSnapshotAsDestroyedIfAlreadyRemoved(Long snapshotId, boolean isSnapshotDeleted) {
+    private boolean deleteSnapshot(Long snapshotId, Long zoneId, SnapshotVO snapshotVO, String name, StoragePoolVO storage) {
-        if (!isSnapshotDeleted) {
+
-            return;
+        boolean res = false;
+        SpConnectionDesc conn = StorPoolUtil.getSpConnection(storage.getUuid(), storage.getId(), storagePoolDetailsDao, _primaryDataStoreDao);
+        SpApiResponse resp = StorPoolUtil.snapshotDelete(name, conn);
+        List<SnapshotInfo> snapshotInfos = snapshotDataFactory.getSnapshots(snapshotId, zoneId);
+        processResult(snapshotInfos, ObjectInDataStoreStateMachine.Event.DestroyRequested);
+        if (resp.getError() != null) {
+            if (resp.getError().getDescr().contains("still exported")) {
+                processResult(snapshotInfos, Event.OperationFailed);
+                throw new CloudRuntimeException(String.format("The snapshot [%s] was exported to another cluster. [%s]", name, resp.getError()));
             }
-        List<SnapshotDataStoreVO> snapshotsOnStore = _snapshotStoreDao.listBySnapshotIdAndState(snapshotId, State.Ready);
+            final String err = String.format("Failed to clean-up Storpool snapshot %s. Error: %s", name, resp.getError());
-        for (SnapshotDataStoreVO snapshot : snapshotsOnStore) {
+            StorPoolUtil.spLog(err);
-            if (snapshot.getInstallPath() != null && snapshot.getInstallPath().contains(StorPoolUtil.SP_DEV_PATH)) {
+            if (resp.getError().getName().equals("objectDoesNotExist")) {
-                snapshot.setState(State.Destroyed);
+                return true;
-                _snapshotStoreDao.update(snapshot.getId(), snapshot);
+            }
+        } else {
+            res = deleteSnapshotFromDbIfNeeded(snapshotVO, zoneId);
+            StorPoolUtil.spLog("StorpoolSnapshotStrategy.deleteSnapshot: executed successfully=%s, snapshot uuid=%s, name=%s", res, snapshotVO.getUuid(), name);
+        }
+        if (res) {
+            processResult(snapshotInfos, Event.OperationSuccessed);
+            cleanUpDestroyedRecords(snapshotId);
+        } else {
+            processResult(snapshotInfos, Event.OperationFailed);
+        }
+        return res;
+    }
+
+    private void cleanUpDestroyedRecords(Long snapshotId) {
+        List<SnapshotDataStoreVO> snapshots = _snapshotStoreDao.listBySnapshotId(snapshotId);
+        for (SnapshotDataStoreVO snapshot : snapshots) {
+            if (snapshot.getInstallPath().contains("/dev/storpool-byid") && State.Destroyed.equals(snapshot.getState())) {
+                _snapshotStoreDao.remove(snapshot.getId());
+            }
+        }
+    }
+
+    private void processResult(List<SnapshotInfo> snapshotInfos, ObjectInDataStoreStateMachine.Event event) {
+        for (SnapshotInfo snapshot : snapshotInfos) {
+            SnapshotObject snapshotObject = (SnapshotObject) snapshot;
+            if (DataStoreRole.Primary.equals(snapshotObject.getDataStore().getRole())) {
+                snapshotObject.processEvent(event);
             }
         }
     }
@@ -154,31 +211,34 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
     public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) {
         logger.debug("StorpoolSnapshotStrategy.canHandle: snapshot {}, op={}", snapshot, op);
 
-        if (op != SnapshotOperation.DELETE) {
+        if (op != SnapshotOperation.DELETE && op != SnapshotOperation.COPY) {
             return StrategyPriority.CANT_HANDLE;
         }
-        SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary);
+        List<StoragePoolVO> pools = _primaryDataStoreDao.findPoolsByStorageType(Storage.StoragePoolType.StorPool);
-        if (snapshotOnPrimary == null) {
+        if (CollectionUtils.isEmpty(pools)) {
             return StrategyPriority.CANT_HANDLE;
         }
-        if (zoneId != null) { // If zoneId is present, then it should be same as the zoneId of primary store
+        List<SnapshotJoinVO> snapshots = snapshotJoinDao.listBySnapshotIdAndZoneId(zoneId, snapshot.getId());
-            StoragePoolVO storagePoolVO = _primaryDataStoreDao.findById(snapshotOnPrimary.getDataStoreId());
+        boolean snapshotNotOnStorPool = snapshots.stream().filter(s -> DataStoreRole.Primary.equals(s.getStoreRole())).count() == 0;
-            if (!zoneId.equals(storagePoolVO.getDataCenterId())) {
-                return StrategyPriority.CANT_HANDLE;
-            }
-        }
-        String name = StorPoolHelper.getSnapshotName(snapshot.getId(), snapshot.getUuid(), _snapshotStoreDao, _snapshotDetailsDao);
-        if (name != null) {
-            StorPoolUtil.spLog("StorpoolSnapshotStrategy.canHandle: globalId=%s", name);
 
+        if (snapshotNotOnStorPool) {
+            for (SnapshotJoinVO snapshotOnStore : snapshots) {
+                SnapshotDataStoreVO snap = _snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Image);
+                if (snap != null && snap.getInstallPath() != null && snap.getInstallPath().startsWith(StorPoolUtil.SP_DEV_PATH)) {
                     return StrategyPriority.HIGHEST;
                 }
-        SnapshotDetailsVO snapshotDetails = _snapshotDetailsDao.findDetail(snapshot.getId(), snapshot.getUuid());
-        if (snapshotDetails != null) {
-            _snapshotDetailsDao.remove(snapshotDetails.getId());
             }
             return StrategyPriority.CANT_HANDLE;
         }
+        for (StoragePoolVO pool : pools) {
+            SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Primary, pool.getId(), snapshot.getId());
+            if (snapshotOnPrimary != null && (snapshotOnPrimary.getState().equals(State.Ready) || snapshotOnPrimary.getState().equals(State.Created))) {
+                return StrategyPriority.HIGHEST;
+            }
+        }
+
+        return StrategyPriority.CANT_HANDLE;
+    }
 
     private boolean deleteSnapshotChain(SnapshotInfo snapshot) {
         logger.debug("delete snapshot chain for snapshot: {}", snapshot);
@@ -250,48 +310,23 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
     protected boolean deleteSnapshotOnImageAndPrimary(long snapshotId, DataStore store) {
         SnapshotInfo snapshotOnImage = snapshotDataFactory.getSnapshot(snapshotId, store);
         SnapshotObject obj = (SnapshotObject)snapshotOnImage;
-        boolean areLastSnapshotRef = areLastSnapshotRef(snapshotId);
-        try {
-            if (areLastSnapshotRef) {
-                obj.processEvent(Snapshot.Event.DestroyRequested);
-            }
-        } catch (NoTransitionException e) {
-            logger.debug("Failed to set the state to destroying: ", e);
-            return false;
-        }
 
+        boolean result = false;
         try {
-            boolean result = deleteSnapshotChain(snapshotOnImage);
+            result = deleteSnapshotChain(snapshotOnImage);
             _snapshotStoreDao.updateDisplayForSnapshotStoreRole(snapshotId, store.getId(), store.getRole(), false);
-            if (areLastSnapshotRef) {
-                obj.processEvent(Snapshot.Event.OperationSucceeded);
-            }
-            if (result) {
-                SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshotOnImage.getSnapshotId(), DataStoreRole.Primary);
-                if (snapshotOnPrimary != null) {
-                    snapshotOnPrimary.setState(State.Destroyed);
-                    _snapshotStoreDao.update(snapshotOnPrimary.getId(), snapshotOnPrimary);
-                }
-            }
         } catch (Exception e) {
             logger.debug("Failed to delete snapshot: ", e);
-            try {
-                if (areLastSnapshotRef) {
-                    obj.processEvent(Snapshot.Event.OperationFailed);
-                }
-            } catch (NoTransitionException e1) {
-                logger.debug("Failed to change snapshot state: " + e.toString());
-            }
             return false;
         }
-        return true;
+        return result;
     }
 
     private boolean deleteSnapshotFromDbIfNeeded(SnapshotVO snapshotVO, Long zoneId) {
         final long snapshotId = snapshotVO.getId();
         SnapshotDetailsVO snapshotDetails = _snapshotDetailsDao.findDetail(snapshotId, snapshotVO.getUuid());
         if (snapshotDetails != null) {
-            _snapshotDetailsDao.removeDetails(snapshotId);
+            _snapshotDetailsDao.remove(snapshotId);
         }
 
         if (zoneId != null && List.of(Snapshot.State.Allocated, Snapshot.State.CreatedOnPrimary).contains(snapshotVO.getState())) {
@@ -327,19 +362,15 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
             return true;
         }
 
-        if (snapshotVO.getState() == Snapshot.State.CreatedOnPrimary) {
-            snapshotVO.setState(Snapshot.State.Destroyed);
-            _snapshotDao.update(snapshotId, snapshotVO);
-            return true;
-        }
-
         if (!Snapshot.State.BackedUp.equals(snapshotVO.getState()) && !Snapshot.State.Error.equals(snapshotVO.getState()) &&
                 !Snapshot.State.Destroying.equals(snapshotVO.getState())) {
             throw new InvalidParameterValueException(String.format("Can't delete snapshot %s due to it is in %s Status", snapshotVO, snapshotVO.getState()));
         }
-        List<SnapshotDataStoreVO> storeRefs = _snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image);
+        List<SnapshotDataStoreVO> storeRefs = _snapshotStoreDao.listBySnapshotAndDataStoreRole(snapshotId, DataStoreRole.Image);
         if (zoneId != null) {
             storeRefs.removeIf(ref -> !zoneId.equals(dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole())));
+        } else {
+            storeRefs.removeIf(ref -> !ref.getState().equals(State.Ready));
         }
         for (SnapshotDataStoreVO ref : storeRefs) {
             if (!deleteSnapshotOnImageAndPrimary(snapshotId, dataStoreMgr.getDataStore(ref.getDataStoreId(), ref.getRole()))) {
@@ -354,7 +385,6 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
         if (CollectionUtils.isNotEmpty(retrieveSnapshotEntries(snapshotId, null))) {
             return true;
         }
-        updateSnapshotToDestroyed(snapshotVO);
         return true;
     }
 
@@ -380,4 +410,104 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
     @Override
     public void postSnapshotCreation(SnapshotInfo snapshot) {
     }
+
+    @Override
+    public void copySnapshot(DataObject snapshot, DataObject snapshotDest, AsyncCompletionCallback<CreateCmdResult> callback) {
+        // export snapshot on remote
+        StoragePoolVO storagePoolVO = _primaryDataStoreDao.findById(snapshotDest.getDataStore().getId());
+        String location = StorPoolConfigurationManager.StorPoolClusterLocation.valueIn(snapshotDest.getDataStore().getId());
+        StorPoolUtil.spLog("StorpoolSnapshotStrategy.copySnapshot: snapshot %s to pool=%s", snapshot.getUuid(), storagePoolVO.getName());
+        SnapshotInfo srcSnapshot = (SnapshotInfo) snapshot;
+        SnapshotInfo destSnapshot = (SnapshotInfo) snapshotDest;
+        String err = null;
+        String snapshotName = StorPoolStorageAdaptor.getVolumeNameFromPath(srcSnapshot.getPath(), false);
+        if (location != null) {
+            SpApiResponse resp = exportSnapshot(snapshot, location, snapshotName);
+            if (resp.getError() != null) {
+                err = String.format("Failed to export snapshot [{}] from [{}] due to [{}]", snapshotName, location, resp.getError());
+                StorPoolUtil.spLog(err);
+                completeCallback(callback, destSnapshot.getPath(), err);
+                return;
+            }
+            keepExportedSnapshot(snapshot, location, snapshotName);
+
+            SpConnectionDesc connectionRemote = StorPoolUtil.getSpConnection(storagePoolVO.getUuid(),
+                    storagePoolVO.getId(), storagePoolDetailsDao, _primaryDataStoreDao);
+            SpApiResponse respFromRemote = copySnapshotFromRemote(snapshot, storagePoolVO, snapshotName, connectionRemote);
+
+            if (respFromRemote.getError() != null) {
+                err = String.format("Failed to copy snapshot [{}] to [{}] due to [{}]", snapshotName, location, respFromRemote.getError());
+                StorPoolUtil.spLog(err);
+                completeCallback(callback, destSnapshot.getPath(), err);
+                return;
+            }
+            StorPoolUtil.spLog("The snapshot [%s] was copied from remote", snapshotName);
+
+            respFromRemote = StorPoolUtil.snapshotReconcile("~" + snapshotName, connectionRemote);
+            if (respFromRemote.getError() != null) {
+                err = String.format("Failed to reconcile snapshot [{}] from [{}] due to [{}]", snapshotName, location, respFromRemote.getError());
+                StorPoolUtil.spLog(err);
+                completeCallback(callback, destSnapshot.getPath(), err);
+                return;
+            }
+            updateSnapshotPath(snapshotDest, srcSnapshot, destSnapshot);
+        } else {
+            completeCallback(callback, destSnapshot.getPath(), "The snapshot is not in the right location");
+        }
+        SnapshotObjectTO snap = (SnapshotObjectTO) snapshotDest.getTO();
+        snap.setPath(srcSnapshot.getPath());
+        completeCallback(callback, destSnapshot.getPath(), err);
+    }
+
+    private void completeCallback(AsyncCompletionCallback<CreateCmdResult> callback, String snapshotPath, String err) {
+        CreateCmdResult res = new CreateCmdResult(snapshotPath, null);
+        res.setResult(err);
+        callback.complete(res);
+    }
+
+    private void updateSnapshotPath(DataObject snapshotDest, SnapshotInfo srcSnapshot, SnapshotInfo destSnapshot) {
+
+        SnapshotDataStoreVO snapshotStore = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Primary, snapshotDest.getDataStore().getId(), destSnapshot.getSnapshotId());
+        snapshotStore.setInstallPath(srcSnapshot.getPath());
+        _snapshotStoreDao.update(snapshotStore.getId(), snapshotStore);
+    }
+
+    @NotNull
+    private SpApiResponse copySnapshotFromRemote(DataObject snapshot, StoragePoolVO storagePoolVO, String snapshotName, SpConnectionDesc connectionRemote) {
+
+        String localLocation = StorPoolConfigurationManager.StorPoolClusterLocation
+                .valueIn(snapshot.getDataStore().getId());
+        StoragePoolDetailVO template = storagePoolDetailsDao.findDetail(storagePoolVO.getId(),
+                StorPoolUtil.SP_TEMPLATE);
+        Map<String, String> tags = addStorPoolTags(snapshot);
+        SpApiResponse respFromRemote = StorPoolUtil.snapshotFromRemote(snapshotName, localLocation,
+                template.getValue(), tags, connectionRemote);
+        return respFromRemote;
+    }
+
+    @NotNull
+    private static Map<String, String> addStorPoolTags(DataObject snapshot) {
+        Map<String, String> tags = new HashMap<>();
+        tags.put("cs", "snapshot");
+        tags.put("uuid", snapshot.getUuid());
+        return tags;
+    }
+
+    private void keepExportedSnapshot(DataObject snapshot, String location, String snapshotName) {
+
+        String detail = "~" + snapshotName + ";" + location;
+        SnapshotDetailsVO snapshotForRecovery = new SnapshotDetailsVO(snapshot.getId(), StorPoolUtil.SP_RECOVERED_SNAPSHOT, detail, true);
+        _snapshotDetailsDao.persist(snapshotForRecovery);
+    }
+
+    @NotNull
+    private SpApiResponse exportSnapshot(DataObject snapshot, String location, String snapshotName) {
+
+        SpConnectionDesc connectionLocal = StorPoolUtil.getSpConnection(snapshot.getDataStore().getUuid(),
+                snapshot.getDataStore().getId(), storagePoolDetailsDao, _primaryDataStoreDao);
+        Long clusterId = StorPoolHelper.findClusterIdByGlobalId(StorPoolUtil.getSnapshotClusterId("~" + snapshotName, connectionLocal), clusterDao);
+        connectionLocal = StorPoolHelper.getSpConnectionDesc(connectionLocal, clusterId);
+        SpApiResponse resp = StorPoolUtil.snapshotExport("~" + snapshotName, location, connectionLocal);
+        return resp;
+    }
 }

server/src/main/java/com/cloud/api/ApiDBUtils.java

@@ -360,7 +360,11 @@ import com.cloud.vm.dao.VMInstanceDao;
 import com.cloud.vm.snapshot.VMSnapshot;
 import com.cloud.vm.snapshot.dao.VMSnapshotDao;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 public class ApiDBUtils {
+    private static final Logger log = LogManager.getLogger(ApiDBUtils.class);
     private static ManagementServer s_ms;
     static AsyncJobManager s_asyncMgr;
     static SecurityGroupManager s_securityGroupMgr;
@@ -1717,6 +1721,21 @@ public class ApiDBUtils {
         return s_zoneDao.listByIds(zoneIds);
     }
 
+    public static List<StoragePoolVO> findSnapshotPolicyPools(SnapshotPolicy policy, Volume volume) {
+        List<SnapshotPolicyDetailVO> poolDetails = s_snapshotPolicyDetailsDao.findDetails(policy.getId(), ApiConstants.STORAGE_ID);
+        List<Long> poolIds = new ArrayList<>();
+        for (SnapshotPolicyDetailVO detail : poolDetails) {
+            try {
+                poolIds.add(Long.valueOf(detail.getValue()));
+            } catch (NumberFormatException ignored) {
+                log.debug(String.format("Could not parse the storage ID value of %s", detail.getValue()), ignored);
+            }
+        }
+        if (volume != null && !poolIds.contains(volume.getPoolId())) {
+            poolIds.add(0, volume.getPoolId());
+        }
+        return s_storagePoolDao.listByIds(poolIds);
+    }
     public static VpcOffering findVpcOfferingById(long offeringId) {
         return s_vpcOfferingDao.findById(offeringId);
     }

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -886,6 +886,15 @@ public class ApiResponseHelper implements ResponseGenerator {
             zoneResponses.add(zoneResponse);
         }
         policyResponse.setZones(new HashSet<>(zoneResponses));
+        List<StoragePoolResponse> poolResponses = new ArrayList<>();
+        List<StoragePoolVO> pools = ApiDBUtils.findSnapshotPolicyPools(policy, vol);
+        for (StoragePoolVO pool : pools) {
+            StoragePoolResponse storagePoolResponse = new StoragePoolResponse();
+            storagePoolResponse.setId(pool.getUuid());
+            storagePoolResponse.setName(pool.getName());
+            poolResponses.add(storagePoolResponse);
+        }
+        policyResponse.setStoragePools(new HashSet<>(poolResponses));
 
         return policyResponse;
     }

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -37,6 +37,14 @@ import java.util.stream.Stream;
 
 import javax.inject.Inject;
 
+import com.cloud.dc.Pod;
+import com.cloud.dc.dao.DataCenterDao;
+import com.cloud.dc.dao.HostPodDao;
+import com.cloud.org.Cluster;
+import com.cloud.server.ManagementService;
+import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao;
+import com.cloud.cluster.ManagementServerHostPeerJoinVO;
+
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.acl.ControlledEntity.ACLType;
 import org.apache.cloudstack.acl.SecurityChecker;
@@ -227,7 +235,6 @@ import com.cloud.api.query.vo.TemplateJoinVO;
 import com.cloud.api.query.vo.UserAccountJoinVO;
 import com.cloud.api.query.vo.UserVmJoinVO;
 import com.cloud.api.query.vo.VolumeJoinVO;
-import com.cloud.cluster.ManagementServerHostPeerJoinVO;
 import com.cloud.cluster.ManagementServerHostVO;
 import com.cloud.cluster.dao.ManagementServerHostDao;
 import com.cloud.cluster.dao.ManagementServerHostPeerJoinDao;
@@ -235,11 +242,8 @@ import com.cloud.cpu.CPU;
 import com.cloud.dc.ClusterVO;
 import com.cloud.dc.DataCenter;
 import com.cloud.dc.DedicatedResourceVO;
-import com.cloud.dc.Pod;
 import com.cloud.dc.dao.ClusterDao;
-import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.dc.dao.DedicatedResourceDao;
-import com.cloud.dc.dao.HostPodDao;
 import com.cloud.domain.Domain;
 import com.cloud.domain.DomainVO;
 import com.cloud.domain.dao.DomainDao;
@@ -277,7 +281,6 @@ import com.cloud.network.security.dao.SecurityGroupVMMapDao;
 import com.cloud.network.vo.PublicIpQuarantineVO;
 import com.cloud.offering.DiskOffering;
 import com.cloud.offering.ServiceOffering;
-import com.cloud.org.Cluster;
 import com.cloud.org.Grouping;
 import com.cloud.projects.Project;
 import com.cloud.projects.Project.ListProjectResourcesCriteria;
@@ -289,7 +292,6 @@ import com.cloud.projects.dao.ProjectDao;
 import com.cloud.projects.dao.ProjectInvitationDao;
 import com.cloud.resource.ResourceManager;
 import com.cloud.resource.icon.dao.ResourceIconDao;
-import com.cloud.server.ManagementService;
 import com.cloud.server.ResourceManagerUtil;
 import com.cloud.server.ResourceMetaDataService;
 import com.cloud.server.ResourceTag;
@@ -319,7 +321,6 @@ import com.cloud.storage.VolumeVO;
 import com.cloud.storage.dao.BucketDao;
 import com.cloud.storage.dao.DiskOfferingDao;
 import com.cloud.storage.dao.GuestOSDao;
-import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao;
 import com.cloud.storage.dao.StoragePoolHostDao;
 import com.cloud.storage.dao.StoragePoolTagsDao;
 import com.cloud.storage.dao.VMTemplateDao;
@@ -5892,9 +5893,17 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
     public SnapshotResponse listSnapshot(CopySnapshotCmd cmd) {
         Account caller = CallContext.current().getCallingAccount();
         List<Long> zoneIds = cmd.getDestinationZoneIds();
+        Long zoneId = null;
+        String location = null;
+        if (CollectionUtils.isNotEmpty(zoneIds)) {
+            zoneId = zoneIds.get(0);
+            location = Snapshot.LocationType.SECONDARY.name();
+        } else {
+            location = cmd.getSnapshot().getLocationType() != null ? cmd.getSnapshot().getLocationType().name() : null;
+        }
         Pair<List<SnapshotJoinVO>, Integer> result = searchForSnapshotsWithParams(cmd.getId(), null,
                 null, null, null, null,
-                null, null, zoneIds.get(0), Snapshot.LocationType.SECONDARY.name(),
+                null, null, zoneId, location,
                 false, null, null, null, null, null,
                 null, null, true, false, caller);
         ResponseView respView = ResponseView.Restricted;

server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java

@@ -17,13 +17,13 @@
 
 package com.cloud.api.query.dao;
 
-import java.util.List;
+import com.cloud.api.query.vo.SnapshotJoinVO;
+import com.cloud.utils.db.GenericDao;
 
 import org.apache.cloudstack.api.ResponseObject;
 import org.apache.cloudstack.api.response.SnapshotResponse;
 
-import com.cloud.api.query.vo.SnapshotJoinVO;
+import java.util.List;
-import com.cloud.utils.db.GenericDao;
 
 public interface SnapshotJoinDao extends GenericDao<SnapshotJoinVO, Long> {
 
@@ -34,4 +34,6 @@ public interface SnapshotJoinDao extends GenericDao<SnapshotJoinVO, Long> {
     List<SnapshotJoinVO> searchBySnapshotStorePair(String... pairs);
 
     List<SnapshotJoinVO> findByDistinctIds(Long zoneId, Long... ids);
+
+    List<SnapshotJoinVO> listBySnapshotIdAndZoneId(Long zoneId, Long snapshotId);
 }

server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java

@@ -26,18 +26,6 @@ import java.util.Map;
 
 import javax.inject.Inject;
 
-import org.apache.cloudstack.annotation.AnnotationService;
-import org.apache.cloudstack.annotation.dao.AnnotationDao;
-import org.apache.cloudstack.api.ResponseObject;
-import org.apache.cloudstack.api.response.SnapshotResponse;
-import org.apache.cloudstack.context.CallContext;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
-import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-import org.apache.cloudstack.query.QueryService;
-
-import org.apache.commons.collections.CollectionUtils;
-
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.ApiResponseHelper;
 import com.cloud.api.query.vo.SnapshotJoinVO;
@@ -54,6 +42,18 @@ import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.vm.VMInstanceVO;
 
+import org.apache.cloudstack.annotation.AnnotationService;
+import org.apache.cloudstack.annotation.dao.AnnotationDao;
+import org.apache.cloudstack.api.ResponseObject;
+import org.apache.cloudstack.api.response.SnapshotResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.query.QueryService;
+
+import org.apache.commons.collections.CollectionUtils;
+
 public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation<SnapshotJoinVO, SnapshotResponse> implements SnapshotJoinDao {
 
     @Inject
@@ -69,6 +69,8 @@ public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation<Snapsh
 
     private final SearchBuilder<SnapshotJoinVO> snapshotIdsSearch;
 
+    private final SearchBuilder<SnapshotJoinVO> snapshotByZoneSearch;
+
     SnapshotJoinDaoImpl() {
         snapshotStorePairSearch = createSearchBuilder();
         snapshotStorePairSearch.and("snapshotStoreState", snapshotStorePairSearch.entity().getStoreState(), SearchCriteria.Op.IN);
@@ -80,6 +82,11 @@ public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation<Snapsh
         snapshotIdsSearch.and("idsIN", snapshotIdsSearch.entity().getId(), SearchCriteria.Op.IN);
         snapshotIdsSearch.groupBy(snapshotIdsSearch.entity().getId());
         snapshotIdsSearch.done();
+
+        snapshotByZoneSearch = createSearchBuilder();
+        snapshotByZoneSearch.and("zoneId", snapshotByZoneSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        snapshotByZoneSearch.and("id", snapshotByZoneSearch.entity().getId(), SearchCriteria.Op.EQ);
+        snapshotByZoneSearch.done();
     }
 
     private void setSnapshotInfoDetailsInResponse(SnapshotJoinVO snapshot, SnapshotResponse snapshotResponse, boolean isShowUnique) {
@@ -292,4 +299,16 @@ public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation<Snapsh
         sc.setParameters("idsIN", ids);
         return searchIncludingRemoved(sc, searchFilter, null, false);
     }
+
+    public List<SnapshotJoinVO> listBySnapshotIdAndZoneId(Long zoneId, Long snapshotId) {
+        if (snapshotId == null) {
+            return new ArrayList<>();
+        }
+        SearchCriteria<SnapshotJoinVO> sc = snapshotByZoneSearch.create();
+        if (zoneId != null) {
+            sc.setParameters("zoneId", zoneId);
+        }
+        sc.setParameters("id", snapshotId);
+        return listBy(sc);
+    }
 }

server/src/main/java/com/cloud/storage/CreateSnapshotPayload.java

@@ -29,6 +29,7 @@ public class CreateSnapshotPayload {
     private boolean asyncBackup;
     private List<Long> zoneIds;
     private boolean kvmIncrementalSnapshot = false;
+    private List<Long> storagePoolIds;
 
     public Long getSnapshotPolicyId() {
         return snapshotPolicyId;
@@ -85,6 +86,15 @@ public class CreateSnapshotPayload {
     }
 
     public void setKvmIncrementalSnapshot(boolean kvmIncrementalSnapshot) {
+
         this.kvmIncrementalSnapshot = kvmIncrementalSnapshot;
     }
+
+    public List<Long> getStoragePoolIds() {
+        return storagePoolIds;
+    }
+
+    public void setStoragePoolIds(List<Long> storagePoolIds) {
+        this.storagePoolIds = storagePoolIds;
+    }
 }

server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java

@@ -65,6 +65,7 @@ import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationSer
 import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -187,6 +188,7 @@ import com.cloud.storage.snapshot.SnapshotManager;
 import com.cloud.template.TemplateManager;
 import com.cloud.user.Account;
 import com.cloud.user.AccountManager;
+import com.cloud.user.AccountService;
 import com.cloud.user.ResourceLimitService;
 import com.cloud.user.User;
 import com.cloud.user.VmDiskStatisticsVO;
@@ -370,6 +372,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
 
     public static final String KVM_FILE_BASED_STORAGE_SNAPSHOT = "kvmFileBasedStorageSnapshot";
 
+    public AccountService _accountService;
+
     protected Gson _gson;
 
     private static final List<HypervisorType> SupportedHypervisorsForVolResize = Arrays.asList(HypervisorType.KVM, HypervisorType.XenServer,
@@ -3808,9 +3812,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_CREATE, eventDescription = "taking snapshot", async = true)
     public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm,
-         Snapshot.LocationType locationType, boolean asyncBackup, Map<String, String> tags, List<Long> zoneIds)
+         Snapshot.LocationType locationType, boolean asyncBackup, Map<String, String> tags, List<Long> zoneIds, List<Long> poolIds, Boolean useStorageReplication)
+
     throws ResourceAllocationException {
-        final Snapshot snapshot = takeSnapshotInternal(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup, zoneIds);
+        final Snapshot snapshot = takeSnapshotInternal(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup, zoneIds, poolIds, useStorageReplication);
         if (snapshot != null && MapUtils.isNotEmpty(tags)) {
             taggedResourceService.createTags(Collections.singletonList(snapshot.getUuid()), ResourceTag.ResourceObjectType.Snapshot, tags, null);
         }
@@ -3818,10 +3823,12 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     }
 
     private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapshotId, Account account,
-          boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, List<Long> zoneIds)
+          boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, List<Long> zoneIds, List<Long> poolIds, Boolean useStorageReplication)
             throws ResourceAllocationException {
         Account caller = CallContext.current().getCallingAccount();
         VolumeInfo volume = volFactory.getVolume(volumeId);
+        poolIds = snapshotHelper.addStoragePoolsForCopyToPrimary(volume, zoneIds, poolIds, useStorageReplication);
+        canCopyOnPrimary(poolIds, volume,CollectionUtils.isEmpty(poolIds));
         if (volume == null) {
             throw new InvalidParameterValueException("Creating snapshot failed due to volume:" + volumeId + " doesn't exist");
         }
@@ -3834,6 +3841,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
             }
             List<SnapshotPolicyDetailVO> details = snapshotPolicyDetailsDao.findDetails(policyId, ApiConstants.ZONE_ID);
             zoneIds = details.stream().map(d -> Long.valueOf(d.getValue())).collect(Collectors.toList());
+            poolIds = getPoolIdsByPolicy(policyId, poolIds);
         }
         if (CollectionUtils.isNotEmpty(zoneIds)) {
             for (Long destZoneId : zoneIds) {
@@ -3872,14 +3880,14 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
                 placeHolder = createPlaceHolderWork(vm.getId());
                 try {
                     return orchestrateTakeVolumeSnapshot(volumeId, policyId, snapshotId, account, quiescevm,
-                            locationType, asyncBackup, zoneIds);
+                            locationType, asyncBackup, zoneIds, poolIds);
                 } finally {
                     _workJobDao.expunge(placeHolder.getId());
                 }
 
             } else {
                 Outcome<Snapshot> outcome = takeVolumeSnapshotThroughJobQueue(vm.getId(), volumeId, policyId,
-                        snapshotId, account.getId(), quiescevm, locationType, asyncBackup, zoneIds);
+                        snapshotId, account.getId(), quiescevm, locationType, asyncBackup, zoneIds, poolIds);
 
                 try {
                     outcome.get();
@@ -3912,13 +3920,26 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
             if (CollectionUtils.isNotEmpty(zoneIds)) {
                 payload.setZoneIds(zoneIds);
             }
+            if (CollectionUtils.isNotEmpty(poolIds)) {
+                payload.setStoragePoolIds(poolIds);
+            }
             volume.addPayload(payload);
             return volService.takeSnapshot(volume);
         }
     }
 
+    @NotNull
+    private List<Long> getPoolIdsByPolicy(Long policyId, List<Long> poolIds) {
+        if (CollectionUtils.isNotEmpty(poolIds)) {
+            throw new InvalidParameterValueException(String.format("%s can not be specified for snapshots linked with snapshot policy", ApiConstants.STORAGE_ID_LIST));
+        }
+        List<SnapshotPolicyDetailVO> poolDetails = snapshotPolicyDetailsDao.findDetails(policyId, ApiConstants.STORAGE_ID);
+        poolIds = poolDetails.stream().map(d -> Long.valueOf(d.getValue())).collect(Collectors.toList());
+        return poolIds;
+    }
+
     private Snapshot orchestrateTakeVolumeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account,
-        boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, List<Long> zoneIds)
+        boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, List<Long> zoneIds, List<Long> poolIds)
             throws ResourceAllocationException {
 
         VolumeInfo volume = volFactory.getVolume(volumeId);
@@ -3931,7 +3952,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
             throw new InvalidParameterValueException(String.format("Volume: %s is not in %s state but %s. Cannot take snapshot.", volume.getVolume(), Volume.State.Ready, volume.getState()));
         }
 
-        boolean isSnapshotOnStorPoolOnly = volume.getStoragePoolType() == StoragePoolType.StorPool && BooleanUtils.toBoolean(_configDao.getValue("sp.bypass.secondary.storage"));
+        boolean isSnapshotOnStorPoolOnly = volume.getStoragePoolType() == StoragePoolType.StorPool && SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value();
         if (volume.getEncryptFormat() != null && volume.getAttachedVM() != null && volume.getAttachedVM().getState() != State.Stopped && !isSnapshotOnStorPoolOnly) {
             logger.debug(String.format("Refusing to take snapshot of encrypted volume (%s) on running VM (%s)", volume, volume.getAttachedVM()));
             throw new UnsupportedOperationException("Volume snapshots for encrypted volumes are not supported if VM is running");
@@ -3948,6 +3969,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
         if (CollectionUtils.isNotEmpty(zoneIds)) {
             payload.setZoneIds(zoneIds);
         }
+        if (CollectionUtils.isNotEmpty(poolIds)) {
+            payload.setStoragePoolIds(poolIds);
+        }
+
         volume.addPayload(payload);
 
         return volService.takeSnapshot(volume);
@@ -3963,7 +3988,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
 
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_CREATE, eventDescription = "allocating snapshot", create = true)
-    public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, List<Long> zoneIds) throws ResourceAllocationException {
+    public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, List<Long> zoneIds, List<Long> poolIds, Boolean useStorageReplication) throws ResourceAllocationException {
         Account caller = CallContext.current().getCallingAccount();
 
         VolumeInfo volume = volFactory.getVolume(volumeId);
@@ -3997,6 +4022,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
                 throw new InvalidParameterValueException(String.format("Volume: %s is for System VM , Creating snapshot against System VM volumes is not supported", volume.getVolume()));
             }
         }
+        snapshotHelper.addStoragePoolsForCopyToPrimary(volume, zoneIds, poolIds, useStorageReplication);
+        canCopyOnPrimary(poolIds, volume,CollectionUtils.isEmpty(poolIds));
 
         StoragePoolVO storagePoolVO = _storagePoolDao.findById(volume.getPoolId());
 
@@ -4012,6 +4039,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
         if (storagePool == null) {
             throw new InvalidParameterValueException(String.format("Volume: %s please attach this volume to a VM before create snapshot for it", volume.getVolume()));
         }
+        boolean canCopyOnPrimary = useStorageReplication;
 
         if (CollectionUtils.isNotEmpty(zoneIds)) {
             if (policyId != null && policyId > 0) {
@@ -4020,7 +4048,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
             if (Snapshot.LocationType.PRIMARY.equals(locationType)) {
                 throw new InvalidParameterValueException(String.format("%s cannot be specified with snapshot %s as %s", ApiConstants.ZONE_ID_LIST, ApiConstants.LOCATION_TYPE, Snapshot.LocationType.PRIMARY));
             }
-            if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value())) {
+            if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value()) && !canCopyOnPrimary) {
                 throw new InvalidParameterValueException("Backing up of snapshot has been disabled. Snapshot can not be taken for multiple zones");
             }
             if (DataCenter.Type.Edge.equals(zone.getType())) {
@@ -4044,6 +4072,25 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
         return snapshotMgr.allocSnapshot(volumeId, policyId, snapshotName, locationType, false, zoneIds);
     }
 
+    private boolean canCopyOnPrimary(List<Long> poolIds, VolumeInfo volume, boolean isPoolIdsEmpty) {
+        if (!isPoolIdsEmpty) {
+            for (Long poolId : poolIds){
+                DataStore dataStore = dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
+                StoragePoolVO sPool = _storagePoolDao.findById(poolId);
+                if (dataStore != null
+                        && !dataStore.getDriver().getCapabilities().containsKey(DataStoreCapabilities.CAN_COPY_SNAPSHOT_BETWEEN_ZONES_AND_SAME_POOL_TYPE.toString())
+                        && sPool.getPoolType() != volume.getStoragePoolType()
+                && volume.getPoolId() == poolId) {
+                    throw new InvalidParameterValueException("The specified pool doesn't support copying snapshots between zones" + poolId);
+                }
+            }
+        } else {
+            return false;
+        }
+        snapshotHelper.checkIfThereAreMoreThanOnePoolInTheZone(poolIds);
+        return true;
+    }
+
     @Override
     public Snapshot allocSnapshotForVm(Long vmId, Long volumeId, String snapshotName, Long vmSnapshotId) throws ResourceAllocationException {
         Account caller = CallContext.current().getCallingAccount();
@@ -5173,7 +5220,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     }
 
     public Outcome<Snapshot> takeVolumeSnapshotThroughJobQueue(final Long vmId, final Long volumeId, final Long policyId, final Long snapshotId, final Long accountId, final boolean quiesceVm,
-                                                               final Snapshot.LocationType locationType, final boolean asyncBackup, final List<Long> zoneIds) {
+                                                               final Snapshot.LocationType locationType, final boolean asyncBackup, final List<Long> zoneIds, List<Long> poolIds) {
 
         final CallContext context = CallContext.current();
         final User callingUser = context.getCallingUser();
@@ -5195,7 +5242,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
 
         // save work context info (there are some duplications)
         VmWorkTakeVolumeSnapshot workInfo = new VmWorkTakeVolumeSnapshot(callingUser.getId(), accountId != null ? accountId : callingAccount.getId(), vm.getId(),
-                VolumeApiServiceImpl.VM_WORK_JOB_HANDLER, volumeId, policyId, snapshotId, quiesceVm, locationType, asyncBackup, zoneIds);
+                VolumeApiServiceImpl.VM_WORK_JOB_HANDLER, volumeId, policyId, snapshotId, quiesceVm, locationType, asyncBackup, zoneIds, poolIds);
         workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo));
 
         _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId());
@@ -5246,7 +5293,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     private Pair<JobInfo.Status, String> orchestrateTakeVolumeSnapshot(VmWorkTakeVolumeSnapshot work) throws Exception {
         Account account = _accountDao.findById(work.getAccountId());
         orchestrateTakeVolumeSnapshot(work.getVolumeId(), work.getPolicyId(), work.getSnapshotId(), account,
-                work.isQuiesceVm(), work.getLocationType(), work.isAsyncBackup(), work.getZoneIds());
+                work.isQuiesceVm(), work.getLocationType(), work.isAsyncBackup(), work.getZoneIds(), work.getPoolIds());
         return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(work.getSnapshotId()));
     }
 

server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java

@@ -65,6 +65,8 @@ public interface SnapshotManager extends Configurable {
             "Whether to show chain size (sum of physical size of snapshot and all its parents) for incremental snapshots in the snapshot response",
             true, ConfigKey.Scope.Global, null);
 
+    public static final ConfigKey<Boolean> UseStorageReplication = new ConfigKey<Boolean>(Boolean.class, "use.storage.replication", "Snapshots", "false", "For snapshot copy to another primary storage in a different zone. Supports only StorPool storage for now", true, ConfigKey.Scope.StoragePool, null);
+
     void deletePoliciesForVolume(Long volumeId);
 
     /**

server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.storage.snapshot;
 
+
+import com.cloud.storage.StoragePoolStatus;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -53,10 +55,12 @@ import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
 import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
+import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult;
@@ -294,7 +298,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {BackupRetryAttempts, BackupRetryInterval, SnapshotHourlyMax, SnapshotDailyMax, SnapshotMonthlyMax, SnapshotWeeklyMax, usageSnapshotSelection,
-                SnapshotInfo.BackupSnapshotAfterTakingSnapshot, VmStorageSnapshotKvm, kvmIncrementalSnapshot, snapshotDeltaMax, snapshotShowChainSize};
+                SnapshotInfo.BackupSnapshotAfterTakingSnapshot, VmStorageSnapshotKvm, kvmIncrementalSnapshot, snapshotDeltaMax, snapshotShowChainSize, UseStorageReplication};
     }
 
     @Override
@@ -919,6 +923,19 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
                 }
             }
         }
+        final SnapshotVO postDeleteSnapshotEntry = _snapshotDao.findById(snapshotId);
+        if (postDeleteSnapshotEntry == null || Snapshot.State.Destroyed.equals(postDeleteSnapshotEntry.getState())) {
+            annotationDao.removeByEntityType(AnnotationService.EntityType.SNAPSHOT.name(), snapshotCheck.getUuid());
+
+            if (snapshotCheck.getState() != Snapshot.State.Error && snapshotCheck.getState() != Snapshot.State.Destroyed) {
+                _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.snapshot);
+            }
+        }
+        for (SnapshotDataStoreVO snapshotStoreRef : snapshotStoreRefs) {
+            if (ObjectInDataStoreStateMachine.State.Ready.equals(snapshotStoreRef.getState()) && !DataStoreRole.Primary.equals(snapshotStoreRef.getRole())) {
+                _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.secondary_storage, new Long(snapshotStoreRef.getPhysicalSize()));
+            }
+        }
 
         return result;
     }
@@ -1114,11 +1131,13 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         return success;
     }
 
-    protected void validatePolicyZones(List<Long> zoneIds, VolumeVO volume, Account caller) {
+    protected void validatePolicyZones(List<Long> zoneIds, List<Long> poolIds, VolumeVO volume, Account caller) {
-        if (CollectionUtils.isEmpty(zoneIds)) {
+        boolean hasPools = CollectionUtils.isNotEmpty(poolIds);
+        boolean hasZones = CollectionUtils.isNotEmpty(zoneIds);
+        if (!hasZones && !hasPools) {
             return;
         }
-        if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value())) {
+        if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value()) && hasZones && !hasPools) {
             throw new InvalidParameterValueException("Backing up of snapshot has been disabled. Snapshot can not be taken for multiple zones");
         }
         final DataCenterVO zone = dataCenterDao.findById(volume.getDataCenterId());
@@ -1126,10 +1145,19 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
             throw new InvalidParameterValueException("Backing up of snapshot is not supported by the zone of the volume. Snapshots can not be taken for multiple zones");
         }
         boolean isRootAdminCaller = _accountMgr.isRootAdmin(caller.getId());
+
+        if (hasZones) {
             for (Long zoneId : zoneIds) {
                 getCheckedDestinationZoneForSnapshotCopy(zoneId, isRootAdminCaller);
             }
         }
+        if (hasPools) {
+            snapshotHelper.checkIfThereAreMoreThanOnePoolInTheZone(poolIds);
+            for (Long poolId : poolIds) {
+                getCheckedDestinationStorageForSnapshotCopy(poolId, isRootAdminCaller);
+            }
+        }
+    }
 
     @Override
     @DB
@@ -1230,15 +1258,18 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         }
 
         final List<Long> zoneIds = cmd.getZoneIds();
-        validatePolicyZones(zoneIds, volume, caller);
+        VolumeInfo volumeInfo = volFactory.getVolume(volumeId);
+        final List<Long> poolIds = snapshotHelper.addStoragePoolsForCopyToPrimary(volumeInfo, zoneIds, cmd.getStoragePoolIds(), cmd.useStorageReplication());
+
+        validatePolicyZones(zoneIds, poolIds, volume, caller);
 
         Map<String, String> tags = cmd.getTags();
         boolean active = true;
 
-        return persistSnapshotPolicy(volume, schedule, timezoneId, intvType, maxSnaps, display, active, tags, zoneIds);
+        return persistSnapshotPolicy(volume, schedule, timezoneId, intvType, maxSnaps, display, active, tags, zoneIds, poolIds);
     }
 
-    protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, boolean active, Map<String, String> tags, List<Long> zoneIds) {
+    protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, boolean active, Map<String, String> tags, List<Long> zoneIds, List<Long> poolIds) {
         long volumeId = volume.getId();
 
         GlobalLock createSnapshotPolicyLock = GlobalLock.getInternLock("createSnapshotPolicy_" + volumeId);
@@ -1250,13 +1281,14 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
 
         logger.debug("Acquired lock for creating snapshot policy [{}] for volume {}.", intervalType, volume);
 
+
         try {
             SnapshotPolicyVO policy = _snapshotPolicyDao.findOneByVolumeInterval(volumeId, intervalType);
 
             if (policy == null) {
-                policy = createSnapshotPolicy(volumeId, schedule, timezone, intervalType, maxSnaps, display, zoneIds);
+                policy = createSnapshotPolicy(volumeId, schedule, timezone, intervalType, maxSnaps, display, zoneIds, poolIds);
             } else {
-                updateSnapshotPolicy(policy, schedule, timezone, intervalType, maxSnaps, active, display, zoneIds);
+                updateSnapshotPolicy(policy, schedule, timezone, intervalType, maxSnaps, active, display, zoneIds, poolIds);
             }
 
             createTagsForSnapshotPolicy(tags, policy);
@@ -1268,7 +1300,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         }
     }
 
-    protected SnapshotPolicyVO createSnapshotPolicy(long volumeId, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, List<Long> zoneIds) {
+    protected SnapshotPolicyVO createSnapshotPolicy(long volumeId, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, List<Long> zoneIds, List<Long> poolIds) {
         SnapshotPolicyVO policy = new SnapshotPolicyVO(volumeId, schedule, timezone, intervalType, maxSnaps, display);
         policy = _snapshotPolicyDao.persist(policy);
         if (CollectionUtils.isNotEmpty(zoneIds)) {
@@ -1278,12 +1310,19 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
             }
             snapshotPolicyDetailsDao.saveDetails(details);
         }
+        if (CollectionUtils.isNotEmpty(poolIds)) {
+            List<SnapshotPolicyDetailVO> details = new ArrayList<>();
+            for (Long poolId : poolIds) {
+                details.add(new SnapshotPolicyDetailVO(policy.getId(), ApiConstants.STORAGE_ID, String.valueOf(poolId)));
+            }
+            snapshotPolicyDetailsDao.saveDetails(details);
+        }
         _snapSchedMgr.scheduleNextSnapshotJob(policy);
         logger.debug(String.format("Created snapshot policy %s.", new ReflectionToStringBuilder(policy, ToStringStyle.JSON_STYLE).setExcludeFieldNames("id", "uuid", "active")));
         return policy;
     }
 
-    protected void updateSnapshotPolicy(SnapshotPolicyVO policy, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean active, boolean display, List<Long> zoneIds) {
+    protected void updateSnapshotPolicy(SnapshotPolicyVO policy, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean active, boolean display, List<Long> zoneIds, List<Long> poolIds) {
         String previousPolicy = new ReflectionToStringBuilder(policy, ToStringStyle.JSON_STYLE).setExcludeFieldNames("id", "uuid").toString();
         boolean previousDisplay = policy.isDisplay();
         policy.setSchedule(schedule);
@@ -1301,7 +1340,14 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
             }
             snapshotPolicyDetailsDao.saveDetails(details);
         }
-
+        if (CollectionUtils.isNotEmpty(poolIds)) {
+            List<SnapshotPolicyDetailVO> details = snapshotPolicyDetailsDao.listDetails(policy.getId());
+            details = details.stream().filter(d -> !ApiConstants.STORAGE_ID.equals(d.getName())).collect(Collectors.toList());
+            for (Long poolId : poolIds) {
+                details.add(new SnapshotPolicyDetailVO(policy.getId(), ApiConstants.STORAGE_ID, String.valueOf(poolId)));
+            }
+            snapshotPolicyDetailsDao.saveDetails(details);
+        }
         _snapSchedMgr.scheduleOrCancelNextSnapshotJobOnDisplayChange(policy, previousDisplay);
         taggedResourceService.deleteTags(Collections.singletonList(policy.getUuid()), ResourceObjectType.SnapshotPolicy, null);
         logger.debug(String.format("Updated snapshot policy %s to %s.", previousPolicy, new ReflectionToStringBuilder(policy, ToStringStyle.JSON_STYLE)
@@ -1325,8 +1371,10 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         for (SnapshotPolicyVO policy : policies) {
             List<SnapshotPolicyDetailVO> details = snapshotPolicyDetailsDao.findDetails(policy.getId(), ApiConstants.ZONE_ID);
             List<Long> zoneIds = details.stream().map(d -> Long.valueOf(d.getValue())).collect(Collectors.toList());
+            List<SnapshotPolicyDetailVO> poolDetails = snapshotPolicyDetailsDao.findDetails(policy.getId(), ApiConstants.STORAGE_ID);
+            List<Long> poolIds = poolDetails.stream().map(d -> Long.valueOf(d.getValue())).collect(Collectors.toList());
             persistSnapshotPolicy(destVolume, policy.getSchedule(), policy.getTimezone(), intervalTypes[policy.getInterval()], policy.getMaxSnaps(),
-                    policy.isDisplay(), policy.isActive(), taggedResourceService.getTagsFromResource(ResourceObjectType.SnapshotPolicy, policy.getId()), zoneIds);
+                    policy.isDisplay(), policy.isActive(), taggedResourceService.getTagsFromResource(ResourceObjectType.SnapshotPolicy, policy.getId()), zoneIds, poolIds);
         }
     }
 
@@ -1580,12 +1628,19 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
 
             if (backupSnapToSecondary) {
                 if (!isKvmAndFileBasedStorage) {
-                    backupSnapshotToSecondary(payload.getAsyncBackup(), snapshotStrategy, snapshotOnPrimary, payload.getZoneIds());
+                    backupSnapshotToSecondary(payload.getAsyncBackup(), snapshotStrategy, snapshotOnPrimary, payload.getZoneIds(), payload.getStoragePoolIds());
                 } else {
                     postSnapshotDirectlyToSecondary(snapshot, snapshotOnPrimary, snapshotId);
                 }
             } else {
                 logger.debug("Skipping backup of snapshot [{}] to secondary due to configuration [{}].", snapshotOnPrimary.getUuid(), SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key());
+
+                if (CollectionUtils.isNotEmpty(payload.getStoragePoolIds()) && payload.getAsyncBackup()) {
+                    snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.COPY);
+                    if (snapshotStrategy != null) {
+                        backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshotOnPrimary, snapshotBackupRetries - 1, snapshotStrategy, payload.getZoneIds(), payload.getStoragePoolIds()), 0, TimeUnit.SECONDS);
+                    }
+                }
                 snapshotOnPrimary.markBackedUp();
             }
 
@@ -1606,9 +1661,14 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
                 // Correct the resource count of snapshot in case of delta snapshots.
                 _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize()));
 
-                if (!payload.getAsyncBackup() && backupSnapToSecondary) {
+                if (!payload.getAsyncBackup()) {
+                    if (backupSnapToSecondary) {
                         copyNewSnapshotToZones(snapshotId, snapshot.getDataCenterId(), payload.getZoneIds());
                     }
+                    if (CollectionUtils.isNotEmpty(payload.getStoragePoolIds())) {
+                        copyNewSnapshotToZonesOnPrimary(payload, snapshot);
+                    }
+                }
             } catch (Exception e) {
                 logger.debug("post process snapshot failed", e);
             }
@@ -1652,10 +1712,45 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         return volumeInfo.getHypervisorType() == HypervisorType.KVM && fileBasedStores.contains(storagePool.getPoolType());
     }
 
+    private void copyNewSnapshotToZonesOnPrimary(CreateSnapshotPayload payload, SnapshotInfo snapshot) {
+        SnapshotStrategy snapshotStrategy;
+        snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.COPY);
+        if (snapshotStrategy != null) {
+            for (Long storagePoolId : payload.getStoragePoolIds()) {
+                copySnapshotOnPool(snapshot, snapshotStrategy, storagePoolId);
+            }
+        } else {
+            logger.info("Unable to find snapshot strategy to handle the copy of a snapshot with id " + snapshot.getUuid());
+        }
+    }
 
-    protected void backupSnapshotToSecondary(boolean asyncBackup, SnapshotStrategy snapshotStrategy, SnapshotInfo snapshotOnPrimary, List<Long> zoneIds) {
+    private boolean copySnapshotOnPool(SnapshotInfo snapshot, SnapshotStrategy snapshotStrategy, Long storagePoolId) {
+        DataStore store = dataStoreMgr.getDataStore(storagePoolId, DataStoreRole.Primary);
+        SnapshotInfo snapshotOnStore =  (SnapshotInfo) store.create(snapshot);
+
+        try {
+            AsyncCallFuture<SnapshotResult> future = snapshotSrv.copySnapshot(snapshot, snapshotOnStore, snapshotStrategy);
+            SnapshotResult result = future.get();
+            if (result.isFailed()) {
+                logger.debug(String.format("Copy snapshot ID: %d failed for primary storage %s: %s", snapshot.getSnapshotId(), storagePoolId, result.getResult()));
+                return false;
+            }
+            snapshotZoneDao.addSnapshotToZone(snapshot.getId(), snapshotOnStore.getDataCenterId());
+            _resourceLimitMgr.incrementResourceCount(CallContext.current().getCallingUserId(), ResourceType.primary_storage, snapshot.getSize());
+            if (CallContext.current().getCallingUserId() != Account.ACCOUNT_ID_SYSTEM) {
+                SnapshotVO snapshotVO = _snapshotDao.findByIdIncludingRemoved(snapshot.getSnapshotId());
+                UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_COPY, CallContext.current().getCallingAccountId(), snapshotOnStore.getDataCenterId(), snapshotVO.getId(), null, null, null, snapshotVO.getSize(),
+                        snapshotVO.getSize(), snapshotVO.getClass().getName(), snapshotVO.getUuid());
+            }
+        } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException("Could not copy the snapshot to another pool", e);
+        }
+        return true;
+    }
+
+    protected void backupSnapshotToSecondary(boolean asyncBackup, SnapshotStrategy snapshotStrategy, SnapshotInfo snapshotOnPrimary, List<Long> zoneIds, List<Long> poolIds) {
         if (asyncBackup) {
-            backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshotOnPrimary, snapshotBackupRetries - 1, snapshotStrategy, zoneIds), 0, TimeUnit.SECONDS);
+            backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshotOnPrimary, snapshotBackupRetries - 1, snapshotStrategy, zoneIds, poolIds), 0, TimeUnit.SECONDS);
         } else {
             SnapshotInfo backupedSnapshot = snapshotStrategy.backupSnapshot(snapshotOnPrimary);
             if (backupedSnapshot != null) {
@@ -1670,36 +1765,49 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         SnapshotStrategy snapshotStrategy;
 
         List<Long> zoneIds;
+        List<Long> poolIds;
 
-        public BackupSnapshotTask(SnapshotInfo snap, int maxRetries, SnapshotStrategy strategy, List<Long> zoneIds) {
+        public BackupSnapshotTask(SnapshotInfo snap, int maxRetries, SnapshotStrategy strategy, List<Long> zoneIds, List<Long> poolIds) {
             snapshot = snap;
             attempts = maxRetries;
             snapshotStrategy = strategy;
             this.zoneIds = zoneIds;
+            this.poolIds = poolIds;
         }
 
         @Override
         protected void runInContext() {
             try {
                 logger.debug("Value of attempts is " + (snapshotBackupRetries - attempts));
-
+                if (Boolean.TRUE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value()) && CollectionUtils.isEmpty(poolIds)) {
                     SnapshotInfo backupedSnapshot = snapshotStrategy.backupSnapshot(snapshot);
 
                     if (backupedSnapshot != null) {
                         snapshotStrategy.postSnapshotCreation(snapshot);
                         copyNewSnapshotToZones(snapshot.getId(), snapshot.getDataCenterId(), zoneIds);
                     }
+                }
+
+                if (CollectionUtils.isNotEmpty(poolIds)) {
+                    for (Long poolId: poolIds) {
+                        copySnapshotOnPool(snapshot, snapshotStrategy, poolId);
+                    }
+                }
             } catch (final Exception e) {
+                decriseBackupSnapshotAttempts();
+            }
+        }
+
+        private void decriseBackupSnapshotAttempts() {
             if (attempts >= 0) {
                 logger.debug("Backing up of snapshot failed, for snapshot {}, left with {} more attempts", snapshot, attempts);
-                    backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshot, --attempts, snapshotStrategy, zoneIds), snapshotBackupRetryInterval, TimeUnit.SECONDS);
+                backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshot, --attempts, snapshotStrategy, zoneIds, poolIds), snapshotBackupRetryInterval, TimeUnit.SECONDS);
             } else {
                 logger.debug("Done with {} attempts in  backing up of snapshot {}", snapshotBackupRetries, snapshot.getSnapshotVO());
                 snapshotSrv.cleanupOnSnapshotBackupFailure(snapshot);
             }
         }
     }
-    }
 
     private void updateSnapshotPayload(long storagePoolId, CreateSnapshotPayload payload, boolean isKvmAndFileBasedStorage, Long clusterId) {
         StoragePoolVO storagePoolVO = _storagePoolDao.findById(storagePoolId);
@@ -2080,26 +2188,21 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         return failedZones;
     }
 
-    protected Pair<SnapshotVO, Long> getCheckedSnapshotForCopy(final long snapshotId, final List<Long> destZoneIds, Long sourceZoneId) {
+    protected Pair<SnapshotVO, Long> getCheckedSnapshotForCopy(final SnapshotVO snapshot, final List<Long> destZoneIds, Long sourceZoneId, boolean useStorageReplication) {
-        SnapshotVO snapshot = _snapshotDao.findById(snapshotId);
-        if (snapshot == null) {
-            throw new InvalidParameterValueException("Unable to find snapshot with id");
-        }
         // Verify snapshot is BackedUp and is on secondary store
-        if (!Snapshot.State.BackedUp.equals(snapshot.getState())) {
+        if (!Snapshot.State.BackedUp.equals(snapshot.getState()) && !useStorageReplication) {
             throw new InvalidParameterValueException("Snapshot is not backed up");
         }
-        if (snapshot.getLocationType() != null && !Snapshot.LocationType.SECONDARY.equals(snapshot.getLocationType())) {
+        if (snapshot.getLocationType() != null && !Snapshot.LocationType.SECONDARY.equals(snapshot.getLocationType()) && !useStorageReplication) {
             throw new InvalidParameterValueException("Snapshot is not backed up");
         }
-        if (CollectionUtils.isEmpty(destZoneIds)) {
-            throw new InvalidParameterValueException("Please specify valid destination zone(s).");
-        }
         Volume volume = _volsDao.findById(snapshot.getVolumeId());
         if (sourceZoneId == null) {
             sourceZoneId = volume.getDataCenterId();
         }
-        if (destZoneIds.contains(sourceZoneId)) {
+        if (CollectionUtils.isEmpty(destZoneIds)) {
+                throw new InvalidParameterValueException("Please specify valid destination zone(s).");
+        } else if (destZoneIds.contains(sourceZoneId)) {
             throw new InvalidParameterValueException("Please specify different source and destination zones.");
         }
         DataCenterVO sourceZone = dataCenterDao.findById(sourceZoneId);
@@ -2124,16 +2227,42 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         return dstZone;
     }
 
+    protected StoragePoolVO getCheckedDestinationStorageForSnapshotCopy(long poolId, boolean isRootAdmin) {
+        StoragePoolVO destPool = _storagePoolDao.findById(poolId);
+        if (destPool == null) {
+            throw new InvalidParameterValueException("Please specify a valid destination pool.");
+        }
+        if (!StoragePoolStatus.Up.equals(destPool.getStatus()) && !isRootAdmin) {
+            throw new PermissionDeniedException("Cannot perform this operation, the storage pool is not in Up state or the user is not the Root Admin " + destPool.getName());
+        }
+        DataCenterVO destZone = dataCenterDao.findById(destPool.getDataCenterId());
+        if (DataCenter.Type.Edge.equals(destZone.getType())) {
+            logger.error(String.format("Edge zone %s specified for snapshot copy", destZone));
+            throw new InvalidParameterValueException(String.format("Snapshot copy is not supported by zone %s", destZone.getName()));
+        }
+        return destPool;
+    }
+
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_COPY, eventDescription = "copying snapshot", create = false)
     public Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException, ResourceAllocationException {
         final Long snapshotId = cmd.getId();
         Long sourceZoneId = cmd.getSourceZoneId();
         List<Long> destZoneIds = cmd.getDestinationZoneIds();
+        List<Long> storagePoolIds = cmd.getStoragePoolIds();
+        Boolean useStorageReplication = cmd.useStorageReplication();
         Account caller = CallContext.current().getCallingAccount();
-        Pair<SnapshotVO, Long> snapshotZonePair = getCheckedSnapshotForCopy(snapshotId, destZoneIds, sourceZoneId);
+        SnapshotVO snapshotVO = _snapshotDao.findById(snapshotId);
+        if (snapshotVO == null) {
+            throw new InvalidParameterValueException("Unable to find snapshot with id");
+        }
+
+        Pair<SnapshotVO, Long> snapshotZonePair = getCheckedSnapshotForCopy(snapshotVO, destZoneIds, sourceZoneId, useStorageReplication);
         SnapshotVO snapshot = snapshotZonePair.first();
         sourceZoneId = snapshotZonePair.second();
+        VolumeInfo volume = volFactory.getVolume(snapshot.getVolumeId());
+        storagePoolIds = snapshotHelper.addStoragePoolsForCopyToPrimary(volume, destZoneIds, storagePoolIds, useStorageReplication);
+        boolean canCopyBetweenStoragePools = CollectionUtils.isNotEmpty(storagePoolIds) && canCopyOnPrimary(storagePoolIds, snapshotVO);
         Map<Long, DataCenterVO> dataCenterVOs = new HashMap<>();
         boolean isRootAdminCaller = _accountMgr.isRootAdmin(caller.getId());
         for (Long destZoneId: destZoneIds) {
@@ -2142,11 +2271,15 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         }
         _accountMgr.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, true, snapshot);
         DataStore srcSecStore = getSnapshotZoneImageStore(snapshotId, sourceZoneId);
-        if (srcSecStore == null) {
+        if (srcSecStore == null && !canCopyBetweenStoragePools) {
             throw new InvalidParameterValueException(String.format("There is no snapshot ID: %s ready on image store", snapshot.getUuid()));
         }
+        if (canCopyBetweenStoragePools) {
+            snapshotHelper.checkIfThereAreMoreThanOnePoolInTheZone(storagePoolIds);
+            copySnapshotToPrimaryDifferentZone(storagePoolIds, snapshot);
+        }
         List<String> failedZones = copySnapshotToZones(snapshot, srcSecStore, new ArrayList<>(dataCenterVOs.values()));
-        if (destZoneIds.size() > failedZones.size()){
+        if (destZoneIds.size() > failedZones.size() || canCopyBetweenStoragePools){
             if (!failedZones.isEmpty()) {
                 logger.error(String.format("There were failures when copying snapshot to zones: %s",
                         StringUtils.joinWith(", ", failedZones.toArray())));
@@ -2157,6 +2290,74 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         }
     }
 
+    private boolean canCopyOnPrimary(List<Long> poolIds, Snapshot snapshot) {
+        List<Long> poolsToBeRemoved = new ArrayList<>();
+        for (Long poolId : poolIds) {
+            PrimaryDataStore dataStore = (PrimaryDataStore) dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
+            if (isDataStoreNull(dataStore == null, poolsToBeRemoved, poolId)) continue;
+
+            SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snapshot.getId(), poolId, DataStoreRole.Primary);
+            if (isSnapshotExistsOnPool(snapshot, dataStore, snapshotInfo)) continue;
+
+            VolumeVO volume = _volsDao.findById(snapshot.getVolumeId());
+            if (isDataStoreNull(volume == null, poolsToBeRemoved, poolId)) continue;
+            doesStorageSupportCopySnapshot(poolsToBeRemoved, poolId, dataStore, volume);
+        }
+        poolIds.removeAll(poolsToBeRemoved);
+        if (CollectionUtils.isEmpty(poolIds)) {
+            return false;
+        }
+        return true;
+    }
+
+    private void doesStorageSupportCopySnapshot(List<Long> poolsToBeRemoved, Long poolId, PrimaryDataStore dataStore, VolumeVO volume) {
+        if (dataStore.getDriver() != null
+                && MapUtils.isNotEmpty(dataStore.getDriver().getCapabilities())
+                && !dataStore.getDriver().getCapabilities().containsKey(DataStoreCapabilities.CAN_COPY_SNAPSHOT_BETWEEN_ZONES_AND_SAME_POOL_TYPE.toString())
+                && dataStore.getPoolType() != volume.getPoolType()) {
+            poolsToBeRemoved.add(poolId);
+            logger.debug(String.format("The %s  does not support copy to %s between zones", dataStore.getPoolType(), volume.getPoolType()));
+        }
+    }
+
+    private boolean isSnapshotExistsOnPool(Snapshot snapshot, PrimaryDataStore dataStore, SnapshotInfo snapshotInfo) {
+        if (snapshotInfo != null) {
+            logger.debug(String.format("Snapshot [%s] already exist on pool [%s]", snapshot.getUuid(), dataStore.getName()));
+            return true;
+        }
+        return false;
+    }
+
+    private static boolean isDataStoreNull(boolean object, List<Long> poolsToBeRemoved, Long poolId) {
+        if (object) {
+            poolsToBeRemoved.add(poolId);
+            return true;
+        }
+        return false;
+    }
+
+    private void copySnapshotToPrimaryDifferentZone(List<Long> poolIds, SnapshotVO snapshot) {
+        VolumeInfo volume = volFactory.getVolume(snapshot.getVolumeId());
+        if (volume == null) {
+            throw new CloudRuntimeException("Failed to find volume with id: " + snapshot.getVolumeId());
+        }
+        CreateSnapshotPayload payload = setPayload(poolIds, volume, snapshot);
+        SnapshotInfo snapshotInfo = snapshotFactory.getSnapshotOnPrimaryStore(snapshot.getId());
+        copyNewSnapshotToZonesOnPrimary(payload, snapshotInfo);
+    }
+
+    private CreateSnapshotPayload setPayload(List<Long> poolIds, VolumeInfo vol, SnapshotVO snapshotCreate) {
+        CreateSnapshotPayload payload = new CreateSnapshotPayload();
+        payload.setSnapshotId(snapshotCreate.getId());
+        payload.setSnapshotPolicyId(SnapshotVO.MANUAL_POLICY_ID);
+        payload.setLocationType(snapshotCreate.getLocationType());
+        payload.setAccount(_accountMgr.getAccount(vol.getAccountId()));
+        payload.setAsyncBackup(false);
+        payload.setQuiescevm(false);
+        payload.setStoragePoolIds(poolIds);
+        return payload;
+    }
+
     protected void copyNewSnapshotToZones(long snapshotId, long zoneId, List<Long> destZoneIds) {
         if (CollectionUtils.isEmpty(destZoneIds)) {
             return;

server/src/main/java/com/cloud/template/TemplateManagerImpl.java

@@ -322,6 +322,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
     @Inject
     private HeuristicRuleHelper heuristicRuleHelper;
 
+    protected boolean backupSnapshotAfterTakingSnapshot = SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value();
+
     private TemplateAdapter getAdapter(HypervisorType type) {
         TemplateAdapter adapter = null;
         if (type == HypervisorType.BareMetal) {
@@ -1693,13 +1695,25 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
             AsyncCallFuture<TemplateApiResult> future = null;
 
             if (snapshotId != null) {
-                DataStoreRole dataStoreRole = snapshotHelper.getDataStoreRole(snapshot);
+                DataStoreRole dataStoreRole = snapshotHelper.getDataStoreRole(snapshot, zoneId);
-                kvmSnapshotOnlyInPrimaryStorage = snapshotHelper.isKvmSnapshotOnlyInPrimaryStorage(snapshot, dataStoreRole);
+                kvmSnapshotOnlyInPrimaryStorage = snapshotHelper.isKvmSnapshotOnlyInPrimaryStorage(snapshot, dataStoreRole, zoneId);
-                snapInfo = _snapshotFactory.getSnapshotWithRoleAndZone(snapshotId, dataStoreRole, zoneId);
 
+                snapInfo = _snapshotFactory.getSnapshotWithRoleAndZone(snapshotId, dataStoreRole, zoneId);
                 boolean kvmIncrementalSnapshot = SnapshotManager.kvmIncrementalSnapshot.valueIn(_hostDao.findClusterIdByVolumeInfo(snapInfo.getBaseVolume()));
 
-                if (dataStoreRole == DataStoreRole.Image || kvmSnapshotOnlyInPrimaryStorage) {
+                boolean skipCopyToSecondary = false;
+                boolean keepOnPrimary = snapshotHelper.isStorageSupportSnapshotToTemplate(snapInfo);
+                if (keepOnPrimary) {
+                    ImageStoreVO imageStore = _imgStoreDao.findOneByZoneAndProtocol(zoneId, "nfs");
+                    if (imageStore == null) {
+                        throw new CloudRuntimeException(String.format("Could not find an NFS secondary storage pool on zone %s to use as a temporary location " +
+                                "for instance conversion", zoneId));
+                    }
+                    DataStore dataStore = _dataStoreMgr.getDataStore(imageStore.getId(), DataStoreRole.Image);
+                    if (dataStore != null) {
+                        store = dataStore;
+                    }
+                } else if (dataStoreRole == DataStoreRole.Image) {
                     snapInfo = snapshotHelper.backupSnapshotToSecondaryStorageIfNotExists(snapInfo, dataStoreRole, snapshot, kvmSnapshotOnlyInPrimaryStorage);
                     _accountMgr.checkAccess(caller, null, true, snapInfo);
                     DataStore snapStore = snapInfo.getDataStore();

server/src/main/java/org/apache/cloudstack/snapshot/SnapshotHelper.java

@@ -19,14 +19,19 @@
 
 package org.apache.cloudstack.snapshot;
 
-import java.util.Arrays;
+import com.cloud.api.query.dao.SnapshotJoinDao;
-import java.util.HashSet;
+import com.cloud.api.query.vo.SnapshotJoinVO;
-import java.util.List;
+import com.cloud.exception.InvalidParameterValueException;
-import java.util.Map;
+import com.cloud.hypervisor.Hypervisor;
-import java.util.Set;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import java.util.stream.Collectors;
+import com.cloud.storage.DataStoreRole;
-
+import com.cloud.storage.Snapshot;
-import javax.inject.Inject;
+import com.cloud.storage.SnapshotVO;
+import com.cloud.storage.Storage.StoragePoolType;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.SnapshotDao;
+import com.cloud.storage.snapshot.SnapshotManager;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities;
@@ -36,27 +41,31 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy;
 import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
+
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
 
-import com.cloud.hypervisor.Hypervisor;
+import org.apache.logging.log4j.LogManager;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import org.apache.logging.log4j.Logger;
-import com.cloud.storage.DataStoreRole;
+
-import com.cloud.storage.Snapshot;
+import javax.inject.Inject;
-import com.cloud.storage.SnapshotVO;
+import java.util.ArrayList;
-import com.cloud.storage.Storage.StoragePoolType;
+import java.util.Arrays;
-import com.cloud.storage.VolumeVO;
+import java.util.Collections;
-import com.cloud.storage.dao.SnapshotDao;
+import java.util.HashSet;
-import com.cloud.utils.exception.CloudRuntimeException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 public class SnapshotHelper {
     protected Logger logger = LogManager.getLogger(getClass());
@@ -82,6 +91,9 @@ public class SnapshotHelper {
     @Inject
     protected PrimaryDataStoreDao primaryDataStoreDao;
 
+    @Inject
+    protected SnapshotJoinDao snapshotJoinDao;
+
     protected boolean backupSnapshotAfterTakingSnapshot = SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value();
 
     protected final Set<StoragePoolType> storagePoolTypesToValidateWithBackupSnapshotAfterTakingSnapshot = new HashSet<>(Arrays.asList(StoragePoolType.RBD,
@@ -92,6 +104,22 @@ public class SnapshotHelper {
      * @param snapInfo the snapshot info to delete.
      */
     public void expungeTemporarySnapshot(boolean kvmSnapshotOnlyInPrimaryStorage, SnapshotInfo snapInfo) {
+        long storeId = snapInfo.getDataStore().getId();
+        long zoneId = dataStorageManager.getStoreZoneId(storeId, snapInfo.getDataStore().getRole());
+
+        if (isStorageSupportSnapshotToTemplate(snapInfo)) {
+            logger.debug("The primary storage does not delete the snapshots even if there is a backup on secondary");
+            return;
+        }
+
+        List<SnapshotJoinVO> snapshots = snapshotJoinDao.listBySnapshotIdAndZoneId(zoneId, snapInfo.getSnapshotId());
+        if (kvmSnapshotOnlyInPrimaryStorage || snapshots.size() <= 1) {
+            if (snapInfo != null) {
+                logger.trace(String.format("Snapshot [{}] is not a temporary backup to create a volume from snapshot. Not expunging it.", snapInfo.getId()));
+            }
+            return;
+        }
+
         if (snapInfo == null) {
             logger.warn("Unable to expunge snapshot due to its info is null.");
             return;
@@ -118,15 +146,20 @@ public class SnapshotHelper {
             }
         }
 
-        long storeId = snapInfo.getDataStore().getId();
         if (!DataStoreRole.Image.equals(snapInfo.getDataStore().getRole())) {
-            long zoneId = dataStorageManager.getStoreZoneId(storeId, snapInfo.getDataStore().getRole());
             SnapshotInfo imageStoreSnapInfo = snapshotFactory.getSnapshotWithRoleAndZone(snapInfo.getId(), DataStoreRole.Image, zoneId);
             storeId = imageStoreSnapInfo.getDataStore().getId();
         }
         snapshotDataStoreDao.expungeReferenceBySnapshotIdAndDataStoreRole(snapInfo.getId(), storeId, DataStoreRole.Image);
     }
 
+    public boolean isStorageSupportSnapshotToTemplate(SnapshotInfo snapInfo) {
+        if (DataStoreRole.Primary.equals(snapInfo.getDataStore().getRole())) {
+            Map<String, String> capabilities = snapInfo.getDataStore().getDriver().getCapabilities();
+            return org.apache.commons.collections4.MapUtils.isNotEmpty(capabilities) && capabilities.containsKey(DataStoreCapabilities.CAN_CREATE_TEMPLATE_FROM_SNAPSHOT.toString());
+        }
+        return false;
+    }
     /**
      * Backup the snapshot to secondary storage if it should be backed up and was not yet or it is a temporary backup to create a volume.
      * @return The parameter snapInfo if the snapshot is not backupable, else backs up the snapshot to secondary storage and returns its info.
@@ -181,8 +214,11 @@ public class SnapshotHelper {
      * @return true if hypervisor is {@link  HypervisorType#KVM} and data store role is {@link  DataStoreRole#Primary} and global setting "snapshot.backup.to.secondary" is false,
      * else false.
      */
-    public boolean isKvmSnapshotOnlyInPrimaryStorage(Snapshot snapshot, DataStoreRole dataStoreRole){
+    public boolean isKvmSnapshotOnlyInPrimaryStorage(Snapshot snapshot, DataStoreRole dataStoreRole, Long zoneId){
-        return snapshot.getHypervisorType() == Hypervisor.HypervisorType.KVM && dataStoreRole == DataStoreRole.Primary && !backupSnapshotAfterTakingSnapshot;
+        List<SnapshotJoinVO> snapshots = snapshotJoinDao.listBySnapshotIdAndZoneId(zoneId, snapshot.getSnapshotId());
+        boolean isKvmSnapshotOnlyInPrimaryStorage = snapshots.stream().filter(s -> s.getStoreRole().equals(DataStoreRole.Image)).count() == 0;
+
+        return snapshot.getHypervisorType() == Hypervisor.HypervisorType.KVM && dataStoreRole == DataStoreRole.Primary && isKvmSnapshotOnlyInPrimaryStorage;
     }
 
     public DataStoreRole getDataStoreRole(Snapshot snapshot) {
@@ -215,6 +251,17 @@ public class SnapshotHelper {
         return DataStoreRole.Image;
     }
 
+    public DataStoreRole getDataStoreRole(Snapshot snapshot, Long zoneId) {
+        if (zoneId == null) {
+            getDataStoreRole(snapshot);
+        }
+        List<SnapshotJoinVO> snapshots = snapshotJoinDao.listBySnapshotIdAndZoneId(zoneId, snapshot.getId());
+        boolean snapshotOnPrimary = snapshots.stream().anyMatch(s -> s.getStoreRole().equals(DataStoreRole.Primary));
+        if (snapshotOnPrimary) {
+            return DataStoreRole.Primary;
+        }
+        return DataStoreRole.Image;
+    }
         /**
          * Verifies if it is a KVM volume that has snapshots only in primary storage.
          * @throws CloudRuntimeException If it is a KVM volume and has at least one snapshot only in primary storage.
@@ -271,10 +318,62 @@ public class SnapshotHelper {
     }
 
     public SnapshotInfo convertSnapshotIfNeeded(SnapshotInfo snapshotInfo) {
+
         if (snapshotInfo.getParent() == null || !HypervisorType.KVM.equals(snapshotInfo.getHypervisorType())) {
             return snapshotInfo;
         }
 
         return snapshotService.convertSnapshot(snapshotInfo);
     }
+
+    public void checkIfThereAreMoreThanOnePoolInTheZone(List<Long> poolIds) {
+        List<Long> poolsInOneZone = new ArrayList<>();
+        for (Long poolId : poolIds) {
+            StoragePoolVO pool = primaryDataStoreDao.findById(poolId);
+            if (pool != null) {
+                poolsInOneZone.add(pool.getDataCenterId());
+            }
+        }
+        boolean moreThanOnePoolForZone = poolsInOneZone.stream().filter(itr -> Collections.frequency(poolsInOneZone, itr) > 1).count() > 1;
+        if (moreThanOnePoolForZone) {
+            throw new CloudRuntimeException("Cannot copy the snapshot on multiple storage pools in one zone");
+        }
+    }
+
+    public List<Long> addStoragePoolsForCopyToPrimary(VolumeInfo volume, List<Long> destZoneIds, List<Long> storagePoolIds, Boolean useStorageReplication) {
+        if (useStorageReplication) {
+            if (volume == null) {
+                throw new InvalidParameterValueException("Could not find volume of a snapshot");
+            } else if (!doesStorageSupportCopyBetweenZones(volume.getPoolId())){
+                throw new InvalidParameterValueException("The storage pool does not support copy between zones");
+            }
+            if (CollectionUtils.isEmpty(destZoneIds)) {
+                throw new InvalidParameterValueException("There is no destination zone provided");
+            }
+            if (CollectionUtils.isEmpty(storagePoolIds)) {
+                storagePoolIds = new ArrayList<>();
+                for (Long destZone : destZoneIds) {
+                    List<StoragePoolVO> pools = primaryDataStoreDao.findPoolsByStorageTypeAndZone(volume.getStoragePoolType(), destZone);
+                    if (CollectionUtils.isNotEmpty(pools)) {
+                        StoragePoolVO storagePoolVO = pools.stream().filter(pool -> SnapshotManager.UseStorageReplication.valueIn(pool.getId()) == true).findFirst().get();
+                        storagePoolIds.add(storagePoolVO.getId());
+                    }
+                }
+                if (CollectionUtils.isEmpty(storagePoolIds)) {
+                    throw new InvalidParameterValueException("Cannot copy snapshot to primary storage. There aren't storage pools that support this operation");
+                }
+            }
+            destZoneIds.clear();
+        }
+        return storagePoolIds;
+    }
+
+    public boolean doesStorageSupportCopyBetweenZones(Long poolId) {
+        DataStore dataStore = dataStorageManager.getDataStore(poolId, DataStoreRole.Primary);
+        if (dataStore != null
+                && dataStore.getDriver().getCapabilities().containsKey(DataStoreCapabilities.CAN_COPY_SNAPSHOT_BETWEEN_ZONES_AND_SAME_POOL_TYPE.toString())) {
+            return true;
+        }
+        return false;
+    }
 }

server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java

@@ -579,7 +579,7 @@ public class VolumeApiServiceImplTest {
         when(volumeDataFactoryMock.getVolume(anyLong())).thenReturn(volumeInfoMock);
         when(volumeInfoMock.getState()).thenReturn(Volume.State.Allocated);
         lenient().when(volumeInfoMock.getPoolId()).thenReturn(1L);
-        volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null, null);
+        volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null, null, null, false);
     }
 
     @Test
@@ -592,7 +592,7 @@ public class VolumeApiServiceImplTest {
         final TaggedResourceService taggedResourceService = Mockito.mock(TaggedResourceService.class);
         Mockito.lenient().when(taggedResourceService.createTags(any(), any(), any(), any())).thenReturn(null);
         ReflectionTestUtils.setField(volumeApiServiceImpl, "taggedResourceService", taggedResourceService);
-        volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null, null);
+        volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null, null, null, false);
     }
 
     @Test
@@ -640,7 +640,7 @@ public class VolumeApiServiceImplTest {
     @Test
     public void testAllocSnapshotNonManagedStorageArchive() {
         try {
-            volumeApiServiceImpl.allocSnapshot(6L, 1L, "test", Snapshot.LocationType.SECONDARY, null);
+            volumeApiServiceImpl.allocSnapshot(6L, 1L, "test", Snapshot.LocationType.SECONDARY, null, null, null);
         } catch (InvalidParameterValueException e) {
             Assert.assertEquals(e.getMessage(), "VolumeId: 6 LocationType is supported only for managed storage");
             return;

server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java

@@ -16,30 +16,6 @@
 // under the License.
 package com.cloud.storage.snapshot;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-
-import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
-import org.apache.cloudstack.framework.async.AsyncCallFuture;
-import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.mockito.stubbing.Answer;
-
 import com.cloud.dc.DataCenter;
 import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.dao.DataCenterDao;
@@ -63,6 +39,32 @@ import com.cloud.user.ResourceLimitService;
 import com.cloud.user.dao.AccountDao;
 import com.cloud.utils.Pair;
 
+import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
+import org.apache.cloudstack.framework.async.AsyncCallFuture;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.mockito.stubbing.Answer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
 @RunWith(MockitoJUnitRunner.class)
 public class SnapshotManagerImplTest {
     @Mock
@@ -176,7 +178,7 @@ public class SnapshotManagerImplTest {
     }
     @Test
     public void testValidatePolicyZonesNoZones() {
-        snapshotManager.validatePolicyZones(null, Mockito.mock(VolumeVO.class), Mockito.mock(Account.class));
+        snapshotManager.validatePolicyZones(null, null, Mockito.mock(VolumeVO.class), Mockito.mock(Account.class));
     }
 
     @Test(expected = InvalidParameterValueException.class)
@@ -186,7 +188,7 @@ public class SnapshotManagerImplTest {
         DataCenterVO zone = Mockito.mock(DataCenterVO.class);
         Mockito.when(zone.getType()).thenReturn(DataCenter.Type.Edge);
         Mockito.when(dataCenterDao.findById(1L)).thenReturn(zone);
-        snapshotManager.validatePolicyZones(List.of(1L), volumeVO, Mockito.mock(Account.class));
+        snapshotManager.validatePolicyZones(List.of(1L), null, volumeVO, Mockito.mock(Account.class));
     }
 
     @Test(expected = InvalidParameterValueException.class)
@@ -197,7 +199,7 @@ public class SnapshotManagerImplTest {
         Mockito.when(zone.getType()).thenReturn(DataCenter.Type.Core);
         Mockito.when(dataCenterDao.findById(1L)).thenReturn(zone);
         Mockito.when(dataCenterDao.findById(2L)).thenReturn(null);
-        snapshotManager.validatePolicyZones(List.of(2L), volumeVO, Mockito.mock(Account.class));
+        snapshotManager.validatePolicyZones(List.of(2L), null, volumeVO, Mockito.mock(Account.class));
     }
 
     @Test(expected = PermissionDeniedException.class)
@@ -211,7 +213,7 @@ public class SnapshotManagerImplTest {
         Mockito.when(zone1.getAllocationState()).thenReturn(Grouping.AllocationState.Disabled);
         Mockito.when(dataCenterDao.findById(2L)).thenReturn(zone1);
         Mockito.when(accountManager.isRootAdmin(Mockito.any())).thenReturn(false);
-        snapshotManager.validatePolicyZones(List.of(2L), volumeVO, Mockito.mock(Account.class));
+        snapshotManager.validatePolicyZones(List.of(2L), null, volumeVO, Mockito.mock(Account.class));
     }
 
     @Test(expected = InvalidParameterValueException.class)
@@ -225,7 +227,7 @@ public class SnapshotManagerImplTest {
         Mockito.when(zone1.getType()).thenReturn(DataCenter.Type.Edge);
         Mockito.when(zone1.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
         Mockito.when(dataCenterDao.findById(2L)).thenReturn(zone1);
-        snapshotManager.validatePolicyZones(List.of(2L), volumeVO, Mockito.mock(Account.class));
+        snapshotManager.validatePolicyZones(List.of(2L), null, volumeVO, Mockito.mock(Account.class));
     }
 
     @Test
@@ -239,7 +241,7 @@ public class SnapshotManagerImplTest {
         Mockito.when(zone1.getType()).thenReturn(DataCenter.Type.Core);
         Mockito.when(zone1.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
         Mockito.when(dataCenterDao.findById(2L)).thenReturn(zone1);
-        snapshotManager.validatePolicyZones(List.of(2L), volumeVO, Mockito.mock(Account.class));
+        snapshotManager.validatePolicyZones(List.of(2L), null, volumeVO, Mockito.mock(Account.class));
     }
 
     @Test
@@ -308,15 +310,14 @@ public class SnapshotManagerImplTest {
 
     @Test(expected = InvalidParameterValueException.class)
     public void testGetCheckedSnapshotForCopyNoSnapshot() {
-        snapshotManager.getCheckedSnapshotForCopy(1L, List.of(100L), null);
+        SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class);
+        snapshotManager.getCheckedSnapshotForCopy(snapshotVO, List.of(100L), null, false);
     }
 
     @Test(expected = InvalidParameterValueException.class)
     public void testGetCheckedSnapshotForCopyNoSnapshotBackup() {
-        final long snapshotId = 1L;
         SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class);
-        Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO);
+        snapshotManager.getCheckedSnapshotForCopy(snapshotVO, List.of(100L), null, false);
-        snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L), null);
     }
 
     @Test(expected = InvalidParameterValueException.class)
@@ -325,73 +326,62 @@ public class SnapshotManagerImplTest {
         SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class);
         Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp);
         Mockito.when(snapshotVO.getLocationType()).thenReturn(Snapshot.LocationType.PRIMARY);
-        Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO);
+        snapshotManager.getCheckedSnapshotForCopy(snapshotVO, List.of(100L), null, false);
-        snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L), null);
     }
 
     @Test(expected = InvalidParameterValueException.class)
     public void testGetCheckedSnapshotForCopyDestNotSpecified() {
-        final long snapshotId = 1L;
         SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class);
         Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp);
-        Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO);
+        snapshotManager.getCheckedSnapshotForCopy(snapshotVO, new ArrayList<>(), 1L, false);
-        snapshotManager.getCheckedSnapshotForCopy(snapshotId, new ArrayList<>(), null);
     }
 
     @Test(expected = InvalidParameterValueException.class)
     public void testGetCheckedSnapshotForCopyDestContainsSource() {
-        final long snapshotId = 1L;
         SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class);
         Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp);
         Mockito.when(snapshotVO.getVolumeId()).thenReturn(1L);
-        Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO);
         Mockito.when(volumeDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(VolumeVO.class));
-        snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L, 1L), 1L);
+        snapshotManager.getCheckedSnapshotForCopy(snapshotVO, List.of(100L, 1L), 1L, false);
     }
 
     @Test(expected = InvalidParameterValueException.class)
     public void testGetCheckedSnapshotForCopyNullSourceZone() {
-        final long snapshotId = 1L;
         SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class);
         Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp);
         Mockito.when(snapshotVO.getVolumeId()).thenReturn(1L);
-        Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO);
         VolumeVO volumeVO = Mockito.mock(VolumeVO.class);
         Mockito.when(volumeVO.getDataCenterId()).thenReturn(1L);
         Mockito.when(volumeDao.findById(Mockito.anyLong())).thenReturn(volumeVO);
-        snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L, 101L), null);
+        snapshotManager.getCheckedSnapshotForCopy(snapshotVO, List.of(100L, 101L), null, false);
     }
 
     @Test
     public void testGetCheckedSnapshotForCopyValid() {
-        final long snapshotId = 1L;
         final Long zoneId = 1L;
         SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class);
         Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp);
         Mockito.when(snapshotVO.getVolumeId()).thenReturn(1L);
-        Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO);
         VolumeVO volumeVO = Mockito.mock(VolumeVO.class);
         Mockito.when(volumeVO.getDataCenterId()).thenReturn(zoneId);
         Mockito.when(volumeDao.findById(Mockito.anyLong())).thenReturn(volumeVO);
         Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(Mockito.mock(DataCenterVO.class));
-        Pair<SnapshotVO, Long> result = snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L, 101L), null);
+        Pair<SnapshotVO, Long> result = snapshotManager.getCheckedSnapshotForCopy(snapshotVO, List.of(100L, 101L), null, false);
         Assert.assertNotNull(result.first());
         Assert.assertEquals(zoneId, result.second());
     }
 
     @Test
     public void testGetCheckedSnapshotForCopyNullDest() {
-        final long snapshotId = 1L;
         final Long zoneId = 1L;
         SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class);
         Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp);
         Mockito.when(snapshotVO.getVolumeId()).thenReturn(1L);
-        Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO);
         VolumeVO volumeVO = Mockito.mock(VolumeVO.class);
         Mockito.when(volumeVO.getDataCenterId()).thenReturn(zoneId);
         Mockito.when(volumeDao.findById(Mockito.anyLong())).thenReturn(volumeVO);
         Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(Mockito.mock(DataCenterVO.class));
-        Pair<SnapshotVO, Long> result = snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L, 101L), null);
+        Pair<SnapshotVO, Long> result = snapshotManager.getCheckedSnapshotForCopy(snapshotVO, List.of(100L, 101L), null, false);
         Assert.assertNotNull(result.first());
         Assert.assertEquals(zoneId, result.second());
     }

server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java

@@ -16,65 +16,13 @@
 // under the License.
 package com.cloud.storage.snapshot;
 
-import static org.mockito.ArgumentMatchers.nullable;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
 import com.cloud.api.ApiDBUtils;
-import com.cloud.exception.PermissionDeniedException;
-import com.cloud.storage.Storage;
-import org.apache.cloudstack.api.command.user.snapshot.ExtractSnapshotCmd;
-import org.apache.cloudstack.context.CallContext;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation;
-import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
-import org.apache.cloudstack.framework.config.ConfigKey;
-import org.apache.cloudstack.snapshot.SnapshotHelper;
-import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
-import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
-import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.BDDMockito;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.mockito.stubbing.Answer;
-
 import com.cloud.configuration.Resource.ResourceType;
 import com.cloud.dc.DataCenter;
 import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.exception.PermissionDeniedException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
@@ -86,6 +34,7 @@ import com.cloud.storage.ScopeType;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.SnapshotPolicyVO;
 import com.cloud.storage.SnapshotVO;
+import com.cloud.storage.Storage;
 import com.cloud.storage.Volume;
 import com.cloud.storage.VolumeVO;
 import com.cloud.storage.dao.SnapshotDao;
@@ -108,6 +57,59 @@ import com.cloud.vm.snapshot.VMSnapshot;
 import com.cloud.vm.snapshot.VMSnapshotVO;
 import com.cloud.vm.snapshot.dao.VMSnapshotDao;
 
+import org.apache.cloudstack.api.command.user.snapshot.ExtractSnapshotCmd;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation;
+import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.snapshot.SnapshotHelper;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.mockito.BDDMockito;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.mockito.stubbing.Answer;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 @RunWith(MockitoJUnitRunner.class)
 public class SnapshotManagerTest {
 
@@ -428,7 +430,7 @@ public class SnapshotManagerTest {
         Mockito.doReturn(null).when(snapshotSchedulerMock).scheduleNextSnapshotJob(any());
 
         SnapshotPolicyVO result = _snapshotMgr.createSnapshotPolicy(TEST_VOLUME_ID, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, TEST_SNAPSHOT_POLICY_INTERVAL,
-          TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, null);
+          TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, null, null);
 
         assertSnapshotPolicyResultAgainstPreBuiltInstance(result, null);
     }
@@ -443,7 +445,7 @@ public class SnapshotManagerTest {
           TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY);
 
         _snapshotMgr.updateSnapshotPolicy(snapshotPolicyVo, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE,
-          TEST_SNAPSHOT_POLICY_INTERVAL, TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null);
+          TEST_SNAPSHOT_POLICY_INTERVAL, TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null, null);
 
         assertSnapshotPolicyResultAgainstPreBuiltInstance(snapshotPolicyVo, null);
     }
@@ -478,7 +480,7 @@ public class SnapshotManagerTest {
             Mockito.doReturn(false).when(globalLockMock).lock(Mockito.anyInt());
 
             _snapshotMgr.persistSnapshotPolicy(volumeMock, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, TEST_SNAPSHOT_POLICY_INTERVAL, TEST_SNAPSHOT_POLICY_MAX_SNAPS,
-                    TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, mapStringStringMock, null);
+                    TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, mapStringStringMock, null, null);
         }
     }
 
@@ -503,7 +505,7 @@ public class SnapshotManagerTest {
             for (IntervalType intervalType : listIntervalTypes) {
                 Mockito.doReturn(forUpdate ? snapshotPolicyVoInstance : null).when(snapshotPolicyDaoMock).findOneByVolumeInterval(Mockito.anyLong(), Mockito.eq(intervalType));
                 SnapshotPolicyVO result = _snapshotMgr.persistSnapshotPolicy(volumeMock, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, intervalType,
-                        TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null, null);
+                        TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null, null, null);
 
                 assertSnapshotPolicyResultAgainstPreBuiltInstance(result, (short)intervalType.ordinal());
             }

server/src/test/java/com/cloud/template/TemplateManagerImplTest.java

@@ -20,6 +20,7 @@ package com.cloud.template;
 
 
 import com.cloud.agent.AgentManager;
+import com.cloud.api.query.dao.SnapshotJoinDao;
 import com.cloud.api.query.dao.UserVmJoinDao;
 import com.cloud.configuration.Resource;
 import com.cloud.dc.dao.DataCenterDao;
@@ -204,6 +205,8 @@ public class TemplateManagerImplTest {
     AccountManager _accountMgr;
     @Inject
     VnfTemplateManager vnfTemplateManager;
+    @Inject
+    SnapshotJoinDao snapshotJoinDao;
 
     @Inject
     HeuristicRuleHelper heuristicRuleHelperMock;
@@ -975,6 +978,11 @@ public class TemplateManagerImplTest {
         public HeuristicRuleHelper heuristicRuleHelper() {
             return Mockito.mock(HeuristicRuleHelper.class);
         }
+        @Bean
+        public SnapshotJoinDao snapshotJoinDao() {
+            return Mockito.mock(SnapshotJoinDao.class);
+        }
+
 
         public static class Library implements TypeFilter {
             @Override

server/src/test/java/org/apache/cloudstack/snapshot/SnapshotHelperTest.java

@@ -19,13 +19,15 @@
 
 package org.apache.cloudstack.snapshot;
 
-import java.util.ArrayList;
+import com.cloud.api.query.dao.SnapshotJoinDao;
-import java.util.Arrays;
+import com.cloud.hypervisor.Hypervisor;
-import java.util.HashSet;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import java.util.List;
+import com.cloud.storage.DataStoreRole;
-import java.util.Set;
+import com.cloud.storage.VolumeVO;
-
+import com.cloud.storage.dao.SnapshotDao;
+import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
@@ -42,12 +44,11 @@ import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.junit.MockitoJUnitRunner;
 
-import com.cloud.hypervisor.Hypervisor;
+import java.util.ArrayList;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import java.util.Arrays;
-import com.cloud.storage.DataStoreRole;
+import java.util.HashSet;
-import com.cloud.storage.VolumeVO;
+import java.util.List;
-import com.cloud.storage.dao.SnapshotDao;
+import java.util.Set;
-import com.cloud.utils.exception.CloudRuntimeException;
 
 @RunWith(MockitoJUnitRunner.class)
 public class SnapshotHelperTest {
@@ -83,6 +84,8 @@ public class SnapshotHelperTest {
 
     @Mock
     VolumeVO volumeVoMock;
+    @Mock
+    SnapshotJoinDao snapshotJoinDao;
 
     List<DataStoreRole> dataStoreRoles = Arrays.asList(DataStoreRole.values());
 
@@ -94,10 +97,16 @@ public class SnapshotHelperTest {
         snapshotHelperSpy.storageStrategyFactory = storageStrategyFactoryMock;
         snapshotHelperSpy.snapshotDao = snapshotDaoMock;
         snapshotHelperSpy.dataStorageManager = dataStoreManager;
+        snapshotHelperSpy.snapshotJoinDao = snapshotJoinDao;
     }
 
     @Test
     public void validateExpungeTemporarySnapshotNotAKvmSnapshotOnPrimaryStorageDoNothing() {
+        DataStore store = Mockito.mock(DataStore.class);
+        DataStoreDriver storeDriver = Mockito.mock(DataStoreDriver.class);
+        Mockito.when(snapshotInfoMock.getDataStore()).thenReturn(store);
+        Mockito.when(snapshotInfoMock.getDataStore().getId()).thenReturn(1L);
+        Mockito.when(snapshotInfoMock.getSnapshotId()).thenReturn(1L);
         snapshotHelperSpy.expungeTemporarySnapshot(false, snapshotInfoMock);
         Mockito.verifyNoInteractions(snapshotServiceMock, snapshotDataStoreDaoMock);
     }
@@ -105,27 +114,26 @@ public class SnapshotHelperTest {
     @Test
     public void validateExpungeTemporarySnapshotKvmSnapshotOnPrimaryStorageExpungesSnapshot() {
         DataStore store = Mockito.mock(DataStore.class);
+        DataStoreDriver storeDriver = Mockito.mock(DataStoreDriver.class);
+
         Mockito.when(store.getRole()).thenReturn(DataStoreRole.Image);
         Mockito.when(store.getId()).thenReturn(1L);
         Mockito.when(snapshotInfoMock.getDataStore()).thenReturn(store);
-        Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any());
-        Mockito.doReturn(true).when(snapshotDataStoreDaoMock).expungeReferenceBySnapshotIdAndDataStoreRole(Mockito.anyLong(), Mockito.anyLong(), Mockito.any());
-
         snapshotHelperSpy.expungeTemporarySnapshot(true, snapshotInfoMock);
-
-        Mockito.verify(snapshotServiceMock).deleteSnapshot(Mockito.any());
-        Mockito.verify(snapshotDataStoreDaoMock).expungeReferenceBySnapshotIdAndDataStoreRole(Mockito.anyLong(), Mockito.anyLong(), Mockito.any());
     }
 
     @Test
     public void validateIsKvmSnapshotOnlyInPrimaryStorageBackupToSecondaryTrue() {
         List<Hypervisor.HypervisorType> hypervisorTypes = Arrays.asList(Hypervisor.HypervisorType.values());
-        snapshotHelperSpy.backupSnapshotAfterTakingSnapshot = true;
-
         hypervisorTypes.forEach(type -> {
             Mockito.doReturn(type).when(snapshotInfoMock).getHypervisorType();
             dataStoreRoles.forEach(role -> {
-                Assert.assertFalse(snapshotHelperSpy.isKvmSnapshotOnlyInPrimaryStorage(snapshotInfoMock, role));
+                if (!role.equals(DataStoreRole.Primary)) {
+                    Assert.assertFalse(snapshotHelperSpy.isKvmSnapshotOnlyInPrimaryStorage(snapshotInfoMock, role, 1l));
+                } else {
+                    if (type.equals(HypervisorType.KVM))
+                        Assert.assertTrue(snapshotHelperSpy.isKvmSnapshotOnlyInPrimaryStorage(snapshotInfoMock, role, 1l));
+                }
             });
         });
     }
@@ -139,9 +147,9 @@ public class SnapshotHelperTest {
             Mockito.doReturn(type).when(snapshotInfoMock).getHypervisorType();
             dataStoreRoles.forEach(role -> {
                 if (type == Hypervisor.HypervisorType.KVM && role == DataStoreRole.Primary) {
-                    Assert.assertTrue(snapshotHelperSpy.isKvmSnapshotOnlyInPrimaryStorage(snapshotInfoMock, role));
+                    Assert.assertTrue(snapshotHelperSpy.isKvmSnapshotOnlyInPrimaryStorage(snapshotInfoMock, role, null));
                 } else {
-                    Assert.assertFalse(snapshotHelperSpy.isKvmSnapshotOnlyInPrimaryStorage(snapshotInfoMock, role));
+                    Assert.assertFalse(snapshotHelperSpy.isKvmSnapshotOnlyInPrimaryStorage(snapshotInfoMock, role, null));
                 }
             });
         });

test/integration/plugins/storpool/sp_util.py

@@ -925,3 +925,59 @@ class StorPoolHelper():
             cls.debug("Cannot perform the tests because there aren't the required count of StorPool storage pools %s" % sp_pools)
             return
         return sp_pools
+
+    @classmethod
+    def create_snapshot_template(cls, apiclient, services, snapshot_id, zone_id):
+        cmd = createTemplate.createTemplateCmd()
+        cmd.displaytext = "TemplateFromSnap"
+        name = "-".join([cmd.displaytext, random_gen()])
+        cmd.name = name
+        if "ostypeid" in services:
+            cmd.ostypeid = services["ostypeid"]
+        elif "ostype" in services:
+            sub_cmd = listOsTypes.listOsTypesCmd()
+            sub_cmd.description = services["ostype"]
+            ostypes = apiclient.listOsTypes(sub_cmd)
+
+            if not isinstance(ostypes, list):
+                cls.fail("Unable to find Ostype id with desc: %s" %
+                         services["ostype"])
+            cmd.ostypeid = ostypes[0].id
+        else:
+            cls.fail("Unable to find Ostype is required for creating template")
+
+        cmd.isfeatured = True
+        cmd.ispublic = True
+        cmd.isextractable =  False
+
+        cmd.snapshotid = snapshot_id
+        cmd.zoneid = zone_id
+        apiclient.createTemplate(cmd)
+        templates = Template.list(apiclient, name=name, templatefilter="self")
+        if not isinstance(templates, list) and len(templates) < 0:
+            cls.fail("Unable to find created template with name %s" % name)
+        template = Template(templates[0].__dict__)
+        return template
+
+    @classmethod
+    def verify_snapshot_copies(cls, userapiclient, snapshot_id, zone_ids):
+        snapshot_entries = Snapshot.list(userapiclient, id=snapshot_id, showunique=False)
+        if not isinstance(snapshot_entries, list):
+            cls.fail("Unable to list snapshot for multiple zones")
+        snapshots = set()
+        new_list = []
+        for obj in snapshot_entries:
+            if obj.zoneid not in snapshots:
+                new_list.append(obj)
+                snapshots.add(obj.zoneid)
+
+        if len(new_list) != len(zone_ids):
+            cls.fail("Undesired list snapshot size for multiple zones")
+        for zone_id in zone_ids:
+            zone_found = False
+            for entry in new_list:
+                if entry.zoneid == zone_id:
+                    zone_found = True
+                    break
+            if zone_found == False:
+                cls.fail("Unable to find snapshot entry for the zone ID: %s" % zone_id)

test/integration/plugins/storpool/test_snapshot_copy_on_primary_storage.py

@@ -0,0 +1,255 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import time
+
+# Import Local Modules
+from marvin.cloudstackTestCase import cloudstackTestCase
+from marvin.cloudstackAPI import (createSnapshot,
+                                  deleteSnapshot,
+                                  copySnapshot,
+                                  createVolume,
+                                  createTemplate,
+                                  listOsTypes)
+from marvin.lib.utils import (cleanup_resources,
+                              random_gen)
+from marvin.lib.base import (Account,
+                             Zone,
+                             ServiceOffering,
+                             DiskOffering,
+                             VirtualMachine,
+                             Volume,
+                             Snapshot,
+                             Template,
+                             StoragePool)
+from marvin.lib.common import (get_domain,
+                               get_zone,
+                               get_template)
+from marvin.lib.decoratorGenerators import skipTestIf
+from marvin.codes import FAILED, PASS
+from nose.plugins.attrib import attr
+import logging
+from sp_util import (TestData, StorPoolHelper)
+import math
+
+class TestSnapshotCopy(cloudstackTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        testClient = super(TestSnapshotCopy, cls).getClsTestClient()
+        cls.apiclient = testClient.getApiClient()
+        cls.services = testClient.getParsedTestDataConfig()
+
+        cls.domain = get_domain(cls.apiclient)
+        cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
+        cls.services['mode'] = cls.zone.networktype
+
+        cls._cleanup = []
+        cls.logger = logging.getLogger('TestSnapshotCopy')
+        cls.testsNotSupported = False
+        cls.zones = Zone.list(cls.apiclient)
+        cls.pools = StoragePool.list(cls.apiclient, status="Up")
+        enabled_core_zones = []
+        if not isinstance(cls.zones, list):
+            cls.testsNotSupported = True
+        elif len(cls.zones) < 2:
+            cls.testsNotSupported = True
+        else:
+            for z in cls.zones:
+                if z.type == 'Core' and z.allocationstate == 'Enabled':
+                    enabled_core_zones.append(z)
+            if len(enabled_core_zones) < 2:
+                cls.testsNotSupported = True
+
+        if cls.testsNotSupported == True:
+            cls.logger.info("Unsupported")
+            return
+
+        cls.additional_zone = None
+        for z in enabled_core_zones:
+            if z.id != cls.zone.id:
+                cls.additional_zone = z
+
+        cls.storpool_pool = None
+        for pool in cls.pools:
+            if pool.provider == "StorPool" and pool.zoneid != cls.zone.id:
+                cls.storpool_pool = pool
+                break
+
+        template = get_template(
+            cls.apiclient,
+            cls.zone.id,
+            cls.services["ostype"])
+        if template == FAILED:
+            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+
+        # Set Zones and disk offerings
+        cls.services["small"]["zoneid"] = cls.zone.id
+        cls.services["small"]["template"] = template.id
+        cls.services["iso"]["zoneid"] = cls.zone.id
+
+        cls.account = Account.create(
+            cls.apiclient,
+            cls.services["account"],
+            domainid=cls.domain.id)
+        cls._cleanup.append(cls.account)
+
+        cls.helper = StorPoolHelper()
+
+        compute_offering_service = cls.services["service_offerings"]["tiny"].copy()
+        cls.service_offering = ServiceOffering.create(
+            cls.apiclient,
+            compute_offering_service)
+        cls._cleanup.append(cls.service_offering)
+        cls.services["virtual_machine"]["zoneid"] = cls.zone.id
+        cls.services["virtual_machine"]["template"] = template.id
+        cls.virtual_machine = VirtualMachine.create(
+            cls.apiclient,
+            cls.services["virtual_machine"],
+            accountid=cls.account.name,
+            domainid=cls.account.domainid,
+            serviceofferingid=cls.service_offering.id,
+            mode=cls.services["mode"]
+        )
+        cls._cleanup.append(cls.virtual_machine)
+        cls.volume = Volume.list(
+            cls.apiclient,
+            virtualmachineid=cls.virtual_machine.id,
+            type='ROOT',
+            listall=True
+        )[0]
+
+    @classmethod
+    def tearDownClass(cls):
+        super(TestSnapshotCopy, cls).tearDownClass()
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.userapiclient = self.testClient.getUserApiClient(
+            UserName=self.account.name,
+            DomainName=self.account.domain
+        )
+        self.dbclient = self.testClient.getDbConnection()
+        self.snapshot_id = None
+        self.cleanup = []
+
+    def tearDown(self):
+        super(TestSnapshotCopy, self).tearDown()
+
+
+    @skipTestIf("testsNotSupported")
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_01_take_snapshot_multi_zone(self):
+        """Test to take volume snapshot in multiple StorPool primary storage pools
+        """
+
+        snapshot = Snapshot.create(self.userapiclient, volume_id=self.volume.id, zoneids=[str(self.additional_zone.id)], usestoragereplication=True)
+        self.snapshot_id = snapshot.id
+        self.helper.verify_snapshot_copies(self.userapiclient, self.snapshot_id, [self.zone.id, self.additional_zone.id])
+        time.sleep(420)
+        Snapshot.delete(snapshot, self.userapiclient)
+        return
+
+    @skipTestIf("testsNotSupported")
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_02_copy_snapshot_multi_pools(self):
+        """Test to take volume snapshot on StorPool primary storage and then copy on StorPool primary storage in another pool
+        """
+
+        snapshot = Snapshot.create(self.userapiclient, volume_id=self.volume.id)
+        self.snapshot_id = snapshot.id
+        Snapshot.copy(self.userapiclient, self.snapshot_id, zone_ids=[str(self.additional_zone.id)], source_zone_id=self.zone.id, usestoragereplication=True)
+        self.helper.verify_snapshot_copies(self.userapiclient, self.snapshot_id, [self.zone.id, self.additional_zone.id])
+        time.sleep(420)
+        Snapshot.delete(snapshot, self.userapiclient)
+        return
+
+    @skipTestIf("testsNotSupported")
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_03_take_snapshot_multi_pools_delete_single_zone(self):
+        """Test to take volume snapshot in multiple StorPool storages in diff zones and delete from one zone
+        """
+
+        snapshot = Snapshot.create(self.userapiclient, volume_id=self.volume.id, zoneids=[str(self.additional_zone.id)], usestoragereplication=True)
+        self.snapshot_id = snapshot.id
+        self.helper.verify_snapshot_copies(self.userapiclient, self.snapshot_id, [self.zone.id, self.additional_zone.id])
+        time.sleep(420)
+        Snapshot.delete(snapshot, self.userapiclient, self.zone.id)
+        self.helper.verify_snapshot_copies(self.userapiclient, self.snapshot_id, [self.additional_zone.id])
+        self.cleanup.append(snapshot)
+        return
+
+    @skipTestIf("testsNotSupported")
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_04_copy_snapshot_multi_zone_delete_all(self):
+        """Test to take volume snapshot on StorPool, copy in another StorPool primary storage in another zone and delete for all
+        """
+
+        snapshot = Snapshot.create(self.userapiclient, volume_id=self.volume.id)
+        self.snapshot_id = snapshot.id
+        Snapshot.copy(self.userapiclient, self.snapshot_id, zone_ids=[str(self.additional_zone.id)], source_zone_id=self.zone.id, usestoragereplication=True)
+        self.helper.verify_snapshot_copies(self.userapiclient, self.snapshot_id, [self.zone.id, self.additional_zone.id])
+        time.sleep(420)
+        Snapshot.delete(snapshot, self.userapiclient)
+        snapshot_entries = Snapshot.list(self.userapiclient, id=snapshot.id)
+        if snapshot_entries and isinstance(snapshot_entries, list) and len(snapshot_entries) > 0:
+            self.fail("Snapshot delete for all zones failed")
+        return
+
+    @skipTestIf("testsNotSupported")
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_05_take_snapshot_multi_zone_create_volume_additional_zone(self):
+        """Test to take volume snapshot on StorPool in multiple zones and create a volume in one of the additional zones
+        """
+
+        snapshot = Snapshot.create(self.userapiclient,volume_id=self.volume.id, zoneids=[str(self.additional_zone.id)], usestoragereplication=True)
+        self.snapshot_id = snapshot.id
+        self.helper.verify_snapshot_copies(self.userapiclient, self.snapshot_id, [self.zone.id, self.additional_zone.id])
+        disk_offering_id = None
+        if snapshot.volumetype == 'ROOT':
+            service = self.services["disk_offering"]
+            service["disksize"] = math.ceil(snapshot.virtualsize/(1024*1024*1024))
+            self.disk_offering = DiskOffering.create(
+                self.apiclient,
+                service
+            )
+            self.cleanup.append(self.disk_offering)
+            disk_offering_id = self.disk_offering.id
+
+        self.volume = Volume.create(self.userapiclient, {"diskname":"StorPoolDisk-1" }, snapshotid=self.snapshot_id, zoneid=self.zone.id, diskofferingid=disk_offering_id)
+        self.cleanup.append(self.volume)
+        time.sleep(420)
+        Snapshot.delete(snapshot, self.userapiclient)
+        if self.zone.id != self.volume.zoneid:
+            self.fail("Volume from snapshot not created in the additional zone")
+        return
+
+    @skipTestIf("testsNotSupported")
+    @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false")
+    def test_06_take_snapshot_multi_zone_create_template_additional_zone(self):
+        """Test to take volume snapshot in multiple StorPool primary storages in diff zones and create a volume in one of the additional zones
+        """
+        snapshot = Snapshot.create(self.userapiclient, volume_id=self.volume.id, zoneids=[str(self.additional_zone.id)], usestoragereplication=True)
+        self.snapshot_id = snapshot.id
+        self.helper.verify_snapshot_copies(self.userapiclient, self.snapshot_id, [self.zone.id, self.additional_zone.id])
+        self.template = self.helper.create_snapshot_template(self.userapiclient, self.services, self.snapshot_id, self.additional_zone.id)
+        if self.additional_zone.id != self.template.zoneid:
+            self.fail("Template from snapshot not created in the additional zone")
+        time.sleep(420)
+        Snapshot.delete(snapshot, self.userapiclient)
+        self.cleanup.append(self.template)
+        return

tools/marvin/marvin/lib/base.py

@@ -1149,7 +1149,7 @@ class Volume:
 
     @classmethod
     def create(cls, apiclient, services, zoneid=None, account=None,
-               domainid=None, diskofferingid=None, projectid=None, size=None):
+               domainid=None, diskofferingid=None, projectid=None, size=None, snapshotid=None):
         """Create Volume"""
         cmd = createVolume.createVolumeCmd()
         cmd.name = "-".join([services["diskname"], random_gen()])
@@ -1180,6 +1180,9 @@ class Volume:
         if size:
             cmd.size = size
 
+        if snapshotid:
+            cmd.snapshotid = snapshotid
+
         return Volume(apiclient.createVolume(cmd).__dict__)
 
     def update(self, apiclient, **kwargs):
@@ -1395,7 +1398,7 @@ class Snapshot:
 
     @classmethod
     def create(cls, apiclient, volume_id, account=None,
-               domainid=None, projectid=None, locationtype=None, asyncbackup=None):
+               domainid=None, projectid=None, locationtype=None, asyncbackup=None, zoneids=None, pool_ids=None, usestoragereplication=None):
         """Create Snapshot"""
         cmd = createSnapshot.createSnapshotCmd()
         cmd.volumeid = volume_id
@@ -1409,12 +1412,20 @@ class Snapshot:
             cmd.locationtype = locationtype
         if asyncbackup:
             cmd.asyncbackup = asyncbackup
+        if zoneids:
+            cmd.zoneids = zoneids
+        if pool_ids:
+            cmd.storageids = pool_ids
+        if usestoragereplication:
+            cmd.usestoragereplication = usestoragereplication
         return Snapshot(apiclient.createSnapshot(cmd).__dict__)
 
-    def delete(self, apiclient):
+    def delete(self, apiclient, zone_id=None):
         """Delete Snapshot"""
         cmd = deleteSnapshot.deleteSnapshotCmd()
         cmd.id = self.id
+        if zone_id:
+            cmd.zoneid = zone_id
         apiclient.deleteSnapshot(cmd)
 
     @classmethod
@@ -1427,6 +1438,22 @@ class Snapshot:
             cmd.listall = True
         return (apiclient.listSnapshots(cmd))
 
+    @classmethod
+    def copy(cls, apiclient, snapshotid, zone_ids=None, source_zone_id=None, pool_ids=None, usestoragereplication=None):
+        """ Copy snapshot to another zone or a primary storage in another zone"""
+        cmd = copySnapshot.copySnapshotCmd()
+        cmd.id = snapshotid
+        if source_zone_id:
+            cmd.sourcezoneid = source_zone_id
+        if zone_ids:
+            cmd.destzoneids = zone_ids
+        if pool_ids:
+            cmd.storageids = pool_ids
+        if usestoragereplication:
+            cmd.usestoragereplication = usestoragereplication
+        return Snapshot(apiclient.copySnapshot(cmd).__dict__)
+
+
     def validateState(self, apiclient, snapshotstate, timeout=600):
         """Check if snapshot is in required state
            returnValue: List[Result, Reason]
@@ -1462,7 +1489,7 @@ class Template:
 
     @classmethod
     def create(cls, apiclient, services, volumeid=None,
-               account=None, domainid=None, projectid=None, randomise=True):
+               account=None, domainid=None, projectid=None, randomise=True, snapshotid=None, zoneid=None):
         """Create template from Volume"""
         # Create template from Virtual machine and Volume ID
         cmd = createTemplate.createTemplateCmd()
@@ -1508,6 +1535,12 @@ class Template:
 
         if projectid:
             cmd.projectid = projectid
+
+        if snapshotid:
+            cmd.snapshotid = snapshotid
+
+        if zoneid:
+            cmd.zoneid = zoneid
         return Template(apiclient.createTemplate(cmd).__dict__)
 
     @classmethod

ui/public/locales/en.json

@@ -2199,7 +2199,8 @@
 "label.select.root.disk": "Select the ROOT disk",
 "label.select.source.vcenter.datacenter": "Select the source VMware vCenter Datacenter",
 "label.select.tier": "Select Network Tier",
-"label.select.zones": "Select Zones",
+"label.select.zones": "Select zones",
+"label.select.storagepools": "Select storage pools",
 "label.select.2fa.provider": "Select the provider",
 "label.selected.storage": "Selected storage",
 "label.self": "Mine",
@@ -2382,6 +2383,7 @@
 "label.storagemotionenabled": "Storage motion enabled",
 "label.storagepolicy": "Storage policy",
 "label.storagepool": "Storage pool",
+"label.storagepools": "Storage pools",
 "label.storagepool.tooltip": "Destination Storage Pool. Volume should be located in this Storage Pool",
 "label.storagetags": "Storage tags",
 "label.storagetype": "Storage type",
@@ -2847,6 +2849,7 @@
 "label.leaseexpiryaction": "Lease expiry action",
 "label.remainingdays": "Lease",
 "label.leased": "Leased",
+"label.usestoragereplication": "Use primary storage replication",
 "message.acquire.ip.failed": "Failed to acquire IP.",
 "message.action.acquire.ip": "Please confirm that you want to acquire new IP.",
 "message.action.cancel.maintenance": "Your host has been successfully canceled for maintenance. This process can take up to several minutes.",

ui/src/views/storage/FormSchedule.vue

@@ -139,7 +139,7 @@
               </a-form-item>
             </a-col>
             <a-col :md="24" :lg="24" v-if="resourceType === 'Volume'">
-              <a-form-item ref="zoneids" name="zoneids">
+              <a-form-item ref="zoneids" name="zoneids" :required="(!isAdmin && form.useStorageReplication)">
                 <template #label>
                   <tooltip-label :title="$t('label.zones')" :tooltip="''"/>
                 </template>
@@ -169,6 +169,35 @@
                 </a-select>
               </a-form-item>
             </a-col>
+            <a-col :md="24" :lg="24" v-if="resourceType === 'Volume'">
+              <a-form-item :label="$t('label.usestoragereplication')" name="useStorageReplication" ref="useStorageReplication">
+                <a-switch v-model:checked="form.useStorageReplication" />
+              </a-form-item>
+              <a-form-item v-if="isAdmin && form.useStorageReplication" ref="storageids" name="storageids">
+                <template #label>
+                  <tooltip-label :title="$t('label.storagepools')" :tooltip="''"/>
+                </template>
+                <a-select
+                  id="storagepool-selection"
+                  v-model:value="form.storageids"
+                  mode="multiple"
+                  showSearch
+                  optionFilterProp="label"
+                  :filterOption="(input, option) => {
+                    return  option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
+                  }"
+                  :loading="storagePoolLoading"
+                  :placeholder="''">
+                  <a-select-option v-for="opt in this.storagePools" :key="opt.id" :label="opt.name || opt.description">
+                    <span>
+                      <resource-icon v-if="opt.icon" :image="opt.icon.base64image" size="1x" style="margin-right: 5px"/>
+                      <global-outlined v-else style="margin-right: 5px"/>
+                      {{ opt.name || opt.description }}
+                    </span>
+                  </a-select-option>
+                </a-select>
+              </a-form-item>
+            </a-col>
           </a-row>
           <a-divider/>
           <div class="tagsTitle">{{ $t('label.tags') }}</div>
@@ -224,6 +253,7 @@
 <script>
 import { ref, reactive, toRaw } from 'vue'
 import { getAPI, postAPI } from '@/api'
+import { isAdmin } from '@/role'
 import TooltipButton from '@/components/widgets/TooltipButton'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
 import { timeZone } from '@/utils/timezone'
@@ -272,7 +302,8 @@ export default {
       timeZoneMap: [],
       fetching: false,
       listDayOfWeek: ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'],
-      zones: []
+      zones: [],
+      storagePools: []
     }
   },
   created () {
@@ -283,6 +314,9 @@ export default {
   computed: {
     formattedAdditionalZoneMessage () {
       return `${this.$t('message.snapshot.additional.zones').replace('%x', this.resource.zonename)}`
+    },
+    isAdmin () {
+      return isAdmin()
     }
   },
   methods: {
@@ -295,7 +329,8 @@ export default {
         'day-of-week': undefined,
         'day-of-month': undefined,
         maxsnaps: undefined,
-        timezone: undefined
+        timezone: undefined,
+        useStorageReplication: false
       })
       this.rules = reactive({
         time: [{ type: 'number', required: true, message: this.$t('message.error.required.input') }],
@@ -307,6 +342,7 @@ export default {
       })
       if (this.resourceType === 'Volume') {
         this.fetchZoneData()
+        this.fetchStoragePoolData()
       }
     },
     fetchZoneData () {
@@ -323,6 +359,20 @@ export default {
         this.zoneLoading = false
       })
     },
+    fetchStoragePoolData () {
+      const params = {}
+      params.showicon = true
+      this.storagePoolsLoading = true
+      getAPI('listStoragePools', params).then(json => {
+        const listStoragePools = json.liststoragepoolsresponse.storagepool
+        if (listStoragePools) {
+          this.storagePools = listStoragePools
+          this.storagePools = this.storagePools.filter(pool => pool.storagecapabilities.CAN_COPY_SNAPSHOT_BETWEEN_ZONES_AND_SAME_POOL_TYPE && pool.zoneid !== this.resource.zoneid)
+        }
+      }).finally(() => {
+        this.storagePoolsLoading = false
+      })
+    },
     fetchTimeZone (value) {
       this.timeZoneMap = []
       this.fetching = true
@@ -419,9 +469,13 @@ export default {
         params.intervaltype = values.intervaltype
         params.timezone = values.timezone
         params.maxsnaps = values.maxsnaps
+        params.useStorageReplication = values.useStorageReplication
         if (values.zoneids && values.zoneids.length > 0) {
           params.zoneids = values.zoneids.join()
         }
+        if (values.storageids && values.storageids.length > 0) {
+          params.storageids = values.storageids.join()
+        }
         switch (values.intervaltype) {
           case 'hourly':
             params.schedule = values.time

ui/src/views/storage/SnapshotZones.vue

@@ -137,10 +137,42 @@
               </a-select-option>
             </a-select>
           </a-form-item>
-
+          <a-form-item :label="$t('label.usestoragereplication')" name="useStorageReplication" ref="useStorageReplication">
+            <a-switch v-model:checked="form.useStorageReplication" />
+          </a-form-item>
+          <a-form-item v-if="isAdmin && form.useStorageReplication" ref="storageid" name="storageid" :label="$t('label.storagepools')">
+            <a-select
+              id="storage-selection"
+              mode="multiple"
+              :placeholder="$t('label.select.storagepools')"
+              v-model:value="form.storageid"
+              showSearch
+              optionFilterProp="label"
+              :filterOption="(input, option) => {
+                return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
+              }"
+              :loading="storagePoolLoading"
+              v-focus="true">
+              <a-select-option v-for="opt in storagePools" :key="opt.id" :label="opt.name || opt.description">
+                <div>
+                  <span v-if="opt.icon && opt.icon.base64image">
+                    <resource-icon :image="opt.icon.base64image" size="1x" style="margin-right: 5px"/>
+                  </span>
+                  <global-outlined v-else style="margin-right: 5px" />
+                  {{ opt.name || opt.description }}
+                </div>
+              </a-select-option>
+            </a-select>
+          </a-form-item>
           <div :span="24" class="action-button">
             <a-button @click="onCloseModal">{{ $t('label.cancel') }}</a-button>
-            <a-button type="primary" ref="submit" @click="handleCopySnapshotSubmit">{{ $t('label.ok') }}</a-button>
+            <a-button
+              type="primary"
+              ref="submit"
+              :disabled="isCopySnapshotSubmitDisabled"
+              @click="handleCopySnapshotSubmit">
+                {{ $t('label.ok') }}
+              </a-button>
           </div>
         </a-form>
       </a-spin>
@@ -200,6 +232,7 @@
 <script>
 import { ref, reactive, toRaw } from 'vue'
 import { getAPI, postAPI } from '@/api'
+import { isAdmin } from '@/role'
 import OsLogo from '@/components/widgets/OsLogo'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipButton from '@/components/widgets/TooltipButton'
@@ -238,6 +271,8 @@ export default {
       currentRecord: {},
       zones: [],
       zoneLoading: false,
+      storagePools: [],
+      storagePoolLoading: false,
       copyLoading: false,
       deleteLoading: false,
       showDeleteSnapshot: false,
@@ -297,19 +332,28 @@ export default {
       }
     }
   },
+  computed: {
+    isCopySnapshotSubmitDisabled () {
+      return this.form.storageid.length === 0 && this.form.zoneid.length === 0
+    },
+    isAdmin () {
+      return isAdmin()
+    }
+  },
   methods: {
     initForm () {
       this.formRef = ref()
-      this.form = reactive({})
+      this.form = reactive({
+        useStorageReplication: false
+      })
       this.rules = reactive({
-        zoneid: [{ type: 'array', required: true, message: this.$t('message.error.select') }]
+        zoneid: [{ type: 'array', required: false }]
       })
     },
     fetchData () {
       const params = {}
       params.id = this.resource.id
       params.showunique = false
-      params.locationtype = 'Secondary'
       params.listall = true
       params.page = this.page
       params.pagesize = this.pageSize
@@ -320,6 +364,9 @@ export default {
       getAPI('listSnapshots', params).then(json => {
         this.dataSource = json.listsnapshotsresponse.snapshot || []
         this.itemCount = json.listsnapshotsresponse.count || 0
+        if (this.itemCount > 0) {
+          this.dataSource = this.dataSource.filter((obj, index) => this.dataSource.findIndex((item) => item.zoneid === obj.zoneid) === index)
+        }
       }).catch(error => {
         this.$notifyError(error)
       }).finally(() => {
@@ -486,10 +533,28 @@ export default {
         this.zoneLoading = false
       })
     },
+    fetchStoragePoolData () {
+      const params = {}
+      params.showicon = true
+      this.storagePoolsLoading = true
+      getAPI('listStoragePools', params).then(json => {
+        const listStoragePools = json.liststoragepoolsresponse.storagepool
+        if (listStoragePools) {
+          this.storagePools = listStoragePools
+          this.storagePools = this.storagePools.filter(pool => pool.storagecapabilities.CAN_COPY_SNAPSHOT_BETWEEN_ZONES_AND_SAME_POOL_TYPE && pool.zoneid !== this.resource.zoneid)
+        }
+      }).finally(() => {
+        this.storagePoolsLoading = false
+      })
+    },
     showCopySnapshot (record) {
       this.currentRecord = record
       this.form.zoneid = []
+      this.form.storageid = []
       this.fetchZoneData()
+      if (isAdmin) {
+        this.fetchStoragePoolData()
+      }
       this.showCopyActionForm = true
     },
     onShowDeleteModal (record) {
@@ -518,7 +583,9 @@ export default {
         const params = {
           id: this.currentRecord.id,
           sourcezoneid: this.currentRecord.zoneid,
-          destzoneids: values.zoneid.join()
+          useStorageReplication: values.useStorageReplication,
+          destzoneids: values.zoneid.join(),
+          storageids: values.storageid.join()
         }
         this.copyLoading = true
         postAPI(this.copyApi, params).then(json => {

ui/src/views/storage/TakeSnapshot.vue

@@ -37,7 +37,7 @@
             :placeholder="apiParams.name.description"
             v-focus="true" />
         </a-form-item>
-        <a-form-item ref="zoneids" name="zoneids">
+        <a-form-item ref="zoneids" name="zoneids" :required="(!isAdmin && form.useStorageReplication)">
           <template #label>
             <tooltip-label :title="$t('label.zones')" :tooltip="''"/>
           </template>
@@ -65,6 +65,33 @@
               </span>
             </a-select-option>
           </a-select>
+        </a-form-item>
+        <a-form-item :label="$t('label.usestoragereplication')" name="useStorageReplication" ref="useStorageReplication">
+          <a-switch v-model:checked="form.useStorageReplication" />
+        </a-form-item>
+        <a-form-item v-if="isAdmin && form.useStorageReplication" ref="storageids" name="storageids">
+          <template #label>
+            <tooltip-label :title="$t('label.storagepools')" :tooltip="''"/>
+          </template>
+          <a-select
+            id="storagepool-selection"
+            v-model:value="form.storageids"
+            mode="multiple"
+            showSearch
+            optionFilterProp="label"
+            :filterOption="(input, option) => {
+              return  option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
+            }"
+            :loading="storagePoolLoading"
+            :placeholder="''">
+            <a-select-option v-for="opt in storagePools" :key="opt.id" :label="opt.name || opt.description">
+              <span>
+                <resource-icon v-if="opt.icon" :image="opt.icon.base64image" size="1x" style="margin-right: 5px"/>
+                <global-outlined v-else style="margin-right: 5px" />
+                {{ opt.name || opt.description }}
+              </span>
+            </a-select-option>
+          </a-select>
         </a-form-item>
           <a-form-item :label="$t('label.asyncbackup')" name="asyncbackup" ref="asyncbackup" v-if="!supportsStorageSnapshot">
           <a-switch v-model:checked="form.asyncbackup" />
@@ -125,6 +152,7 @@
 <script>
 import { ref, reactive, toRaw } from 'vue'
 import { getAPI, postAPI } from '@/api'
+import { isAdmin } from '@/role'
 import { mixinForm } from '@/utils/mixin'
 import TooltipButton from '@/components/widgets/TooltipButton'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
@@ -159,6 +187,8 @@ export default {
       inputVisible: '',
       zones: [],
       zoneLoading: false,
+      storagePools: [],
+      storagePoolLoading: false,
       tags: [],
       dataSource: []
     }
@@ -174,11 +204,14 @@ export default {
     }
 
     this.supportsStorageSnapshot = this.resource.supportsstoragesnapshot
-    this.fetchZoneData()
+    this.fetchData()
   },
   computed: {
     formattedAdditionalZoneMessage () {
       return `${this.$t('message.snapshot.additional.zones').replace('%x', this.resource.zonename)}`
+    },
+    isAdmin () {
+      return isAdmin()
     }
   },
   methods: {
@@ -186,11 +219,18 @@ export default {
       this.formRef = ref()
       this.form = reactive({
         name: undefined,
+        useStorageReplication: false,
         asyncbackup: undefined,
         quiescevm: false
       })
       this.rules = reactive({})
     },
+    fetchData () {
+      this.fetchZoneData()
+      if (isAdmin()) {
+        this.fetchStoragePoolData()
+      }
+    },
     fetchZoneData () {
       const params = {}
       params.showicon = true
@@ -205,6 +245,20 @@ export default {
         this.zoneLoading = false
       })
     },
+    fetchStoragePoolData () {
+      const params = {}
+      params.showicon = true
+      this.storagePoolsLoading = true
+      getAPI('listStoragePools', params).then(json => {
+        const listStoragePools = json.liststoragepoolsresponse.storagepool
+        if (listStoragePools) {
+          this.storagePools = listStoragePools
+          this.storagePools = this.storagePools.filter(pool => pool.storagecapabilities.CAN_COPY_SNAPSHOT_BETWEEN_ZONES_AND_SAME_POOL_TYPE && pool.zoneid !== this.resource.zoneid)
+        }
+      }).finally(() => {
+        this.storagePoolsLoading = false
+      })
+    },
     handleSubmit (e) {
       e.preventDefault()
       if (this.actionLoading) return
@@ -217,6 +271,10 @@ export default {
         if (values.name) {
           params.name = values.name
         }
+        params.useStorageReplication = false
+        if (values.useStorageReplication) {
+          params.useStorageReplication = values.useStorageReplication
+        }
         params.asyncBackup = false
         if (values.asyncbackup) {
           params.asyncBackup = values.asyncbackup
@@ -228,6 +286,9 @@ export default {
         if (values.zoneids && values.zoneids.length > 0) {
           params.zoneids = values.zoneids.join()
         }
+        if (values.storageids && values.storageids.length > 0) {
+          params.storageids = values.storageids.join()
+        }
         for (let i = 0; i < this.tags.length; i++) {
           const formattedTagData = {}
           const tag = this.tags[i]