support for data migration of incremental snaps on xen (#4395)

* support for handling incremental snaps (on DB entries) on xen

* Addressed comments

* Update NfsSecondaryStorageResource.java

adjusted space in comment/ log

Co-authored-by: Pearl Dsilva <pearl.dsilva@shapeblue.com>

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java

@@ -36,6 +36,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.storage.ImageStoreService;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
@@ -47,6 +48,7 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
 import com.cloud.host.HostVO;
 import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.SnapshotVO;
 import com.cloud.storage.VMTemplateVO;
@@ -79,7 +81,6 @@ public class DataMigrationUtility {
     HostDao hostDao;
     @Inject
     SnapshotDao snapshotDao;
-
     /**
      *  This function verifies if the given image store contains data objects that are not in any of the following states:
      *  "Ready" "Allocated", "Destroying", "Destroyed", "Failed". If this is the case, and if the migration policy is complete,
@@ -115,7 +116,7 @@ public class DataMigrationUtility {
     protected Long getFileSize(DataObject file, Map<DataObject, Pair<List<SnapshotInfo>, Long>> snapshotChain) {
         Long size = file.getSize();
         Pair<List<SnapshotInfo>, Long> chain = snapshotChain.get(file);
-        if (file instanceof SnapshotInfo && chain.first() != null) {
+        if (file instanceof SnapshotInfo && chain.first() != null && !chain.first().isEmpty()) {
             size = chain.second();
         }
         return size;
@@ -178,7 +179,8 @@ public class DataMigrationUtility {
         List<TemplateDataStoreVO> templates = templateDataStoreDao.listByStoreId(srcDataStore.getId());
         for (TemplateDataStoreVO template : templates) {
             VMTemplateVO templateVO = templateDao.findById(template.getTemplateId());
-            if (template.getState() == ObjectInDataStoreStateMachine.State.Ready && !templateVO.isPublicTemplate()) {
+            if (template.getState() == ObjectInDataStoreStateMachine.State.Ready && templateVO != null && !templateVO.isPublicTemplate() &&
+                    templateVO.getHypervisorType() != Hypervisor.HypervisorType.Simulator) {
                 files.add(templateFactory.getTemplate(template.getTemplateId(), srcDataStore));
             }
         }
@@ -194,7 +196,9 @@ public class DataMigrationUtility {
         List<SnapshotDataStoreVO> snapshots = snapshotDataStoreDao.listByStoreId(srcDataStore.getId(), DataStoreRole.Image);
         for (SnapshotDataStoreVO snapshot : snapshots) {
             SnapshotVO snapshotVO = snapshotDao.findById(snapshot.getSnapshotId());
-            if (snapshot.getState() == ObjectInDataStoreStateMachine.State.Ready && snapshot.getParentSnapshotId() == 0 ) {
+            if (snapshot.getState() == ObjectInDataStoreStateMachine.State.Ready &&
+                    snapshotVO != null && snapshotVO.getHypervisorType() != Hypervisor.HypervisorType.Simulator
+                    && snapshot.getParentSnapshotId() == 0 ) {
                 SnapshotInfo snap = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), DataStoreRole.Image);
                 files.add(snap);
             }
@@ -230,7 +234,10 @@ public class DataMigrationUtility {
         List<VolumeDataStoreVO> volumes = volumeDataStoreDao.listByStoreId(srcDataStore.getId());
         for (VolumeDataStoreVO volume : volumes) {
             if (volume.getState() == ObjectInDataStoreStateMachine.State.Ready) {
-                files.add(volumeFactory.getVolume(volume.getVolumeId(), srcDataStore));
+                VolumeInfo volumeInfo = volumeFactory.getVolume(volume.getVolumeId(), srcDataStore);
+                if (volumeInfo != null && volumeInfo.getHypervisorType() != Hypervisor.HypervisorType.Simulator) {
+                    files.add(volumeInfo);
+                }
             }
         }
         return files;

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java

@@ -393,7 +393,6 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra
             if (meanStdDevCurrent > threshold && storageCapacityBelowThreshold(storageCapacities, destDatastoreId)) {
                 return true;
             }
-            return  true;
         } else {
             if (storageCapacityBelowThreshold(storageCapacities, destDatastoreId)) {
                 return true;
@@ -404,7 +403,8 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra
 
     private boolean storageCapacityBelowThreshold(Map<Long, Pair<Long, Long>> storageCapacities, Long destStoreId) {
         Pair<Long, Long> imageStoreCapacity = storageCapacities.get(destStoreId);
-        if (imageStoreCapacity != null && (imageStoreCapacity.first() / (imageStoreCapacity.second() * 1.0)) <= imageStoreCapacityThreshold) {
+        long usedCapacity = imageStoreCapacity.second() - imageStoreCapacity.first();
+        if (imageStoreCapacity != null && (usedCapacity / (imageStoreCapacity.second() * 1.0)) <= imageStoreCapacityThreshold) {
             s_logger.debug("image store: " + destStoreId + " has sufficient capacity to proceed with migration of file");
             return true;
         }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java

@@ -291,4 +291,8 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
     public void setVolumeId(Long volumeId) {
         this.volumeId = volumeId;
     }
+
+    public void setCreated(Date created) {
+        this.created = created;
+    }
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java

@@ -402,4 +402,7 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
         this.extractUrlCreated = extractUrlCreated;
     }
 
+    public void setCreated(Date created) {
+        this.created = created;
+    }
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java

@@ -381,4 +381,8 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
     public void setExtractUrlCreated(Date extractUrlCreated) {
         this.extractUrlCreated = extractUrlCreated;
     }
+
+    public void setCreated(Date created) {
+        this.created = created;
+    }
 }

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java

@@ -17,6 +17,7 @@
 
 package org.apache.cloudstack.storage.image;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
@@ -88,11 +89,29 @@ public class SecondaryStorageServiceImpl implements SecondaryStorageService {
         DataObject destDataObject = null;
         try {
             if (srcDataObject instanceof SnapshotInfo && snapshotChain != null && snapshotChain.containsKey(srcDataObject)) {
+                List<String> parentSnapshotPaths = new ArrayList<>();
                 for (SnapshotInfo snapshotInfo : snapshotChain.get(srcDataObject).first()) {
+                    if (!parentSnapshotPaths.isEmpty() && parentSnapshotPaths.contains(snapshotInfo.getPath())) {
+                        parentSnapshotPaths.add(snapshotInfo.getPath());
+                        SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, srcDatastore.getId(), snapshotInfo.getSnapshotId());
+                        if (snapshotStore == null) {
+                            res.setResult("Failed to find snapshot " + snapshotInfo.getUuid() + " on store: " + srcDatastore.getName());
+                            res.setSuccess(false);
+                            future.complete(res);
+                            break;
+                        }
+                        snapshotStore.setDataStoreId(destDatastore.getId());
+                        snapshotStoreDao.update(snapshotStore.getId(), snapshotStore);
+                        continue;
+                    }
+                    parentSnapshotPaths.add(snapshotInfo.getPath());
                     destDataObject = destDatastore.create(snapshotInfo);
                     snapshotInfo.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested);
                     destDataObject.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested);
                     migrateJob(future, snapshotInfo, destDataObject, destDatastore);
+                    if (future.get() != null && future.get().isFailed()) {
+                        break;
+                    }
                 }
             } else {
                 // Check if template in destination store, if yes, do not proceed
@@ -163,26 +182,13 @@ public class SecondaryStorageServiceImpl implements SecondaryStorageService {
                 if (destData != null) {
                     destData.getDataStore().delete(destData);
                 }
-
             } else {
                 if (destData instanceof  VolumeInfo) {
                     ((VolumeInfo) destData).processEventOnly(ObjectInDataStoreStateMachine.Event.OperationSuccessed, answer);
                 } else {
                     destData.processEvent(ObjectInDataStoreStateMachine.Event.OperationSuccessed, answer);
                 }
-                if (destData instanceof SnapshotInfo) {
+                updateDataObject(srcData, destData);
-                    SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySourceSnapshot(srcData.getId(), DataStoreRole.Image);
-                    SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findBySnapshot(srcData.getId(), DataStoreRole.Image);
-                    destSnapshotStore.setPhysicalSize(snapshotStore.getPhysicalSize());
-                    snapshotStoreDao.update(destSnapshotStore.getId(), destSnapshotStore);
-                }
-
-                if (destData instanceof VolumeInfo) {
-                    VolumeDataStoreVO srcVolume = volumeDataStoreDao.findByStoreVolume(srcData.getDataStore().getId(), srcData.getId());
-                    VolumeDataStoreVO destVolume = volumeDataStoreDao.findByStoreVolume(destData.getDataStore().getId(), destData.getId());
-                    destVolume.setPhysicalSize(srcVolume.getPhysicalSize());
-                    volumeDataStoreDao.update(destVolume.getId(), destVolume);
-                }
                 s_logger.debug("Deleting source data");
                 srcData.getDataStore().delete(srcData);
                 s_logger.debug("Successfully migrated "+srcData.getUuid());
@@ -198,6 +204,37 @@ public class SecondaryStorageServiceImpl implements SecondaryStorageService {
         return null;
     }
 
+    private void updateDataObject(DataObject srcData, DataObject destData) {
+        if (destData instanceof SnapshotInfo) {
+            SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySourceSnapshot(srcData.getId(), DataStoreRole.Image);
+            SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findBySnapshot(srcData.getId(), DataStoreRole.Image);
+            if (snapshotStore != null && destSnapshotStore != null) {
+                destSnapshotStore.setPhysicalSize(snapshotStore.getPhysicalSize());
+                destSnapshotStore.setCreated(snapshotStore.getCreated());
+                if (snapshotStore.getParentSnapshotId() != destSnapshotStore.getParentSnapshotId()) {
+                    destSnapshotStore.setParentSnapshotId(snapshotStore.getParentSnapshotId());
+                }
+                snapshotStoreDao.update(destSnapshotStore.getId(), destSnapshotStore);
+            }
+        } else if (destData instanceof VolumeInfo) {
+            VolumeDataStoreVO srcVolume = volumeDataStoreDao.findByStoreVolume(srcData.getDataStore().getId(), srcData.getId());
+            VolumeDataStoreVO destVolume = volumeDataStoreDao.findByStoreVolume(destData.getDataStore().getId(), destData.getId());
+            if (srcVolume != null && destVolume != null) {
+                destVolume.setPhysicalSize(srcVolume.getPhysicalSize());
+                destVolume.setCreated(srcVolume.getCreated());
+                volumeDataStoreDao.update(destVolume.getId(), destVolume);
+            }
+        } else if (destData instanceof TemplateInfo) {
+            TemplateDataStoreVO srcTemplate = templateStoreDao.findByStoreTemplate(srcData.getDataStore().getId(), srcData.getId());
+            TemplateDataStoreVO destTemplate = templateStoreDao.findByStoreTemplate(destData.getDataStore().getId(), destData.getId());
+            if (srcTemplate != null && destTemplate != null) {
+                destTemplate.setCreated(srcTemplate.getCreated());
+                templateStoreDao.update(destTemplate.getId(), destTemplate);
+            }
+        } else {
+            s_logger.debug("Unsupported data object type");
+        }
+    }
 }
 
 

engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java

@@ -391,17 +391,18 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
     private Answer sendToLeastBusyEndpoint(List<EndPoint> eps, CopyCommand cmd) {
         Answer answer = null;
         EndPoint endPoint = null;
-        Long epId = ssvmWithLeastMigrateJobs();
+        List<Long> epIds = ssvmWithLeastMigrateJobs();
-        if (epId == null) {
+
+        if (epIds.isEmpty()) {
             Collections.shuffle(eps);
             endPoint = eps.get(0);
         } else {
-            List<EndPoint> remainingEps = eps.stream().filter(ep -> ep.getId() != epId ).collect(Collectors.toList());
+            List<EndPoint> remainingEps = eps.stream().filter(ep -> !epIds.contains(ep.getId())).collect(Collectors.toList());
             if (!remainingEps.isEmpty()) {
                 Collections.shuffle(remainingEps);
                 endPoint = remainingEps.get(0);
             } else {
-                endPoint = _defaultEpSelector.getEndPointFromHostId(epId);
+                endPoint = _defaultEpSelector.getEndPointFromHostId(epIds.get(0));
             }
         }
         CommandExecLogVO execLog = new CommandExecLogVO(endPoint.getId(), _secStorageVmDao.findByInstanceName(hostDao.findById(endPoint.getId()).getName()).getId(), "DataMigrationCommand", 1);
@@ -495,27 +496,22 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
         return null;
     }
 
-    private Integer getCopyCmdsCountToSpecificSSVM(Long ssvmId) {
+    private List<Long> ssvmWithLeastMigrateJobs() {
-        return _cmdExecLogDao.getCopyCmdCountForSSVM(ssvmId);
-    }
-
-    private Long ssvmWithLeastMigrateJobs() {
         s_logger.debug("Picking ssvm from the pool with least commands running on it");
-        String query = "select host_id, count(*) from cmd_exec_log group by host_id order by 2 limit 1;";
+        String query = "select host_id, count(*) from cmd_exec_log group by host_id order by 2;";
         TransactionLegacy txn = TransactionLegacy.currentTxn();
 
-        Long epId = null;
+        List<Long> result = new ArrayList<Long>();
         PreparedStatement pstmt = null;
         try {
             pstmt = txn.prepareAutoCloseStatement(query);
             ResultSet rs = pstmt.executeQuery();
-            if (rs.getFetchSize() > 0) {
+            while (rs.next()) {
-                rs.absolute(1);
+                result.add((long) rs.getInt(1));
-                epId = (long) rs.getInt(1);
             }
         } catch (SQLException e) {
             s_logger.debug("SQLException caught", e);
         }
-        return epId;
+        return result;
     }
 }

services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/PremiumSecondaryStorageManagerImpl.java

@@ -191,7 +191,7 @@ public class PremiumSecondaryStorageManagerImpl extends SecondaryStorageManagerI
     private void scaleDownSSVMOnLoad(List<SecondaryStorageVmVO> alreadyRunning, List<CommandExecLogVO> activeCmds,
                                List<CommandExecLogVO> copyCmdsInPipeline)  {
         int halfLimit = Math.round((float) (alreadyRunning.size() * migrateCapPerSSVM) / 2);
-        if ((copyCmdsInPipeline.size() < halfLimit && alreadyRunning.size() * _capacityPerSSVM - activeCmds.size() > (_standbyCapacity + 5)) && alreadyRunning.size() > 1) {
+        if (alreadyRunning.size() > 1 && ( copyCmdsInPipeline.size() < halfLimit && (activeCmds.size() < (((alreadyRunning.size() -1) * _capacityPerSSVM)/2)) )) {
             Collections.reverse(alreadyRunning);
             for(SecondaryStorageVmVO vm : alreadyRunning) {
                 long count = activeCmds.stream().filter(cmd -> cmd.getInstanceId() == vm.getId()).count();

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java

@@ -1300,15 +1300,17 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         try {
             File srcFile = new File(getDir(srcStore.getUrl(), _nfsVersion), srcData.getPath());
             File destFile = new File(getDir(destStore.getUrl(), _nfsVersion), destData.getPath());
-            ImageFormat format = getTemplateFormat(srcFile.getName());
 
             if (srcFile == null) {
-                return new CopyCmdAnswer("Can't find src file:" + srcFile);
+                return new CopyCmdAnswer("Can't find source file at path: "+ srcData.getPath() +" on datastore: "+ srcDataStore.getUuid() +" to initiate file transfer");
             }
+            ImageFormat format = getTemplateFormat(srcFile.getName());
             if (srcData instanceof TemplateObjectTO || srcData instanceof VolumeObjectTO) {
                 File srcDir = null;
                 if (srcFile.isFile() || srcFile.getName().contains(".")) {
                     srcDir = new File(srcFile.getParent());
+                } else if (!srcFile.isDirectory()) {
+                    srcDir = new File(srcFile.getParent());
                 }
                 File destDir = null;
                 if (destFile.isFile()) {
@@ -1351,7 +1353,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
                 if (srcFile.isFile()) {
                     newVol.setPath(destData.getPath() + File.separator + srcFile.getName());
                 } else {
-                    newVol.setPath(destData.getPath());
+                    newVol.setPath(srcData.getPath());
                 }
                 newVol.setSize(getVirtualSize(srcFile, format));
                 retObj = newVol;