From 0d487fc8c93037dc8625e7d811a0fb458825e9c3 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Sun, 18 Oct 2020 02:15:10 +0530 Subject: [PATCH] support for data migration of incremental snaps on xen (#4395) * support for handling incremental snaps (on DB entries) on xen * Addressed comments * Update NfsSecondaryStorageResource.java adjusted space in comment/ log Co-authored-by: Pearl Dsilva --- .../orchestration/DataMigrationUtility.java | 17 +++-- .../orchestration/StorageOrchestrator.java | 4 +- .../datastore/db/SnapshotDataStoreVO.java | 4 ++ .../datastore/db/TemplateDataStoreVO.java | 3 + .../datastore/db/VolumeDataStoreVO.java | 4 ++ .../image/SecondaryStorageServiceImpl.java | 65 +++++++++++++++---- .../image/BaseImageStoreDriverImpl.java | 26 ++++---- .../PremiumSecondaryStorageManagerImpl.java | 2 +- .../resource/NfsSecondaryStorageResource.java | 8 ++- 9 files changed, 93 insertions(+), 40 deletions(-) diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java index 86b7350d138..dd451f59465 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java @@ -36,6 +36,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.storage.ImageStoreService; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; @@ -47,6 +48,7 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; import com.cloud.storage.SnapshotVO; import com.cloud.storage.VMTemplateVO; @@ -79,7 +81,6 @@ public class DataMigrationUtility { HostDao hostDao; @Inject SnapshotDao snapshotDao; - /** * This function verifies if the given image store contains data objects that are not in any of the following states: * "Ready" "Allocated", "Destroying", "Destroyed", "Failed". If this is the case, and if the migration policy is complete, @@ -115,7 +116,7 @@ public class DataMigrationUtility { protected Long getFileSize(DataObject file, Map, Long>> snapshotChain) { Long size = file.getSize(); Pair, Long> chain = snapshotChain.get(file); - if (file instanceof SnapshotInfo && chain.first() != null) { + if (file instanceof SnapshotInfo && chain.first() != null && !chain.first().isEmpty()) { size = chain.second(); } return size; @@ -178,7 +179,8 @@ public class DataMigrationUtility { List templates = templateDataStoreDao.listByStoreId(srcDataStore.getId()); for (TemplateDataStoreVO template : templates) { VMTemplateVO templateVO = templateDao.findById(template.getTemplateId()); - if (template.getState() == ObjectInDataStoreStateMachine.State.Ready && !templateVO.isPublicTemplate()) { + if (template.getState() == ObjectInDataStoreStateMachine.State.Ready && templateVO != null && !templateVO.isPublicTemplate() && + templateVO.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { files.add(templateFactory.getTemplate(template.getTemplateId(), srcDataStore)); } } @@ -194,7 +196,9 @@ public class DataMigrationUtility { List snapshots = snapshotDataStoreDao.listByStoreId(srcDataStore.getId(), DataStoreRole.Image); for (SnapshotDataStoreVO snapshot : snapshots) { SnapshotVO snapshotVO = snapshotDao.findById(snapshot.getSnapshotId()); - if (snapshot.getState() == ObjectInDataStoreStateMachine.State.Ready && snapshot.getParentSnapshotId() == 0 ) { + if (snapshot.getState() == ObjectInDataStoreStateMachine.State.Ready && + snapshotVO != null && snapshotVO.getHypervisorType() != Hypervisor.HypervisorType.Simulator + && snapshot.getParentSnapshotId() == 0 ) { SnapshotInfo snap = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), DataStoreRole.Image); files.add(snap); } @@ -230,7 +234,10 @@ public class DataMigrationUtility { List volumes = volumeDataStoreDao.listByStoreId(srcDataStore.getId()); for (VolumeDataStoreVO volume : volumes) { if (volume.getState() == ObjectInDataStoreStateMachine.State.Ready) { - files.add(volumeFactory.getVolume(volume.getVolumeId(), srcDataStore)); + VolumeInfo volumeInfo = volumeFactory.getVolume(volume.getVolumeId(), srcDataStore); + if (volumeInfo != null && volumeInfo.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { + files.add(volumeInfo); + } } } return files; diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index 85b182a3dc2..ab53fd46488 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -393,7 +393,6 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra if (meanStdDevCurrent > threshold && storageCapacityBelowThreshold(storageCapacities, destDatastoreId)) { return true; } - return true; } else { if (storageCapacityBelowThreshold(storageCapacities, destDatastoreId)) { return true; @@ -404,7 +403,8 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra private boolean storageCapacityBelowThreshold(Map> storageCapacities, Long destStoreId) { Pair imageStoreCapacity = storageCapacities.get(destStoreId); - if (imageStoreCapacity != null && (imageStoreCapacity.first() / (imageStoreCapacity.second() * 1.0)) <= imageStoreCapacityThreshold) { + long usedCapacity = imageStoreCapacity.second() - imageStoreCapacity.first(); + if (imageStoreCapacity != null && (usedCapacity / (imageStoreCapacity.second() * 1.0)) <= imageStoreCapacityThreshold) { s_logger.debug("image store: " + destStoreId + " has sufficient capacity to proceed with migration of file"); return true; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java index 75b1d75dd6a..f36216911b0 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java @@ -291,4 +291,8 @@ public class SnapshotDataStoreVO implements StateObject parentSnapshotPaths = new ArrayList<>(); for (SnapshotInfo snapshotInfo : snapshotChain.get(srcDataObject).first()) { + if (!parentSnapshotPaths.isEmpty() && parentSnapshotPaths.contains(snapshotInfo.getPath())) { + parentSnapshotPaths.add(snapshotInfo.getPath()); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, srcDatastore.getId(), snapshotInfo.getSnapshotId()); + if (snapshotStore == null) { + res.setResult("Failed to find snapshot " + snapshotInfo.getUuid() + " on store: " + srcDatastore.getName()); + res.setSuccess(false); + future.complete(res); + break; + } + snapshotStore.setDataStoreId(destDatastore.getId()); + snapshotStoreDao.update(snapshotStore.getId(), snapshotStore); + continue; + } + parentSnapshotPaths.add(snapshotInfo.getPath()); destDataObject = destDatastore.create(snapshotInfo); snapshotInfo.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested); destDataObject.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested); migrateJob(future, snapshotInfo, destDataObject, destDatastore); + if (future.get() != null && future.get().isFailed()) { + break; + } } } else { // Check if template in destination store, if yes, do not proceed @@ -163,26 +182,13 @@ public class SecondaryStorageServiceImpl implements SecondaryStorageService { if (destData != null) { destData.getDataStore().delete(destData); } - } else { if (destData instanceof VolumeInfo) { ((VolumeInfo) destData).processEventOnly(ObjectInDataStoreStateMachine.Event.OperationSuccessed, answer); } else { destData.processEvent(ObjectInDataStoreStateMachine.Event.OperationSuccessed, answer); } - if (destData instanceof SnapshotInfo) { - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySourceSnapshot(srcData.getId(), DataStoreRole.Image); - SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findBySnapshot(srcData.getId(), DataStoreRole.Image); - destSnapshotStore.setPhysicalSize(snapshotStore.getPhysicalSize()); - snapshotStoreDao.update(destSnapshotStore.getId(), destSnapshotStore); - } - - if (destData instanceof VolumeInfo) { - VolumeDataStoreVO srcVolume = volumeDataStoreDao.findByStoreVolume(srcData.getDataStore().getId(), srcData.getId()); - VolumeDataStoreVO destVolume = volumeDataStoreDao.findByStoreVolume(destData.getDataStore().getId(), destData.getId()); - destVolume.setPhysicalSize(srcVolume.getPhysicalSize()); - volumeDataStoreDao.update(destVolume.getId(), destVolume); - } + updateDataObject(srcData, destData); s_logger.debug("Deleting source data"); srcData.getDataStore().delete(srcData); s_logger.debug("Successfully migrated "+srcData.getUuid()); @@ -198,6 +204,37 @@ public class SecondaryStorageServiceImpl implements SecondaryStorageService { return null; } + private void updateDataObject(DataObject srcData, DataObject destData) { + if (destData instanceof SnapshotInfo) { + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySourceSnapshot(srcData.getId(), DataStoreRole.Image); + SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findBySnapshot(srcData.getId(), DataStoreRole.Image); + if (snapshotStore != null && destSnapshotStore != null) { + destSnapshotStore.setPhysicalSize(snapshotStore.getPhysicalSize()); + destSnapshotStore.setCreated(snapshotStore.getCreated()); + if (snapshotStore.getParentSnapshotId() != destSnapshotStore.getParentSnapshotId()) { + destSnapshotStore.setParentSnapshotId(snapshotStore.getParentSnapshotId()); + } + snapshotStoreDao.update(destSnapshotStore.getId(), destSnapshotStore); + } + } else if (destData instanceof VolumeInfo) { + VolumeDataStoreVO srcVolume = volumeDataStoreDao.findByStoreVolume(srcData.getDataStore().getId(), srcData.getId()); + VolumeDataStoreVO destVolume = volumeDataStoreDao.findByStoreVolume(destData.getDataStore().getId(), destData.getId()); + if (srcVolume != null && destVolume != null) { + destVolume.setPhysicalSize(srcVolume.getPhysicalSize()); + destVolume.setCreated(srcVolume.getCreated()); + volumeDataStoreDao.update(destVolume.getId(), destVolume); + } + } else if (destData instanceof TemplateInfo) { + TemplateDataStoreVO srcTemplate = templateStoreDao.findByStoreTemplate(srcData.getDataStore().getId(), srcData.getId()); + TemplateDataStoreVO destTemplate = templateStoreDao.findByStoreTemplate(destData.getDataStore().getId(), destData.getId()); + if (srcTemplate != null && destTemplate != null) { + destTemplate.setCreated(srcTemplate.getCreated()); + templateStoreDao.update(destTemplate.getId(), destTemplate); + } + } else { + s_logger.debug("Unsupported data object type"); + } + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 965c3322888..2341603803e 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -391,17 +391,18 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { private Answer sendToLeastBusyEndpoint(List eps, CopyCommand cmd) { Answer answer = null; EndPoint endPoint = null; - Long epId = ssvmWithLeastMigrateJobs(); - if (epId == null) { + List epIds = ssvmWithLeastMigrateJobs(); + + if (epIds.isEmpty()) { Collections.shuffle(eps); endPoint = eps.get(0); } else { - List remainingEps = eps.stream().filter(ep -> ep.getId() != epId ).collect(Collectors.toList()); + List remainingEps = eps.stream().filter(ep -> !epIds.contains(ep.getId())).collect(Collectors.toList()); if (!remainingEps.isEmpty()) { Collections.shuffle(remainingEps); endPoint = remainingEps.get(0); } else { - endPoint = _defaultEpSelector.getEndPointFromHostId(epId); + endPoint = _defaultEpSelector.getEndPointFromHostId(epIds.get(0)); } } CommandExecLogVO execLog = new CommandExecLogVO(endPoint.getId(), _secStorageVmDao.findByInstanceName(hostDao.findById(endPoint.getId()).getName()).getId(), "DataMigrationCommand", 1); @@ -495,27 +496,22 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { return null; } - private Integer getCopyCmdsCountToSpecificSSVM(Long ssvmId) { - return _cmdExecLogDao.getCopyCmdCountForSSVM(ssvmId); - } - - private Long ssvmWithLeastMigrateJobs() { + private List ssvmWithLeastMigrateJobs() { s_logger.debug("Picking ssvm from the pool with least commands running on it"); - String query = "select host_id, count(*) from cmd_exec_log group by host_id order by 2 limit 1;"; + String query = "select host_id, count(*) from cmd_exec_log group by host_id order by 2;"; TransactionLegacy txn = TransactionLegacy.currentTxn(); - Long epId = null; + List result = new ArrayList(); PreparedStatement pstmt = null; try { pstmt = txn.prepareAutoCloseStatement(query); ResultSet rs = pstmt.executeQuery(); - if (rs.getFetchSize() > 0) { - rs.absolute(1); - epId = (long) rs.getInt(1); + while (rs.next()) { + result.add((long) rs.getInt(1)); } } catch (SQLException e) { s_logger.debug("SQLException caught", e); } - return epId; + return result; } } diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/PremiumSecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/PremiumSecondaryStorageManagerImpl.java index d21ec614f40..74d1b509e21 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/PremiumSecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/PremiumSecondaryStorageManagerImpl.java @@ -191,7 +191,7 @@ public class PremiumSecondaryStorageManagerImpl extends SecondaryStorageManagerI private void scaleDownSSVMOnLoad(List alreadyRunning, List activeCmds, List copyCmdsInPipeline) { int halfLimit = Math.round((float) (alreadyRunning.size() * migrateCapPerSSVM) / 2); - if ((copyCmdsInPipeline.size() < halfLimit && alreadyRunning.size() * _capacityPerSSVM - activeCmds.size() > (_standbyCapacity + 5)) && alreadyRunning.size() > 1) { + if (alreadyRunning.size() > 1 && ( copyCmdsInPipeline.size() < halfLimit && (activeCmds.size() < (((alreadyRunning.size() -1) * _capacityPerSSVM)/2)) )) { Collections.reverse(alreadyRunning); for(SecondaryStorageVmVO vm : alreadyRunning) { long count = activeCmds.stream().filter(cmd -> cmd.getInstanceId() == vm.getId()).count(); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index dd002a9a236..863376d3381 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -1300,15 +1300,17 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S try { File srcFile = new File(getDir(srcStore.getUrl(), _nfsVersion), srcData.getPath()); File destFile = new File(getDir(destStore.getUrl(), _nfsVersion), destData.getPath()); - ImageFormat format = getTemplateFormat(srcFile.getName()); if (srcFile == null) { - return new CopyCmdAnswer("Can't find src file:" + srcFile); + return new CopyCmdAnswer("Can't find source file at path: "+ srcData.getPath() +" on datastore: "+ srcDataStore.getUuid() +" to initiate file transfer"); } + ImageFormat format = getTemplateFormat(srcFile.getName()); if (srcData instanceof TemplateObjectTO || srcData instanceof VolumeObjectTO) { File srcDir = null; if (srcFile.isFile() || srcFile.getName().contains(".")) { srcDir = new File(srcFile.getParent()); + } else if (!srcFile.isDirectory()) { + srcDir = new File(srcFile.getParent()); } File destDir = null; if (destFile.isFile()) { @@ -1351,7 +1353,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S if (srcFile.isFile()) { newVol.setPath(destData.getPath() + File.separator + srcFile.getName()); } else { - newVol.setPath(destData.getPath()); + newVol.setPath(srcData.getPath()); } newVol.setSize(getVirtualSize(srcFile, format)); retObj = newVol;