refactor data motion service for volumes

engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/CreateCmdResult.java

@@ -18,20 +18,26 @@
  */
 package org.apache.cloudstack.engine.subsystem.api.storage;
 
+import com.cloud.agent.api.Answer;
+
 public class CreateCmdResult extends CommandResult {
     private String path;
-    private Long size;
+    private Answer answer;
-    public CreateCmdResult(String path, Long size) {
+    public CreateCmdResult(String path, Answer answer) {
         super();
         this.path = path;
-        this.size = size;
     }
     
     public String getPath() {
         return this.path;
     }
-    
+
-    public Long getSize() {
+    public Answer getAnswer() {
-        return this.size;
+        return answer;
     }
+
+    public void setAnswer(Answer answer) {
+        this.answer = answer;
+    }
+
 }

engine/api/src/org/apache/cloudstack/storage/to/TemplateObjectTO.java

@@ -28,7 +28,7 @@ public class TemplateObjectTO implements DataTO {
     private  String path;
     private  String uuid;
     private  DiskFormat diskType;
-    private  ImageStoreTO imageDataStore;
+    private  DataStoreTO imageDataStore;
     private  String name;
 
     public TemplateObjectTO() {
@@ -38,7 +38,7 @@ public class TemplateObjectTO implements DataTO {
         this.path = template.getUri();
         this.uuid = template.getUuid();
         //this.diskType = template.getDiskType();
-        this.imageDataStore = new ImageStoreTO((ImageStoreInfo)template.getDataStore());
+        this.imageDataStore = template.getDataStore().getTO();
         this.name = template.getUniqueName();
     }
     
@@ -55,7 +55,7 @@ public class TemplateObjectTO implements DataTO {
         return this.diskType;
     }
     
-    public ImageStoreTO getImageDataStore() {
+    public DataStoreTO getImageDataStore() {
         return this.imageDataStore;
     }
 

engine/api/src/org/apache/cloudstack/storage/to/VolumeObjectTO.java

@@ -18,16 +18,17 @@ package org.apache.cloudstack.storage.to;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataTO;
-import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.disktype.DiskFormat;
 import org.apache.cloudstack.engine.subsystem.api.storage.type.VolumeType;
 
+import com.cloud.agent.api.to.DataStoreTO;
+
 public class VolumeObjectTO implements DataTO {
     private String uuid;
     private VolumeType volumeType;
     private DiskFormat diskType;
-    private PrimaryDataStoreTO dataStore;
+    private DataStoreTO dataStore;
     private String name;
     private long size;
     private String path;
@@ -42,7 +43,7 @@ public class VolumeObjectTO implements DataTO {
         //this.volumeType = volume.getType();
         //this.diskType = volume.getDiskType();
         if (volume.getDataStore() != null) {
-            this.dataStore = new PrimaryDataStoreTO((PrimaryDataStoreInfo)volume.getDataStore());
+            this.dataStore = volume.getDataStore().getTO();
         } else {
             this.dataStore = null;
         }
@@ -66,7 +67,7 @@ public class VolumeObjectTO implements DataTO {
         return this.diskType;
     }
     
-    public PrimaryDataStoreTO getDataStore() {
+    public DataStoreTO getDataStore() {
         return this.dataStore;
     }
     

engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java

@@ -141,47 +141,24 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
 
     @DB
     protected Answer copyVolumeFromImage(DataObject srcData, DataObject destData) {
-        String value = configDao.getValue(Config.RecreateSystemVmEnabled.key());
+        String value = configDao.getValue(Config.CopyVolumeWait.key());
         int _copyvolumewait = NumbersUtil.parseInt(value,
                 Integer.parseInt(Config.CopyVolumeWait.getDefaultValue()));
 
-        VolumeDataStoreVO volumeStoreVO = volumeStoreDao.findByVolume(srcData
+        if (srcData.getDataStore().getRole() != DataStoreRole.ImageCache && destData.getDataStore().getRole() != DataStoreRole.ImageCache) {
-                .getId());
+            //need to copy it to image cache store
-        DataStore srcStore = srcData.getDataStore();
+            DataObject cacheData = cacheMgr.createCacheObject(srcData, destData.getDataStore().getScope());
-        String[] volumePath = volumeStoreVO.getInstallPath().split("/");
+            CopyCommand cmd = new CopyCommand(cacheData.getTO(), destData.getTO(), _copyvolumewait);
-        String volumeUUID = volumePath[volumePath.length - 1].split("\\.")[0];
+            EndPoint ep = selector.select(cacheData, destData);
-        StoragePool destPool = (StoragePool) destData.getDataStore();
+            Answer answer = ep.sendMessage(cmd);
-        CopyVolumeCommand cvCmd = new CopyVolumeCommand(srcData.getId(),
+            return answer;
-                volumeUUID, destPool, srcStore.getUri(), false,
+        } else {
-                _copyvolumewait);
+            //handle copy it to/from cache store
-        CopyVolumeAnswer cvAnswer = null;
+            CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _copyvolumewait);
-        String errMsg = null;
+            EndPoint ep = selector.select(srcData, destData);
-        try {
+            Answer answer = ep.sendMessage(cmd);
-            cvAnswer = (CopyVolumeAnswer) this.storageMgr.sendToPool(destPool,
+            return answer;
-                    cvCmd);
-        } catch (StorageUnavailableException e1) {
-            s_logger.debug("Failed to copy volume " + srcData.getId() + " to "
-                    + destData.getId(), e1);
-            errMsg = e1.toString();
         }
-
-        if (cvAnswer == null || !cvAnswer.getResult()) {
-            errMsg = cvAnswer.getDetails();
-        }
-
-        VolumeVO vol = this.volDao.findById(destData.getId());
-        Transaction txn = Transaction.currentTxn();
-        txn.start();
-        vol.setPath(cvAnswer.getVolumePath());
-        vol.setFolder(destPool.getPath());
-        vol.setPodId(destPool.getPodId());
-        vol.setPoolId(destPool.getId());
-        vol.setPodId(destPool.getPodId());
-
-        this.volDao.update(vol.getId(), vol);
-        volumeStoreDao.remove(volumeStoreVO.getId());
-        txn.commit();
-        return cvAnswer;
     }
 
     private Answer copyTemplate(DataObject srcData, DataObject destData) {
@@ -195,7 +172,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
             Answer answer = ep.sendMessage(cmd);
             return answer;
         } else {
-            //handle copy it to cache store
+            //handle copy it to/from cache store
             CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _primaryStorageDownloadWait);
             EndPoint ep = selector.select(srcData, destData);
             Answer answer = ep.sendMessage(cmd);
@@ -326,56 +303,15 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
     }
 
     protected Answer copyVolumeBetweenPools(DataObject srcData, DataObject destData) {
-        VolumeInfo volume = (VolumeInfo)srcData;
+        String value = configDao.getValue(Config.CopyVolumeWait.key());
-        VolumeInfo destVolume = (VolumeInfo)destData;
-        String secondaryStorageURL = this.templateMgr.getSecondaryStorageURL(volume
-                .getDataCenterId());
-        StoragePool srcPool = (StoragePool)this.dataStoreMgr.getDataStore(volume
-                .getPoolId(), DataStoreRole.Primary);
-
-        StoragePool destPool = (StoragePool)this.dataStoreMgr.getDataStore(destVolume.getPoolId(), DataStoreRole.Primary);
-
-        String value = this.configDao.getValue(Config.CopyVolumeWait.toString());
         int _copyvolumewait = NumbersUtil.parseInt(value,
                 Integer.parseInt(Config.CopyVolumeWait.getDefaultValue()));
-        CopyVolumeCommand cvCmd = new CopyVolumeCommand(volume.getId(),
-                volume.getPath(), srcPool, secondaryStorageURL, true,
-                _copyvolumewait);
-        CopyVolumeAnswer cvAnswer;
-        try {
-            cvAnswer = (CopyVolumeAnswer) this.storageMgr.sendToPool(srcPool, cvCmd);
-        } catch (StorageUnavailableException e1) {
-            throw new CloudRuntimeException(
-                    "Failed to copy the volume from the source primary storage pool to secondary storage.",
-                    e1);
-        }
 
-        if (cvAnswer == null || !cvAnswer.getResult()) {
+        DataObject cacheData = cacheMgr.createCacheObject(srcData, destData.getDataStore().getScope());
-            throw new CloudRuntimeException(
+        CopyCommand cmd = new CopyCommand(cacheData.getTO(), destData.getTO(), _copyvolumewait);
-                    "Failed to copy the volume from the source primary storage pool to secondary storage.");
+        EndPoint ep = selector.select(cacheData, destData);
-        }
+        Answer answer = ep.sendMessage(cmd);
-
+        return answer;
-        String secondaryStorageVolumePath = cvAnswer.getVolumePath();
-
-        cvCmd = new CopyVolumeCommand(volume.getId(),
-                secondaryStorageVolumePath, destPool,
-                secondaryStorageURL, false, _copyvolumewait);
-        try {
-            cvAnswer = (CopyVolumeAnswer) this.storageMgr.sendToPool(destPool, cvCmd);
-        } catch (StorageUnavailableException e1) {
-            throw new CloudRuntimeException(
-                    "Failed to copy the volume from secondary storage to the destination primary storage pool.");
-        }
-
-        if (cvAnswer == null || !cvAnswer.getResult()) {
-            throw new CloudRuntimeException(
-                    "Failed to copy the volume from secondary storage to the destination primary storage pool.");
-        }
-
-        VolumeVO destVol = this.volDao.findById(destVolume.getId());
-        destVol.setPath(cvAnswer.getVolumePath());
-        this.volDao.update(destVol.getId(), destVol);
-        return cvAnswer;
     }
 
     @Override

engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java

@@ -29,10 +29,13 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.disktype.DiskFormat;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
 import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager;
+import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.storage.DownloadAnswer;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Volume;
@@ -52,6 +55,8 @@ public class VolumeObject implements VolumeInfo {
     @Inject
     VolumeDao volumeDao;
     @Inject
+    VolumeDataStoreDao volumeStoreDao;
+    @Inject
     ObjectInDataStoreManager ojbectInStoreMgr;
     private Object payload;
 
@@ -357,6 +362,14 @@ public class VolumeObject implements VolumeInfo {
                vol.setSize(newVol.getSize());
                volumeDao.update(vol.getId(), vol);
            }
+       } else if (this.dataStore.getRole() == DataStoreRole.Image) {
+           if (answer instanceof DownloadAnswer) {
+               DownloadAnswer dwdAnswer = (DownloadAnswer)answer;
+               VolumeDataStoreVO volStore = this.volumeStoreDao.findByStoreVolume(this.dataStore.getId(), this.getId());
+               volStore.setInstallPath(dwdAnswer.getInstallPath());
+               volStore.setChecksum(dwdAnswer.getCheckSum());
+               this.volumeStoreDao.update(volStore.getId(), volStore);
+           }
        }
        
        this.processEvent(event);

engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java

@@ -621,7 +621,7 @@ public class VolumeServiceImpl implements VolumeService {
                 return null;
             }
             srcVolume.processEvent(Event.OperationSuccessed);
-            destVolume.processEvent(Event.OperationSuccessed);
+            destVolume.processEvent(Event.OperationSuccessed, result.getAnswer());
             AsyncCallFuture<VolumeApiResult> destroyFuture = this.expungeVolumeAsync(srcVolume);
             destroyFuture.get();
             future.complete(res);
@@ -639,25 +639,28 @@ public class VolumeServiceImpl implements VolumeService {
     public AsyncCallFuture<VolumeApiResult> registerVolume(VolumeInfo volume, DataStore store) {
 
         AsyncCallFuture<VolumeApiResult> future = new AsyncCallFuture<VolumeApiResult>();
-        VolumeObject vo = (VolumeObject) volume;
+        DataObject volumeOnStore = store.create(volume);
-
+        
-        CreateVolumeContext<VolumeApiResult> context = new CreateVolumeContext<VolumeApiResult>(null, vo, future);
+        volumeOnStore.processEvent(Event.CreateOnlyRequested);
+        
+        CreateVolumeContext<VolumeApiResult> context = new CreateVolumeContext<VolumeApiResult>(null, volumeOnStore, future);
         AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> caller = AsyncCallbackDispatcher.create(this);
         caller.setCallback(caller.getTarget().registerVolumeCallback(null, null))
         .setContext(context);
 
-        dataObjectMgr.createAsync(volume, store, caller, true);
+        store.getDriver().createAsync(volumeOnStore, caller);
         return future;
     }
 
     protected Void registerVolumeCallback(AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> callback, CreateVolumeContext<VolumeApiResult> context) {
         CreateCmdResult result = callback.getResult();
+        
         VolumeObject vo = (VolumeObject)context.volume;
-        /*if (result.isFailed()) {
+        if (result.isFailed()) {
-            vo.stateTransit(Volume.Event.OperationFailed);
+            vo.processEvent(Event.OperationFailed);
         } else {
-            vo.stateTransit(Volume.Event.OperationSucceeded);
+            vo.processEvent(Event.OperationSuccessed, result.getAnswer());
-        }*/
+        }
 
     	_resourceLimitMgr.incrementResourceCount(vo.getAccountId(), ResourceType.secondary_storage,
               	vo.getSize());

plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java

@@ -55,12 +55,15 @@ import org.apache.xmlrpc.XmlRpcException;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
+import com.cloud.agent.api.storage.CopyVolumeAnswer;
 import com.cloud.agent.api.storage.CreateAnswer;
 import com.cloud.agent.api.storage.DeleteVolumeCommand;
 import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer;
 import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.StorageFilerTO;
 import com.cloud.agent.api.to.VolumeTO;
+import com.cloud.exception.InternalErrorException;
 import com.cloud.hypervisor.xen.resource.CitrixResourceBase.SRType;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -774,7 +777,75 @@ public class XenServerStorageResource {
             return new CopyCmdAnswer(e.toString());
         }
     }
- 
+    
+    
+    protected Answer copyVolumeFromImageCacheToPrimary(DataTO srcData, DataTO destData, int wait) {
+        Connection conn = hypervisorResource.getConnection();
+        VolumeObjectTO srcVolume = (VolumeObjectTO)srcData;
+        VolumeObjectTO destVolume = (VolumeObjectTO)destData;
+        PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)destVolume.getDataStore();
+        DataStoreTO srcStore = srcVolume.getDataStore();
+
+        if (srcStore instanceof NfsTO) {
+            NfsTO nfsStore = (NfsTO)srcStore;
+            try {
+                SR primaryStoragePool = hypervisorResource.getStorageRepository(conn, primaryStore.getUuid());
+                String srUuid = primaryStoragePool.getUuid(conn);
+                String volumePath = nfsStore.getUrl() + ":" + srcVolume.getPath();
+                String uuid = copy_vhd_from_secondarystorage(conn, volumePath, srUuid, wait );
+                VolumeObjectTO newVol = new VolumeObjectTO();
+                newVol.setPath(uuid);
+                newVol.setSize(srcVolume.getSize());
+
+                return new CopyCmdAnswer(newVol);
+            } catch (Exception e) {
+                String msg = "Catch Exception " + e.getClass().getName() + " due to " + e.toString();
+                s_logger.warn(msg, e);
+                return new CopyCmdAnswer(e.toString());
+            }
+        }
+        
+        s_logger.debug("unsupported protocol");
+        return new CopyCmdAnswer("unsupported protocol"); 
+    }
+
+    protected Answer copyVolumeFromPrimaryToSecondary(DataTO srcData, DataTO destData, int wait) {
+        Connection conn = hypervisorResource.getConnection();
+        VolumeObjectTO srcVolume = (VolumeObjectTO)srcData;
+        VolumeObjectTO destVolume = (VolumeObjectTO)destData;
+        DataStoreTO destStore = destVolume.getDataStore();
+
+        if (destStore instanceof NfsTO) {
+            SR secondaryStorage = null;
+            try {
+                NfsTO nfsStore = (NfsTO)destStore;
+                URI uri = new URI(nfsStore.getUrl());
+                // Create the volume folder
+                if (!hypervisorResource.createSecondaryStorageFolder(conn, uri.getHost() + ":" + uri.getPath(), destVolume.getPath())) {
+                    throw new InternalErrorException("Failed to create the volume folder.");
+                }
+
+                // Create a SR for the volume UUID folder
+                secondaryStorage = hypervisorResource.createNfsSRbyURI(conn, new URI(nfsStore.getUrl() + destVolume.getPath()), false);
+                // Look up the volume on the source primary storage pool
+                VDI srcVdi = getVDIbyUuid(conn, srcVolume.getPath());
+                // Copy the volume to secondary storage
+                VDI destVdi = hypervisorResource.cloudVDIcopy(conn, srcVdi, secondaryStorage, wait);
+                String destVolumeUUID = destVdi.getUuid(conn);
+
+                VolumeObjectTO newVol = new VolumeObjectTO();
+                newVol.setPath(destVolumeUUID);
+                newVol.setSize(srcVolume.getSize());
+                return new CopyCmdAnswer(newVol);
+            } catch (Exception e) {
+                s_logger.debug("Failed to copy volume to secondary: " + e.toString());
+                return new CopyCmdAnswer("Failed to copy volume to secondary: " + e.toString()); 
+            } finally {
+                hypervisorResource.removeSR(conn, secondaryStorage);
+            }
+        }
+        return new CopyCmdAnswer("unsupported protocol"); 
+    }
     
     protected Answer execute(CopyCommand cmd) {
         DataTO srcData = cmd.getSrcTO();
@@ -788,6 +859,11 @@ public class XenServerStorageResource {
         } else if (srcData.getObjectType() == DataObjectType.TEMPLATE && srcDataStore.getRole() == DataStoreRole.Primary && destDataStore.getRole() == DataStoreRole.Primary) {
             //clone template to a volume
             return cloneVolumeFromBaseTemplate(srcData, destData);
+        } else if (srcData.getObjectType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.ImageCache) {
+            //copy volume from image cache to primary
+            return copyVolumeFromImageCacheToPrimary(srcData, destData, cmd.getWait());
+        } else if (srcData.getObjectType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Primary) {
+            return copyVolumeFromPrimaryToSecondary(srcData, destData, cmd.getWait());
         }
 
         return new Answer(cmd, false, "not implemented yet");

plugins/storage/image/sample/src/org/apache/cloudstack/storage/datastore/driver/SampleImageStoreDriverImpl.java

@@ -101,7 +101,7 @@ public class SampleImageStoreDriverImpl implements ImageStoreDriver {
             if (answer.getResult()) {
                 //update imagestorevo
 
-                result = new CreateCmdResult(answer.getPath(), answer.getSize());
+                result = new CreateCmdResult(answer.getPath(), null);
             } else {
                 result.setResult(answer.getDetails());
             }