CLOUDSTACK-2637: fix upload volume

api/src/com/cloud/storage/Volume.java

@@ -44,6 +44,7 @@ public interface Volume extends ControlledEntity, Identity, InternalIdentity, Ba
         Destroying("The volume is destroying, and can't be recovered."),  
         UploadOp ("The volume upload operation is in progress or in short the volume is on secondary storage"),
         Uploading("volume is uploading"),
+        Copying("volume is copying from image store to primary, in case it's an uploaded volume"),
         Uploaded("volume is uploaded");
 
         String _description;
@@ -73,9 +74,9 @@ public interface Volume extends ControlledEntity, Identity, InternalIdentity, Ba
             s_fsm.addTransition(Resizing, Event.OperationSucceeded, Ready);
             s_fsm.addTransition(Resizing, Event.OperationFailed, Ready);          
             s_fsm.addTransition(Allocated, Event.UploadRequested, UploadOp);
-            s_fsm.addTransition(Uploaded, Event.CopyRequested, Creating);// CopyRequested for volume from sec to primary storage            
+            s_fsm.addTransition(Uploaded, Event.CopyRequested, Copying);            
-            s_fsm.addTransition(Creating, Event.CopySucceeded, Ready);
+            s_fsm.addTransition(Copying, Event.OperationSucceeded, Ready);
-            s_fsm.addTransition(Creating, Event.CopyFailed, Uploaded);// Copying volume from sec to primary failed.  
+            s_fsm.addTransition(Copying, Event.OperationFailed, Uploaded);
             s_fsm.addTransition(UploadOp, Event.DestroyRequested, Destroy);
             s_fsm.addTransition(Ready, Event.DestroyRequested, Destroy);
             s_fsm.addTransition(Destroy, Event.ExpungingRequested, Expunging);

core/src/com/cloud/agent/api/ComputeChecksumCommand.java

@@ -17,24 +17,32 @@
 package com.cloud.agent.api;
 
 import com.cloud.agent.api.storage.ssCommand;
+import com.cloud.agent.api.to.DataStoreTO;
 
 
 public class ComputeChecksumCommand extends ssCommand {
-
+	private DataStoreTO store;
-
     private String templatePath;
     public ComputeChecksumCommand() {
         super();
     }
 
-    public ComputeChecksumCommand(String secUrl, String templatePath) {
+    public ComputeChecksumCommand(DataStoreTO store, String templatePath) {
-        super(secUrl);
         this.templatePath = templatePath;
+        this.setStore(store);
     }
 
     public String getTemplatePath() {
         return templatePath;
     }
 
+	public DataStoreTO getStore() {
+		return store;
+	}
+
+	public void setStore(DataStoreTO store) {
+		this.store = store;
+	}
+
 
 }

engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java

@@ -20,6 +20,7 @@ package org.apache.cloudstack.storage.image.store;
 
 import java.util.Date;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
 
 import javax.inject.Inject;
 
@@ -31,10 +32,13 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
+import org.apache.cloudstack.framework.async.AsyncCallFuture;
+import org.apache.cloudstack.storage.command.CommandResult;
 import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
 import org.apache.cloudstack.storage.image.ImageStoreDriver;
 import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
+import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.storage.DataStoreRole;
@@ -44,6 +48,7 @@ import com.cloud.utils.storage.encoding.EncodingType;
 
 
 public class ImageStoreImpl implements ImageStoreEntity {
+	private static final Logger s_logger = Logger.getLogger(ImageStoreImpl.class);
     @Inject
     VMTemplateDao imageDao;
     @Inject
@@ -145,8 +150,19 @@ public class ImageStoreImpl implements ImageStoreEntity {
 
     @Override
     public boolean delete(DataObject obj) {
-        // TODO Auto-generated method stub
+    	AsyncCallFuture<CommandResult> future = new AsyncCallFuture<CommandResult>();
-        return false;
+    	this.driver.deleteAsync(obj, future);
+    	try {
+			future.get();
+		} catch (InterruptedException e) {
+			s_logger.debug("failed delete obj", e);
+			return false;
+		} catch (ExecutionException e) {
+			s_logger.debug("failed delete obj", e);
+			return false;
+		}
+    	objectInStoreMgr.delete(obj);
+        return true;
     }
 
     @Override

engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java

@@ -79,12 +79,6 @@ public class MockLocalNfsSecondaryStorageResource extends
 
     }
 
-    @Override
-    public String getRootDir(ssCommand cmd) {
-        return "/mnt";
-
-    }
-
     @Override
     public String getRootDir(String secUrl){
         return "/mnt";

engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java

@@ -193,15 +193,13 @@ public class VolumeObject implements VolumeInfo {
             }
             if (this.dataStore.getRole() == DataStoreRole.Image) {
                 objectInStoreMgr.update(this, event);
-                if (this.volumeVO.getState() == Volume.State.Migrating) {
+                if (this.volumeVO.getState() == Volume.State.Migrating || this.volumeVO.getState() == Volume.State.Copying || this.volumeVO.getState() == Volume.State.Uploaded) {
                 	return;
                 }
-                if (event == ObjectInDataStoreStateMachine.Event.CreateRequested) {
+                if (event == ObjectInDataStoreStateMachine.Event.CreateOnlyRequested) {
                     volEvent = Volume.Event.UploadRequested;
-                } else if (event == ObjectInDataStoreStateMachine.Event.OperationSuccessed) {
+                } else if (event == ObjectInDataStoreStateMachine.Event.MigrationRequested) {
-                    volEvent = Volume.Event.CopySucceeded;
+                	volEvent = Volume.Event.CopyRequested;
-                } else if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) {
-                    volEvent = Volume.Event.CopyFailed;
                 }
             } else {
                 if (event == ObjectInDataStoreStateMachine.Event.CreateRequested ||
@@ -231,7 +229,8 @@ public class VolumeObject implements VolumeInfo {
             throw new CloudRuntimeException("Failed to update state:" + e.toString());
         } finally{
             // in case of OperationFailed, expunge the entry
-            if ( event == ObjectInDataStoreStateMachine.Event.OperationFailed){
+            if ( event == ObjectInDataStoreStateMachine.Event.OperationFailed && (this.volumeVO.getState() != Volume.State.Copying 
+            		&& this.volumeVO.getState() != Volume.State.Uploaded)){
                 objectInStoreMgr.delete(this);
             }
         }

engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java

@@ -587,9 +587,71 @@ public class VolumeServiceImpl implements VolumeService {
         }
 
     }
+    
+    protected AsyncCallFuture<VolumeApiResult> copyVolumeFromImageToPrimary(VolumeInfo srcVolume, DataStore destStore) {
+    	AsyncCallFuture<VolumeApiResult> future = new AsyncCallFuture<VolumeApiResult>();
+    	VolumeApiResult res = new VolumeApiResult(srcVolume);
+    	VolumeInfo destVolume = null;
+    	try {
+    		destVolume = (VolumeInfo)destStore.create(srcVolume);
+    		destVolume.processEvent(Event.CopyingRequested);
+    		srcVolume.processEvent(Event.CopyingRequested);
+
+    		CopyVolumeContext<VolumeApiResult> context = new CopyVolumeContext<VolumeApiResult>(null, future, srcVolume,
+    				destVolume,
+    				destStore);
+    		AsyncCallbackDispatcher<VolumeServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher.create(this);
+    		caller.setCallback(caller.getTarget().copyVolumeFromImageToPrimaryCallback(null, null))
+    		.setContext(context);
+
+    		motionSrv.copyAsync(srcVolume, destVolume, caller);
+    		return future;
+    	} catch (Exception e) {
+    		s_logger.error("failed to copy volume from image store", e);
+    		if (destVolume != null) {
+    			destVolume.processEvent(Event.OperationFailed);
+    		}
+    		
+    		srcVolume.processEvent(Event.OperationFailed);
+    		res.setResult(e.toString());
+    		future.complete(res);
+    		return future;
+    	}
+    }
+
+    protected Void copyVolumeFromImageToPrimaryCallback(AsyncCallbackDispatcher<VolumeServiceImpl, CopyCommandResult> callback, CopyVolumeContext<VolumeApiResult> context) {
+    	VolumeInfo srcVolume = context.srcVolume;
+    	VolumeInfo destVolume = context.destVolume;
+    	CopyCommandResult result = callback.getResult();
+    	AsyncCallFuture<VolumeApiResult> future = context.future;
+    	VolumeApiResult res = new VolumeApiResult(destVolume);
+    	try {
+    		if (res.isFailed()) {
+    			destVolume.processEvent(Event.OperationFailed);
+    			srcVolume.processEvent(Event.OperationFailed);
+    			res.setResult(result.getResult());
+    			future.complete(res);
+    		}
+
+    		srcVolume.processEvent(Event.OperationSuccessed);
+    		destVolume.processEvent(Event.OperationSuccessed, result.getAnswer());
+    		srcVolume.getDataStore().delete(srcVolume);
+    		future.complete(res);
+    	} catch (Exception e) {
+    		res.setResult(e.toString());
+    		future.complete(res);
+    	}
+    	return null;
+    }
+    
     @Override
     public AsyncCallFuture<VolumeApiResult> copyVolume(VolumeInfo srcVolume,
             DataStore destStore) {
+    	
+    	if (srcVolume.getState() == Volume.State.Uploaded) {
+    		return copyVolumeFromImageToPrimary(srcVolume, destStore);
+    	}
+    	
         AsyncCallFuture<VolumeApiResult> future = new AsyncCallFuture<VolumeApiResult>();
         VolumeApiResult res = new VolumeApiResult(srcVolume);
         try {
@@ -825,8 +887,8 @@ public class VolumeServiceImpl implements VolumeService {
 
         CreateVolumeContext<VolumeApiResult> context = new CreateVolumeContext<VolumeApiResult>(null, volumeOnStore, future);
         AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> caller = AsyncCallbackDispatcher.create(this);
-        caller.setCallback(caller.getTarget().registerVolumeCallback(null, null))
+        caller.setCallback(caller.getTarget().registerVolumeCallback(null, null));
-        .setContext(context);
+        caller.setContext(context);
 
         store.getDriver().createAsync(volumeOnStore, caller);
         return future;
@@ -834,19 +896,25 @@ public class VolumeServiceImpl implements VolumeService {
 
     protected Void registerVolumeCallback(AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> callback, CreateVolumeContext<VolumeApiResult> context) {
         CreateCmdResult result = callback.getResult();
+        try {
+        	VolumeObject vo = (VolumeObject)context.volume;
+        	if (result.isFailed()) {
+        		vo.processEvent(Event.OperationFailed);
+        	} else {
+        		vo.processEvent(Event.OperationSuccessed, result.getAnswer());
+        	}
 
-        VolumeObject vo = (VolumeObject)context.volume;
+        	_resourceLimitMgr.incrementResourceCount(vo.getAccountId(), ResourceType.secondary_storage,
-        if (result.isFailed()) {
+        			vo.getSize());
-            vo.processEvent(Event.OperationFailed);
+        	VolumeApiResult res = new VolumeApiResult(vo);
-        } else {
+        	context.future.complete(res);
-            vo.processEvent(Event.OperationSuccessed, result.getAnswer());
+        	return null;
+        } catch (Exception e) {
+        	s_logger.error("register volume failed: ", e);
+        	VolumeApiResult res = new VolumeApiResult(null);
+        	context.future.complete(res);
+        	return null;
         }
-
-    	_resourceLimitMgr.incrementResourceCount(vo.getAccountId(), ResourceType.secondary_storage,
-              	vo.getSize());
-        VolumeApiResult res = new VolumeApiResult(vo);
-        context.future.complete(res);
-        return null;
     }
 
 
@@ -961,7 +1029,7 @@ public class VolumeServiceImpl implements VolumeService {
                             String url = _volumeStoreDao.findByVolume(volume.getId()).getDownloadUrl();
                             _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(volume.getAccountId()),
                                     com.cloud.configuration.Resource.ResourceType.secondary_storage,
-                                    volInfo.getSize() - UriUtils.getRemoteSize(url));
+                                    volInfo.getSize() - volInfo.getPhysicalSize());
                         } catch (ResourceAllocationException e) {
                             s_logger.warn(e.getMessage());
                             _alertMgr.sendAlert(_alertMgr.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, volume.getDataCenterId(),
@@ -995,13 +1063,14 @@ public class VolumeServiceImpl implements VolumeService {
         }
 
         //Delete volumes which are not present on DB.
+        /*
         for (Long uniqueName : volumeInfos.keySet()) {
             TemplateProp vInfo = volumeInfos.get(uniqueName);
             expungeVolumeAsync(volFactory.getVolume(vInfo.getId(), store));
 
             String description = "Deleted volume " + vInfo.getTemplateName() + " on image store " + storeId;
             s_logger.info(description);
-        }
+        }*/
 
     }
 

plugins/hypervisors/simulator/src/com/cloud/resource/AgentStorageResource.java

@@ -109,7 +109,7 @@ public class AgentStorageResource extends AgentResourceBase implements Secondary
     }
 
     @Override
-    public String getRootDir(ssCommand cmd) {
+    public String getRootDir(String url) {
         // TODO Auto-generated method stub
         return null;
     }

plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java

@@ -212,7 +212,7 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
     		}
 
 
-    		CreateCmdResult result = new CreateCmdResult(null, null);
+    		CreateCmdResult result = new CreateCmdResult(null, answer);
     		caller.complete(result);
     	}
     	return null;
@@ -247,8 +247,7 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
                 throw new CloudRuntimeException(
                         "Please specify a volume that is not currently being uploaded.");
             }
-            _volumeStoreDao.remove(volumeStore.getId());
+          
-            volumeDao.remove(vol.getId());
             CommandResult result = new CommandResult();
             callback.complete(result);
             return;

server/src/com/cloud/storage/VolumeManagerImpl.java

@@ -737,12 +737,6 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager {
                 .getDomainId());
         volume.setFormat(ImageFormat.valueOf(format));
         volume = _volsDao.persist(volume);
-        try {
-            stateTransitTo(volume, Event.UploadRequested);
-        } catch (NoTransitionException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
-        }
         UserContext.current().setEventDetails("Volume Id: " + volume.getId());
 
         // Increment resource count during allocation; if actual creation fails,

server/src/com/cloud/template/TemplateManagerImpl.java

@@ -636,11 +636,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
 
     @Override
     public String getChecksum(DataStore store, String templatePath) {
-
-        String secUrl = store.getUri();
         EndPoint ep = _epSelector.select(store);
         ComputeChecksumCommand cmd = new ComputeChecksumCommand(
-                secUrl, templatePath);
+                store.getTO(), templatePath);
         Answer answer = ep.sendMessage(cmd);
         if (answer != null && answer.getResult()) {
             return answer.getDetails();

services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalSecondaryStorageResource.java

@@ -21,12 +21,11 @@ import java.util.Map;
 
 import javax.naming.ConfigurationException;
 
-import org.apache.log4j.Logger;
-
 import org.apache.cloudstack.storage.command.DownloadCommand;
 import org.apache.cloudstack.storage.command.DownloadProgressCommand;
 import org.apache.cloudstack.storage.template.DownloadManager;
 import org.apache.cloudstack.storage.template.DownloadManagerImpl;
+import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.CheckHealthAnswer;
@@ -42,7 +41,6 @@ import com.cloud.agent.api.StartupCommand;
 import com.cloud.agent.api.StartupStorageCommand;
 import com.cloud.agent.api.storage.ListTemplateAnswer;
 import com.cloud.agent.api.storage.ListTemplateCommand;
-import com.cloud.agent.api.storage.ssCommand;
 import com.cloud.agent.api.to.NfsTO;
 import com.cloud.host.Host;
 import com.cloud.host.Host.Type;
@@ -74,7 +72,7 @@ public class LocalSecondaryStorageResource extends ServerResourceBase implements
 
 
     @Override
-    public String getRootDir(ssCommand cmd){
+    public String getRootDir(String url){
         return getRootDir();
 
     }

services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java

@@ -998,7 +998,12 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
     private Answer execute(ComputeChecksumCommand cmd) {
 
         String relativeTemplatePath = cmd.getTemplatePath();
-        String parent = getRootDir(cmd);
+        DataStoreTO store = cmd.getStore();
+        if (!(store instanceof NfsTO)) {
+        	return new Answer(cmd, false, "can't handle non nfs data store");
+        }
+        NfsTO nfsStore = (NfsTO)store;
+        String parent = getRootDir(nfsStore.getUrl());
 
         if (relativeTemplatePath.startsWith(File.separator)) {
             relativeTemplatePath = relativeTemplatePath.substring(1);
@@ -1720,12 +1725,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
 
-    @Override
-    public String getRootDir(ssCommand cmd) {
-        return getRootDir(cmd.getSecUrl());
-
-    }
-
     protected long getUsedSize(String rootDir) {
         return _storage.getUsedSpace(rootDir);
     }

services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageResource.java

@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 package org.apache.cloudstack.storage.resource;
-import com.cloud.agent.api.storage.ssCommand;
 import com.cloud.resource.ServerResource;
 /**
  *
@@ -23,6 +22,6 @@ import com.cloud.resource.ServerResource;
  */
 public interface SecondaryStorageResource extends ServerResource {
     
-    public String getRootDir(ssCommand cmd);
+    public String getRootDir(String cmd);
 
 }

services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java

@@ -692,11 +692,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         String installPathPrefix = cmd.getInstallPath();
         // for NFS, we need to get mounted path
         if (dstore instanceof NfsTO) {
-            if (ResourceType.TEMPLATE == resourceType) {
+        	installPathPrefix = resource.getRootDir(((NfsTO) dstore).getUrl()) + File.separator + installPathPrefix;
-                installPathPrefix = resource.getRootDir(cmd) + File.separator + installPathPrefix;
-            } else {
-                installPathPrefix = resource.getRootDir(cmd) + File.separator + installPathPrefix;
-            }
         }
         String user = null;
         String password = null;