generalize storage orchestration: DataObjectManager will handle all the object life cycle management

engine/storage/image/src/org/apache/cloudstack/storage/image/ImageServiceImpl.java

@@ -77,7 +77,7 @@ public class ImageServiceImpl implements ImageService {
         ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(template.getId(), template.getType(), store.getId(), store.getRole());
         TemplateInfo templateOnStore = null;
         if (obj == null) {
-            templateOnStore = objectInDataStoreMgr.create(template, store);
+            templateOnStore = (TemplateInfo)objectInDataStoreMgr.create(template, store);
             obj = objectInDataStoreMgr.findObject(template.getId(), template.getType(), store.getId(), store.getRole());
         } else {
             CommandResult result = new CommandResult();
@@ -87,7 +87,7 @@ public class ImageServiceImpl implements ImageService {
         }
         
         try {
-            objectInDataStoreMgr.update(templateOnStore, Event.CreateOnlyRequested);
+            objectInDataStoreMgr.update(obj, Event.CreateOnlyRequested);
         } catch (NoTransitionException e) {
             s_logger.debug("failed to transit", e);
             CommandResult result = new CommandResult();
@@ -125,8 +125,7 @@ public class ImageServiceImpl implements ImageService {
             future.complete(result);
             return null;
         }
-        
+        ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(templateOnStore.getId(), templateOnStore.getType(), templateOnStore.getDataStore().getId(), templateOnStore.getDataStore().getRole());
-        ObjectInDataStoreVO obj = context.obj;
         obj.setInstallPath(callbackResult.getPath());
         
         if (callbackResult.getSize() != null) {
@@ -134,7 +133,7 @@ public class ImageServiceImpl implements ImageService {
         }
         
         try {
-            objectInDataStoreMgr.update(templateOnStore, Event.OperationSuccessed);
+            objectInDataStoreMgr.update(obj, Event.OperationSuccessed);
         } catch (NoTransitionException e) {
             s_logger.debug("Failed to transit state", e);
             result.setResult(e.toString());
@@ -143,6 +142,7 @@ public class ImageServiceImpl implements ImageService {
         }
         
         template.setImageStoreId(templateOnStore.getDataStore().getId());
+        template.setSize(callbackResult.getSize());
         try {
             template.stateTransit(TemplateEvent.OperationSucceeded);
         } catch (NoTransitionException e) {

engine/storage/image/src/org/apache/cloudstack/storage/image/driver/DefaultImageDataStoreDriverImpl.java

@@ -34,11 +34,15 @@ import org.apache.cloudstack.storage.command.CreateObjectAnswer;
 import org.apache.cloudstack.storage.command.CreateObjectCommand;
 import org.apache.cloudstack.storage.endpoint.EndPointSelector;
 import org.apache.cloudstack.storage.image.ImageDataStoreDriver;
+import org.apache.cloudstack.storage.image.db.ImageDataDao;
+import org.apache.cloudstack.storage.image.db.ImageDataVO;
 
 //http-read-only based image store
 public class DefaultImageDataStoreDriverImpl implements ImageDataStoreDriver {
     @Inject
     EndPointSelector selector;
+    @Inject
+    ImageDataDao imageDataDao;
     public DefaultImageDataStoreDriverImpl() {
     }
 
@@ -81,10 +85,13 @@ public class DefaultImageDataStoreDriverImpl implements ImageDataStoreDriver {
             CreateObjectCommand createCmd = new CreateObjectCommand(data.getUri());
             CreateObjectAnswer answer = (CreateObjectAnswer)ep.sendMessage(createCmd);
             if (answer.getResult()) {
+                //update imagestorevo
+               
                 result = new CreateCmdResult(answer.getPath(), answer.getSize());
             } else {
                 result.setResult(answer.getDetails());
             }
+            
         }
         
         callback.complete(result);

engine/storage/image/src/org/apache/cloudstack/storage/image/store/HttpDataStoreImpl.java

@@ -38,6 +38,7 @@ import org.apache.cloudstack.storage.image.db.ImageDataStoreVO;
 import org.apache.cloudstack.storage.snapshot.SnapshotInfo;
 
 import com.cloud.utils.component.ComponentContext;
+import com.cloud.utils.storage.encoding.EncodingType;
 
 
 public class HttpDataStoreImpl implements ImageDataStore {
@@ -100,11 +101,9 @@ public class HttpDataStoreImpl implements ImageDataStore {
 
     @Override
     public String getUri() {
-        return this.imageDataStoreVO.getProtocol() + "://" + "?role=" + this.getRole();
+        return this.imageDataStoreVO.getProtocol() + "://" + "?" + EncodingType.ROLE + "=" + this.getRole();
     }
 
-
-
     @Override
     public Scope getScope() {
         // TODO Auto-generated method stub

engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java

@@ -18,8 +18,6 @@
  */
 package org.apache.cloudstack.storage.image.store;
 
-import java.io.File;
-
 import javax.inject.Inject;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType;
@@ -37,9 +35,11 @@ import org.apache.log4j.Logger;
 
 import com.cloud.utils.component.ComponentContext;
 import com.cloud.utils.fsm.NoTransitionException;
+import com.cloud.utils.storage.encoding.EncodingType;
 
 public class TemplateObject implements TemplateInfo {
-    private static final Logger s_logger = Logger.getLogger(TemplateObject.class);
+    private static final Logger s_logger = Logger
+            .getLogger(TemplateObject.class);
     private ImageDataVO imageVO;
     private DataStore dataStore;
     @Inject
@@ -67,6 +67,10 @@ public class TemplateObject implements TemplateInfo {
         this.imageVO.setImageDataStoreId(id);
     }
     
+    public void setSize(Long size) {
+        this.imageVO.setSize(size);
+    }
+
     public ImageDataVO getImage() {
         return this.imageVO;
     }
@@ -89,14 +93,28 @@ public class TemplateObject implements TemplateInfo {
 
     @Override
     public String getUri() {
+        ImageDataVO image = imageDao.findById(this.imageVO.getId());
         if (this.dataStore == null) {
-            return this.imageVO.getUrl();
+            return image.getUrl();
         } else {
-            ObjectInDataStoreVO obj = ojbectInStoreMgr.findObject(this.imageVO.getId(), DataObjectType.TEMPLATE, this.dataStore.getId(), this.dataStore.getRole());
+            ObjectInDataStoreVO obj = ojbectInStoreMgr.findObject(
-            if (obj.getState() != ObjectInDataStoreStateMachine.State.Ready) {
+                    this.imageVO.getId(), DataObjectType.TEMPLATE,
-                return this.dataStore.getUri() + File.separator + "&objType=" + DataObjectType.TEMPLATE + "&size=" + this.imageVO.getSize() + "&path=" + this.imageVO.getUrl(); 
+                    this.dataStore.getId(), this.dataStore.getRole());
+            StringBuilder builder = new StringBuilder();
+            if (obj.getState() == ObjectInDataStoreStateMachine.State.Ready
+                    || obj.getState() == ObjectInDataStoreStateMachine.State.Copying) {
+                
+                builder.append(this.dataStore.getUri());
+                builder.append("&" + EncodingType.OBJTYPE + "=" + DataObjectType.TEMPLATE);
+                builder.append("&" + EncodingType.PATH + "=" + obj.getInstallPath());
+                builder.append("&" + EncodingType.SIZE + "=" + image.getSize());
+                return builder.toString();
             } else {
-                return this.dataStore.getUri() + File.separator + "&objType=" + DataObjectType.TEMPLATE + "&path=" + obj.getInstallPath();
+                builder.append(this.dataStore.getUri());
+                builder.append("&" + EncodingType.OBJTYPE + "=" + DataObjectType.TEMPLATE);
+                builder.append("&" + EncodingType.SIZE + "=" + image.getSize());
+                builder.append("&" + EncodingType.PATH + "=" + image.getUrl());
+                return builder.toString();
             }
         }
     }
@@ -104,9 +122,11 @@ public class TemplateObject implements TemplateInfo {
     @Override
     public Long getSize() {
         if (this.dataStore == null) {
-            return null;
+            return this.imageVO.getSize();
         }
-        ObjectInDataStoreVO obj = ojbectInStoreMgr.findObject(this.imageVO.getId(), DataObjectType.TEMPLATE, this.dataStore.getId(), this.dataStore.getRole());
+        ObjectInDataStoreVO obj = ojbectInStoreMgr.findObject(
+                this.imageVO.getId(), DataObjectType.TEMPLATE,
+                this.dataStore.getId(), this.dataStore.getRole());
         return obj.getSize();
     }
 
@@ -121,6 +141,9 @@ public class TemplateObject implements TemplateInfo {
     }
 
     public boolean stateTransit(TemplateEvent e) throws NoTransitionException {
-        return imageMgr.getStateMachine().transitTo(this.imageVO, e, null, imageDao);
+        boolean result= imageMgr.getStateMachine().transitTo(this.imageVO, e, null,
+                imageDao);
+        this.imageVO = imageDao.findById(this.imageVO.getId());
+        return result;
     }
 }

engine/storage/imagemotion/src/org/apache/cloudstack/storage/image/motion/DefaultImageMotionStrategy.java

@@ -93,7 +93,7 @@ public class DefaultImageMotionStrategy implements ImageMotionStrategy {
                 || srcStore.getRole() == DataStoreRole.ImageCache) {
             return true;
         }
-        return false;
+        return true;
     }
 
     @Override
@@ -124,13 +124,14 @@ public class DefaultImageMotionStrategy implements ImageMotionStrategy {
     
     protected Void copyAsyncCallback(AsyncCallbackDispatcher<DefaultImageMotionStrategy, Answer> callback, CreateTemplateContext<CopyCommandResult> context) {
         AsyncCompletionCallback<CopyCommandResult> parentCall = context.getParentCallback();
-        CopyCmdAnswer answer = (CopyCmdAnswer)callback.getResult();
+        Answer answer = (Answer)callback.getResult();
         if (!answer.getResult()) {
             CopyCommandResult result = new CopyCommandResult("");
             result.setResult(answer.getDetails());
             parentCall.complete(result);
         } else {
-            CopyCommandResult result = new CopyCommandResult(answer.getPath());
+            CopyCmdAnswer ans = (CopyCmdAnswer)answer;
+            CopyCommandResult result = new CopyCommandResult(ans.getPath());
             parentCall.complete(result);
         }
         return null;

engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java

@@ -19,7 +19,6 @@ package org.apache.cloudstack.storage.test;
 import org.apache.cloudstack.acl.APIChecker;
 import org.apache.cloudstack.engine.service.api.OrchestrationService;
 import org.apache.cloudstack.storage.HostEndpointRpcServer;
-import org.apache.cloudstack.storage.endpoint.DefaultEndPointSelector;
 import org.apache.cloudstack.storage.endpoint.EndPointSelector;
 import org.mockito.Mockito;
 import org.springframework.context.annotation.Bean;
@@ -28,6 +27,8 @@ import com.cloud.agent.AgentManager;
 import com.cloud.cluster.ClusteredAgentRebalanceService;
 import com.cloud.cluster.agentlb.dao.HostTransferMapDao;
 import com.cloud.cluster.agentlb.dao.HostTransferMapDaoImpl;
+import com.cloud.configuration.dao.ConfigurationDao;
+import com.cloud.configuration.dao.ConfigurationDaoImpl;
 import com.cloud.dc.dao.ClusterDao;
 import com.cloud.dc.dao.ClusterDaoImpl;
 import com.cloud.dc.dao.DataCenterDao;
@@ -39,6 +40,8 @@ import com.cloud.dc.dao.DcDetailsDaoImpl;
 import com.cloud.dc.dao.HostPodDao;
 import com.cloud.dc.dao.HostPodDaoImpl;
 import com.cloud.dc.dao.PodVlanDaoImpl;
+import com.cloud.domain.dao.DomainDao;
+import com.cloud.domain.dao.DomainDaoImpl;
 import com.cloud.host.dao.HostDao;
 import com.cloud.host.dao.HostDetailsDao;
 import com.cloud.host.dao.HostDetailsDaoImpl;
@@ -47,7 +50,10 @@ import com.cloud.host.dao.HostTagsDaoImpl;
 import com.cloud.server.auth.UserAuthenticator;
 import com.cloud.storage.dao.StoragePoolHostDao;
 import com.cloud.storage.dao.StoragePoolHostDaoImpl;
-import com.cloud.utils.component.ComponentContext;
+import com.cloud.storage.dao.VMTemplateDetailsDao;
+import com.cloud.storage.dao.VMTemplateDetailsDaoImpl;
+import com.cloud.storage.dao.VMTemplateZoneDao;
+import com.cloud.storage.dao.VMTemplateZoneDaoImpl;
 
 public class ChildTestConfiguration extends TestConfiguration {
 	
@@ -105,11 +111,22 @@ public class ChildTestConfiguration extends TestConfiguration {
 	@Bean StoragePoolHostDao storagePoolHostDao() {
 	    return new StoragePoolHostDaoImpl();
 	}
-	
+	@Bean VMTemplateZoneDao templateZoneDao() {
+	    return new VMTemplateZoneDaoImpl();
+	}
+	@Bean VMTemplateDetailsDao templateDetailsDao() {
+	    return new VMTemplateDetailsDaoImpl();
+	}
+	@Bean ConfigurationDao configDao() {
+	    return new ConfigurationDaoImpl();
+	}
 	@Bean
 	public AgentManager agentMgr() {
 		return new DirectAgentManagerSimpleImpl();
 	}
+	@Bean DomainDao domainDao() {
+	    return new DomainDaoImpl();
+	}
 	
     @Bean
     public HostEndpointRpcServer rpcServer() {

engine/storage/integration-test/test/org/apache/cloudstack/storage/test/MockRpcCallBack.java

@@ -19,14 +19,15 @@ package org.apache.cloudstack.storage.test;
 import javax.inject.Inject;
 
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
+import org.apache.log4j.Logger;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
 import com.cloud.utils.db.DB;
-import com.cloud.utils.db.Transaction;
 
 public class MockRpcCallBack implements Runnable {
+    private static final Logger s_logger = Logger.getLogger(MockRpcCallBack.class);
     @Inject
     AgentManager agentMgr;
     private Command cmd;
@@ -50,12 +51,9 @@ public class MockRpcCallBack implements Runnable {
     public void run() {
         try {
             Answer answer = agentMgr.send(hostId, cmd);
-            
             callback.complete(answer);
         } catch (Throwable e) {
-            //s_logger.debug("send command failed:" + e.toString());
+            s_logger.debug("send command failed:", e);
-        } finally {
-            int i =1;
         }
     }
     

engine/storage/integration-test/test/org/apache/cloudstack/storage/test/volumeServiceTest.java

@@ -25,12 +25,11 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.ExecutionException;
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import org.apache.cloudstack.engine.cloud.entity.api.TemplateEntity;
-import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity;
 import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
 import org.apache.cloudstack.engine.subsystem.api.storage.CommandResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@@ -38,11 +37,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreRole;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
-import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.ScopeType;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.type.RootDisk;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.storage.HypervisorHostEndPoint;
+import org.apache.cloudstack.storage.datastore.VolumeDataFactory;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreVO;
 import org.apache.cloudstack.storage.datastore.provider.DataStoreProvider;
@@ -54,6 +54,7 @@ import org.apache.cloudstack.storage.image.TemplateInfo;
 import org.apache.cloudstack.storage.image.db.ImageDataDao;
 import org.apache.cloudstack.storage.image.db.ImageDataVO;
 import org.apache.cloudstack.storage.volume.VolumeService;
+import org.apache.cloudstack.storage.volume.VolumeService.VolumeApiResult;
 import org.apache.cloudstack.storage.volume.db.VolumeDao2;
 import org.apache.cloudstack.storage.volume.db.VolumeVO;
 import org.mockito.Mockito;
@@ -109,12 +110,14 @@ public class volumeServiceTest extends CloudStackTestNGBase {
 	EndPointSelector selector;
 	@Inject
 	ImageDataFactory imageDataFactory;
+	@Inject
+	VolumeDataFactory volumeFactory;
 	Long dcId;
 	Long clusterId;
 	Long podId;
 	HostVO host;
 	String primaryName = "my primary data store";
-	PrimaryDataStoreInfo primaryStore;
+	DataStore primaryStore;
 	
     @Test(priority = -1)
 	public void setUp() {
@@ -199,6 +202,7 @@ public class volumeServiceTest extends CloudStackTestNGBase {
                 host.getPrivateIpAddress()));
         Mockito.when(selector.selectAll(Mockito.any(DataStore.class))).thenReturn(eps);
         Mockito.when(selector.select(Mockito.any(DataObject.class))).thenReturn(eps.get(0));
+        Mockito.when(selector.select(Mockito.any(DataObject.class), Mockito.any(DataObject.class))).thenReturn(eps.get(0));
     }
 
 	private ImageDataVO createImageData() {
@@ -227,13 +231,14 @@ public class volumeServiceTest extends CloudStackTestNGBase {
 		return image;
 	}
 
-	private TemplateEntity createTemplate() {
+	private TemplateInfo createTemplate() {
 		try {
 		    DataStore store = createImageStore();
 		    ImageDataVO image = createImageData();
 		    TemplateInfo template = imageDataFactory.getTemplate(image.getId(), store);
 		    AsyncCallFuture<CommandResult> future = imageService.createTemplateAsync(template, store);
 		    future.get();
+		    template = imageDataFactory.getTemplate(image.getId(), store);
 			/*imageProviderMgr.configure("image Provider", new HashMap<String, Object>());
 			ImageDataVO image = createImageData();
 			ImageDataStoreProvider defaultProvider = imageProviderMgr.getProvider("DefaultProvider");
@@ -242,19 +247,19 @@ public class volumeServiceTest extends CloudStackTestNGBase {
 			imageService.registerTemplate(image.getId(), store.getImageDataStoreId());
 			TemplateEntity te = imageService.getTemplateEntity(image.getId());
 			return te;*/
-		    return null;
+		    return template;
 		} catch (Exception e) {
 		    Assert.fail("failed", e);
 		    return null;
 		}
 	}
 
-	@Test
+	//@Test
 	public void createTemplateTest() {
 		createTemplate();
 	}
 	
-	//@Test
+	@Test
 	public void testCreatePrimaryStorage() {
 	    DataStoreProvider provider = dataStoreProviderMgr.getDataStoreProvider("default primary data store provider");
         Map<String, String> params = new HashMap<String, String>();
@@ -278,9 +283,9 @@ public class volumeServiceTest extends CloudStackTestNGBase {
         params.put("providerId", String.valueOf(provider.getId()));
         
         DataStoreLifeCycle lifeCycle = provider.getLifeCycle();
-        DataStore store = lifeCycle.initialize(params);
+        this.primaryStore  = lifeCycle.initialize(params);
         ClusterScope scope = new ClusterScope(clusterId, podId, dcId);
-        lifeCycle.attachCluster(store, scope);
+        lifeCycle.attachCluster(this.primaryStore, scope);
 	}
 	
 	private DataStore createImageStore() {
@@ -296,15 +301,14 @@ public class volumeServiceTest extends CloudStackTestNGBase {
         DataStore store = lifeCycle.initialize(params);
         return store;
 	}
-	@Test
+	//@Test
 	public void testcreateImageStore() {
 	    createImageStore();
 	}
 	
 
-	@Test
+	public DataStore createPrimaryDataStore() {
-	public PrimaryDataStoreInfo createPrimaryDataStore() {
+		try {
-		try {/*
 		    DataStoreProvider provider = dataStoreProviderMgr.getDataStoreProvider("default primary data store provider");
 		    Map<String, String> params = new HashMap<String, String>();
 		    URI uri = new URI(this.getPrimaryStorageUrl());
@@ -324,7 +328,7 @@ public class volumeServiceTest extends CloudStackTestNGBase {
 		    DataStore store = lifeCycle.initialize(params);
 		    ClusterScope scope = new ClusterScope(clusterId, podId, dcId);
 		    lifeCycle.attachCluster(store, scope);
-		    */
+		    
 		    /*
 		    PrimaryDataStoreProvider provider = primaryDataStoreProviderMgr.getDataStoreProvider("default primary data store provider");
 		    primaryDataStoreProviderMgr.configure("primary data store mgr", new HashMap<String, Object>());
@@ -349,7 +353,7 @@ public class volumeServiceTest extends CloudStackTestNGBase {
 			lc.attachCluster(scope);
 			return primaryDataStoreInfo;
 			*/
-		    return null;
+		    return store;
 		} catch (Exception e) {
 			return null;
 		}
@@ -362,23 +366,70 @@ public class volumeServiceTest extends CloudStackTestNGBase {
 		return volume;
 	}
 
-	//@Test(priority=2)
+	@Test(priority=2)
 	public void createVolumeFromTemplate() {
-	    primaryStore = createPrimaryDataStore();
+	    DataStore primaryStore = this.primaryStore;
-		TemplateEntity te = createTemplate();
+		TemplateInfo te = createTemplate();
 		VolumeVO volume = createVolume(te.getId(), primaryStore.getId());
-		VolumeEntity ve = volumeService.getVolumeEntity(volume.getId());
+		VolumeInfo vol = volumeFactory.getVolume(volume.getId(), primaryStore);
 		//ve.createVolumeFromTemplate(primaryStore.getId(), new VHD(), te);
-		ve.destroy();
+		AsyncCallFuture<VolumeApiResult> future = volumeService.createVolumeFromTemplateAsync(vol, primaryStore.getId(), te);
+		try {
+            future.get();
+        } catch (InterruptedException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (ExecutionException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
 	}
 	
 	//@Test(priority=3) 
 	public void createDataDisk() {
-	    primaryStore = createPrimaryDataStore();
+	    DataStore primaryStore = this.primaryStore;
 	    VolumeVO volume = createVolume(null, primaryStore.getId());
-	    VolumeEntity ve = volumeService.getVolumeEntity(volume.getId());
+	    VolumeInfo vol = volumeFactory.getVolume(volume.getId(), primaryStore);
-	    //ve.createVolume(primaryStore.getId(), new VHD());
+	    AsyncCallFuture<VolumeApiResult> future = volumeService.createVolumeAsync(vol, primaryStore.getId());
-	    ve.destroy();
+	    try {
+            future.get();
+        } catch (InterruptedException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (ExecutionException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+	}
+	
+	//@Test(priority=3) 
+	public void createAndDeleteDataDisk() {
+	    DataStore primaryStore = this.primaryStore;
+	    VolumeVO volume = createVolume(null, primaryStore.getId());
+	    VolumeInfo vol = volumeFactory.getVolume(volume.getId(), primaryStore);
+	    AsyncCallFuture<VolumeApiResult> future = volumeService.createVolumeAsync(vol, primaryStore.getId());
+	    try {
+	        future.get();
+	    } catch (InterruptedException e) {
+	        // TODO Auto-generated catch block
+	        e.printStackTrace();
+	    } catch (ExecutionException e) {
+	        // TODO Auto-generated catch block
+	        e.printStackTrace();
+	    }
+	    
+	    //delete the volume
+	    vol = volumeFactory.getVolume(volume.getId(), primaryStore);
+	    future = volumeService.deleteVolumeAsync(vol);
+	    try {
+            future.get();
+        } catch (InterruptedException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (ExecutionException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
 	}
 
 	//@Test(priority=3)

engine/storage/src/org/apache/cloudstack/storage/command/CopyCmd.java

@@ -16,25 +16,24 @@
 // under the License.
 package org.apache.cloudstack.storage.command;
 
-import org.apache.cloudstack.storage.to.ImageOnPrimayDataStoreTO;
-
 import com.cloud.agent.api.Command;
 
 public class CopyCmd extends Command implements StorageSubSystemCommand {
+    private String srcUri;
+    private String destUri;
 
-    private ImageOnPrimayDataStoreTO imageTO;
+    public CopyCmd(String srcUri, String destUri) {
-
-    protected CopyCmd() {
         super();
+        this.srcUri = srcUri;
+        this.destUri = destUri;
     }
     
-    public CopyCmd(String destUri, String srcUri) {
+    public String getDestUri() {
-        super();
+        return this.destUri;
-       // this.imageTO = image;
     }
     
-    public ImageOnPrimayDataStoreTO getImage() {
+    public String getSrcUri() {
-        return this.imageTO;
+        return this.srcUri;
     }
 
     @Override

engine/storage/src/org/apache/cloudstack/storage/datastore/DataObjectManager.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.storage.datastore;
+
+import org.apache.cloudstack.engine.subsystem.api.storage.CommandResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
+
+public interface DataObjectManager {
+    public void createAsync(DataObject data, DataStore store, AsyncCompletionCallback<CreateCmdResult> callback, boolean noCopy);
+    /*
+     * Only create internal state, without actually send down create command. 
+     * It's up to device driver decides whether to create object before copying
+     */
+    public DataObject createInternalStateOnly(DataObject data, DataStore store);
+    public void update(DataObject data, String path, Long size);
+    public void copyAsync(DataObject srcData, DataObject destData, AsyncCompletionCallback<CreateCmdResult> callback);
+    public void deleteAsync(DataObject data, AsyncCompletionCallback<CommandResult> callback);
+}

engine/storage/src/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java

@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.storage.datastore;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.engine.subsystem.api.storage.CommandResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
+import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
+import org.apache.cloudstack.framework.async.AsyncRpcConext;
+import org.apache.cloudstack.storage.db.ObjectInDataStoreVO;
+import org.apache.cloudstack.storage.motion.DataMotionService;
+import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine;
+import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine.Event;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.fsm.NoTransitionException;
+
+@Component
+public class DataObjectManagerImpl implements DataObjectManager {
+    private static final Logger s_logger = Logger
+            .getLogger(DataObjectManagerImpl.class);
+    @Inject
+    ObjectInDataStoreManager objectInDataStoreMgr;
+    @Inject
+    DataMotionService motionSrv;
+    protected long waitingTime = 1800; // half an hour
+    protected long waitingRetries = 10;
+    
+    protected DataObject waitingForCreated(DataObject dataObj,
+            DataStore dataStore) {
+        long retries = this.waitingRetries;
+        ObjectInDataStoreVO obj = null;
+        do {
+            try {
+                Thread.sleep(waitingTime);
+            } catch (InterruptedException e) {
+                s_logger.debug("sleep interrupted", e);
+                throw new CloudRuntimeException("sleep interrupted", e);
+            }
+
+            obj = objectInDataStoreMgr.findObject(dataObj.getId(),
+                    dataObj.getType(), dataStore.getId(), dataStore.getRole());
+            if (obj == null) {
+                s_logger.debug("can't find object in db, maybe it's cleaned up already, exit waiting");
+                break;
+            }
+            if (obj.getState() == ObjectInDataStoreStateMachine.State.Ready) {
+                break;
+            }
+            retries--;
+        } while (retries > 0);
+
+        if (obj == null || retries <= 0) {
+            s_logger.debug("waiting too long for template downloading, marked it as failed");
+            throw new CloudRuntimeException(
+                    "waiting too long for template downloading, marked it as failed");
+        }
+        return objectInDataStoreMgr.get(dataObj, dataStore);
+    }
+    class CreateContext<T> extends AsyncRpcConext<T> {
+        final DataObject objInStrore;
+
+        public CreateContext(AsyncCompletionCallback<T> callback,
+                DataObject objInStore) {
+            super(callback);
+           this.objInStrore = objInStore;
+        }
+
+    }
+    
+    @Override
+    public void createAsync(DataObject data, DataStore store,
+            AsyncCompletionCallback<CreateCmdResult> callback, boolean noCopy) {
+        ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(
+                data.getId(), data.getType(), store.getId(),
+                store.getRole());
+        DataObject objInStore = null;
+        boolean freshNewTemplate = false;
+        if (obj == null) {
+            try {
+                objInStore = objectInDataStoreMgr.create(
+                        data, store);
+                freshNewTemplate = true;
+            } catch (Throwable e) {
+                obj = objectInDataStoreMgr.findObject(data.getId(),
+                        data.getType(), store.getId(), store.getRole());
+                if (obj == null) {
+                    CreateCmdResult result = new CreateCmdResult(
+                            null, null);
+                    result.setSucess(false);
+                    result.setResult(e.toString());
+                    callback.complete(result);
+                    return;
+                }
+            }
+        }
+
+        if (!freshNewTemplate
+                && obj.getState() != ObjectInDataStoreStateMachine.State.Ready) {
+            try {
+                objInStore = waitingForCreated(
+                        data, store);
+            } catch (Exception e) {
+                CreateCmdResult result = new CreateCmdResult(null, null);
+                result.setSucess(false);
+                result.setResult(e.toString());
+                callback.complete(result);
+                return;
+            }
+
+            CreateCmdResult result = new CreateCmdResult(
+                    null, null);
+            callback.complete(result);
+            return;
+        }
+
+        try {
+            ObjectInDataStoreStateMachine.Event event = null;
+            if (noCopy) {
+                event = ObjectInDataStoreStateMachine.Event.CreateOnlyRequested;
+            } else {
+                event = ObjectInDataStoreStateMachine.Event.CreateRequested;
+            }
+            objectInDataStoreMgr.update(objInStore,
+                    event);
+        } catch (NoTransitionException e) {
+            try {
+                objectInDataStoreMgr.update(objInStore,
+                        ObjectInDataStoreStateMachine.Event.OperationFailed);
+            } catch (NoTransitionException e1) {
+                s_logger.debug("state transation failed", e1);
+            }
+            CreateCmdResult result = new CreateCmdResult(null, null);
+            result.setSucess(false);
+            result.setResult(e.toString());
+            callback.complete(result);
+            return;
+        }
+
+        CreateContext<CreateCmdResult> context = new CreateContext<CreateCmdResult>(
+                callback, objInStore);
+        AsyncCallbackDispatcher<DataObjectManagerImpl, CreateCmdResult> caller = AsyncCallbackDispatcher
+                .create(this);
+        caller.setCallback(
+                caller.getTarget().createAsynCallback(null, null))
+                .setContext(context);
+
+        store.getDriver().createAsync(objInStore, caller);
+        return;
+    }
+    
+    protected Void createAsynCallback(AsyncCallbackDispatcher<DataObjectManagerImpl, CreateCmdResult> callback,
+            CreateContext<CreateCmdResult> context) {
+        CreateCmdResult result = callback.getResult();
+        DataObject objInStrore = context.objInStrore;
+        CreateCmdResult upResult = new CreateCmdResult(
+                null, null);
+        if (result.isFailed()) {
+            upResult.setResult(result.getResult());
+            context.getParentCallback().complete(upResult);
+            return null;
+        }
+
+        ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(
+                objInStrore.getId(), objInStrore
+                        .getType(), objInStrore.getDataStore()
+                        .getId(), objInStrore.getDataStore()
+                        .getRole());
+
+        obj.setInstallPath(result.getPath());
+        obj.setSize(result.getSize());
+        try {
+            objectInDataStoreMgr.update(obj,
+                    ObjectInDataStoreStateMachine.Event.OperationSuccessed);
+        } catch (NoTransitionException e) {
+            try {
+                objectInDataStoreMgr.update(obj,
+                        ObjectInDataStoreStateMachine.Event.OperationFailed);
+            } catch (NoTransitionException e1) {
+                s_logger.debug("failed to change state", e1);
+            }
+
+            upResult.setResult(e.toString());
+            context.getParentCallback().complete(upResult);
+            return null;
+        }
+        
+        context.getParentCallback().complete(result);
+        return null;
+    }
+    
+    class CopyContext<T> extends AsyncRpcConext<T> {
+        DataObject destObj;
+        DataObject srcObj;
+
+        public CopyContext(AsyncCompletionCallback<T> callback,
+                DataObject srcObj,
+                DataObject destObj) {
+            super(callback);
+            this.srcObj = srcObj;
+            this.destObj = destObj;
+        }
+    }
+
+    @Override
+    public void copyAsync(DataObject srcData, DataObject destData,
+            AsyncCompletionCallback<CreateCmdResult> callback) {
+        try {
+            objectInDataStoreMgr.update(destData,
+                    ObjectInDataStoreStateMachine.Event.CopyingRequested);
+        } catch (NoTransitionException e) {
+            s_logger.debug("failed to change state", e);
+            try {
+                objectInDataStoreMgr.update(destData,
+                        ObjectInDataStoreStateMachine.Event.OperationFailed);
+            } catch (NoTransitionException e1) {
+
+            }
+            CreateCmdResult res = new CreateCmdResult(null, null);
+            res.setResult("Failed to change state: " + e.toString());
+            callback.complete(res);
+        }
+
+        CopyContext<CreateCmdResult> anotherCall = new CopyContext<CreateCmdResult>(
+                callback, srcData, destData);
+        AsyncCallbackDispatcher<DataObjectManagerImpl, CopyCommandResult> caller = AsyncCallbackDispatcher
+                .create(this);
+        caller.setCallback(caller.getTarget().copyCallback(null, null))
+                .setContext(anotherCall);
+
+        motionSrv.copyAsync(srcData, destData, caller);
+    }
+    
+    protected Void copyCallback(
+            AsyncCallbackDispatcher<DataObjectManagerImpl, CopyCommandResult> callback,
+            CopyContext<CreateCmdResult> context) {
+        CopyCommandResult result = callback.getResult();
+        DataObject destObj = context.destObj;
+        ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(
+                destObj.getId(), destObj
+                        .getType(), destObj.getDataStore()
+                        .getId(), destObj.getDataStore()
+                        .getRole());
+        if (result.isFailed()) {
+            try {
+                objectInDataStoreMgr.update(obj, Event.OperationFailed);
+            } catch (NoTransitionException e) {
+                s_logger.debug("Failed to update copying state", e);
+            }
+            CreateCmdResult res = new CreateCmdResult(
+                    null, null);
+            res.setResult(result.getResult());
+            context.getParentCallback().complete(res);
+        }
+
+        obj.setInstallPath(result.getPath());
+
+        try {
+            objectInDataStoreMgr.update(obj,
+                    ObjectInDataStoreStateMachine.Event.OperationSuccessed);
+        } catch (NoTransitionException e) {
+            s_logger.debug("Failed to update copying state: ", e);
+            try {
+                objectInDataStoreMgr.update(destObj,
+                        ObjectInDataStoreStateMachine.Event.OperationFailed);
+            } catch (NoTransitionException e1) {
+            }
+            CreateCmdResult res = new CreateCmdResult(
+                    null, null);
+            res.setResult("Failed to update copying state: " + e.toString());
+            context.getParentCallback().complete(res);
+        }
+        CreateCmdResult res = new CreateCmdResult(
+                result.getPath(), null);
+        context.getParentCallback().complete(res);
+        return null;
+    }
+
+    
+    class DeleteContext <T> extends AsyncRpcConext<T> {
+        private final DataObject obj;
+        public DeleteContext(AsyncCompletionCallback<T> callback, DataObject obj) {
+            super(callback);
+            this.obj = obj;
+        }
+        
+    }
+    @Override
+    public void deleteAsync(DataObject data,
+            AsyncCompletionCallback<CommandResult> callback) {
+        ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(
+                data.getId(), data.getType(), data.getDataStore().getId(),
+                data.getDataStore().getRole());
+        try {
+            objectInDataStoreMgr.update(obj, Event.DestroyRequested);
+        } catch (NoTransitionException e) {
+            s_logger.debug("destroy failed", e);
+            CreateCmdResult res = new CreateCmdResult(
+                    null, null);
+            callback.complete(res);
+        }
+        
+        DeleteContext<CommandResult> context = new DeleteContext<CommandResult>(
+                callback, data);
+        AsyncCallbackDispatcher<DataObjectManagerImpl, CommandResult> caller = AsyncCallbackDispatcher
+                .create(this);
+        caller.setCallback(
+                caller.getTarget().deleteAsynCallback(null, null))
+                .setContext(context);
+
+        data.getDataStore().getDriver().deleteAsync(data, caller);
+        return;
+    }
+    
+    protected Void deleteAsynCallback(AsyncCallbackDispatcher<DataObjectManagerImpl, CommandResult> callback,
+            DeleteContext<CommandResult> context) {
+        DataObject destObj = context.obj;
+        ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(
+                destObj.getId(), destObj
+                        .getType(), destObj.getDataStore()
+                        .getId(), destObj.getDataStore()
+                        .getRole());
+        
+        CommandResult res = callback.getResult();
+        if (res.isFailed()) {
+            try {
+                objectInDataStoreMgr.update(obj, Event.OperationFailed);
+            } catch (NoTransitionException e) {
+               s_logger.debug("delete failed", e);
+            }
+            
+        } else {
+            try {
+                objectInDataStoreMgr.update(obj, Event.OperationSuccessed);
+            } catch (NoTransitionException e) {
+               s_logger.debug("delete failed", e);
+            }
+        }
+        
+        context.getParentCallback().complete(res);
+        return null;
+    }
+
+    @Override
+    public DataObject createInternalStateOnly(DataObject data, DataStore store) {
+        ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(
+                data.getId(), data.getType(), store.getId(),
+                store.getRole());
+        DataObject objInStore = null;
+        if (obj == null) {
+            objInStore = objectInDataStoreMgr.create(
+                    data, store);
+        }
+        try {
+            ObjectInDataStoreStateMachine.Event event = null;
+            event = ObjectInDataStoreStateMachine.Event.CreateRequested;
+            objectInDataStoreMgr.update(objInStore,
+                    event);
+            
+            objectInDataStoreMgr.update(objInStore, ObjectInDataStoreStateMachine.Event.OperationSuccessed);
+        } catch (NoTransitionException e) {
+            s_logger.debug("Failed to update state", e);
+            throw new CloudRuntimeException("Failed to update state", e);
+        }
+        
+        return objInStore;
+    }
+
+    @Override
+    public void update(DataObject data, String path, Long size) {
+        ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(
+                data.getId(), data.getType(), data.getDataStore().getId(),
+                data.getDataStore().getRole());
+        
+        obj.setInstallPath(path);
+        obj.setSize(size);
+        objectInDataStoreMgr.update(obj);
+    }
+}

engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java

@@ -22,18 +22,21 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreRole;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.storage.db.ObjectInDataStoreVO;
-import org.apache.cloudstack.storage.image.TemplateInfo;
 import org.apache.cloudstack.storage.snapshot.SnapshotInfo;
-import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine;
 import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine.Event;
 
 import com.cloud.utils.fsm.NoTransitionException;
 
 public interface ObjectInDataStoreManager {
-    public TemplateInfo create(TemplateInfo template, DataStore dataStore);
+    public DataObject create(DataObject template, DataStore dataStore);
     public VolumeInfo create(VolumeInfo volume, DataStore dataStore);
     public SnapshotInfo create(SnapshotInfo snapshot, DataStore dataStore);
     public ObjectInDataStoreVO findObject(long objectId, DataObjectType type,
             long dataStoreId, DataStoreRole role);
+    public DataObject get(DataObject dataObj, DataStore store);
     public boolean update(DataObject vo, Event event) throws NoTransitionException;
+    boolean update(ObjectInDataStoreVO obj, Event event)
+            throws NoTransitionException;
+    
+    boolean update(ObjectInDataStoreVO obj);
 }

engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java

@@ -26,7 +26,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.storage.db.ObjectInDataStoreDao;
 import org.apache.cloudstack.storage.db.ObjectInDataStoreVO;
 import org.apache.cloudstack.storage.image.ImageDataFactory;
-import org.apache.cloudstack.storage.image.TemplateInfo;
 import org.apache.cloudstack.storage.snapshot.SnapshotInfo;
 import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine;
 import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine.Event;
@@ -83,17 +82,23 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager {
     }
 
     @Override
-    public TemplateInfo create(TemplateInfo template, DataStore dataStore) {
+    public DataObject create(DataObject obj, DataStore dataStore) {
+        
         ObjectInDataStoreVO vo = new ObjectInDataStoreVO();
         vo.setDataStoreId(dataStore.getId());
         vo.setDataStoreRole(dataStore.getRole());
-        vo.setObjectId(template.getId());
+        vo.setObjectId(obj.getId());
+        vo.setSize(obj.getSize());
 
-        vo.setObjectType(template.getType());
+        vo.setObjectType(obj.getType());
         vo = objectDataStoreDao.persist(vo);
 
-        return imageFactory.getTemplate(template.getId(), dataStore);
+        if (obj.getType() == DataObjectType.TEMPLATE) {
-
+            return imageFactory.getTemplate(obj.getId(), dataStore);
+        } else if (obj.getType() == DataObjectType.VOLUME) {
+            return volumeFactory.getVolume(obj.getId(), dataStore); 
+        }
+        throw new CloudRuntimeException("unknown type");
     }
 
     @Override
@@ -145,4 +150,27 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager {
                 objectDataStoreDao);
 
     }
+    
+    @Override
+    public boolean update(ObjectInDataStoreVO obj, Event event)
+            throws NoTransitionException {
+        return this.stateMachines.transitTo(obj, event, null,
+                objectDataStoreDao);
+
+    }
+
+    @Override
+    public DataObject get(DataObject dataObj, DataStore store) {
+        if (dataObj.getType() == DataObjectType.TEMPLATE) {
+            return imageFactory.getTemplate(dataObj.getId(), store);
+        } else if (dataObj.getType() == DataObjectType.VOLUME) {
+            return volumeFactory.getVolume(dataObj.getId(), store); 
+        }
+        throw new CloudRuntimeException("unknown type");
+    }
+
+    @Override
+    public boolean update(ObjectInDataStoreVO obj) {
+        return objectDataStoreDao.update(obj.getId(), obj);
+    }
 }

engine/storage/src/org/apache/cloudstack/storage/db/ObjectInDataStoreDaoImpl.java

@@ -16,31 +16,43 @@
 // under the License.
 package org.apache.cloudstack.storage.db;
 import java.util.Date;
+import java.util.Map;
 
-import org.springframework.stereotype.Component;
+import javax.naming.ConfigurationException;
 
 import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine.Event;
 import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine.State;
-import org.apache.cloudstack.storage.volume.db.VolumeVO;
 import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
 
+import com.cloud.storage.VolumeVO;
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
-import com.cloud.utils.db.SearchCriteria2;
-import com.cloud.utils.db.SearchCriteriaService;
 import com.cloud.utils.db.UpdateBuilder;
+import com.cloud.utils.db.SearchCriteria.Op;
 
 @Component
 public class ObjectInDataStoreDaoImpl extends GenericDaoBase<ObjectInDataStoreVO, Long> implements ObjectInDataStoreDao {
     private static final Logger s_logger = Logger.getLogger(ObjectInDataStoreDaoImpl.class);
+    private SearchBuilder<ObjectInDataStoreVO> updateStateSearch;
+    @Override
+    public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
+        updateStateSearch = this.createSearchBuilder();
+        updateStateSearch.and("id", updateStateSearch.entity().getId(), Op.EQ);
+        updateStateSearch.and("state", updateStateSearch.entity().getState(), Op.EQ);
+        updateStateSearch.and("updatedCount", updateStateSearch.entity().getUpdatedCount(), Op.EQ);
+        updateStateSearch.done();
+        return true;
+    }
     @Override
     public boolean updateState(State currentState, Event event,
             State nextState, ObjectInDataStoreVO vo, Object data) {
         Long oldUpdated = vo.getUpdatedCount();
         Date oldUpdatedTime = vo.getUpdated();
     
-        SearchCriteria<ObjectInDataStoreVO> sc = this.createSearchCriteria();
+        
+        SearchCriteria<ObjectInDataStoreVO> sc = updateStateSearch.create();
         sc.setParameters("id", vo.getId());
         sc.setParameters("state", currentState);
         sc.setParameters("updatedCount", vo.getUpdatedCount());

engine/storage/src/org/apache/cloudstack/storage/image/db/ImageDataDaoImpl.java

@@ -68,7 +68,9 @@ import com.cloud.utils.db.JoinBuilder;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.SearchCriteria.Func;
+import com.cloud.utils.db.SearchCriteria.Op;
 import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.UpdateBuilder;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 @Component
@@ -76,18 +78,19 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
     private static final Logger s_logger = Logger.getLogger(VMTemplateDaoImpl.class);
 
     @Inject
-    VMTemplateZoneDao _templateZoneDao = null;
+    VMTemplateZoneDao templateZoneDao;
     @Inject
-    VMTemplateDetailsDao _templateDetailsDao = null;
+    VMTemplateDetailsDao templateDetailsDao;
 
     @Inject
-    ConfigurationDao _configDao = null;
+    ConfigurationDao configDao;
     @Inject
-    HostDao _hostDao = null;
+    HostDao hostDao;
     @Inject
-    DomainDao _domainDao = null;
+    DomainDao domainDao;
     @Inject
-    DataCenterDao _dcDao = null;
+    DataCenterDao dcDao;
+
     private final String SELECT_TEMPLATE_HOST_REF = "SELECT t.id, h.data_center_id, t.unique_name, t.name, t.public, t.featured, t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, "
             + "t.checksum, t.display_text, t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type FROM vm_template t";
 
@@ -110,6 +113,7 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
     private SearchBuilder<ImageDataVO> PublicIsoSearch;
     private SearchBuilder<ImageDataVO> UserIsoSearch;
     private GenericSearchBuilder<ImageDataVO, Long> CountTemplatesByAccount;
+    private SearchBuilder<ImageDataVO> updateStateSearch;
 
     //ResourceTagsDaoImpl _tagsDao = ComponentInject.inject(ResourceTagsDaoImpl.class);
     @Inject
@@ -326,7 +330,7 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
 
         tmpltTypeHyperSearch = createSearchBuilder();
         tmpltTypeHyperSearch.and("templateType", tmpltTypeHyperSearch.entity().getTemplateType(), SearchCriteria.Op.EQ);
-        SearchBuilder<HostVO> hostHyperSearch = _hostDao.createSearchBuilder();
+        SearchBuilder<HostVO> hostHyperSearch = hostDao.createSearchBuilder();
         hostHyperSearch.and("type", hostHyperSearch.entity().getType(), SearchCriteria.Op.EQ);
         hostHyperSearch.and("zoneId", hostHyperSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
         hostHyperSearch.groupBy(hostHyperSearch.entity().getHypervisorType());
@@ -348,7 +352,7 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
         AccountIdSearch.and("publicTemplate", AccountIdSearch.entity().isPublicTemplate(), SearchCriteria.Op.EQ);
         AccountIdSearch.done();
 
-        SearchBuilder<VMTemplateZoneVO> tmpltZoneSearch = _templateZoneDao.createSearchBuilder();
+        SearchBuilder<VMTemplateZoneVO> tmpltZoneSearch = templateZoneDao.createSearchBuilder();
         tmpltZoneSearch.and("removed", tmpltZoneSearch.entity().getRemoved(), SearchCriteria.Op.NULL);
         tmpltZoneSearch.and("zoneId", tmpltZoneSearch.entity().getZoneId(), SearchCriteria.Op.EQ);
 
@@ -367,6 +371,11 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
         CountTemplatesByAccount.and("removed", CountTemplatesByAccount.entity().getRemoved(), SearchCriteria.Op.NULL);
         CountTemplatesByAccount.done();
 
+        updateStateSearch = this.createSearchBuilder();
+        updateStateSearch.and("id", updateStateSearch.entity().getId(), Op.EQ);
+        updateStateSearch.and("state", updateStateSearch.entity().getState(), Op.EQ);
+        updateStateSearch.and("updatedCount", updateStateSearch.entity().getUpdatedCount(), Op.EQ);
+        updateStateSearch.done();
         return result;
     }
 
@@ -572,7 +581,7 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
                 for (Account account : permittedAccounts) {
                     // accountType = account.getType();
                     // accountId = Long.toString(account.getId());
-                    DomainVO accountDomain = _domainDao.findById(account.getDomainId());
+                    DomainVO accountDomain = domainDao.findById(account.getDomainId());
 
                     // get all parent domain ID's all the way till root domain
                     DomainVO domainTreeNode = accountDomain;
@@ -580,7 +589,7 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
                         relatedDomainIds.append(domainTreeNode.getId());
                         relatedDomainIds.append(",");
                         if (domainTreeNode.getParent() != null) {
-                            domainTreeNode = _domainDao.findById(domainTreeNode.getParent());
+                            domainTreeNode = domainDao.findById(domainTreeNode.getParent());
                         } else {
                             break;
                         }
@@ -588,7 +597,7 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
 
                     // get all child domain ID's
                     if (isAdmin(account.getType())) {
-                        List<DomainVO> allChildDomains = _domainDao.findAllChildren(accountDomain.getPath(), accountDomain.getId());
+                        List<DomainVO> allChildDomains = domainDao.findAllChildren(accountDomain.getPath(), accountDomain.getId());
                         for (DomainVO childDomain : allChildDomains) {
                             relatedDomainIds.append(childDomain.getId());
                             relatedDomainIds.append(",");
@@ -779,7 +788,7 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
     }
 
     private String getOrderByLimit(Long pageSize, Long startIndex) {
-        Boolean isAscending = Boolean.parseBoolean(_configDao.getValue("sortkey.algorithm"));
+        Boolean isAscending = Boolean.parseBoolean(configDao.getValue("sortkey.algorithm"));
         isAscending = (isAscending == null ? true : isAscending);
 
         String sql;
@@ -806,17 +815,17 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
                 throw new CloudRuntimeException("Failed to persist the template " + tmplt);
             }
             if (tmplt.getDetails() != null) {
-                _templateDetailsDao.persist(tmplt.getId(), tmplt.getDetails());
+                templateDetailsDao.persist(tmplt.getId(), tmplt.getDetails());
             }
         }
-        VMTemplateZoneVO tmpltZoneVO = _templateZoneDao.findByZoneTemplate(zoneId, tmplt.getId());
+        VMTemplateZoneVO tmpltZoneVO = templateZoneDao.findByZoneTemplate(zoneId, tmplt.getId());
         if (tmpltZoneVO == null) {
             tmpltZoneVO = new VMTemplateZoneVO(zoneId, tmplt.getId(), new Date());
-            _templateZoneDao.persist(tmpltZoneVO);
+            templateZoneDao.persist(tmpltZoneVO);
         } else {
             tmpltZoneVO.setRemoved(null);
             tmpltZoneVO.setLastUpdated(new Date());
-            _templateZoneDao.update(tmpltZoneVO.getId(), tmpltZoneVO);
+            templateZoneDao.update(tmpltZoneVO.getId(), tmpltZoneVO);
         }
         txn.commit();
 
@@ -931,7 +940,36 @@ public class ImageDataDaoImpl extends GenericDaoBase<ImageDataVO, Long> implemen
     @Override
     public boolean updateState(TemplateState currentState, TemplateEvent event,
             TemplateState nextState, ImageDataVO vo, Object data) {
-        // TODO Auto-generated method stub
+        Long oldUpdated = vo.getUpdatedCount();
-        return false;
+        Date oldUpdatedTime = vo.getUpdated();
+    
+        
+        SearchCriteria<ImageDataVO> sc = updateStateSearch.create();
+        sc.setParameters("id", vo.getId());
+        sc.setParameters("state", currentState);
+        sc.setParameters("updatedCount", vo.getUpdatedCount());
+
+        vo.incrUpdatedCount();
+
+        UpdateBuilder builder = getUpdateBuilder(vo);
+        builder.set(vo, "state", nextState);
+        builder.set(vo, "updated", new Date());
+
+        int rows = update((ImageDataVO) vo, sc);
+        if (rows == 0 && s_logger.isDebugEnabled()) {
+            ImageDataVO dbVol = findByIdIncludingRemoved(vo.getId());
+            if (dbVol != null) {
+                StringBuilder str = new StringBuilder("Unable to update ").append(vo.toString());
+                str.append(": DB Data={id=").append(dbVol.getId()).append("; state=").append(dbVol.getState()).append("; updatecount=").append(dbVol.getUpdatedCount()).append(";updatedTime=")
+                        .append(dbVol.getUpdated());
+                str.append(": New Data={id=").append(vo.getId()).append("; state=").append(nextState).append("; event=").append(event).append("; updatecount=").append(vo.getUpdatedCount())
+                        .append("; updatedTime=").append(vo.getUpdated());
+                str.append(": stale Data={id=").append(vo.getId()).append("; state=").append(currentState).append("; event=").append(event).append("; updatecount=").append(oldUpdated)
+                        .append("; updatedTime=").append(oldUpdatedTime);
+            } else {
+                s_logger.debug("Unable to update objectIndatastore: id=" + vo.getId() + ", as there is no such object exists in the database anymore");
+            }
+        }
+        return rows > 0;
     }
 }

engine/storage/src/org/apache/cloudstack/storage/image/db/ImageDataVO.java

@@ -136,11 +136,18 @@ public class ImageDataVO implements Identity, StateObject<TemplateState> {
     private long imageDataStoreId;
     
     @Column(name = "size")
-    private long size;
+    private Long size;
     
     @Column(name = "state")
     private TemplateState state;
     
+    @Column(name="update_count", updatable = true)
+    protected long updatedCount;
+    
+    @Column(name = "updated")
+    @Temporal(value = TemporalType.TIMESTAMP)
+    Date updated;
+
     @Transient
     Map details;
 
@@ -408,11 +415,11 @@ public class ImageDataVO implements Identity, StateObject<TemplateState> {
         this.imageDataStoreId = dataStoreId;
     }
     
-    public void setSize(long size) {
+    public void setSize(Long size) {
         this.size = size;
     }
     
-    public long getSize() {
+    public Long getSize() {
         return this.size;
     }
     
@@ -420,4 +427,24 @@ public class ImageDataVO implements Identity, StateObject<TemplateState> {
         return this.state;
     }
     
+    public long getUpdatedCount() {
+        return this.updatedCount;
+    }
+    
+    public void incrUpdatedCount() {
+        this.updatedCount++;
+    }
+
+    public void decrUpdatedCount() {
+        this.updatedCount--;
+    }
+    
+    public Date getUpdated() {
+        return updated;
+    }
+    
+    public void setUpdated(Date updated) {
+        this.updated = updated;
+    }
+
 }

engine/storage/volume/src/org/apache/cloudstack/storage/datastore/DefaultPrimaryDataStore.java

@@ -25,6 +25,7 @@ import javax.inject.Inject;
 import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State;
 import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreRole;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle;
@@ -36,6 +37,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.disktype.DiskFormat;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreVO;
 import org.apache.cloudstack.storage.datastore.provider.DataStoreProvider;
+import org.apache.cloudstack.storage.db.ObjectInDataStoreVO;
 import org.apache.cloudstack.storage.image.ImageDataFactory;
 import org.apache.cloudstack.storage.image.TemplateInfo;
 import org.apache.cloudstack.storage.snapshot.SnapshotDataFactory;
@@ -48,6 +50,7 @@ import org.apache.log4j.Logger;
 
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.utils.component.ComponentContext;
+import com.cloud.utils.storage.encoding.EncodingType;
 
 public class DefaultPrimaryDataStore implements PrimaryDataStore {
     private static final Logger s_logger = Logger
@@ -126,11 +129,16 @@ public class DefaultPrimaryDataStore implements PrimaryDataStore {
     public String getUri() {
         String path = this.pdsv.getPath();
         path.replaceFirst("/*", "");
-        return this.pdsv.getPoolType() + ":" + File.separator + File.separator
+        StringBuilder builder = new StringBuilder();
-                + this.pdsv.getHostAddress() + File.separator
+        builder.append(this.pdsv.getPoolType());
-                + this.pdsv.getPath() + File.separator
+        builder.append("://");
-                + "?role=" + this.getRole()
+        builder.append(this.pdsv.getHostAddress());
-                + "&storeUuid=" + this.pdsv.getUuid();
+        builder.append(File.separator);
+        builder.append(this.pdsv.getPath());
+        builder.append(File.separator);
+        builder.append("?" + EncodingType.ROLE + "=" + this.getRole());
+        builder.append("&" + EncodingType.STOREUUID + "=" + this.pdsv.getUuid());
+        return builder.toString();
     }
 
     @Override
@@ -212,6 +220,10 @@ public class DefaultPrimaryDataStore implements PrimaryDataStore {
 
     @Override
     public TemplateInfo getTemplate(long templateId) {
+        ObjectInDataStoreVO obj = objectInStoreMgr.findObject(templateId, DataObjectType.TEMPLATE, this.getId(), this.getRole());
+        if (obj == null) {
+            return null;
+        }
         return imageDataFactory.getTemplate(templateId, this);
     }
 

engine/storage/volume/src/org/apache/cloudstack/storage/datastore/driver/DefaultPrimaryDataStoreDriverImpl.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.storage.datastore.driver;
 
+import java.net.URISyntaxException;
 import java.util.Set;
 
 import javax.inject.Inject;
@@ -29,17 +30,20 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
 import org.apache.cloudstack.framework.async.AsyncRpcConext;
-import org.apache.cloudstack.storage.command.CreateObjectCommand;
 import org.apache.cloudstack.storage.command.CreateObjectAnswer;
+import org.apache.cloudstack.storage.command.CreateObjectCommand;
 import org.apache.cloudstack.storage.command.DeleteCommand;
+import org.apache.cloudstack.storage.datastore.DataObjectManager;
 import org.apache.cloudstack.storage.endpoint.EndPointSelector;
 import org.apache.cloudstack.storage.snapshot.SnapshotInfo;
 import org.apache.cloudstack.storage.volume.PrimaryDataStoreDriver;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
-import com.cloud.storage.StoragePoolHostVO;
 import com.cloud.storage.dao.StoragePoolHostDao;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.storage.encoding.DecodedDataObject;
+import com.cloud.utils.storage.encoding.Decoder;
 
 
 public class DefaultPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver {
@@ -48,6 +52,8 @@ public class DefaultPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
     EndPointSelector selector;
     @Inject
     StoragePoolHostDao storeHostDao;
+    @Inject
+    DataObjectManager dataObjMgr;
     public DefaultPrimaryDataStoreDriverImpl() {
         
     }
@@ -158,7 +164,7 @@ public class DefaultPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
         EndPoint ep = selector.select(vol);
         CreateObjectCommand createCmd = new CreateObjectCommand(vol.getUri());
         
-        CreateVolumeContext<CreateCmdResult> context = null;
+        CreateVolumeContext<CreateCmdResult> context = new CreateVolumeContext<CreateCmdResult>(callback, vol);
         AsyncCallbackDispatcher<DefaultPrimaryDataStoreDriverImpl, Answer> caller = AsyncCallbackDispatcher.create(this);
         caller.setContext(context)
             .setCallback(caller.getTarget().createAsyncCallback(null, null));
@@ -168,8 +174,28 @@ public class DefaultPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
 
     @Override
     public String grantAccess(DataObject object, EndPoint ep) {
-        StoragePoolHostVO poolHost = storeHostDao.findByPoolHost(object.getDataStore().getId(), ep.getId());
+        //StoragePoolHostVO poolHost = storeHostDao.findByPoolHost(object.getDataStore().getId(), ep.getId());
-        return object.getUri() + "&storagePath=" + poolHost.getLocalPath();
+        
+        String uri = object.getUri();
+        try {
+            DecodedDataObject obj = Decoder.decode(uri);
+            if (obj.getPath() == null) {
+                //create an obj
+                EndPoint newEp = selector.select(object);
+                CreateObjectCommand createCmd = new CreateObjectCommand(uri);
+                CreateObjectAnswer answer = (CreateObjectAnswer)ep.sendMessage(createCmd);
+                if (answer.getResult()) {
+                    dataObjMgr.update(object, answer.getPath(), answer.getSize());
+                } else {
+                    s_logger.debug("failed to create object" + answer.getDetails());
+                    throw new CloudRuntimeException("failed to create object" + answer.getDetails());
+                }
+            }
+            
+            return object.getUri();
+        } catch (URISyntaxException e) {
+           throw new CloudRuntimeException("uri parsed error", e);
+        }
     }
 
     @Override

engine/storage/volume/src/org/apache/cloudstack/storage/volume/TemplateInstallStrategyImpl.java

@@ -106,8 +106,8 @@ public class TemplateInstallStrategyImpl implements TemplateInstallStrategy {
         boolean freshNewTemplate = false;
         if (obj == null) {
             try {
-                templateOnPrimaryStoreObj = objectInDataStoreMgr.create(
+                /*templateOnPrimaryStoreObj = objectInDataStoreMgr.create(
-                        template, store);
+                        template, store);*/
                 freshNewTemplate = true;
             } catch (Throwable e) {
                 obj = objectInDataStoreMgr.findObject(template.getId(),
@@ -201,12 +201,13 @@ public class TemplateInstallStrategyImpl implements TemplateInstallStrategy {
                         .getRole());
 
         obj.setInstallPath(result.getPath());
+        obj.setSize(result.getSize());
         try {
-            objectInDataStoreMgr.update(templateOnPrimaryStoreObj,
+            objectInDataStoreMgr.update(obj,
                     ObjectInDataStoreStateMachine.Event.OperationSuccessed);
         } catch (NoTransitionException e) {
             try {
-                objectInDataStoreMgr.update(templateOnPrimaryStoreObj,
+                objectInDataStoreMgr.update(obj,
                         ObjectInDataStoreStateMachine.Event.OperationFailed);
             } catch (NoTransitionException e1) {
                 s_logger.debug("failed to change state", e1);
@@ -273,7 +274,7 @@ public class TemplateInstallStrategyImpl implements TemplateInstallStrategy {
         CreateBaseImageResult res = new CreateBaseImageResult(
                 templateOnPrimaryStoreObj);
         try {
-            objectInDataStoreMgr.update(templateOnPrimaryStoreObj,
+            objectInDataStoreMgr.update(obj,
                     ObjectInDataStoreStateMachine.Event.OperationSuccessed);
         } catch (NoTransitionException e) {
             s_logger.debug("Failed to update copying state: ", e);

engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java

@@ -44,7 +44,8 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory {
         VolumeVO volumeVO = volumeDao.findById(volumeId);
         ObjectInDataStoreVO obj = objMap.findObject(volumeId, DataObjectType.VOLUME, store.getId(), store.getRole());
         if (obj == null) {
-            return null;
+            VolumeObject vol = VolumeObject.getVolumeObject(null, volumeVO);
+            return vol;
         }
         VolumeObject vol = VolumeObject.getVolumeObject(store, volumeVO);
         return vol;

engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java

@@ -16,8 +16,6 @@
 // under the License.
 package org.apache.cloudstack.storage.volume;
 
-import java.io.File;
-
 import javax.inject.Inject;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType;
@@ -35,6 +33,7 @@ import com.cloud.utils.component.ComponentContext;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.fsm.NoTransitionException;
 import com.cloud.utils.fsm.StateMachine2;
+import com.cloud.utils.storage.encoding.EncodingType;
 
 public class VolumeObject implements VolumeInfo {
     private static final Logger s_logger = Logger.getLogger(VolumeObject.class);
@@ -48,14 +47,18 @@ public class VolumeObject implements VolumeInfo {
     @Inject
     ObjectInDataStoreManager ojbectInStoreMgr;
 
-    private VolumeObject(DataStore dataStore, VolumeVO volumeVO) {
+    protected VolumeObject() {
+      
+    }
+    
+    protected void configure(DataStore dataStore, VolumeVO volumeVO) {
         this.volumeVO = volumeVO;
         this.dataStore = dataStore;
     }
 
     public static VolumeObject getVolumeObject(DataStore dataStore, VolumeVO volumeVO) {
-        VolumeObject vo = new VolumeObject(dataStore, volumeVO);
+        VolumeObject vo = ComponentContext.inject(VolumeObject.class);
-        vo = ComponentContext.inject(vo);
+        vo.configure(dataStore, volumeVO);
         return vo;
     }
 
@@ -121,9 +124,14 @@ public class VolumeObject implements VolumeInfo {
         }
         ObjectInDataStoreVO obj = ojbectInStoreMgr.findObject(this.volumeVO.getId(), DataObjectType.VOLUME, this.dataStore.getId(), this.dataStore.getRole());
         if (obj.getState() != ObjectInDataStoreStateMachine.State.Ready) {
-            return this.dataStore.getUri() + File.separator + "&objType=" + DataObjectType.VOLUME + "&size=" + this.volumeVO.getSize() + "&name=" + this.volumeVO.getName();
+            return this.dataStore.getUri() + 
+                    "&" + EncodingType.OBJTYPE + "=" + DataObjectType.VOLUME + 
+                    "&" + EncodingType.SIZE + "=" + this.volumeVO.getSize() + 
+                    "&" + EncodingType.NAME + "=" + this.volumeVO.getName();
         } else {
-            return this.dataStore.getUri() + File.separator + "&objType=" + DataObjectType.VOLUME + "&path=" + obj.getInstallPath();
+            return this.dataStore.getUri() +
+                    "&" + EncodingType.OBJTYPE + "=" + DataObjectType.VOLUME + 
+                    "&" + EncodingType.PATH + "=" + obj.getInstallPath();
         }
     }
 

engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java

@@ -22,7 +22,8 @@ import javax.inject.Inject;
 
 import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity;
 import org.apache.cloudstack.engine.subsystem.api.storage.CommandResult;
-import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
@@ -31,6 +32,7 @@ import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
 import org.apache.cloudstack.framework.async.AsyncRpcConext;
+import org.apache.cloudstack.storage.datastore.DataObjectManager;
 import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager;
 import org.apache.cloudstack.storage.datastore.PrimaryDataStore;
 import org.apache.cloudstack.storage.datastore.PrimaryDataStoreProviderManager;
@@ -55,6 +57,8 @@ public class VolumeServiceImpl implements VolumeService {
     @Inject
     ObjectInDataStoreManager objectInDataStoreMgr;
     @Inject
+    DataObjectManager dataObjectMgr;
+    @Inject
     ImageMotionService imageMotion;
     @Inject
     TemplateInstallStrategy templateInstallStrategy;
@@ -85,6 +89,8 @@ public class VolumeServiceImpl implements VolumeService {
         
     }
     
+    
+    
     @Override
     public AsyncCallFuture<VolumeApiResult> createVolumeAsync(VolumeInfo volume, long dataStoreId) {
         PrimaryDataStore dataStore = dataStoreMgr.getPrimaryDataStore(dataStoreId);
@@ -107,16 +113,16 @@ public class VolumeServiceImpl implements VolumeService {
         vo.stateTransit(Volume.Event.CreateRequested);
 
         CreateVolumeContext<VolumeApiResult> context = new CreateVolumeContext<VolumeApiResult>(null, vo, future);
-        AsyncCallbackDispatcher<VolumeServiceImpl, CommandResult> caller = AsyncCallbackDispatcher.create(this);
+        AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> caller = AsyncCallbackDispatcher.create(this);
         caller.setCallback(caller.getTarget().createVolumeCallback(null, null))
         .setContext(context);
         
-        //dataStore.createVolumeAsync(vo, null, caller);
+        dataObjectMgr.createAsync(volume, dataStore, caller, true);
         return future;
     }
     
-    protected Void createVolumeCallback(AsyncCallbackDispatcher<VolumeServiceImpl, CommandResult> callback, CreateVolumeContext<VolumeApiResult> context) {
+    protected Void createVolumeCallback(AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> callback, CreateVolumeContext<VolumeApiResult> context) {
-        CommandResult result = callback.getResult();
+        CreateCmdResult result = callback.getResult();
         VolumeObject vo = context.getVolume();
         VolumeApiResult volResult = new VolumeApiResult(vo);
         if (result.isSuccess()) {
@@ -172,7 +178,7 @@ public class VolumeServiceImpl implements VolumeService {
         caller.setCallback(caller.getTarget().deleteVolumeCallback(null, null))
             .setContext(context);
         
-        dataStore.getDriver().deleteAsync(volume, caller);
+        dataObjectMgr.deleteAsync(volume, caller);
         return future;
     }
     
@@ -279,21 +285,23 @@ public class VolumeServiceImpl implements VolumeService {
     
     @DB
     protected void createBaseImageAsync(VolumeInfo volume, PrimaryDataStore dataStore, TemplateInfo template, AsyncCallFuture<VolumeApiResult> future) {
-        CreateBaseImageContext<VolumeApiResult> context = new CreateBaseImageContext<VolumeApiResult>(null, volume, 
+        CreateBaseImageContext<CreateCmdResult> context = new CreateBaseImageContext<CreateCmdResult>(null, volume, 
                 dataStore,
                 template,
                 future);
         
-        AsyncCallbackDispatcher<VolumeServiceImpl, CreateBaseImageResult> caller = AsyncCallbackDispatcher.create(this);
+        AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> caller = AsyncCallbackDispatcher.create(this);
-        caller.setCallback(caller.getTarget().createBaseImageCallback(null, null))
+        caller.setCallback(caller.getTarget().copyBaseImageCallback(null, null))
         .setContext(context);
+        DataObject templateOnPrimaryStoreObj = dataObjectMgr.createInternalStateOnly(template, dataStore);
      
-        templateInstallStrategy.installAsync(template, dataStore, caller);
+        dataObjectMgr.copyAsync(context.srcTemplate, templateOnPrimaryStoreObj, caller);
+        return;
     }
     
     @DB
-    protected Void createBaseImageCallback(AsyncCallbackDispatcher<VolumeServiceImpl, CreateBaseImageResult> callback, CreateBaseImageContext<VolumeApiResult> context) {
+    protected Void copyBaseImageCallback(AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> callback, CreateBaseImageContext<VolumeApiResult> context) {
-        CreateBaseImageResult result = callback.getResult();
+        CreateCmdResult result = callback.getResult();
         VolumeApiResult res = new VolumeApiResult(context.getVolume());
         
         AsyncCallFuture<VolumeApiResult> future = context.getFuture();
@@ -302,23 +310,26 @@ public class VolumeServiceImpl implements VolumeService {
             future.complete(res);
             return null;
         }
+        DataObject templateOnPrimaryStoreObj = objectInDataStoreMgr.get(context.srcTemplate, context.dataStore);
 
-        //now create volume on base image
+        createVolumeFromBaseImageAsync(context.volume, templateOnPrimaryStoreObj, context.dataStore, future);
-        TemplateInfo templateOnPrimaryStoreObj = result.template;
-        VolumeInfo volume = context.getVolume();
-        PrimaryDataStore pd = context.getDataStore();
-
-        createVolumeFromBaseImageAsync(volume, templateOnPrimaryStoreObj, pd, future);
         return null;
     }
     
     private class CreateVolumeFromBaseImageContext<T> extends AsyncRpcConext<T> {
         private final VolumeObject vo;
         private final AsyncCallFuture<VolumeApiResult> future;
-        public CreateVolumeFromBaseImageContext(AsyncCompletionCallback<T> callback, VolumeObject vo, AsyncCallFuture<VolumeApiResult> future) {
+        private final DataStore primaryStore;
+        private final DataObject templateOnStore;
+        public CreateVolumeFromBaseImageContext(AsyncCompletionCallback<T> callback, VolumeObject vo, 
+                DataStore primaryStore,
+                DataObject templateOnStore,
+                AsyncCallFuture<VolumeApiResult> future) {
             super(callback);
             this.vo = vo;
             this.future = future;
+            this.primaryStore = primaryStore;
+            this.templateOnStore = templateOnStore;
         }
         
         public VolumeObject getVolumeObject() {
@@ -331,7 +342,7 @@ public class VolumeServiceImpl implements VolumeService {
     }
     
     @DB
-    protected void createVolumeFromBaseImageAsync(VolumeInfo volume, TemplateInfo templateOnPrimaryStore, PrimaryDataStore pd, AsyncCallFuture<VolumeApiResult> future) {
+    protected void createVolumeFromBaseImageAsync(VolumeInfo volume, DataObject templateOnPrimaryStore, PrimaryDataStore pd, AsyncCallFuture<VolumeApiResult> future) {
         VolumeObject vo = (VolumeObject) volume;
         try {
             vo.stateTransit(Volume.Event.CreateRequested);
@@ -342,19 +353,22 @@ public class VolumeServiceImpl implements VolumeService {
             return;
         }
 
-        CreateVolumeFromBaseImageContext<VolumeApiResult> context = new CreateVolumeFromBaseImageContext<VolumeApiResult>(null, vo, future);
+        CreateVolumeFromBaseImageContext<VolumeApiResult> context = new CreateVolumeFromBaseImageContext<VolumeApiResult>(null, vo, pd, templateOnPrimaryStore, future);
-        AsyncCallbackDispatcher<VolumeServiceImpl, CopyCommandResult> caller =  AsyncCallbackDispatcher.create(this);
+        AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> caller =  AsyncCallbackDispatcher.create(this);
-        caller.setCallback(caller.getTarget().createVolumeFromBaseImageCallback(null, null))
+        caller.setCallback(caller.getTarget().copyBaseImageCallBack(null, null))
         .setContext(context);
 
-       pd.getDriver().copyAsync(volume, templateOnPrimaryStore, caller);
+        DataObject volumeOnPrimaryStorage = dataObjectMgr.createInternalStateOnly(volume, pd);
+        dataObjectMgr.copyAsync(context.templateOnStore, volumeOnPrimaryStorage, caller);
+        return;
     }
     
     @DB
-    public Object createVolumeFromBaseImageCallback(AsyncCallbackDispatcher<VolumeServiceImpl, CopyCommandResult> callback, CreateVolumeFromBaseImageContext<VolumeApiResult> context) {
+    public Void copyBaseImageCallBack(AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> callback, CreateVolumeFromBaseImageContext<VolumeApiResult> context) {
         VolumeObject vo = context.getVolumeObject();
-        CopyCommandResult result = callback.getResult();
+        CreateCmdResult result = callback.getResult();
         VolumeApiResult volResult = new VolumeApiResult(vo);
+       
         if (result.isSuccess()) {
             if (result.getPath() != null) {
                 vo.setPath(result.getPath());

plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java

@@ -28,22 +28,18 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.UUID;
 
 import org.apache.cloudstack.storage.command.AttachPrimaryDataStoreAnswer;
 import org.apache.cloudstack.storage.command.AttachPrimaryDataStoreCmd;
 import org.apache.cloudstack.storage.command.CopyCmd;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
+import org.apache.cloudstack.storage.command.CreateObjectAnswer;
 import org.apache.cloudstack.storage.command.CreateObjectCommand;
 import org.apache.cloudstack.storage.command.CreatePrimaryDataStoreCmd;
-import org.apache.cloudstack.storage.command.CreateObjectAnswer;
 import org.apache.cloudstack.storage.command.CreateVolumeFromBaseImageCommand;
 import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
 import org.apache.cloudstack.storage.datastore.protocol.DataStoreProtocol;
 import org.apache.cloudstack.storage.to.ImageOnPrimayDataStoreTO;
-import org.apache.cloudstack.storage.to.NfsPrimaryDataStoreTO;
-import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
-import org.apache.cloudstack.storage.to.TemplateTO;
 import org.apache.cloudstack.storage.to.VolumeTO;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
@@ -57,8 +53,10 @@ import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.storage.DeleteVolumeCommand;
 import com.cloud.hypervisor.xen.resource.CitrixResourceBase.SRType;
-import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.storage.encoding.DataStore;
+import com.cloud.utils.storage.encoding.DecodedDataObject;
+import com.cloud.utils.storage.encoding.Decoder;
 import com.xensource.xenapi.Connection;
 import com.xensource.xenapi.Host;
 import com.xensource.xenapi.PBD;
@@ -137,28 +135,23 @@ public class XenServerStorageResource {
     }
     protected CreateObjectAnswer execute(CreateObjectCommand cmd) {
         String uriString = cmd.getObjectUri();
-        Map<String, String> params = null;
+        DecodedDataObject obj = null;
 
-        try {
-            URI uri = new URI(uriString);
-            params = getParameters(uri);
-        } catch (URISyntaxException e1) {
-            s_logger.debug("uri exception", e1);
-            return new CreateObjectAnswer(cmd, false, e1.toString()); 
-        }
-        
-        if (params.get("objType").equalsIgnoreCase("template")) {
-            return getTemplateSize(cmd, params.get("path"));
-        }
-        
-        long size = Long.parseLong(params.get("size"));
-        String name = params.get("name");
-        String storeUuid = params.get("storagePath");
         Connection conn = hypervisorResource.getConnection();
         VDI vdi = null;
         boolean result = false;
         String errorMsg = null;
+        
         try {
+            obj = Decoder.decode(uriString);
+            DataStore store = obj.getStore(); 
+            if (obj.getObjType().equalsIgnoreCase("template") && store.getRole().equalsIgnoreCase("image")) {
+                return getTemplateSize(cmd, obj.getPath());
+            }
+            
+            long size = obj.getSize();
+            String name = obj.getName();
+            String storeUuid = store.getUuid();
             SR primaryDataStoreSR = getSRByNameLabel(conn, storeUuid);
             vdi = createVdi(conn, name, primaryDataStoreSR, size);
             VDI.Record record = vdi.getRecord(conn);
@@ -173,6 +166,9 @@ public class XenServerStorageResource {
         } catch (XmlRpcException e) {
             s_logger.debug("Failed to create volume", e);
             errorMsg = e.toString();
+        } catch (URISyntaxException e) {
+            s_logger.debug("Failed to create volume", e);
+            errorMsg = e.toString();
         } finally {
             if (!result && vdi != null) {
                 try {
@@ -227,13 +223,13 @@ public class XenServerStorageResource {
         }
     }
     
-    protected SR getNfsSR(Connection conn, URI uri) {
+    protected SR getNfsSR(Connection conn, DataStore store) {
         Map<String, String> deviceConfig = new HashMap<String, String>();
-        Map<String, String> params = getParameters(uri);
+
-        String uuid = params.get("storeUuid");
+        String uuid = store.getUuid();
         try {
-            String server = uri.getHost();
+            String server = store.getServer();
-            String serverpath = uri.getPath();
+            String serverpath = store.getPath();
             
             serverpath = serverpath.replace("//", "/");
             Set<SR> srs = SR.getAll(conn);
@@ -410,25 +406,17 @@ public class XenServerStorageResource {
     protected Answer execute(CreatePrimaryDataStoreCmd cmd) {
         Connection conn = hypervisorResource.getConnection();
         String storeUrl = cmd.getDataStore();
-        String scheme = null;
-        String type = null;
-        URI storeUri = null;
-        try {
-            storeUri = new URI(storeUrl);
-        } catch(URISyntaxException e) {
-            return new Answer(cmd, false, e.toString());
-        }
-        
-        scheme = storeUri.getScheme();
 
         try {
-            if (scheme.equalsIgnoreCase("nfs")) {
+            DecodedDataObject obj = Decoder.decode(storeUrl);
-                SR sr = getNfsSR(conn, storeUri);
+            DataStore store = obj.getStore();
-            } else if (scheme.equalsIgnoreCase("iscsi")) {
+            if (store.getScheme().equalsIgnoreCase("nfs")) {
+                SR sr = getNfsSR(conn, store);
+            } else if (store.getScheme().equalsIgnoreCase("iscsi")) {
                 //getIscsiSR(conn, dataStore);
-            } else if (scheme.equalsIgnoreCase("presetup")) {
+            } else if (store.getScheme().equalsIgnoreCase("presetup")) {
             } else {
-                return new Answer(cmd, false, "The pool type: " + scheme + " is not supported.");
+                return new Answer(cmd, false, "The pool type: " + store.getScheme() + " is not supported.");
             }
             return new Answer(cmd, true, "success");
         } catch (Exception e) {
@@ -509,31 +497,30 @@ public class XenServerStorageResource {
        
     }
     
-    protected Answer directDownloadHttpTemplate(CopyCmd cmd, TemplateTO template, PrimaryDataStoreTO primarDataStore) {
+    protected Answer directDownloadHttpTemplate(CopyCmd cmd, DecodedDataObject srcObj, DecodedDataObject destObj) {
-        String primaryStoreUuid = primarDataStore.getUuid();
         Connection conn = hypervisorResource.getConnection();
         SR poolsr = null;
         VDI vdi = null;
         boolean result = false;
         try {
+            if (destObj.getPath() == null) {
+                //need to create volume at first
                 
-            Set<SR> srs = SR.getByNameLabel(conn, primaryStoreUuid);
+            }
+            vdi = VDI.getByUuid(conn, destObj.getPath());
+            if (vdi == null) {
+                throw new CloudRuntimeException("can't find volume: " + destObj.getPath());
+            }
+            String destStoreUuid = destObj.getStore().getUuid();
+            Set<SR> srs = SR.getByNameLabel(conn, destStoreUuid);
             if (srs.size() != 1) {
-                throw new CloudRuntimeException("storage uuid: " + primaryStoreUuid + " is not unique");
+                throw new CloudRuntimeException("storage uuid: " + destStoreUuid + " is not unique");
             }
             poolsr = srs.iterator().next();
-            VDI.Record vdir = new VDI.Record();
+            VDI.Record vdir = vdi.getRecord(conn);
-            vdir.nameLabel = "Base-Image-" + UUID.randomUUID().toString();
-            vdir.SR = poolsr;
-            vdir.type = Types.VdiType.USER;
-            
-            vdir.virtualSize = getTemplateSize(conn, template.getPath());
-            vdi = VDI.create(conn, vdir);
-            
-            vdir = vdi.getRecord(conn);
             String vdiLocation = vdir.location;
             String pbdLocation = null;
-            if (primarDataStore.getType().equalsIgnoreCase(DataStoreProtocol.NFS.toString())) {
+            if (destObj.getStore().getScheme().equalsIgnoreCase(DataStoreProtocol.NFS.toString())) {
                 pbdLocation = "/run/sr-mount/" + poolsr.getUuid(conn);
             } else {
                 Set<PBD> pbds = poolsr.getPBDs(conn);
@@ -551,7 +538,7 @@ public class XenServerStorageResource {
             String vdiPath = pbdLocation + "/" + vdiLocation + ".vhd";
             //download a url into vdipath
             //downloadHttpToLocalFile(vdiPath, template.getPath());
-            hypervisorResource.callHostPlugin(conn, "storagePlugin", "downloadTemplateFromUrl", "destPath", vdiPath, "srcUrl", template.getPath());
+            hypervisorResource.callHostPlugin(conn, "storagePlugin", "downloadTemplateFromUrl", "destPath", vdiPath, "srcUrl", srcObj.getPath());
             result = true;
             return new CopyCmdAnswer(cmd, vdi.getUuid(conn));
         } catch (BadServerResponse e) {
@@ -581,16 +568,10 @@ public class XenServerStorageResource {
     protected Answer execute(AttachPrimaryDataStoreCmd cmd) {
         String dataStoreUri = cmd.getDataStore();
         Connection conn = hypervisorResource.getConnection();
-        Map<String, String> params = null;
         try {
-            try {
+            DecodedDataObject obj = Decoder.decode(dataStoreUri);
-                URI uri = new URI(dataStoreUri);
+            DataStore store = obj.getStore();
-                params = getParameters(uri);
+            SR sr = hypervisorResource.getStorageRepository(conn, store.getUuid());
-            } catch (URISyntaxException e1) {
-                s_logger.debug("uri exception", e1);
-                return new CreateObjectAnswer(cmd, false, e1.toString()); 
-            }
-            SR sr = hypervisorResource.getStorageRepository(conn, params.get("storeUuid"));
             hypervisorResource.setupHeartbeatSr(conn, sr, false);
             long capacity = sr.getPhysicalSize(conn);
             long available = capacity - sr.getPhysicalUtilisation(conn);
@@ -616,10 +597,18 @@ public class XenServerStorageResource {
     }
     
     protected Answer execute(CopyCmd cmd) {
-        ImageOnPrimayDataStoreTO imageTO = cmd.getImage();
+        DecodedDataObject srcObj = null;
-        TemplateTO template = imageTO.getTemplate();
+        DecodedDataObject destObj = null;
-        if (template.getPath().startsWith("http")) {
+        try {
-            return directDownloadHttpTemplate(cmd, template, imageTO.getPrimaryDataStore());
+            srcObj = Decoder.decode(cmd.getSrcUri());
+            destObj = Decoder.decode(cmd.getDestUri());
+        } catch (URISyntaxException e) {
+            return new Answer(cmd, false, e.toString());
+        }
+       
+        
+        if (srcObj.getPath().startsWith("http")) {
+            return directDownloadHttpTemplate(cmd, srcObj, destObj);
         } else {
             return new Answer(cmd, false, "not implemented yet");
             /*

server/src/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java

@@ -37,7 +37,7 @@ public class VMTemplateDetailsDaoImpl extends GenericDaoBase<VMTemplateDetailVO,
     protected final SearchBuilder<VMTemplateDetailVO> TemplateSearch;
     protected final SearchBuilder<VMTemplateDetailVO> DetailSearch;
     
-	protected VMTemplateDetailsDaoImpl() {
+	public VMTemplateDetailsDaoImpl() {
 		TemplateSearch = createSearchBuilder();
 		TemplateSearch.and("templateId", TemplateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
 		TemplateSearch.done();

setup/db/4.1-new-db-schema.sql

@@ -18,6 +18,8 @@
 alter table vm_template add image_data_store_id bigint unsigned;
 alter table vm_template add size bigint unsigned;
 alter table vm_template add state varchar(255);
+alter table vm_template add update_count bigint unsigned;
+alter table vm_template add updated  datetime;
 alter table storage_pool add storage_provider_id bigint unsigned; 
 alter table storage_pool add scope varchar(255); 
 alter table storage_pool modify id bigint unsigned AUTO_INCREMENT UNIQUE NOT NULL;
@@ -67,7 +69,7 @@ CREATE TABLE  `cloud`.`object_datastore_ref` (
   `size` bigint unsigned COMMENT 'the size of the template on the pool',
   `state` varchar(255) NOT NULL,
   `update_count` bigint unsigned NOT NULL,
-  `updated` DATETIME NOT NULL,
+  `updated` DATETIME,
   PRIMARY KEY  (`id`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;