Make copyFromS3ToNFS flow work.

core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java

@@ -231,9 +231,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
 
-    protected Answer copyFromS3ToNfs(CopyCommand cmd, DataTO srcData, S3TO s3,
+    protected Answer copyFromS3ToNfs(CopyCommand cmd, DataTO srcData, S3TO s3, DataTO destData, NfsTO destImageStore) {
-
-    DataTO destData, NfsTO destImageStore) {
         final String storagePath = destImageStore.getUrl();
         final String destPath = destData.getPath();
 
@@ -288,8 +286,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
             return new CopyCmdAnswer(errMsg);
         }
     }
-    
+
-   
+
     protected Answer copySnapshotToTemplateFromNfsToNfsXenserver(CopyCommand cmd, SnapshotObjectTO srcData, NfsTO srcDataStore, TemplateObjectTO destData, NfsTO destDataStore) {
         String srcMountPoint = this.getRootDir(srcDataStore.getUrl());
         String snapshotPath = srcData.getPath();
@@ -330,7 +328,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
             loc.save();
 
             TemplateObjectTO newTemplate = new TemplateObjectTO();
-            newTemplate.setPath(destData.getPath() + File.separator + templateName); 
+            newTemplate.setPath(destData.getPath() + File.separator + templateName);
             newTemplate.setFormat(ImageFormat.VHD);
             return new CopyCmdAnswer(newTemplate);
         } catch (ConfigurationException e) {
@@ -346,16 +344,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
 
         return new CopyCmdAnswer(errMsg);
     }
-    
+
     protected Answer copySnapshotToTemplateFromNfsToNfs(CopyCommand cmd, SnapshotObjectTO srcData, NfsTO srcDataStore, TemplateObjectTO destData, NfsTO destDataStore) {
-        
+
         if (srcData.getHypervisorType() == HypervisorType.XenServer) {
             return copySnapshotToTemplateFromNfsToNfsXenserver(cmd, srcData, srcDataStore, destData, destDataStore);
         }
 
         return new CopyCmdAnswer("");
     }
-    
+
     protected Answer createTemplateFromSnapshot(CopyCommand cmd) {
         DataTO srcData = cmd.getSrcTO();
         DataTO destData = cmd.getDestTO();
@@ -366,7 +364,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
                 s_logger.debug("only support nfs storage as src, when create template from snapshot");
                 return Answer.createUnsupportedCommandAnswer(cmd);
             }
-        
+
             if (destDataStore instanceof NfsTO){
                 return copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO)srcData, (NfsTO)srcDataStore, (TemplateObjectTO)destData, (NfsTO)destDataStore);
             }
@@ -380,12 +378,18 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         DataTO destData = cmd.getDestTO();
         DataStoreTO srcDataStore = srcData.getDataStore();
         DataStoreTO destDataStore = destData.getDataStore();
-        
+
         if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.TEMPLATE) {
             return createTemplateFromSnapshot(cmd);
         }
 
-     
+        if (srcDataStore instanceof S3TO && destDataStore instanceof NfsTO && destDataStore.getRole() == DataStoreRole.ImageCache){
+            S3TO s3 = (S3TO)srcDataStore;
+            NfsTO destImageStore = (NfsTO)destDataStore;
+            return this.copyFromS3ToNfs(cmd, srcData, s3, destData, destImageStore);
+        }
+
+
         return Answer.createUnsupportedCommandAnswer(cmd);
     }
 
@@ -1729,7 +1733,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         if (_configIpFirewallScr != null) {
             s_logger.info("_configIpFirewallScr found in " + _configIpFirewallScr);
         }
-        
+
         createTemplateFromSnapshotXenScript = Script.findScript(getDefaultScriptsDir(), "create_privatetemplate_from_snapshot_xen.sh");
         if (createTemplateFromSnapshotXenScript == null) {
             throw new ConfigurationException("create_privatetemplate_from_snapshot_xen.sh not found in " + getDefaultScriptsDir());

engine/api/src/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java

@@ -30,5 +30,6 @@ public interface ImageStoreDao extends GenericDao<ImageStoreVO, Long> {
     public ImageStoreVO findByName(String name);
     public List<ImageStoreVO> findByProvider(String provider);
     public List<ImageStoreVO> findByScope(ZoneScope scope);
+    public List<ImageStoreVO> findImageCacheByScope(ZoneScope scope);
 
 }

engine/storage/cache/src/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java

@@ -70,13 +70,13 @@ public class StorageCacheManagerImpl implements StorageCacheManager, Manager {
     @Override
     public void setName(String name) {
         // TODO Auto-generated method stub
-        
+
     }
 
     @Override
     public void setConfigParams(Map<String, Object> params) {
         // TODO Auto-generated method stub
-        
+
     }
 
     @Override
@@ -94,7 +94,7 @@ public class StorageCacheManagerImpl implements StorageCacheManager, Manager {
     @Override
     public void setRunLevel(int level) {
         // TODO Auto-generated method stub
-        
+
     }
 
     @Override
@@ -113,9 +113,9 @@ public class StorageCacheManagerImpl implements StorageCacheManager, Manager {
         // TODO Auto-generated method stub
         return true;
     }
-    
 
-    
+
+
     private class CreateCacheObjectContext<T> extends AsyncRpcConext<T> {
         final AsyncCallFuture<CopyCommandResult> future;
         /**
@@ -125,29 +125,34 @@ public class StorageCacheManagerImpl implements StorageCacheManager, Manager {
             super(callback);
             this.future = future;
         }
-        
+
     }
 
 	@Override
 	public DataObject createCacheObject(DataObject data, Scope scope) {
 		DataStore cacheStore = this.getCacheStorage(scope);
 		DataObject objOnCacheStore = cacheStore.create(data);
+
 		AsyncCallFuture<CopyCommandResult> future = new AsyncCallFuture<CopyCommandResult>();
+		/*
 		CreateCacheObjectContext<CopyCommandResult> context = new CreateCacheObjectContext<CopyCommandResult>(null, future);
-		AsyncCallbackDispatcher<StorageCacheManagerImpl, CopyCommandResult> caller = AsyncCallbackDispatcher.create(this); 
+		AsyncCallbackDispatcher<StorageCacheManagerImpl, CopyCommandResult> caller = AsyncCallbackDispatcher.create(this);
 		caller.setContext(context);
-		
+		caller.setCallback(future);
+		*/
+
 		CopyCommandResult result = null;
 		try {
 		    objOnCacheStore.processEvent(Event.CreateOnlyRequested);
-		    
+
-		    dataMotionSvr.copyAsync(data, objOnCacheStore, caller);
+		    dataMotionSvr.copyAsync(data, objOnCacheStore, future);
 		    result = future.get();
-		    
+
 		    if (result.isFailed()) {
 		        cacheStore.delete(data);
 		    } else {
 		        objOnCacheStore.processEvent(Event.OperationSuccessed, result.getAnswer());
+		        return objOnCacheStore;
 		    }
         } catch (InterruptedException e) {
             s_logger.debug("create cache storage failed: " + e.toString());
@@ -160,19 +165,19 @@ public class StorageCacheManagerImpl implements StorageCacheManager, Manager {
                 cacheStore.delete(data);
             }
         }
-		
+
 		return null;
 	}
-	
+
 	@Override
     public DataObject getCacheObject(DataObject data, Scope scope) {
         DataStore cacheStore = this.getCacheStorage(scope);
         DataObject objOnCacheStore = cacheStore.create(data);
-        
+
         return objOnCacheStore;
     }
-	
+
-	protected Void createCacheObjectCallBack(AsyncCallbackDispatcher<StorageCacheManagerImpl, CopyCommandResult> callback, 
+	protected Void createCacheObjectCallBack(AsyncCallbackDispatcher<StorageCacheManagerImpl, CopyCommandResult> callback,
 	        CreateCacheObjectContext<CopyCommandResult> context) {
 	    AsyncCallFuture<CopyCommandResult> future = context.future;
 	    future.complete(callback.getResult());

engine/storage/image/src/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java

@@ -124,17 +124,13 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager
 
     @Override
     public List<DataStore> listImageCacheStores(Scope scope) {
-        List<DataStore> imageStores = new ArrayList<DataStore>();
         if (scope.getScopeType() != ScopeType.ZONE) {
             s_logger.debug("only support zone wide image cache stores");
-            return imageStores;
+            return null;
         }
-        SearchCriteriaService<ImageStoreVO, ImageStoreVO> sc = SearchCriteria2.create(ImageStoreVO.class);
+        List<ImageStoreVO> stores = dataStoreDao.findImageCacheByScope(new ZoneScope(scope.getScopeId()));
-        sc.addAnd(sc.getEntity().getScope(), Op.EQ, ScopeType.ZONE);
+        List<DataStore> imageStores = new ArrayList<DataStore>();
-        sc.addAnd(sc.getEntity().getDataCenterId(), Op.EQ, scope.getScopeId());
+        for (ImageStoreVO store : stores) {
-        sc.addAnd(sc.getEntity().getRole(), Op.EQ, DataStoreRole.ImageCache);
-        List<ImageStoreVO> cacheStores = sc.list();
-        for (ImageStoreVO store : cacheStores) {
             imageStores.add(getImageStore(store.getId()));
         }
         return imageStores;

engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java

@@ -85,8 +85,7 @@ public class ImageStoreImpl implements ImageStoreEntity {
 
     @Override
     public DataStoreRole getRole() {
-        // TODO Auto-generated method stub
+         return this.imageDataStoreVO.getRole();
-        return DataStoreRole.Image;
     }
     @Override
     public long getId() {

engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java

@@ -186,7 +186,7 @@ public class TemplateObject implements TemplateInfo {
                     templEvent = TemplateEvent.OperationFailed;
                 }
 
-                if (templEvent != null) {
+                if (templEvent != null && this.getDataStore().getRole() == DataStoreRole.Image) {
                     this.stateTransit(templEvent);
                 }
             }
@@ -223,9 +223,11 @@ public class TemplateObject implements TemplateInfo {
         			templateStoreRef.setDownloadPercent(100);
         			templateStoreRef.setDownloadState(Status.DOWNLOADED);
         			templateStoreDao.update(templateStoreRef.getId(), templateStoreRef);
-        			VMTemplateVO templateVO = this.imageDao.findById(this.getId());
+                    if (this.getDataStore().getRole() == DataStoreRole.Image) {
-        			templateVO.setFormat(newTemplate.getFormat());
+                        VMTemplateVO templateVO = this.imageDao.findById(this.getId());
-        			this.imageDao.update(templateVO.getId(), templateVO);
+                        templateVO.setFormat(newTemplate.getFormat());
+                        this.imageDao.update(templateVO.getId(), templateVO);
+                    }
         		}
 
         		TemplateEvent templEvent = null;
@@ -239,7 +241,7 @@ public class TemplateObject implements TemplateInfo {
         		    templEvent = TemplateEvent.OperationFailed;
         		}
 
-        		if (templEvent != null) {
+        		if (templEvent != null && this.getDataStore().getRole() == DataStoreRole.Image) {
         		    this.stateTransit(templEvent);
         		}
         	}

engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java

@@ -65,6 +65,11 @@ public class MockLocalNfsSecondaryStorageResource extends
 
     }
 
+    @Override
+    public String getRootDir(String secUrl){
+        return "/mnt";
+    }
+
     @Override
     public Answer executeRequest(Command cmd) {
         if (cmd instanceof DownloadSystemTemplateCommand){

engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java

@@ -1,6 +1,5 @@
 package org.apache.cloudstack.storage.test;
 
-import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
@@ -12,11 +11,14 @@ import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
+import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.storage.LocalHostEndpoint;
 import org.apache.cloudstack.storage.MockLocalNfsSecondaryStorageResource;
@@ -28,15 +30,14 @@ import org.mockito.Mockito;
 import org.springframework.test.context.ContextConfiguration;
 import org.testng.annotations.Test;
 import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.assertNotNull;
 import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.DataCenter.NetworkType;
 import com.cloud.dc.dao.DataCenterDao;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage;
 import com.cloud.storage.VMTemplateVO;
-import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.TemplateType;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.download.DownloadMonitorImpl;
@@ -65,6 +66,8 @@ public class S3TemplateTest extends CloudStackTestNGBase {
 	DownloadMonitorImpl downloadMonitor;
 	@Inject
 	ImageStoreHelper imageStoreHelper;
+	@Inject
+	StorageCacheManager cacheMgr;
 	long dcId;
 	long templateId;
 
@@ -77,7 +80,6 @@ public class S3TemplateTest extends CloudStackTestNGBase {
 		dc = dcDao.persist(dc);
 		dcId = dc.getId();
 
-
         // add s3 image store
         Map<String, Object> sParams = new HashMap<String, Object>();
         sParams.put("name", "test");
@@ -92,6 +94,17 @@ public class S3TemplateTest extends CloudStackTestNGBase {
         sDetails.put(ApiConstants.S3_END_POINT, this.getS3EndPoint());
         this.imageStoreHelper.createImageStore(sParams, sDetails);
 
+        // add nfs cache storage
+        Map<String, Object> cParams = new HashMap<String, Object>();
+        cParams.put("name", "testCache");
+        cParams.put("protocol", "nfs");
+        cParams.put("providerName", DataStoreProvider.NFS_IMAGE);
+        cParams.put("scope", ScopeType.ZONE);
+        cParams.put("role", DataStoreRole.ImageCache);
+        cParams.put("url", this.getSecondaryStorage());
+        cParams.put("zoneId", dcId);
+        this.imageStoreHelper.createImageStore(cParams);
+
 
 		VMTemplateVO image = new VMTemplateVO();
 		image.setTemplateType(TemplateType.SYSTEM);
@@ -119,9 +132,10 @@ public class S3TemplateTest extends CloudStackTestNGBase {
 		ep.setResource(new MockLocalNfsSecondaryStorageResource());
 		Mockito.when(epSelector.select(Mockito.any(DataObject.class))).thenReturn(ep);
 		Mockito.when(epSelector.select(Mockito.any(DataStore.class))).thenReturn(ep);
+        Mockito.when(epSelector.select(Mockito.any(DataObject.class), Mockito.any(DataObject.class))).thenReturn(ep);
 	}
 
-	@Test
+	@Test(priority = 1)
 	public void registerTemplate() {
 		TemplateInfo template = templateFactory.getTemplate(templateId);
 		DataStore store = dataStoreMgr.getImageStore(dcId);
@@ -141,4 +155,11 @@ public class S3TemplateTest extends CloudStackTestNGBase {
 		}
 	}
 
+    @Test(priority = 2)
+    public void copyTemplateToCache() {
+        TemplateInfo template = templateFactory.getTemplate(templateId);
+        DataObject cacheObj = this.cacheMgr.createCacheObject(template, new ZoneScope(dcId));
+        assertNotNull(cacheObj, "failed to create cache object");
+    }
+
 }

engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreHelper.java

@@ -48,18 +48,22 @@ public class ImageStoreHelper {
             return store;
         }
         store = new ImageStoreVO();
-        store.setName((String)params.get("name"));
         store.setProtocol((String)params.get("protocol"));
         store.setProviderName((String)params.get("providerName"));
         store.setScope((ScopeType)params.get("scope"));
+        store.setDataCenterId((Long)params.get("zoneId"));
         String uuid = (String)params.get("uuid");
         if (uuid != null) {
             store.setUuid(uuid);
         } else {
             store.setUuid(UUID.randomUUID().toString());
         }
+        store.setName((String)params.get("name"));
+        if ( store.getName() == null ){
+            store.setName(store.getUuid());
+        }
         store.setUrl((String)params.get("url"));
-        store.setRole(DataStoreRole.getRole((String)params.get("role")));
+        store.setRole((DataStoreRole)params.get("role"));
         store = imageStoreDao.persist(store);
         return store;
     }

engine/storage/src/org/apache/cloudstack/storage/image/db/ImageStoreDaoImpl.java

@@ -87,5 +87,13 @@ public class ImageStoreDaoImpl extends GenericDaoBase<ImageStoreVO, Long> implem
         return listBy(sc);
     }
 
+    @Override
+    public List<ImageStoreVO> findImageCacheByScope(ZoneScope scope) {
+        SearchCriteria<ImageStoreVO> sc = createSearchCriteria();
+        sc.addAnd("role", SearchCriteria.Op.EQ, DataStoreRole.ImageCache);
+        sc.addAnd("scope", SearchCriteria.Op.EQ, ScopeType.ZONE);
+        sc.addAnd("dcId", SearchCriteria.Op.EQ, scope.getScopeId());
+        return listBy(sc);
+    }
 
 }

plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java

@@ -126,7 +126,7 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
     public DataStoreTO getStoreTO(DataStore store) {
         ImageStoreImpl nfsStore = (ImageStoreImpl)store;
         NfsTO nfsTO = new NfsTO();
-        nfsTO.setRole(DataStoreRole.Image);
+        nfsTO.setRole(store.getRole());
         nfsTO.setUrl(nfsStore.getUri());
         return nfsTO;
     }