make template download testable

engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java

@@ -20,4 +20,5 @@ package org.apache.cloudstack.engine.subsystem.api.storage;
 
 public interface TemplateInfo extends DataObject {
     public String getUniqueName();
+    public String getInstallPath();
 }

engine/api/src/org/apache/cloudstack/storage/to/TemplateObjectTO.java

@@ -26,6 +26,7 @@ import com.cloud.agent.api.to.DataStoreTO;
 
 public class TemplateObjectTO implements DataTO {
     private  String path;
+    private String origUrl;
     private  String uuid;
     private  DiskFormat diskType;
     private  DataStoreTO imageDataStore;
@@ -35,8 +36,9 @@ public class TemplateObjectTO implements DataTO {
         
     }
     public TemplateObjectTO(TemplateInfo template) {
-        this.path = template.getUri();
+        this.path = template.getInstallPath();
         this.uuid = template.getUuid();
+        this.origUrl = template.getUri();
         //this.diskType = template.getDiskType();
         this.imageDataStore = template.getDataStore().getTO();
         this.name = template.getUniqueName();
@@ -84,4 +86,10 @@ public class TemplateObjectTO implements DataTO {
     public void setName(String name) {
         this.name = name;
     }
+	public String getOrigUrl() {
+		return origUrl;
+	}
+	public void setOrigUrl(String origUrl) {
+		this.origUrl = origUrl;
+	}
 }

engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java

@@ -108,12 +108,9 @@ public class TemplateObject implements TemplateInfo {
     @Override
     public String getUri() {
         VMTemplateVO image = imageDao.findById(this.imageVO.getId());
-        if (this.dataStore == null) {
+       
             return image.getUrl();
-        } else {
+       
-            DataObjectInStore obj = ojbectInStoreMgr.findObject(this, this.dataStore);
-           return obj.getInstallPath();
-        }
     }
 
     @Override
@@ -218,4 +215,10 @@ public class TemplateObject implements TemplateInfo {
 
         return to;
     }
+
+	@Override
+	public String getInstallPath() {
+		 DataObjectInStore obj = ojbectInStoreMgr.findObject(this, this.dataStore);
+         return obj.getInstallPath();
+	}
 }

engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java

@@ -54,6 +54,7 @@ import com.cloud.dc.dao.DcDetailsDaoImpl;
 import com.cloud.dc.dao.HostPodDaoImpl;
 import com.cloud.dc.dao.PodVlanDaoImpl;
 import com.cloud.domain.dao.DomainDaoImpl;
+import com.cloud.host.dao.HostDao;
 import com.cloud.host.dao.HostDaoImpl;
 import com.cloud.host.dao.HostDetailsDaoImpl;
 import com.cloud.host.dao.HostTagsDaoImpl;
@@ -77,13 +78,16 @@ import com.cloud.storage.dao.VMTemplateZoneDaoImpl;
 import com.cloud.storage.dao.VolumeDaoImpl;
 import com.cloud.storage.dao.VolumeHostDaoImpl;
 import com.cloud.storage.download.DownloadMonitor;
+import com.cloud.storage.download.DownloadMonitorImpl;
 import com.cloud.storage.s3.S3Manager;
+import com.cloud.storage.secondary.SecondaryStorageVmManager;
 import com.cloud.storage.snapshot.SnapshotManager;
 import com.cloud.storage.swift.SwiftManager;
 import com.cloud.tags.dao.ResourceTagsDaoImpl;
 import com.cloud.template.TemplateManager;
 import com.cloud.user.AccountManager;
 import com.cloud.user.ResourceLimitService;
+import com.cloud.user.dao.AccountDaoImpl;
 import com.cloud.user.dao.UserDaoImpl;
 import com.cloud.utils.component.SpringComponentScanUtils;
 import com.cloud.vm.VirtualMachineManager;
@@ -93,6 +97,7 @@ import com.cloud.vm.dao.NicDaoImpl;
 import com.cloud.vm.dao.SecondaryStorageVmDaoImpl;
 import com.cloud.vm.dao.UserVmDaoImpl;
 import com.cloud.vm.dao.UserVmDetailsDaoImpl;
+import com.cloud.vm.dao.VMInstanceDao;
 import com.cloud.vm.dao.VMInstanceDaoImpl;
 import com.cloud.vm.snapshot.dao.VMSnapshotDaoImpl;
 @Configuration
@@ -111,7 +116,6 @@ import com.cloud.vm.snapshot.dao.VMSnapshotDaoImpl;
         HostPodDaoImpl.class,
         VMTemplateZoneDaoImpl.class,
         VMTemplateDetailsDaoImpl.class,
-        HostDaoImpl.class,
         HostDetailsDaoImpl.class,
         HostTagsDaoImpl.class,
         HostTransferMapDaoImpl.class,
@@ -137,15 +141,29 @@ import com.cloud.vm.snapshot.dao.VMSnapshotDaoImpl;
         UserDaoImpl.class,
         DataCenterDaoImpl.class,
         StoragePoolDetailsDaoImpl.class,
-        DomainDaoImpl.class
+        DomainDaoImpl.class,
-
+        DownloadMonitorImpl.class,
+        AccountDaoImpl.class
 },
 includeFilters={@Filter(value=Library.class, type=FilterType.CUSTOM)},
 useDefaultFilters=false
 )
 public class ChildTestConfiguration extends TestConfiguration {
 	
+	@Bean
+	public SecondaryStorageVmManager secondaryStoreageMgr() {
+		return Mockito.mock(SecondaryStorageVmManager.class);
+	}
 
+	@Bean
+	public HostDao hostDao() {
+		return Mockito.spy(new HostDaoImpl());
+	}
+	
+	@Bean
+	public EndPointSelector selector() {
+		return Mockito.mock(EndPointSelector.class);
+	}
 	@Bean
 	public AgentManager agentMgr() {
 		return new DirectAgentManagerSimpleImpl();
@@ -161,11 +179,6 @@ public class ChildTestConfiguration extends TestConfiguration {
         return Mockito.mock(ResourceLimitService.class);
     }
     
-    @Bean
-    public DownloadMonitor downloadMonitor() {
-        return Mockito.mock(DownloadMonitor.class);
-    }
-    
     @Bean
     public AccountManager acctMgt() {
         return Mockito.mock(AccountManager.class);

engine/storage/integration-test/test/org/apache/cloudstack/storage/test/CloudStackTestNGBase.java

@@ -35,6 +35,7 @@ public class CloudStackTestNGBase extends AbstractTestNGSpringContextTests {
     private String templateUrl;
     private String localStorageUuid;
     private String primaryStorageUrl;
+    private String secondaryStorage;
     private Transaction txn;
     
     protected void injectMockito() {
@@ -61,10 +62,10 @@ public class CloudStackTestNGBase extends AbstractTestNGSpringContextTests {
     @BeforeMethod(alwaysRun = true)
     @Parameters({"devcloud-host-uuid", "devcloud-host-gateway", "devcloud-host-cidr", 
         "devcloud-host-ip", "template-url", "devcloud-local-storage-uuid", 
-        "primary-storage-want-to-add"})
+        "primary-storage-want-to-add", "devcloud-secondary-storage"})
     protected void setup(String hostuuid, String gateway, String cidr, 
             String hostIp, String templateUrl, String localStorageUuid,
-            String primaryStorage) {
+            String primaryStorage, String secondaryStorage) {
         this.hostGuid = hostuuid;
         this.hostGateway = gateway;
         this.hostCidr = cidr;
@@ -72,6 +73,7 @@ public class CloudStackTestNGBase extends AbstractTestNGSpringContextTests {
         this.templateUrl = templateUrl;
         this.localStorageUuid = localStorageUuid;
         this.primaryStorageUrl = primaryStorage;
+        this.setSecondaryStorage(secondaryStorage);
     }
     
     protected String getHostGuid() {
@@ -101,4 +103,12 @@ public class CloudStackTestNGBase extends AbstractTestNGSpringContextTests {
     protected String getPrimaryStorageUrl() {
         return this.primaryStorageUrl;
     }
+
+	public String getSecondaryStorage() {
+		return secondaryStorage;
+	}
+
+	public void setSecondaryStorage(String secondaryStorage) {
+		this.secondaryStorage = secondaryStorage;
+	}
 }

engine/storage/integration-test/test/org/apache/cloudstack/storage/test/TemplateTest.java

@@ -0,0 +1,122 @@
+package org.apache.cloudstack.storage.test;
+
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
+import org.apache.cloudstack.framework.async.AsyncCallFuture;
+import org.apache.cloudstack.storage.LocalHostEndpoint;
+import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
+import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
+import org.mockito.Mockito;
+import org.springframework.test.context.ContextConfiguration;
+import org.testng.annotations.Test;
+
+import com.cloud.dc.DataCenterVO;
+import com.cloud.dc.DataCenter.NetworkType;
+import com.cloud.dc.dao.DataCenterDao;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.Storage.TemplateType;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.storage.download.DownloadMonitor;
+import com.cloud.storage.download.DownloadMonitorImpl;
+import com.cloud.utils.component.ComponentContext;
+
+@ContextConfiguration(locations={"classpath:/storageContext.xml"})
+ 
+public class TemplateTest extends CloudStackTestNGBase {
+	@Inject
+	DataCenterDao dcDao;
+	ImageStoreVO imageStore;
+	@Inject
+	ImageStoreDao imageStoreDao;
+	@Inject
+	TemplateService templateSvr;
+	@Inject
+	VMTemplateDao templateDao;
+	@Inject
+	TemplateDataFactory templateFactory;
+	@Inject
+	DataStoreManager dataStoreMgr;
+	@Inject
+	EndPointSelector epSelector;
+	@Inject
+	DownloadMonitorImpl downloadMonitor;
+	long dcId;
+	long templateId;
+	
+	@Test(priority = -1)
+	public void setUp() {
+		ComponentContext.initComponentsLifeCycle();
+		//create data center
+		DataCenterVO dc = new DataCenterVO(UUID.randomUUID().toString(), "test", "8.8.8.8", null, "10.0.0.1", null,  "10.0.0.1/24",
+				null, null, NetworkType.Basic, null, null, true,  true, null, null);
+		dc = dcDao.persist(dc);
+		dcId = dc.getId();
+		
+		imageStore = new ImageStoreVO();
+		imageStore.setName("test");
+		imageStore.setDataCenterId(dcId);
+		imageStore.setProviderName("CloudStack ImageStore Provider");
+		imageStore.setRole(DataStoreRole.Image);
+		imageStore.setUrl(this.getSecondaryStorage());
+		imageStore.setUuid(UUID.randomUUID().toString());
+		imageStore.setProtocol("nfs");
+		imageStore = imageStoreDao.persist(imageStore);
+		
+		VMTemplateVO image = new VMTemplateVO();
+		image.setTemplateType(TemplateType.USER);
+		image.setUrl(this.getTemplateUrl());
+		image.setUniqueName(UUID.randomUUID().toString());
+		image.setName(UUID.randomUUID().toString());
+		image.setPublicTemplate(true);
+		image.setFeatured(true);
+		image.setRequiresHvm(true);
+		image.setBits(64);
+		image.setFormat(Storage.ImageFormat.VHD);
+		image.setEnablePassword(true);
+		image.setEnableSshKey(true);
+		image.setGuestOSId(1);
+		image.setBootable(true);
+		image.setPrepopulate(true);
+		image.setCrossZones(true);
+		image.setExtractable(true);
+
+		
+		//image.setImageDataStoreId(storeId);
+		image = templateDao.persist(image);
+		templateId = image.getId();
+		
+		Mockito.when(epSelector.select(Mockito.any(DataObject.class))).thenReturn(new LocalHostEndpoint());
+		//Mockito.when(downloadMonitor.isTemplateUpdateable(Mockito.anyLong(), Mockito.anyLong())).thenReturn(true);
+	}
+	
+	@Test
+	public void registerTemplate() {
+		TemplateInfo template = templateFactory.getTemplate(templateId);
+		DataStore store = dataStoreMgr.getImageStore(dcId);
+		AsyncCallFuture<TemplateApiResult> future = new AsyncCallFuture<TemplateApiResult>();
+		templateSvr.createTemplateAsync(template, store, future);
+		try {
+			future.get();
+		} catch (InterruptedException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (ExecutionException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+
+}

engine/storage/integration-test/test/org/apache/cloudstack/storage/test/volumeServiceTest.java

@@ -21,6 +21,7 @@ package org.apache.cloudstack.storage.test;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -49,6 +50,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeAp
 import org.apache.cloudstack.engine.subsystem.api.storage.type.RootDisk;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.storage.RemoteHostEndPoint;
+import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
+import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.volume.db.VolumeDao2;
@@ -78,6 +81,7 @@ import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.Storage.TemplateType;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.utils.component.ComponentContext;
@@ -114,6 +118,9 @@ public class volumeServiceTest extends CloudStackTestNGBase {
 	TemplateDataFactory imageDataFactory;
 	@Inject
 	VolumeDataFactory volumeFactory;
+	@Inject
+	ImageStoreDao imageStoreDao;
+	ImageStoreVO imageStore;
 	Long dcId;
 	Long clusterId;
 	Long podId;
@@ -124,12 +131,7 @@ public class volumeServiceTest extends CloudStackTestNGBase {
     @Test(priority = -1)
 	public void setUp() {
         ComponentContext.initComponentsLifeCycle();
-       /* try {
+  
-            dataStoreProviderMgr.configure(null, new HashMap<String, Object>());
-        } catch (ConfigurationException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
-        }*/
         host = hostDao.findByGuid(this.getHostGuid());
         if (host != null) {
             dcId = host.getDataCenterId();
@@ -172,6 +174,15 @@ public class volumeServiceTest extends CloudStackTestNGBase {
 
 		host = hostDao.persist(host);
 		
+		imageStore = new ImageStoreVO();
+		imageStore.setName("test");
+		imageStore.setDataCenterId(dcId);
+		imageStore.setProviderName("CloudStack ImageStore Provider");
+		imageStore.setRole(DataStoreRole.Image);
+		imageStore.setUrl(this.getSecondaryStorage());
+		imageStore.setUuid(UUID.randomUUID().toString());
+		imageStore = imageStoreDao.persist(imageStore);
+
 		//primaryStore = createPrimaryDataStore();
 
 		//CreateVolumeAnswer createVolumeFromImageAnswer = new CreateVolumeAnswer(UUID.randomUUID().toString());
@@ -227,9 +238,11 @@ public class volumeServiceTest extends CloudStackTestNGBase {
 		image.setCrossZones(true);
 		image.setExtractable(true);
 
+		
 		//image.setImageDataStoreId(storeId);
 		image = imageDataDao.persist(image);
 		
+
 		return image;
 	}
 

engine/storage/integration-test/test/resource/storageContext.xml

@@ -59,7 +59,6 @@
   <bean id="dataMotionServiceImpl" class="org.apache.cloudstack.storage.motion.DataMotionServiceImpl" />
   <bean id="dataObjectManagerImpl" class="org.apache.cloudstack.storage.datastore.DataObjectManagerImpl" />
   <bean id="dataStoreManagerImpl" class="org.apache.cloudstack.storage.datastore.DataStoreManagerImpl" />
-  <bean id="defaultEndPointSelector" class="org.apache.cloudstack.storage.endpoint.DefaultEndPointSelector" />
   <bean id="primaryDataStoreProviderManagerImpl" class="org.apache.cloudstack.storage.datastore.manager.PrimaryDataStoreProviderManagerImpl" />
   <bean id="imageStoreProviderManagerImpl" class="org.apache.cloudstack.storage.image.manager.ImageStoreProviderManagerImpl" />  
   <bean id="hypervsiorHostEndPointRpcServer" class="org.apache.cloudstack.storage.HypervsiorHostEndPointRpcServer" />
@@ -88,5 +87,5 @@
   <bean id="s3ImageStoreProviderImpl" class="org.apache.cloudstack.storage.datastore.provider.S3ImageStoreProviderImpl" />
   <bean id="swiftImageStoreProviderImpl" class="org.apache.cloudstack.storage.datastore.provider.SwiftImageStoreProviderImpl" />  
   <bean id="BAREMETAL" class="org.apache.cloudstack.storage.image.format.BAREMETAL" />
-
+  <bean id="storagePoolAutomationImpl" class="com.cloud.storage.StoragePoolAutomationImpl" />
 </beans>

engine/storage/integration-test/test/resource/testng.xml

@@ -27,6 +27,7 @@
   <parameter name="primary-storage-want-to-add" value="nfs://192.168.56.2/opt/storage/primarynfs"/>
   <parameter name="devcloud-local-storage-uuid" value="cd10cac1-4772-92e5-5da6-c2bc16b1ce1b"/>
   <parameter name="devcloud-host-uuid" value="759ee4c9-a15a-297b-67c6-ac267d8aa429"/>
+  <parameter name="devcloud-secondary-storage" value="nfs://192.168.56.10/opt/storage/secondary"/>
  
    <listeners>
     <listener class-name="org.apache.cloudstack.storage.test.TestNGAop" />

engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java

@@ -12,6 +12,7 @@ import com.cloud.agent.Listener;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.storage.DownloadCommand;
 import com.cloud.resource.ServerResource;
 import com.cloud.storage.download.DownloadListener;
 import com.cloud.storage.resource.LocalNfsSecondaryStorageResource;
@@ -38,7 +39,7 @@ public class LocalHostEndpoint implements EndPoint {
 
     @Override
 	public Answer sendMessage(Command cmd) {
-		if (cmd instanceof CopyCommand) {
+		if ((cmd instanceof CopyCommand) || (cmd instanceof DownloadCommand)) {
 			return resource.executeRequest(cmd);
 		}
 		// TODO Auto-generated method stub

server/src/com/cloud/storage/download/DownloadListener.java

@@ -183,8 +183,8 @@ public class DownloadListener implements Listener {
 				if (this.object.getType() == DataObjectType.VOLUME) {
 					dcmd.setResourceType(ResourceType.VOLUME);
 				}
-	            _downloadMonitor.send(_ssAgent.getId(), dcmd, this);
+	            _ssAgent.sendMessageAsyncWithListener(dcmd, this);
-            } catch (AgentUnavailableException e) {
+            } catch (Exception e) {
             	s_logger.debug("Send command failed", e);
 				setDisconnected();
             }

server/src/com/cloud/storage/download/DownloadMonitorImpl.java

@@ -97,14 +97,6 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor
     @Inject
     VolumeDataStoreDao _volumeStoreDao;
     @Inject
-    AlertManager _alertMgr;
-    @Inject
-    protected SwiftManager _swiftMgr;
-    @Inject
-    SecondaryStorageVmManager _ssvmMgr;
-    @Inject
-    StorageManager _storageMgr;
-    @Inject
     VMTemplateDao _templateDao = null;
     @Inject
     private AgentManager _agentMgr;
@@ -113,18 +105,6 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor
     @Inject
     ConfigurationDao _configDao;
     @Inject
-    UserVmManager _vmMgr;
-
-    @Inject
-    TemplateManager templateMgr;
-
-    @Inject
-    protected ResourceLimitService _resourceLimitMgr;
-    @Inject
-    protected UserVmDao _userVmDao;
-    @Inject
-    protected AccountManager _accountMgr;
-    @Inject
     EndPointSelector _epSelector;
     @Inject
     TemplateDataFactory tmplFactory;
@@ -144,9 +124,6 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor
     final Map<VolumeHostVO, DownloadListener> _listenerVolumeMap = new ConcurrentHashMap<VolumeHostVO, DownloadListener>();
     final Map<VolumeDataStoreVO, DownloadListener> _listenerVolMap = new ConcurrentHashMap<VolumeDataStoreVO, DownloadListener>();
 
-    public void send(Long hostId, Command cmd, Listener listener) throws AgentUnavailableException {
-        _agentMgr.send(hostId, new Commands(cmd), listener);
-    }
 
     @Override
     public boolean configure(String name, Map<String, Object> params) {