CLOUDSTACK-6599:

1. Adding the missing Template/Volume URLs expiration functionality
2. Improvement - While deleting the volume during expiration use rm -rf as vmware now contains directoy
3. Improvement - Use standard Answer so that the error gets logged in case deletion of expiration link didnt work fine.
4. Improvement - In case of domain change, expire the old urls

core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLAnswer.java

@@ -1,37 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package com.cloud.agent.api.storage;
-
-import com.cloud.agent.api.Answer;
-
-public class DeleteEntityDownloadURLAnswer extends Answer {
-
-    String resultString;
-    short resultCode;
-    public static final short RESULT_SUCCESS = 1;
-    public static final short RESULT_FAILURE = 0;
-
-    public DeleteEntityDownloadURLAnswer(String resultString, short resultCode) {
-        super();
-        this.resultString = resultString;
-        this.resultCode = resultCode;
-    }
-
-    public DeleteEntityDownloadURLAnswer() {
-    }
-
-}

engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java

@@ -20,6 +20,7 @@ package org.apache.cloudstack.storage.image.datastore;
 
 import java.util.Set;
 
+import com.cloud.storage.Upload;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
@@ -43,4 +44,6 @@ public interface ImageStoreEntity extends DataStore, ImageStore {
     String getMountPoint(); // get the mount point on ssvm.
 
     String createEntityExtractUrl(String installPath, ImageFormat format, DataObject dataObject);  // get the entity download URL
+
+    void deleteExtractUrl(String installPath, String url, Upload.Type volume);
 }

engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java

@@ -74,4 +74,6 @@ public interface TemplateDataStoreDao extends GenericDao<TemplateDataStoreVO, Lo
     List<TemplateDataStoreVO> listOnCache(long templateId);
 
     void updateStoreRoleToCachce(long storeId);
+
+    List<TemplateDataStoreVO> listTemplateDownloadUrls();
 }

engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java

@@ -98,6 +98,13 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
     @Column(name = "url")
     private String downloadUrl;
 
+    @Column(name = "download_url")
+    private String extractUrl;
+
+    @Column(name = "download_url_created")
+    @Temporal(value = TemporalType.TIMESTAMP)
+    private Date extractUrlCreated = null;
+
     @Column(name = "is_copy")
     private boolean isCopy = false;
 
@@ -379,4 +386,20 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
         }
     }
 
+    public String getExtractUrl() {
+        return extractUrl;
+    }
+
+    public void setExtractUrl(String extractUrl) {
+        this.extractUrl = extractUrl;
+    }
+
+    public Date getExtractUrlCreated() {
+        return extractUrlCreated;
+    }
+
+    public void setExtractUrlCreated(Date extractUrlCreated) {
+        this.extractUrlCreated = extractUrlCreated;
+    }
+
 }

engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java

@@ -42,4 +42,6 @@ public interface VolumeDataStoreDao extends GenericDao<VolumeDataStoreVO, Long>,
     List<VolumeDataStoreVO> listDestroyed(long storeId);
 
     void duplicateCacheRecordsOnRegionStore(long storeId);
+
+    List<VolumeDataStoreVO> listVolumeDownloadUrls();
 }

engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java

@@ -102,6 +102,10 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
     @Column(name = "download_url")
     private String extractUrl;
 
+    @Column(name = "download_url_created")
+    @Temporal(value = TemporalType.TIMESTAMP)
+    private Date extractUrlCreated = null;
+
     @Column(name = "destroyed")
     boolean destroyed = false;
 
@@ -369,4 +373,12 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
     public void setExtractUrl(String extractUrl) {
         this.extractUrl = extractUrl;
     }
+
+    public Date getExtractUrlCreated() {
+        return extractUrlCreated;
+    }
+
+    public void setExtractUrlCreated(Date extractUrlCreated) {
+        this.extractUrlCreated = extractUrlCreated;
+    }
 }

engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java

@@ -24,6 +24,7 @@ import java.util.concurrent.ExecutionException;
 
 import javax.inject.Inject;
 
+import com.cloud.storage.Upload;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@@ -203,4 +204,10 @@ public class ImageStoreImpl implements ImageStoreEntity {
         return driver.createEntityExtractUrl(this, installPath, format, dataObject);
     }
 
+    @Override
+    public void deleteExtractUrl(String installPath, String url, Upload.Type entityType) {
+        driver.deleteEntityExtractUrl(this, installPath, url, entityType);
+    }
+
+
 }

engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java

@@ -25,6 +25,7 @@ import java.util.Map;
 
 import javax.inject.Inject;
 
+import com.cloud.storage.Upload;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
@@ -271,4 +272,8 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
     @Override
     public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> callback) {
     }
+
+    @Override
+    public void deleteEntityExtractUrl(DataStore store, String installPath, String url, Upload.Type entityType){
+    }
 }

engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java

@@ -18,6 +18,7 @@
  */
 package org.apache.cloudstack.storage.image;
 
+import com.cloud.storage.Upload;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
@@ -26,4 +27,6 @@ import com.cloud.storage.Storage.ImageFormat;
 
 public interface ImageStoreDriver extends DataStoreDriver {
     String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject);
+
+    void deleteEntityExtractUrl(DataStore store, String installPath, String url, Upload.Type entityType);
 }

engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java

@@ -63,6 +63,7 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
     private SearchBuilder<TemplateDataStoreVO> storeTemplateSearch;
     private SearchBuilder<TemplateDataStoreVO> storeTemplateStateSearch;
     private SearchBuilder<TemplateDataStoreVO> storeTemplateDownloadStatusSearch;
+    private SearchBuilder<TemplateDataStoreVO> downloadTemplateSearch;
 
     @Inject
     private DataStoreManager _storeMgr;
@@ -131,6 +132,11 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         storeTemplateSearch.and("destroyed", storeTemplateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
         storeTemplateSearch.done();
 
+        downloadTemplateSearch = createSearchBuilder();
+        downloadTemplateSearch.and("download_url", downloadTemplateSearch.entity().getExtractUrl(), Op.NNULL);
+        downloadTemplateSearch.and("destroyed", downloadTemplateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
+        downloadTemplateSearch.done();
+
         return true;
     }
 
@@ -488,4 +494,11 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
 
     }
 
+    @Override
+    public List<TemplateDataStoreVO> listTemplateDownloadUrls() {
+        SearchCriteria<TemplateDataStoreVO> sc = downloadTemplateSearch.create();
+        sc.setParameters("destroyed", false);
+        return listBy(sc);
+    }
+
 }

engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java

@@ -50,6 +50,7 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
     private SearchBuilder<VolumeDataStoreVO> storeSearch;
     private SearchBuilder<VolumeDataStoreVO> cacheSearch;
     private SearchBuilder<VolumeDataStoreVO> storeVolumeSearch;
+    private SearchBuilder<VolumeDataStoreVO> downloadVolumeSearch;
 
     @Inject
     DataStoreManager storeMgr;
@@ -85,6 +86,12 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
         updateStateSearch.and("state", updateStateSearch.entity().getState(), Op.EQ);
         updateStateSearch.and("updatedCount", updateStateSearch.entity().getUpdatedCount(), Op.EQ);
         updateStateSearch.done();
+
+        downloadVolumeSearch = createSearchBuilder();
+        downloadVolumeSearch.and("download_url", downloadVolumeSearch.entity().getExtractUrl(), Op.NNULL);
+        downloadVolumeSearch.and("destroyed", downloadVolumeSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
+        downloadVolumeSearch.done();
+
         return true;
     }
 
@@ -253,4 +260,11 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
         }
 
     }
+
+    @Override
+    public List<VolumeDataStoreVO> listVolumeDownloadUrls() {
+        SearchCriteria<VolumeDataStoreVO> sc = downloadVolumeSearch.create();
+        sc.setParameters("destroyed", false);
+        return listBy(sc);
+    }
 }

plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java

@@ -22,6 +22,8 @@ import java.util.UUID;
 
 import javax.inject.Inject;
 
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
+import com.cloud.storage.Upload;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@@ -109,4 +111,28 @@ public class CloudStackImageStoreDriverImpl extends BaseImageStoreDriverImpl {
         return scheme + "://" + hostname + "/userdata/" + uuid;
     }
 
+    @Override
+    public void deleteEntityExtractUrl(DataStore store, String installPath, String downloadUrl, Upload.Type entityType) {
+        // find an endpoint to send command
+        EndPoint ep = _epSelector.select(store);
+
+        // Delete Symlink at ssvm. In case of volume also delete the volume.
+        DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(installPath, entityType, downloadUrl, ((ImageStoreEntity) store).getMountPoint());
+
+        Answer ans = null;
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            ans = new Answer(cmd, false, errMsg);
+        } else {
+            ans = ep.sendMessage(cmd);
+        }
+        if (ans == null || !ans.getResult()) {
+            String errorString = "Unable to delete the url " + downloadUrl + " for path " + installPath + " on ssvm, " + ans.getDetails();
+            s_logger.error(errorString);
+            throw new CloudRuntimeException(errorString);
+        }
+
+    }
+
 }

server/src/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -17,6 +17,7 @@
 package com.cloud.configuration;
 
 import java.net.URI;
+import java.sql.Date;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -36,6 +37,7 @@ import javax.ejb.Local;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.storage.StorageManager;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.acl.SecurityChecker;
@@ -315,6 +317,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
     AffinityGroupDao _affinityGroupDao;
     @Inject
     AffinityGroupService _affinityGroupService;
+    @Inject
+    StorageManager _storageManager;
 
     // FIXME - why don't we have interface for DataCenterLinkLocalIpAddressDao?
     @Inject
@@ -585,6 +589,27 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                     throw new CloudRuntimeException("Failed to update SecondaryStorage offering's use_local_storage option to value:" + useLocalStorage);
                 }
             }
+        }else if (Config.SecStorageSecureCopyCert.key().equalsIgnoreCase(name)) {
+            //FIXME - Ideally there should be a listener model to listen to global config changes and be able to take action gracefully.
+            //Expire the download urls
+            String sqlTemplate = "update template_store_ref set download_url_created=?";
+            String sqlVolume = "update volume_store_ref set download_url_created=?";
+            try {
+                // Change for templates
+                pstmt = txn.prepareAutoCloseStatement(sqlTemplate);
+                pstmt.setDate(1, new Date(-1l));// Set the time before the epoch time.
+                pstmt.executeUpdate();
+                // Change for volumes
+                pstmt = txn.prepareAutoCloseStatement(sqlVolume);
+                pstmt.setDate(1, new Date(-1l));// Set the time before the epoch time.
+                pstmt.executeUpdate();
+                // Cleanup the download urls
+                _storageManager.cleanupDownloadUrls();
+            } catch (Throwable e) {
+                throw new CloudRuntimeException("Failed to clean up download URLs in template_store_ref or volume_store_ref due to exception ", e);
+            }
+
+
         }
 
         txn.commit();

server/src/com/cloud/storage/StorageManager.java

@@ -114,4 +114,6 @@ public interface StorageManager extends StorageService {
     Long getDiskIopsReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering);
 
     Long getDiskIopsWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering);
+
+    void cleanupDownloadUrls();
 }

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -41,6 +41,8 @@ import javax.ejb.Local;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.utils.DateUtil;
+import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
@@ -284,6 +286,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
     boolean _templateCleanupEnabled = true;
     int _storageCleanupInterval;
     int _storagePoolAcquisitionWaitSeconds = 1800; // 30 minutes
+    int _downloadUrlCleanupInterval;
+    int _downloadUrlExpirationInterval;
     // protected BigDecimal _overProvisioningFactor = new BigDecimal(1);
     private long _serverId;
 
@@ -455,6 +459,12 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         s_logger.info("Storage cleanup enabled: " + _storageCleanupEnabled + ", interval: " + _storageCleanupInterval + ", template cleanup enabled: " +
             _templateCleanupEnabled);
 
+        String cleanupInterval = configs.get("extract.url.cleanup.interval");
+        _downloadUrlCleanupInterval = NumbersUtil.parseInt(cleanupInterval, 7200);
+
+        String urlExpirationInterval = configs.get("extract.url.expiration.interval");
+        _downloadUrlExpirationInterval = NumbersUtil.parseInt(urlExpirationInterval, 14400);
+
         String workers = configs.get("expunge.workers");
         int wrks = NumbersUtil.parseInt(workers, 10);
         _executor = Executors.newScheduledThreadPool(wrks, new NamedThreadFactory("StorageManager-Scavenger"));
@@ -507,6 +517,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         } else {
             s_logger.debug("Storage cleanup is not enabled, so the storage cleanup thread is not being scheduled.");
         }
+
+        _executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS);
+
         return true;
     }
 
@@ -1962,6 +1975,80 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         return true;
     }
 
+    protected class DownloadURLGarbageCollector implements Runnable {
+
+        public DownloadURLGarbageCollector() {
+        }
+
+        @Override
+        public void run() {
+            try {
+                s_logger.trace("Download URL Garbage Collection Thread is running.");
+
+                cleanupDownloadUrls();
+
+            } catch (Exception e) {
+                s_logger.error("Caught the following Exception", e);
+            }
+        }
+    }
+
+    @Override
+    public void cleanupDownloadUrls(){
+
+        // Cleanup expired volume URLs
+        List<VolumeDataStoreVO> volumesOnImageStoreList = _volumeStoreDao.listVolumeDownloadUrls();
+        for(VolumeDataStoreVO volumeOnImageStore : volumesOnImageStoreList){
+
+            try {
+                long downloadUrlCurrentAgeInSecs = DateUtil.getTimeDifference(DateUtil.now(), volumeOnImageStore.getExtractUrlCreated());
+                if(downloadUrlCurrentAgeInSecs < _downloadUrlExpirationInterval){  // URL hasnt expired yet
+                    continue;
+                }
+
+                s_logger.debug("Removing download url " + volumeOnImageStore.getExtractUrl() + " for volume id " + volumeOnImageStore.getVolumeId());
+
+                // Remove it from image store
+                ImageStoreEntity secStore = (ImageStoreEntity) _dataStoreMgr.getDataStore(volumeOnImageStore.getDataStoreId(), DataStoreRole.Image);
+                secStore.deleteExtractUrl(volumeOnImageStore.getInstallPath(), volumeOnImageStore.getExtractUrl(), Upload.Type.VOLUME);
+
+                // Now expunge it from DB since this entry was created only for download purpose
+                _volumeStoreDao.expunge(volumeOnImageStore.getId());
+            }catch(Throwable th){
+                s_logger.warn("Caught exception while deleting download url " +volumeOnImageStore.getExtractUrl() +
+                        " for volume id " + volumeOnImageStore.getVolumeId(), th);
+            }
+        }
+
+        // Cleanup expired template URLs
+        List<TemplateDataStoreVO> templatesOnImageStoreList = _templateStoreDao.listTemplateDownloadUrls();
+        for(TemplateDataStoreVO templateOnImageStore : templatesOnImageStoreList){
+
+            try {
+                long downloadUrlCurrentAgeInSecs = DateUtil.getTimeDifference(DateUtil.now(), templateOnImageStore.getExtractUrlCreated());
+                if(downloadUrlCurrentAgeInSecs < _downloadUrlExpirationInterval){  // URL hasnt expired yet
+                    continue;
+                }
+
+                s_logger.debug("Removing download url " + templateOnImageStore.getExtractUrl() + " for template id " + templateOnImageStore.getTemplateId());
+
+                // Remove it from image store
+                ImageStoreEntity secStore = (ImageStoreEntity) _dataStoreMgr.getDataStore(templateOnImageStore.getDataStoreId(), DataStoreRole.Image);
+                secStore.deleteExtractUrl(templateOnImageStore.getInstallPath(), templateOnImageStore.getExtractUrl(), Upload.Type.TEMPLATE);
+
+                // Now remove download details from DB.
+                templateOnImageStore.setExtractUrl(null);
+                templateOnImageStore.setExtractUrlCreated(null);
+                _templateStoreDao.update(templateOnImageStore.getId(), templateOnImageStore);
+            }catch(Throwable th){
+                s_logger.warn("caught exception while deleting download url " +templateOnImageStore.getExtractUrl() +
+                        " for template id " +templateOnImageStore.getTemplateId(), th);
+            }
+        }
+
+
+    }
+
     // get bytesReadRate from service_offering, disk_offering and vm.disk.throttling.bytes_read_rate
     @Override
     public Long getDiskBytesReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) {

server/src/com/cloud/storage/VolumeApiServiceImpl.java

@@ -26,6 +26,7 @@ import java.util.concurrent.ExecutionException;
 
 import javax.inject.Inject;
 
+import com.cloud.utils.DateUtil;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
@@ -1910,6 +1911,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
         String extractUrl = secStore.createEntityExtractUrl(vol.getPath(), vol.getFormat(), vol);
         volumeStoreRef = _volumeStoreDao.findByVolume(volumeId);
         volumeStoreRef.setExtractUrl(extractUrl);
+        volumeStoreRef.setExtractUrlCreated(DateUtil.now());
         _volumeStoreDao.update(volumeStoreRef.getId(), volumeStoreRef);
 
         return extractUrl;

server/src/com/cloud/template/TemplateManagerImpl.java

@@ -31,6 +31,7 @@ import javax.ejb.Local;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.utils.DateUtil;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@@ -437,12 +438,20 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
             throw new InvalidParameterValueException("The " + desc + " has not been downloaded ");
         }
 
+        // Check if the url already exists
+        if(tmpltStoreRef.getExtractUrl() != null){
+            return tmpltStoreRef.getExtractUrl();
+        }
+
         // Handle NFS to S3 object store migration case, we trigger template sync from NFS to S3 during extract template or copy template
         _tmpltSvr.syncTemplateToRegionStore(templateId, tmpltStore);
 
         TemplateInfo templateObject = _tmplFactory.getTemplate(templateId, tmpltStore);
-
-        return tmpltStore.createEntityExtractUrl(templateObject.getInstallPath(), template.getFormat(), templateObject);
+        String extractUrl = tmpltStore.createEntityExtractUrl(tmpltStoreRef.getInstallPath(), template.getFormat(), templateObject);
+        tmpltStoreRef.setExtractUrl(extractUrl);
+        tmpltStoreRef.setExtractUrlCreated(DateUtil.now());
+        _tmplStoreDao.update(tmpltStoreRef.getId(), tmpltStoreRef);
+        return extractUrl;
     }
 
     @Override

server/test/org/apache/cloudstack/networkoffering/ChildTestConfiguration.java

@@ -19,6 +19,7 @@ package org.apache.cloudstack.networkoffering;
 
 import java.io.IOException;
 
+import com.cloud.storage.StorageManager;
 import org.mockito.Mockito;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
@@ -132,7 +133,8 @@ import com.cloud.vm.dao.VMInstanceDaoImpl;
     PortableIpRangeDaoImpl.class, RegionDaoImpl.class, PortableIpDaoImpl.class, AccountGuestVlanMapDaoImpl.class},
                includeFilters = {@Filter(value = ChildTestConfiguration.Library.class, type = FilterType.CUSTOM)},
                useDefaultFilters = false)
-public class ChildTestConfiguration {
+public class
+        ChildTestConfiguration {
 
     @Bean
     public ManagementService managementService() {
@@ -329,6 +331,11 @@ public class ChildTestConfiguration {
         return Mockito.mock(AffinityGroupService.class);
     }
 
+    @Bean
+    public StorageManager storageManager() {
+        return Mockito.mock(StorageManager.class);
+    }
+
     public static class Library implements TypeFilter {
 
         @Override

services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManager.java

@@ -16,11 +16,11 @@
 // under the License.
 package org.apache.cloudstack.storage.template;
 
+import com.cloud.agent.api.Answer;
 import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
 
 import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer;
 import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
-import com.cloud.agent.api.storage.DeleteEntityDownloadURLAnswer;
 import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
 import com.cloud.agent.api.storage.UploadAnswer;
 import com.cloud.agent.api.storage.UploadCommand;
@@ -77,6 +77,6 @@ public interface UploadManager extends Manager {
 
     CreateEntityDownloadURLAnswer handleCreateEntityURLCommand(CreateEntityDownloadURLCommand cmd);
 
-    DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd);
+    Answer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd);
 
 }

services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java

@@ -29,13 +29,13 @@ import java.util.concurrent.Executors;
 
 import javax.naming.ConfigurationException;
 
+import com.cloud.agent.api.Answer;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
 
 import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer;
 import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
-import com.cloud.agent.api.storage.DeleteEntityDownloadURLAnswer;
 import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
 import com.cloud.agent.api.storage.UploadAnswer;
 import com.cloud.agent.api.storage.UploadCommand;
@@ -303,7 +303,7 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager {
     }
 
     @Override
-    public DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd) {
+    public Answer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd) {
 
         //Delete the soft link. Example path = volumes/8/74eeb2c6-8ab1-4357-841f-2e9d06d1f360.vhd
         s_logger.warn("handleDeleteEntityDownloadURLCommand Path:" + cmd.getPath() + " Type:" + cmd.getType().toString());
@@ -318,24 +318,24 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager {
         if (result != null) {
             String errorString = "Error in deleting =" + result;
             s_logger.warn(errorString);
-            return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+            return new Answer(cmd, false, errorString);
         }
 
         // If its a volume also delete the Hard link since it was created only for the purpose of download.
         if (cmd.getType() == Upload.Type.VOLUME) {
             command = new Script("/bin/bash", s_logger);
             command.add("-c");
-            command.add("rm -f /mnt/SecStorage/" + cmd.getParentPath() + File.separator + path);
+            command.add("rm -rf /mnt/SecStorage/" + cmd.getParentPath() + File.separator + path);
             s_logger.warn(" " + parentDir + File.separator + path);
             result = command.execute();
             if (result != null) {
                 String errorString = "Error in linking  err=" + result;
                 s_logger.warn(errorString);
-                return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+                return new Answer(cmd, false, errorString);
             }
         }
 
-        return new DeleteEntityDownloadURLAnswer("", CreateEntityDownloadURLAnswer.RESULT_SUCCESS);
+        return new Answer(cmd, true, "");
     }
 
     private String getInstallPath(String jobId) {

setup/db/db/schema-430to440.sql

@@ -1685,3 +1685,8 @@ alter table `cloud`.`vlan` add column created datetime NULL COMMENT 'date create
 
 alter table `cloud`.`user_ip_address` drop key public_ip_address;
 alter table `cloud`.`user_ip_address` add UNIQUE KEY public_ip_address (public_ip_address,source_network_id, removed);
+
+ALTER TABLE `cloud`.`volume_store_ref` ADD `download_url_created` datetime;
+ALTER TABLE `cloud`.`template_store_ref` ADD `download_url_created` datetime;
+ALTER TABLE `cloud`.`template_store_ref` ADD `download_url` varchar(255);
+

utils/src/com/cloud/utils/DateUtil.java

@@ -261,6 +261,17 @@ public class DateUtil {
         }
     }
 
+    public static long getTimeDifference(Date date1, Date date2){
+
+        Calendar dateCalendar1 = Calendar.getInstance();
+        dateCalendar1.setTime(date1);
+        Calendar dateCalendar2 = Calendar.getInstance();
+        dateCalendar2.setTime(date2);
+
+        return (dateCalendar1.getTimeInMillis() - dateCalendar2.getTimeInMillis() )/1000;
+
+    }
+
     // test only
     public static void main(String[] args) {
         TimeZone localTimezone = Calendar.getInstance().getTimeZone();