Fix the flow of deleteTemplateCmd.

api/src/com/cloud/agent/api/storage/DeleteTemplateCommand.java

@@ -16,17 +16,26 @@
 // under the License.
 package com.cloud.agent.api.storage;
 
+import com.cloud.agent.api.to.DataStoreTO;
+
 
 public class DeleteTemplateCommand extends ssCommand {
+    private DataStoreTO store;
 	private String templatePath;
+	private Long templateId;
+	private Long accountId;
 
 
 	public DeleteTemplateCommand() {
 	}
 
-	public DeleteTemplateCommand(String secUrl, String templatePath) {
+
+	public DeleteTemplateCommand(DataStoreTO store, String secUrl, String templatePath, Long templateId, Long accountId) {
 	    this.setSecUrl(secUrl);
     	this.templatePath = templatePath;
+    	this.templateId = templateId;
+    	this.accountId = accountId;
+    	this.store = store;
     }
 
 	@Override
@@ -37,4 +46,18 @@ public class DeleteTemplateCommand extends ssCommand {
 	public String getTemplatePath() {
 		return templatePath;
 	}
+
+    public Long getTemplateId() {
+        return templateId;
+    }
+
+    public Long getAccountId() {
+        return accountId;
+    }
+
+    public DataStoreTO getDataStore() {
+        return store;
+    }
+
+
 }

core/src/com/cloud/storage/VMTemplateVO.java

@@ -129,9 +129,6 @@ public class VMTemplateVO implements VirtualMachineTemplate, StateObject<Templat
     @Column(name="enable_sshkey")
     private boolean enableSshKey;
 
-    @Column(name = "image_data_store_id")
-    private long imageDataStoreId;
-    
     @Column(name = "size")
     private Long size;
 
@@ -490,13 +487,6 @@ public class VMTemplateVO implements VirtualMachineTemplate, StateObject<Templat
 		enableSshKey = enable;
 	}
 
-	 public Long getImageDataStoreId() {
-	        return this.imageDataStoreId;
-	    }
-
-	    public void setImageDataStoreId(long dataStoreId) {
-	        this.imageDataStoreId = dataStoreId;
-	    }
 
 	    public void setSize(Long size) {
 	        this.size = size;

core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java

@@ -371,7 +371,7 @@ SecondaryStorageResource {
             return new Answer(cmd, false, "Swift is not currently support DownloadCommand");
         }
         else{
-            return new Answer(cmd, false, "Unsupport image data store: " + dstore);
+            return new Answer(cmd, false, "Unsupported image data store: " + dstore);
         }
 
     }
@@ -504,6 +504,8 @@ SecondaryStorageResource {
 
     }
 
+
+
     private Answer execute(final DeleteTemplateFromS3Command cmd) {
 
         final S3TO s3 = cmd.getS3();
@@ -1318,6 +1320,8 @@ SecondaryStorageResource {
     }
 
     protected Answer execute(final DeleteTemplateCommand cmd) {
+        DataStoreTO dstore = cmd.getDataStore();
+        if (dstore instanceof NfsTO) {
             String relativeTemplatePath = cmd.getTemplatePath();
             String parent = getRootDir(cmd);
 
@@ -1347,8 +1351,7 @@ SecondaryStorageResource {
                         found = true;
                     }
                     if (!f.delete()) {
-                    return new Answer(cmd, false, "Unable to delete file " + f.getName() + " under Template path "
+                        return new Answer(cmd, false, "Unable to delete file " + f.getName() + " under Template path " + relativeTemplatePath);
-                            + relativeTemplatePath);
                     }
                 }
                 if (!found) {
@@ -1357,13 +1360,68 @@ SecondaryStorageResource {
                 }
             }
             if (!tmpltParent.delete()) {
-            details = "Unable to delete directory " + tmpltParent.getName() + " under Template path "
+                details = "Unable to delete directory " + tmpltParent.getName() + " under Template path " + relativeTemplatePath;
-                    + relativeTemplatePath;
                 s_logger.debug(details);
                 return new Answer(cmd, false, details);
             }
             return new Answer(cmd, true, null);
         }
+        else if (dstore instanceof S3TO ){
+            final S3TO s3 = (S3TO)dstore;
+            final Long accountId = cmd.getAccountId();
+            final Long templateId = cmd.getTemplateId();
+
+            if (accountId == null || (accountId != null && accountId <= 0)) {
+                final String errorMessage = "No account id specified for S3 template deletion.";
+                s_logger.error(errorMessage);
+                return new Answer(cmd, false, errorMessage);
+            }
+
+            if (templateId == null || (templateId != null && templateId <= 0)) {
+                final String errorMessage = "No template id specified for S3 template deletion.";
+                s_logger.error(errorMessage);
+                return new Answer(cmd, false, errorMessage);
+            }
+
+
+            final String bucket = s3.getBucketName();
+            try {
+                S3Utils.deleteDirectory(s3, bucket,
+                        determineS3TemplateDirectory(templateId, accountId));
+                return new Answer(cmd, true, String.format(
+                        "Deleted template %1%s from bucket %2$s.", templateId,
+                        bucket));
+            } catch (Exception e) {
+                final String errorMessage = String
+                        .format("Failed to delete templaet id %1$s from bucket %2$s due to the following error: %3$s",
+                                templateId, bucket, e.getMessage());
+                s_logger.error(errorMessage, e);
+                return new Answer(cmd, false, errorMessage);
+            }
+        }
+        else if (dstore instanceof SwiftTO){
+            SwiftTO swift = (SwiftTO)dstore;
+            String container = "T-" + cmd.getTemplateId();
+            String object = "";
+
+            try {
+                String result = swiftDelete(swift, container, object);
+                if (result != null) {
+                    String errMsg = "failed to delete object " + container + "/" + object + " , err=" + result;
+                    s_logger.warn(errMsg);
+                    return new Answer(cmd, false, errMsg);
+                }
+                return new Answer(cmd, true, "success");
+            } catch (Exception e) {
+                String errMsg = cmd + " Command failed due to " + e.toString();
+                s_logger.warn(errMsg, e);
+                return new Answer(cmd, false, errMsg);
+            }
+        }
+        else{
+            return new Answer(cmd, false, "Unsupported image data store: " + dstore);
+        }
+    }
 
     protected Answer execute(final DeleteVolumeCommand cmd) {
         String relativeVolumePath = cmd.getVolumePath();

engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java

@@ -32,11 +32,19 @@ public interface TemplateDataStoreDao extends GenericDao<TemplateDataStoreVO, Lo
 
     public List<TemplateDataStoreVO> listByStoreId(long id);
 
+    public List<TemplateDataStoreVO> listDestroyed(long storeId);
+
     public void deletePrimaryRecordsForStore(long id);
 
+    List<TemplateDataStoreVO> listByTemplateStore(long templateId, long storeId);
+
     List<TemplateDataStoreVO> listByTemplateStoreStatus(long templateId, long storeId, State... states);
 
     List<TemplateDataStoreVO> listByTemplateStoreDownloadStatus(long templateId, long storeId, Status... status);
 
     TemplateDataStoreVO findByStoreTemplate(long storeId, long templateId);
+
+    TemplateDataStoreVO findByTemplate(long templateId);
+
+    List<TemplateDataStoreVO> listByTemplate(long templateId);
 }

engine/storage/image/src/org/apache/cloudstack/storage/image/ImageDataFactoryImpl.java

@@ -28,6 +28,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.ImageDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.image.store.TemplateObject;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
@@ -50,6 +52,8 @@ public class ImageDataFactoryImpl implements ImageDataFactory {
     DataStoreManager storeMgr;
     @Inject
     VMTemplatePoolDao templatePoolDao;
+    @Inject
+    TemplateDataStoreDao templateStoreDao;
     @Override
     public TemplateInfo getTemplate(long templateId, DataStore store) {
         VMTemplateVO templ = imageDataDao.findById(templateId);
@@ -77,13 +81,17 @@ public class ImageDataFactoryImpl implements ImageDataFactory {
         TemplateObject tmpl =  TemplateObject.getTemplate(templ, store);
         return tmpl;
     }
+
+    //TODO: this method is problematic, since one template can be stored in multiple image stores.
+    // need to see if we can get rid of this method or change to plural format.
     @Override
     public TemplateInfo getTemplate(long templateId) {
         VMTemplateVO templ = imageDataDao.findById(templateId);
-        if (templ.getImageDataStoreId() == null) {
+        TemplateDataStoreVO tmplStore = templateStoreDao.findByTemplate(templateId);
-            return this.getTemplate(templateId, null);
+        DataStore store = null;
+        if ( tmplStore != null ){
+            store = this.storeMgr.getDataStore(tmplStore.getDataStoreId(), DataStoreRole.Image);
         }
-        DataStore store = this.storeMgr.getDataStore(templ.getImageDataStoreId(), DataStoreRole.Image);
         return this.getTemplate(templateId, store);
     }
     @Override

engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -40,6 +40,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import com.cloud.storage.template.TemplateProp;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
@@ -139,6 +141,28 @@ public class TemplateServiceImpl implements TemplateService {
         }
     }
 
+    class DeleteTemplateContext<T> extends AsyncRpcConext<T> {
+        final TemplateObject template;
+        final AsyncCallFuture<CommandResult> future;
+
+        public DeleteTemplateContext(AsyncCompletionCallback<T> callback, TemplateObject template,
+                AsyncCallFuture<CommandResult> future) {
+            super(callback);
+            this.template = template;
+            this.future = future;
+        }
+
+        public TemplateObject getTemplate() {
+            return template;
+        }
+
+        public AsyncCallFuture<CommandResult> getFuture() {
+            return future;
+        }
+
+
+    }
+
     @Override
     public AsyncCallFuture<CommandResult> createTemplateAsync(
             TemplateInfo template, DataStore store) {
@@ -361,7 +385,8 @@ public class TemplateServiceImpl implements TemplateService {
             List<UserVmVO> userVmUsingIso = _userVmDao.listByIsoId(tInfo.getId());
             //check if there is any Vm using this ISO.
             if (userVmUsingIso == null || userVmUsingIso.isEmpty()) {
-                DeleteTemplateCommand dtCommand = new DeleteTemplateCommand(store.getUri(), tInfo.getInstallPath());
+                VMTemplateVO template = _templateDao.findById(tInfo.getId());
+                DeleteTemplateCommand dtCommand = new DeleteTemplateCommand(store.getTO(), store.getUri(), tInfo.getInstallPath(), template.getId(), template.getAccountId());
                 try {
                     HostVO ssAhost = _ssvmMgr.pickSsvmHost(store);
                     _agentMgr.sendToSecStorage(ssAhost, dtCommand, null);
@@ -458,7 +483,28 @@ public class TemplateServiceImpl implements TemplateService {
     @Override
     public AsyncCallFuture<CommandResult> deleteTemplateAsync(
             TemplateInfo template) {
-        // TODO Auto-generated method stub
+        TemplateObject to = (TemplateObject) template;
+        // update template_store_ref status
+        to.processEvent(ObjectInDataStoreStateMachine.Event.DestroyRequested);
+        AsyncCallFuture<CommandResult> future = new AsyncCallFuture<CommandResult>();
+
+        DeleteTemplateContext<CommandResult> context = new DeleteTemplateContext<CommandResult>(null, to, future);
+        AsyncCallbackDispatcher<TemplateServiceImpl, CommandResult> caller = AsyncCallbackDispatcher.create(this);
+        caller.setCallback(caller.getTarget().deleteTemplateCallback(null, null)).setContext(context);
+        to.getDataStore().getDriver().deleteAsync(to, caller);
+        return future;
+    }
+
+    public Void deleteTemplateCallback(AsyncCallbackDispatcher<TemplateServiceImpl, CommandResult> callback, DeleteTemplateContext<CommandResult> context) {
+        CommandResult result = callback.getResult();
+        TemplateObject vo = context.getTemplate();
+        // we can only update state in template_store_ref table
+         if (result.isSuccess()) {
+            vo.processEvent(Event.OperationSuccessed);
+        } else {
+            vo.processEvent(Event.OperationFailed);
+         }
+        context.getFuture().complete(result);
         return null;
     }
 

engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java

@@ -69,9 +69,6 @@ public class TemplateObject implements TemplateInfo {
         return to;
     }
 
-    public void setImageStoreId(long id) {
-        this.imageVO.setImageDataStoreId(id);
-    }
 
     public void setSize(Long size) {
         this.imageVO.setSize(size);

engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java

@@ -43,9 +43,11 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
     private static final Logger s_logger = Logger.getLogger(TemplateDataStoreDaoImpl.class);
     private SearchBuilder<TemplateDataStoreVO> updateStateSearch;
     private SearchBuilder<TemplateDataStoreVO> storeSearch;
+    private SearchBuilder<TemplateDataStoreVO> templateSearch;
+    private SearchBuilder<TemplateDataStoreVO> storeTemplateSearch;
     private SearchBuilder<TemplateDataStoreVO> storeTemplateStateSearch;
     private SearchBuilder<TemplateDataStoreVO> storeTemplateDownloadStatusSearch;
-    private SearchBuilder<TemplateDataStoreVO> storeTemplateSearch;
+
 
     @Override
     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
@@ -57,12 +59,24 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         storeSearch.and("destroyed", storeSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
         storeSearch.done();
 
+        templateSearch = createSearchBuilder();
+        templateSearch.and("template_id", templateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
+        templateSearch.and("destroyed", templateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
+        templateSearch.done();
+
+
     	updateStateSearch = this.createSearchBuilder();
         updateStateSearch.and("id", updateStateSearch.entity().getId(), Op.EQ);
         updateStateSearch.and("state", updateStateSearch.entity().getState(), Op.EQ);
         updateStateSearch.and("updatedCount", updateStateSearch.entity().getUpdatedCount(), Op.EQ);
         updateStateSearch.done();
 
+        storeTemplateSearch = createSearchBuilder();
+        storeTemplateSearch.and("template_id", storeTemplateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
+        storeTemplateSearch.and("store_id", storeTemplateSearch.entity().getDataStoreId(), SearchCriteria.Op.EQ);
+        storeTemplateSearch.and("destroyed", storeTemplateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
+        storeTemplateSearch.done();
+
         storeTemplateStateSearch = createSearchBuilder();
         storeTemplateStateSearch.and("template_id", storeTemplateStateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
         storeTemplateStateSearch.and("store_id", storeTemplateStateSearch.entity().getDataStoreId(), SearchCriteria.Op.EQ);
@@ -133,6 +147,14 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         return listIncludingRemovedBy(sc);
     }
 
+    @Override
+    public List<TemplateDataStoreVO> listDestroyed(long id) {
+        SearchCriteria<TemplateDataStoreVO> sc = storeSearch.create();
+        sc.setParameters("store_id", id);
+        sc.setParameters("destroyed", true);
+        return listIncludingRemovedBy(sc);
+    }
+
     @Override
     public void deletePrimaryRecordsForStore(long id) {
         SearchCriteria<TemplateDataStoreVO> sc = storeSearch.create();
@@ -144,12 +166,22 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
     }
 
 
+    @Override
+    public List<TemplateDataStoreVO> listByTemplateStore(long templateId, long storeId) {
+        SearchCriteria<TemplateDataStoreVO> sc = storeTemplateSearch.create();
+        sc.setParameters("template_id", templateId);
+        sc.setParameters("store_id", storeId);
+        sc.setParameters("destroyed", false);
+        return search(sc, null);
+    }
+
     @Override
     public List<TemplateDataStoreVO> listByTemplateStoreStatus(long templateId, long storeId, State... states) {
         SearchCriteria<TemplateDataStoreVO> sc = storeTemplateStateSearch.create();
         sc.setParameters("template_id", templateId);
         sc.setParameters("store_id", storeId);
         sc.setParameters("states", (Object[])states);
+        sc.setParameters("destroyed", false);
         return search(sc, null);
     }
 
@@ -157,10 +189,11 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
 
     @Override
     public List<TemplateDataStoreVO> listByTemplateStoreDownloadStatus(long templateId, long storeId, Status... status) {
-        SearchCriteria<TemplateDataStoreVO> sc = storeTemplateStateSearch.create();
+        SearchCriteria<TemplateDataStoreVO> sc = storeTemplateDownloadStatusSearch.create();
         sc.setParameters("template_id", templateId);
         sc.setParameters("store_id", storeId);
         sc.setParameters("downloadState", (Object[])status);
+        sc.setParameters("destroyed", false);
         return search(sc, null);
     }
 
@@ -173,6 +206,22 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         return findOneIncludingRemovedBy(sc);
     }
 
+    @Override
+    public TemplateDataStoreVO findByTemplate(long templateId) {
+        SearchCriteria<TemplateDataStoreVO> sc = templateSearch.create();
+        sc.setParameters("template_id", templateId);
+        sc.setParameters("destroyed", false);
+        return findOneIncludingRemovedBy(sc);
+    }
+
+    @Override
+    public List<TemplateDataStoreVO> listByTemplate(long templateId) {
+        SearchCriteria<TemplateDataStoreVO> sc = templateSearch.create();
+        sc.setParameters("template_id", templateId);
+        sc.setParameters("destroyed", false);
+        return search(sc, null);
+    }
+
 
 
 }

plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java

@@ -34,6 +34,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
 import org.apache.cloudstack.framework.async.AsyncRpcConext;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.image.ImageStoreDriver;
 import org.apache.cloudstack.storage.image.store.ImageStoreImpl;
 import org.apache.cloudstack.storage.image.store.TemplateObject;
@@ -44,17 +46,23 @@ import org.apache.log4j.Logger;
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.DeleteSnapshotBackupCommand;
+import com.cloud.agent.api.storage.DeleteTemplateCommand;
 import com.cloud.agent.api.storage.DeleteVolumeCommand;
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.configuration.Resource.ResourceType;
+import com.cloud.event.EventTypes;
+import com.cloud.event.UsageEventUtils;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.RegisterVolumePayload;
 import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.storage.SnapshotVO;
+import com.cloud.storage.VMTemplateHostVO;
 import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.VMTemplateZoneVO;
@@ -68,9 +76,14 @@ import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.dao.VolumeHostDao;
 import com.cloud.storage.download.DownloadMonitor;
 import com.cloud.storage.s3.S3Manager;
+import com.cloud.storage.secondary.SecondaryStorageVmManager;
 import com.cloud.storage.snapshot.SnapshotManager;
 import com.cloud.storage.swift.SwiftManager;
+import com.cloud.user.Account;
+import com.cloud.user.dao.AccountDao;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.UserVmVO;
+import com.cloud.vm.dao.UserVmDao;
 
 public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
     private static final Logger s_logger = Logger
@@ -92,6 +105,15 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
     private SwiftManager _swiftMgr;
     @Inject
     private S3Manager _s3Mgr;
+    @Inject AccountDao _accountDao;
+    @Inject UserVmDao _userVmDao;
+    @Inject
+    SecondaryStorageVmManager _ssvmMgr;
+    @Inject
+    private AgentManager _agentMgr;
+    @Inject TemplateDataStoreDao _templateStoreDao;
+
+
     @Override
     public String grantAccess(DataObject data, EndPoint ep) {
         // TODO Auto-generated method stub
@@ -189,6 +211,60 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
 
     private void deleteTemplate(DataObject data, AsyncCompletionCallback<CommandResult> callback) {
 
+        TemplateObject templateObj = (TemplateObject) data;
+        VMTemplateVO template = templateObj.getImage();
+        ImageStoreImpl store = (ImageStoreImpl) templateObj.getDataStore();
+        long storeId = store.getId();
+        Long sZoneId = store.getDataCenterId();
+        long templateId = template.getId();
+
+        Account account = _accountDao.findByIdIncludingRemoved(template.getAccountId());
+        String eventType = "";
+
+        if (template.getFormat().equals(ImageFormat.ISO)) {
+            eventType = EventTypes.EVENT_ISO_DELETE;
+        } else {
+            eventType = EventTypes.EVENT_TEMPLATE_DELETE;
+        }
+
+        // TODO: need to understand why we need to mark destroyed in
+        // template_store_ref table here instead of in callback.
+        // Currently I did that in callback, so I removed previous code to mark template_host_ref
+
+        UsageEventUtils.publishUsageEvent(eventType, account.getId(), sZoneId, templateId, null, null, null);
+
+        List<UserVmVO> userVmUsingIso = _userVmDao.listByIsoId(templateId);
+        // check if there is any VM using this ISO.
+        if (userVmUsingIso == null || userVmUsingIso.isEmpty()) {
+            HostVO ssAhost = _ssvmMgr.pickSsvmHost(store);
+            // get installpath of this template on image store
+            TemplateDataStoreVO tmplStore = _templateStoreDao.findByStoreTemplate(storeId, templateId);
+            String installPath = tmplStore.getInstallPath();
+            if (installPath != null) {
+                Answer answer = _agentMgr.sendToSecStorage(ssAhost, new DeleteTemplateCommand(store.getTO(), store.getUri(), installPath, template.getId(), template.getAccountId()));
+
+                if (answer == null || !answer.getResult()) {
+                    s_logger.debug("Failed to deleted template at store: " + store.getName());
+                    CommandResult result = new CommandResult();
+                    result.setSucess(false);
+                    result.setResult("Delete template failed");
+                    callback.complete(result);
+
+                } else {
+                    s_logger.debug("Deleted template at: " + installPath);
+                    CommandResult result = new CommandResult();
+                    result.setSucess(false);
+                    callback.complete(result);
+                }
+
+                VMTemplateZoneVO templateZone = templateZoneDao.findByZoneTemplate(sZoneId, templateId);
+
+                if (templateZone != null) {
+                    templateZoneDao.remove(templateZone.getId());
+                }
+            }
+        }
+
     }
 
     private void deleteSnapshot(DataObject data, AsyncCompletionCallback<CommandResult> callback) {

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -1234,69 +1234,80 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
     public void cleanupSecondaryStorage(boolean recurring) {
         try {
             // Cleanup templates in secondary storage hosts
-            List<HostVO> secondaryStorageHosts = _ssvmMgr
+            List<DataStore> imageStores = this.dataStoreMgr.getImageStoresByScope(new ZoneScope(null));
-                    .listSecondaryStorageHostsInAllZones();
+             for (DataStore store : imageStores) {
-            for (HostVO secondaryStorageHost : secondaryStorageHosts) {
                 try {
-                    long hostId = secondaryStorageHost.getId();
+                    long storeId = store.getId();
-                    List<VMTemplateHostVO> destroyedTemplateHostVOs = _vmTemplateHostDao
+                    List<TemplateDataStoreVO> destroyedTemplateStoreVOs = this._templateStoreDao.listDestroyed(storeId);
-                            .listDestroyed(hostId);
                     s_logger.debug("Secondary storage garbage collector found "
-                            + destroyedTemplateHostVOs.size()
+                            + destroyedTemplateStoreVOs.size()
                             + " templates to cleanup on secondary storage host: "
-                            + secondaryStorageHost.getName());
+                            + store.getName());
-                    for (VMTemplateHostVO destroyedTemplateHostVO : destroyedTemplateHostVOs) {
+                    for (TemplateDataStoreVO destroyedTemplateStoreVO : destroyedTemplateStoreVOs) {
                         if (!_tmpltMgr
-                                .templateIsDeleteable(destroyedTemplateHostVO)) {
+                                .templateIsDeleteable(destroyedTemplateStoreVO)) {
                             if (s_logger.isDebugEnabled()) {
                                 s_logger.debug("Not deleting template at: "
-                                        + destroyedTemplateHostVO);
+                                        + destroyedTemplateStoreVO);
                             }
                             continue;
                         }
 
                         if (s_logger.isDebugEnabled()) {
-                            s_logger.debug("Deleting template host: "
+                            s_logger.debug("Deleting template store: "
-                                    + destroyedTemplateHostVO);
+                                    + destroyedTemplateStoreVO);
                         }
 
-                        String installPath = destroyedTemplateHostVO
+                        VMTemplateVO destroyedTemplate = this._vmTemplateDao.findById(destroyedTemplateStoreVO.getTemplateId());
+                        if ( destroyedTemplate == null ){
+                            s_logger.error("Cannot find template : " + destroyedTemplateStoreVO.getTemplateId() + " from template table");
+                            throw new CloudRuntimeException("Template " + destroyedTemplateStoreVO.getTemplateId() + " is found in secondary storage, but not found in template table");
+                        }
+                        String installPath = destroyedTemplateStoreVO
                                 .getInstallPath();
 
+                        HostVO ssAhost = this._ssvmMgr.pickSsvmHost(store);
                         if (installPath != null) {
+
                             Answer answer = _agentMgr.sendToSecStorage(
-                                    secondaryStorageHost,
+                                    ssAhost,
                                     new DeleteTemplateCommand(
-                                            secondaryStorageHost
+                                            store.getTO(),
-                                                    .getStorageUrl(),
+                                            store.getUri(),
-                                            destroyedTemplateHostVO
+                                            destroyedTemplateStoreVO
-                                                    .getInstallPath()));
+                                                    .getInstallPath(),
+                                            destroyedTemplate.getId(),
+                                            destroyedTemplate.getAccountId()
+
+                                            ));
 
                             if (answer == null || !answer.getResult()) {
                                 s_logger.debug("Failed to delete "
-                                        + destroyedTemplateHostVO
+                                        + destroyedTemplateStoreVO
                                         + " due to "
                                         + ((answer == null) ? "answer is null"
                                                 : answer.getDetails()));
                             } else {
                                 _vmTemplateHostDao
-                                        .remove(destroyedTemplateHostVO.getId());
+                                        .remove(destroyedTemplateStoreVO.getId());
                                 s_logger.debug("Deleted template at: "
-                                        + destroyedTemplateHostVO
+                                        + destroyedTemplateStoreVO
                                                 .getInstallPath());
                             }
                         } else {
-                            _vmTemplateHostDao.remove(destroyedTemplateHostVO
+                            _vmTemplateHostDao.remove(destroyedTemplateStoreVO
                                     .getId());
                         }
                     }
                 } catch (Exception e) {
                     s_logger.warn(
-                            "problem cleaning up templates in secondary storage "
+                            "problem cleaning up templates in secondary storage store "
-                                    + secondaryStorageHost, e);
+                                    + store.getName(), e);
                 }
             }
 
+            List<HostVO> secondaryStorageHosts = _ssvmMgr
+                    .listSecondaryStorageHostsInAllZones();
             // Cleanup snapshot in secondary storage hosts
             for (HostVO secondaryStorageHost : secondaryStorageHosts) {
                 try {

server/src/com/cloud/template/HypervisorTemplateAdapter.java

@@ -38,6 +38,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ImageDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
 import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.AgentManager;
@@ -180,7 +181,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase implements Te
 		}
         for (DataStore imageStore : imageStores) {
             AsyncCallFuture<CommandResult> future = this.imageService
-                    .createTemplateAsync(this.imageFactory.getTemplate(template.getId()), imageStore);
+                    .createTemplateAsync(this.imageFactory.getTemplate(template.getId(), imageStore), imageStore);
             try {
                 future.get();
             } catch (InterruptedException e) {
@@ -202,123 +203,60 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase implements Te
 		boolean success = true;
 
     	VMTemplateVO template = (VMTemplateVO)profile.getTemplate();
-    	Long zoneId = profile.getZoneId();
-    	Long templateId = template.getId();
 
-    	String zoneName;
+        // find all eligible image stores for this template
-    	List<HostVO> secondaryStorageHosts;
+        List<DataStore> imageStores = this.templateMgr.getImageStoreByTemplate(template.getId(), profile.getZoneId());
-    	if (!template.isCrossZones() && zoneId != null) {
+        if ( imageStores == null || imageStores.size() == 0 ){
-    		DataCenterVO zone = _dcDao.findById(zoneId);
+            throw new CloudRuntimeException("Unable to find image store to delete template "+ profile.getTemplate());
-    		zoneName = zone.getName();
-    		secondaryStorageHosts = _ssvmMgr.listSecondaryStorageHostsInOneZone(zoneId);
-    	} else {
-    		zoneName = "(all zones)";
-    		secondaryStorageHosts = _ssvmMgr.listSecondaryStorageHostsInAllZones();
         }
 
-    	s_logger.debug("Attempting to mark template host refs for template: " + template.getName() + " as destroyed in zone: " + zoneName);
+        // Make sure the template is downloaded to all found image stores
-
+        for (DataStore store : imageStores) {
-		// Make sure the template is downloaded to all the necessary secondary storage hosts
+            long storeId = store.getId();
-		for (HostVO secondaryStorageHost : secondaryStorageHosts) {
+            List<TemplateDataStoreVO> templateStores = _tmpltStoreDao.listByTemplateStore(template.getId(), storeId);
-			long hostId = secondaryStorageHost.getId();
+            for (TemplateDataStoreVO templateStore : templateStores) {
-			List<VMTemplateHostVO> templateHostVOs = _tmpltHostDao.listByHostTemplate(hostId, templateId);
+                if (templateStore.getDownloadState() == Status.DOWNLOAD_IN_PROGRESS) {
-			for (VMTemplateHostVO templateHostVO : templateHostVOs) {
-				if (templateHostVO.getDownloadState() == Status.DOWNLOAD_IN_PROGRESS) {
                     String errorMsg = "Please specify a template that is not currently being downloaded.";
-					s_logger.debug("Template: " + template.getName() + " is currently being downloaded to secondary storage host: " + secondaryStorageHost.getName() + "; cant' delete it.");
+                    s_logger.debug("Template: " + template.getName() + " is currently being downloaded to secondary storage host: " + store.getName() + "; cant' delete it.");
                     throw new CloudRuntimeException(errorMsg);
                 }
             }
         }
 
+
+        for (DataStore imageStore : imageStores) {
+            s_logger.info("Delete template from image store: " + imageStore.getName());
+            AsyncCallFuture<CommandResult> future = this.imageService
+                    .deleteTemplateAsync(this.imageFactory.getTemplate(template.getId(), imageStore));
+            try {
+                CommandResult result = future.get();
+                success = result.isSuccess();
+                if ( !success )
+                    break;
+            } catch (InterruptedException e) {
+                s_logger.debug("delete template Failed", e);
+                throw new CloudRuntimeException("delete template Failed", e);
+            } catch (ExecutionException e) {
+                s_logger.debug("delete template Failed", e);
+                throw new CloudRuntimeException("delete template Failed", e);
+            }
+        }
+
+        if (success) {
+            s_logger.info("Delete template from template table");
+            // remove template from vm_templates table
+            if (_tmpltDao.remove(template.getId())) {
+                // Decrement the number of templates and total secondary storage
+                // space used by the account
                 Account account = _accountDao.findByIdIncludingRemoved(template.getAccountId());
-		String eventType = "";
+                _resourceLimitMgr.decrementResourceCount(template.getAccountId(), ResourceType.template);
-
+                _resourceLimitMgr.recalculateResourceCount(template.getAccountId(), account.getDomainId(),
-		if (template.getFormat().equals(ImageFormat.ISO)){
-			eventType = EventTypes.EVENT_ISO_DELETE;
-		} else {
-			eventType = EventTypes.EVENT_TEMPLATE_DELETE;
-		}
-
-		// Iterate through all necessary secondary storage hosts and mark the template on each host as destroyed
-		for (HostVO secondaryStorageHost : secondaryStorageHosts) {
-			long hostId = secondaryStorageHost.getId();
-			long sZoneId = secondaryStorageHost.getDataCenterId();
-			List<VMTemplateHostVO> templateHostVOs = _tmpltHostDao.listByHostTemplate(hostId, templateId);
-			for (VMTemplateHostVO templateHostVO : templateHostVOs) {
-				VMTemplateHostVO lock = _tmpltHostDao.acquireInLockTable(templateHostVO.getId());
-				try {
-					if (lock == null) {
-						s_logger.debug("Failed to acquire lock when deleting templateHostVO with ID: " + templateHostVO.getId());
-						success = false;
-						break;
-					}
-					UsageEventUtils.publishUsageEvent(eventType, account.getId(), sZoneId, templateId, null, null, null);
-                    templateHostVO.setDestroyed(true);
-					_tmpltHostDao.update(templateHostVO.getId(), templateHostVO);
-                    String installPath = templateHostVO.getInstallPath();
-                    List<UserVmVO> userVmUsingIso = _userVmDao.listByIsoId(templateId);
-                    //check if there is any VM using this ISO.
-                    if (userVmUsingIso == null || userVmUsingIso.isEmpty()) {
-                    if (installPath != null) {
-                        Answer answer = _agentMgr.sendToSecStorage(secondaryStorageHost, new DeleteTemplateCommand(secondaryStorageHost.getStorageUrl(), installPath));
-
-                        if (answer == null || !answer.getResult()) {
-                            s_logger.debug("Failed to delete " + templateHostVO + " due to " + ((answer == null) ? "answer is null" : answer.getDetails()));
-                        } else {
-                            _tmpltHostDao.remove(templateHostVO.getId());
-                            s_logger.debug("Deleted template at: " + installPath);
-                        }
-                    } else {
-                        _tmpltHostDao.remove(templateHostVO.getId());
-                    }
-                    }
-					VMTemplateZoneVO templateZone = _tmpltZoneDao.findByZoneTemplate(sZoneId, templateId);
-
-					if (templateZone != null) {
-						_tmpltZoneDao.remove(templateZone.getId());
-					}
-				} finally {
-					if (lock != null) {
-						_tmpltHostDao.releaseFromLockTable(lock.getId());
-					}
-				}
-			}
-
-			if (!success) {
-				break;
-			}
-		}
-
-		s_logger.debug("Successfully marked template host refs for template: " + template.getName() + " as destroyed in zone: " + zoneName);
-
-		// If there are no more non-destroyed template host entries for this template, delete it
-		if (success && (_tmpltHostDao.listByTemplateId(templateId).size() == 0)) {
-			long accountId = template.getAccountId();
-
-			VMTemplateVO lock = _tmpltDao.acquireInLockTable(templateId);
-
-			try {
-				if (lock == null) {
-					s_logger.debug("Failed to acquire lock when deleting template with ID: " + templateId);
-					success = false;
-				} else if (_tmpltDao.remove(templateId)) {
-				    // Decrement the number of templates and total secondary storage space used by the account
-				    _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template);
-				    _resourceLimitMgr.recalculateResourceCount(accountId, account.getDomainId(),
                         ResourceType.secondary_storage.getOrdinal());
             }
-
-			} finally {
-				if (lock != null) {
-					_tmpltDao.releaseFromLockTable(lock.getId());
         }
-			}
-
-			s_logger.debug("Removed template: " + template.getName() + " because all of its template host refs were marked as destroyed.");
-		}
-
         return success;
+
+
 	}
 
 	public TemplateProfile prepareDelete(DeleteTemplateCmd cmd) {

server/src/com/cloud/template/TemplateAdapterBase.java

@@ -28,6 +28,7 @@ import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd;
 import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.log4j.Logger;
 
 import com.cloud.api.ApiDBUtils;
@@ -75,6 +76,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
 	protected @Inject DataCenterDao _dcDao;
 	protected @Inject VMTemplateDao _tmpltDao;
 	protected @Inject VMTemplateHostDao _tmpltHostDao;
+	protected @Inject TemplateDataStoreDao _tmpltStoreDao;
 	protected @Inject VMTemplateZoneDao _tmpltZoneDao;
 	protected @Inject UsageEventDao _usageEventDao;
 	protected @Inject HostDao _hostDao;

server/src/com/cloud/template/TemplateManager.java

@@ -20,6 +20,7 @@ import java.util.List;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 
 import com.cloud.dc.DataCenterVO;
 import com.cloud.exception.InternalErrorException;
@@ -92,6 +93,8 @@ public interface TemplateManager extends TemplateApiService{
 
     boolean templateIsDeleteable(VMTemplateHostVO templateHostRef);
 
+    boolean templateIsDeleteable(TemplateDataStoreVO templateStoreRef);
+
     VMTemplateHostVO prepareISOForCreate(VMTemplateVO template, StoragePool pool);
 
 
@@ -117,4 +120,6 @@ public interface TemplateManager extends TemplateApiService{
 
     String getChecksum(Long hostId, String templatePath);
 
+    List<DataStore> getImageStoreByTemplate(long templateId, Long zoneId);
+
 }

server/src/com/cloud/template/TemplateManagerImpl.java

@@ -65,8 +65,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
+import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
+import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
@@ -192,6 +196,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
     private final static Logger s_logger = Logger.getLogger(TemplateManagerImpl.class);
     @Inject VMTemplateDao _tmpltDao;
     @Inject VMTemplateHostDao _tmpltHostDao;
+    @Inject TemplateDataStoreDao _tmplStoreDao;
     @Inject VMTemplatePoolDao _tmpltPoolDao;
     @Inject VMTemplateZoneDao _tmpltZoneDao;
     @Inject
@@ -253,6 +258,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
     protected ResourceManager _resourceMgr;
     @Inject VolumeManager volumeMgr;
     @Inject VMTemplateHostDao templateHostDao;
+    @Inject ImageStoreDao _imageStoreDao;
 
 
     int _primaryStorageDownloadWait;
@@ -1261,6 +1267,40 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
 	}
 
     @Override
+    public boolean templateIsDeleteable(TemplateDataStoreVO templateStoreRef) {
+        VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templateStoreRef.getTemplateId());
+        long templateId = template.getId();
+        ImageStoreVO imageStore = _imageStoreDao.findById(templateStoreRef.getDataStoreId());
+        long zoneId = imageStore.getDataCenterId();
+        DataCenterVO zone = _dcDao.findById(zoneId);
+
+        // Check if there are VMs running in the template host ref's zone that use the template
+        List<VMInstanceVO> nonExpungedVms = _vmInstanceDao.listNonExpungedByZoneAndTemplate(zoneId, templateId);
+
+        if (!nonExpungedVms.isEmpty()) {
+            s_logger.debug("Template " + template.getName() + " in zone " + zone.getName() + " is not deleteable because there are non-expunged VMs deployed from this template.");
+            return false;
+        }
+        List<UserVmVO> userVmUsingIso = _userVmDao.listByIsoId(templateId);
+        //check if there is any VM using this ISO.
+        if (!userVmUsingIso.isEmpty()) {
+            s_logger.debug("ISO " + template.getName() + " in zone " + zone.getName() + " is not deleteable because it is attached to " + userVmUsingIso.size() + " VMs");
+            return false;
+        }
+        // Check if there are any snapshots for the template in the template host ref's zone
+        List<VolumeVO> volumes = _volumeDao.findByTemplateAndZone(templateId, zoneId);
+        for (VolumeVO volume : volumes) {
+            List<SnapshotVO> snapshots = _snapshotDao.listByVolumeIdVersion(volume.getId(), "2.1");
+            if (!snapshots.isEmpty()) {
+                s_logger.debug("Template " + template.getName() + " in zone " + zone.getName() + " is not deleteable because there are 2.1 snapshots using this template.");
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+	@Override
     @ActionEvent(eventType = EventTypes.EVENT_ISO_DETACH, eventDescription = "detaching ISO", async = true)
 	public boolean detachIso(long vmId)  {
         Account caller = UserContext.current().getCaller();
@@ -1421,12 +1461,14 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
     	if (template.getFormat() == ImageFormat.ISO) {
     		throw new InvalidParameterValueException("Please specify a valid template.");
     	}
+    	/*
         if (cmd.getZoneId() == null && _swiftMgr.isSwiftEnabled()) {
             _swiftMgr.deleteTemplate(cmd);
         }
         if (cmd.getZoneId() == null && _s3Mgr.isS3Enabled()) {
             _s3Mgr.deleteTemplate(cmd.getId(), caller.getAccountId());
         }
+        */
 
     	TemplateAdapter adapter = getAdapter(template.getHypervisorType());
     	TemplateProfile profile = adapter.prepareDelete(cmd);
@@ -2145,4 +2187,24 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
         return size;
     }
 
+    // find image store where this template is located
+    @Override
+    public List<DataStore> getImageStoreByTemplate(long templateId, Long zoneId) {
+        // find all eligible image stores for this zone scope
+        List<DataStore> imageStores = this.dataStoreMgr.getImageStoresByScope(new ZoneScope(zoneId));
+        if ( imageStores == null || imageStores.size() == 0 ){
+            return null;
+        }
+        List<DataStore> stores = new ArrayList<DataStore>();
+        for (DataStore store : imageStores){
+            // check if the template is stored there
+            List<TemplateDataStoreVO> storeTmpl = this._tmplStoreDao.listByTemplateStore(templateId, store.getId());
+            if ( storeTmpl != null && storeTmpl.size() > 0 ){
+                stores.add(store);
+            }
+        }
+        return stores;
+    }
+
+
 }

setup/db/db/schema-410to420.sql

@@ -140,7 +140,7 @@ CREATE TABLE  `cloud`.`template_store_ref` (
   INDEX `i_template_store_ref__template_id`(`template_id`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
 
-ALTER TABLE `cloud`.`vm_template` ADD COLUMN `image_data_store_id` bigint unsigned;
+-- ALTER TABLE `cloud`.`vm_template` ADD COLUMN `image_data_store_id` bigint unsigned;
 
 -- Do we still need these columns? TODO, to delete them, remove FK constraints from snapshots table
 -- ALTER TABLE `cloud`.`snapshots` DROP COLUMN `swift_id`;