diff --git a/api/src/com/cloud/agent/api/storage/DeleteTemplateCommand.java b/api/src/com/cloud/agent/api/storage/DeleteTemplateCommand.java index 69f465c6b16..9b874cf5156 100644 --- a/api/src/com/cloud/agent/api/storage/DeleteTemplateCommand.java +++ b/api/src/com/cloud/agent/api/storage/DeleteTemplateCommand.java @@ -16,17 +16,26 @@ // under the License. package com.cloud.agent.api.storage; +import com.cloud.agent.api.to.DataStoreTO; + public class DeleteTemplateCommand extends ssCommand { + private DataStoreTO store; private String templatePath; + private Long templateId; + private Long accountId; public DeleteTemplateCommand() { } - public DeleteTemplateCommand(String secUrl, String templatePath) { + + public DeleteTemplateCommand(DataStoreTO store, String secUrl, String templatePath, Long templateId, Long accountId) { this.setSecUrl(secUrl); this.templatePath = templatePath; + this.templateId = templateId; + this.accountId = accountId; + this.store = store; } @Override @@ -37,4 +46,18 @@ public class DeleteTemplateCommand extends ssCommand { public String getTemplatePath() { return templatePath; } + + public Long getTemplateId() { + return templateId; + } + + public Long getAccountId() { + return accountId; + } + + public DataStoreTO getDataStore() { + return store; + } + + } diff --git a/core/src/com/cloud/storage/VMTemplateVO.java b/core/src/com/cloud/storage/VMTemplateVO.java index e643d75bf1e..e3339fdd6b4 100755 --- a/core/src/com/cloud/storage/VMTemplateVO.java +++ b/core/src/com/cloud/storage/VMTemplateVO.java @@ -128,19 +128,16 @@ public class VMTemplateVO implements VirtualMachineTemplate, StateObject listByStoreId(long id); + public List listDestroyed(long storeId); + public void deletePrimaryRecordsForStore(long id); + List listByTemplateStore(long templateId, long storeId); + List listByTemplateStoreStatus(long templateId, long storeId, State... states); List listByTemplateStoreDownloadStatus(long templateId, long storeId, Status... status); TemplateDataStoreVO findByStoreTemplate(long storeId, long templateId); + + TemplateDataStoreVO findByTemplate(long templateId); + + List listByTemplate(long templateId); } diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/ImageDataFactoryImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/ImageDataFactoryImpl.java index 37204bf0386..d2c2520c74e 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/ImageDataFactoryImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/ImageDataFactoryImpl.java @@ -28,6 +28,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ImageDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.store.TemplateObject; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -50,6 +52,8 @@ public class ImageDataFactoryImpl implements ImageDataFactory { DataStoreManager storeMgr; @Inject VMTemplatePoolDao templatePoolDao; + @Inject + TemplateDataStoreDao templateStoreDao; @Override public TemplateInfo getTemplate(long templateId, DataStore store) { VMTemplateVO templ = imageDataDao.findById(templateId); @@ -77,13 +81,17 @@ public class ImageDataFactoryImpl implements ImageDataFactory { TemplateObject tmpl = TemplateObject.getTemplate(templ, store); return tmpl; } + + //TODO: this method is problematic, since one template can be stored in multiple image stores. + // need to see if we can get rid of this method or change to plural format. @Override public TemplateInfo getTemplate(long templateId) { VMTemplateVO templ = imageDataDao.findById(templateId); - if (templ.getImageDataStoreId() == null) { - return this.getTemplate(templateId, null); + TemplateDataStoreVO tmplStore = templateStoreDao.findByTemplate(templateId); + DataStore store = null; + if ( tmplStore != null ){ + store = this.storeMgr.getDataStore(tmplStore.getDataStoreId(), DataStoreRole.Image); } - DataStore store = this.storeMgr.getDataStore(templ.getImageDataStoreId(), DataStoreRole.Image); return this.getTemplate(templateId, store); } @Override diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 3dcb86b0939..38381082216 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -40,6 +40,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import com.cloud.storage.template.TemplateProp; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; @@ -139,6 +141,28 @@ public class TemplateServiceImpl implements TemplateService { } } + class DeleteTemplateContext extends AsyncRpcConext { + final TemplateObject template; + final AsyncCallFuture future; + + public DeleteTemplateContext(AsyncCompletionCallback callback, TemplateObject template, + AsyncCallFuture future) { + super(callback); + this.template = template; + this.future = future; + } + + public TemplateObject getTemplate() { + return template; + } + + public AsyncCallFuture getFuture() { + return future; + } + + + } + @Override public AsyncCallFuture createTemplateAsync( TemplateInfo template, DataStore store) { @@ -361,7 +385,8 @@ public class TemplateServiceImpl implements TemplateService { List userVmUsingIso = _userVmDao.listByIsoId(tInfo.getId()); //check if there is any Vm using this ISO. if (userVmUsingIso == null || userVmUsingIso.isEmpty()) { - DeleteTemplateCommand dtCommand = new DeleteTemplateCommand(store.getUri(), tInfo.getInstallPath()); + VMTemplateVO template = _templateDao.findById(tInfo.getId()); + DeleteTemplateCommand dtCommand = new DeleteTemplateCommand(store.getTO(), store.getUri(), tInfo.getInstallPath(), template.getId(), template.getAccountId()); try { HostVO ssAhost = _ssvmMgr.pickSsvmHost(store); _agentMgr.sendToSecStorage(ssAhost, dtCommand, null); @@ -458,7 +483,28 @@ public class TemplateServiceImpl implements TemplateService { @Override public AsyncCallFuture deleteTemplateAsync( TemplateInfo template) { - // TODO Auto-generated method stub + TemplateObject to = (TemplateObject) template; + // update template_store_ref status + to.processEvent(ObjectInDataStoreStateMachine.Event.DestroyRequested); + AsyncCallFuture future = new AsyncCallFuture(); + + DeleteTemplateContext context = new DeleteTemplateContext(null, to, future); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().deleteTemplateCallback(null, null)).setContext(context); + to.getDataStore().getDriver().deleteAsync(to, caller); + return future; + } + + public Void deleteTemplateCallback(AsyncCallbackDispatcher callback, DeleteTemplateContext context) { + CommandResult result = callback.getResult(); + TemplateObject vo = context.getTemplate(); + // we can only update state in template_store_ref table + if (result.isSuccess()) { + vo.processEvent(Event.OperationSuccessed); + } else { + vo.processEvent(Event.OperationFailed); + } + context.getFuture().complete(result); return null; } diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java index 7727f12d91e..15aad4ba3e7 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -69,9 +69,6 @@ public class TemplateObject implements TemplateInfo { return to; } - public void setImageStoreId(long id) { - this.imageVO.setImageDataStoreId(id); - } public void setSize(Long size) { this.imageVO.setSize(size); @@ -85,7 +82,7 @@ public class TemplateObject implements TemplateInfo { public DataStore getDataStore() { return this.dataStore; } - + @Override public String getUniqueName() { return this.imageVO.getUniqueName(); @@ -126,7 +123,7 @@ public class TemplateObject implements TemplateInfo { if (templateHostVO == null) { VMTemplateSwiftVO templateSwiftVO = _swiftMgr.findByTmpltId(templateForVmCreation.getId()); - if (templateSwiftVO != null) { + if (templateSwiftVO != null) { long templateSize = templateSwiftVO.getPhysicalSize(); if (templateSize == 0) { templateSize = templateSwiftVO.getSize(); @@ -172,7 +169,7 @@ public class TemplateObject implements TemplateInfo { throw new CloudRuntimeException("Failed to update state" + e.toString()); } } - + @Override public void processEvent(ObjectInDataStoreStateMachine.Event event, Answer answer) { try { @@ -182,14 +179,14 @@ public class TemplateObject implements TemplateInfo { throw new CloudRuntimeException("Failed to update state" + e.toString()); } } - + @Override public DataTO getTO() { DataTO to = this.dataStore.getDriver().getTO(this); if (to == null) { to = new TemplateObjectTO(this); } - + return to; } } diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java index e34a1aba885..dd8e319323f 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java @@ -43,9 +43,11 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase updateStateSearch; private SearchBuilder storeSearch; + private SearchBuilder templateSearch; + private SearchBuilder storeTemplateSearch; private SearchBuilder storeTemplateStateSearch; private SearchBuilder storeTemplateDownloadStatusSearch; - private SearchBuilder storeTemplateSearch; + @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -57,12 +59,24 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase listDestroyed(long id) { + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store_id", id); + sc.setParameters("destroyed", true); + return listIncludingRemovedBy(sc); + } + @Override public void deletePrimaryRecordsForStore(long id) { SearchCriteria sc = storeSearch.create(); @@ -144,12 +166,22 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase listByTemplateStore(long templateId, long storeId) { + SearchCriteria sc = storeTemplateSearch.create(); + sc.setParameters("template_id", templateId); + sc.setParameters("store_id", storeId); + sc.setParameters("destroyed", false); + return search(sc, null); + } + @Override public List listByTemplateStoreStatus(long templateId, long storeId, State... states) { SearchCriteria sc = storeTemplateStateSearch.create(); sc.setParameters("template_id", templateId); sc.setParameters("store_id", storeId); sc.setParameters("states", (Object[])states); + sc.setParameters("destroyed", false); return search(sc, null); } @@ -157,10 +189,11 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase listByTemplateStoreDownloadStatus(long templateId, long storeId, Status... status) { - SearchCriteria sc = storeTemplateStateSearch.create(); + SearchCriteria sc = storeTemplateDownloadStatusSearch.create(); sc.setParameters("template_id", templateId); sc.setParameters("store_id", storeId); sc.setParameters("downloadState", (Object[])status); + sc.setParameters("destroyed", false); return search(sc, null); } @@ -173,6 +206,22 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase sc = templateSearch.create(); + sc.setParameters("template_id", templateId); + sc.setParameters("destroyed", false); + return findOneIncludingRemovedBy(sc); + } + + @Override + public List listByTemplate(long templateId) { + SearchCriteria sc = templateSearch.create(); + sc.setParameters("template_id", templateId); + sc.setParameters("destroyed", false); + return search(sc, null); + } + } diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java index f90238571dc..85ebd230071 100644 --- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java +++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java @@ -34,6 +34,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcConext; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.ImageStoreDriver; import org.apache.cloudstack.storage.image.store.ImageStoreImpl; import org.apache.cloudstack.storage.image.store.TemplateObject; @@ -44,17 +46,23 @@ import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.DeleteSnapshotBackupCommand; +import com.cloud.agent.api.storage.DeleteTemplateCommand; import com.cloud.agent.api.storage.DeleteVolumeCommand; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; +import com.cloud.configuration.Resource.ResourceType; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.storage.DataStoreRole; import com.cloud.storage.RegisterVolumePayload; import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VMTemplateZoneVO; @@ -68,9 +76,14 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeHostDao; import com.cloud.storage.download.DownloadMonitor; import com.cloud.storage.s3.S3Manager; +import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.storage.swift.SwiftManager; +import com.cloud.user.Account; +import com.cloud.user.dao.AccountDao; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.dao.UserVmDao; public class CloudStackImageStoreDriverImpl implements ImageStoreDriver { private static final Logger s_logger = Logger @@ -92,6 +105,15 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver { private SwiftManager _swiftMgr; @Inject private S3Manager _s3Mgr; + @Inject AccountDao _accountDao; + @Inject UserVmDao _userVmDao; + @Inject + SecondaryStorageVmManager _ssvmMgr; + @Inject + private AgentManager _agentMgr; + @Inject TemplateDataStoreDao _templateStoreDao; + + @Override public String grantAccess(DataObject data, EndPoint ep) { // TODO Auto-generated method stub @@ -189,6 +211,60 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver { private void deleteTemplate(DataObject data, AsyncCompletionCallback callback) { + TemplateObject templateObj = (TemplateObject) data; + VMTemplateVO template = templateObj.getImage(); + ImageStoreImpl store = (ImageStoreImpl) templateObj.getDataStore(); + long storeId = store.getId(); + Long sZoneId = store.getDataCenterId(); + long templateId = template.getId(); + + Account account = _accountDao.findByIdIncludingRemoved(template.getAccountId()); + String eventType = ""; + + if (template.getFormat().equals(ImageFormat.ISO)) { + eventType = EventTypes.EVENT_ISO_DELETE; + } else { + eventType = EventTypes.EVENT_TEMPLATE_DELETE; + } + + // TODO: need to understand why we need to mark destroyed in + // template_store_ref table here instead of in callback. + // Currently I did that in callback, so I removed previous code to mark template_host_ref + + UsageEventUtils.publishUsageEvent(eventType, account.getId(), sZoneId, templateId, null, null, null); + + List userVmUsingIso = _userVmDao.listByIsoId(templateId); + // check if there is any VM using this ISO. + if (userVmUsingIso == null || userVmUsingIso.isEmpty()) { + HostVO ssAhost = _ssvmMgr.pickSsvmHost(store); + // get installpath of this template on image store + TemplateDataStoreVO tmplStore = _templateStoreDao.findByStoreTemplate(storeId, templateId); + String installPath = tmplStore.getInstallPath(); + if (installPath != null) { + Answer answer = _agentMgr.sendToSecStorage(ssAhost, new DeleteTemplateCommand(store.getTO(), store.getUri(), installPath, template.getId(), template.getAccountId())); + + if (answer == null || !answer.getResult()) { + s_logger.debug("Failed to deleted template at store: " + store.getName()); + CommandResult result = new CommandResult(); + result.setSucess(false); + result.setResult("Delete template failed"); + callback.complete(result); + + } else { + s_logger.debug("Deleted template at: " + installPath); + CommandResult result = new CommandResult(); + result.setSucess(false); + callback.complete(result); + } + + VMTemplateZoneVO templateZone = templateZoneDao.findByZoneTemplate(sZoneId, templateId); + + if (templateZone != null) { + templateZoneDao.remove(templateZone.getId()); + } + } + } + } private void deleteSnapshot(DataObject data, AsyncCompletionCallback callback) { diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index a5f82a8daef..cd15167f3fe 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -1234,69 +1234,80 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C public void cleanupSecondaryStorage(boolean recurring) { try { // Cleanup templates in secondary storage hosts - List secondaryStorageHosts = _ssvmMgr - .listSecondaryStorageHostsInAllZones(); - for (HostVO secondaryStorageHost : secondaryStorageHosts) { + List imageStores = this.dataStoreMgr.getImageStoresByScope(new ZoneScope(null)); + for (DataStore store : imageStores) { try { - long hostId = secondaryStorageHost.getId(); - List destroyedTemplateHostVOs = _vmTemplateHostDao - .listDestroyed(hostId); + long storeId = store.getId(); + List destroyedTemplateStoreVOs = this._templateStoreDao.listDestroyed(storeId); s_logger.debug("Secondary storage garbage collector found " - + destroyedTemplateHostVOs.size() + + destroyedTemplateStoreVOs.size() + " templates to cleanup on secondary storage host: " - + secondaryStorageHost.getName()); - for (VMTemplateHostVO destroyedTemplateHostVO : destroyedTemplateHostVOs) { + + store.getName()); + for (TemplateDataStoreVO destroyedTemplateStoreVO : destroyedTemplateStoreVOs) { if (!_tmpltMgr - .templateIsDeleteable(destroyedTemplateHostVO)) { + .templateIsDeleteable(destroyedTemplateStoreVO)) { if (s_logger.isDebugEnabled()) { s_logger.debug("Not deleting template at: " - + destroyedTemplateHostVO); + + destroyedTemplateStoreVO); } continue; } if (s_logger.isDebugEnabled()) { - s_logger.debug("Deleting template host: " - + destroyedTemplateHostVO); + s_logger.debug("Deleting template store: " + + destroyedTemplateStoreVO); } - String installPath = destroyedTemplateHostVO + VMTemplateVO destroyedTemplate = this._vmTemplateDao.findById(destroyedTemplateStoreVO.getTemplateId()); + if ( destroyedTemplate == null ){ + s_logger.error("Cannot find template : " + destroyedTemplateStoreVO.getTemplateId() + " from template table"); + throw new CloudRuntimeException("Template " + destroyedTemplateStoreVO.getTemplateId() + " is found in secondary storage, but not found in template table"); + } + String installPath = destroyedTemplateStoreVO .getInstallPath(); + HostVO ssAhost = this._ssvmMgr.pickSsvmHost(store); if (installPath != null) { + Answer answer = _agentMgr.sendToSecStorage( - secondaryStorageHost, + ssAhost, new DeleteTemplateCommand( - secondaryStorageHost - .getStorageUrl(), - destroyedTemplateHostVO - .getInstallPath())); + store.getTO(), + store.getUri(), + destroyedTemplateStoreVO + .getInstallPath(), + destroyedTemplate.getId(), + destroyedTemplate.getAccountId() + + )); if (answer == null || !answer.getResult()) { s_logger.debug("Failed to delete " - + destroyedTemplateHostVO + + destroyedTemplateStoreVO + " due to " + ((answer == null) ? "answer is null" : answer.getDetails())); } else { _vmTemplateHostDao - .remove(destroyedTemplateHostVO.getId()); + .remove(destroyedTemplateStoreVO.getId()); s_logger.debug("Deleted template at: " - + destroyedTemplateHostVO + + destroyedTemplateStoreVO .getInstallPath()); } } else { - _vmTemplateHostDao.remove(destroyedTemplateHostVO + _vmTemplateHostDao.remove(destroyedTemplateStoreVO .getId()); } } } catch (Exception e) { s_logger.warn( - "problem cleaning up templates in secondary storage " - + secondaryStorageHost, e); + "problem cleaning up templates in secondary storage store " + + store.getName(), e); } } + List secondaryStorageHosts = _ssvmMgr + .listSecondaryStorageHostsInAllZones(); // Cleanup snapshot in secondary storage hosts for (HostVO secondaryStorageHost : secondaryStorageHosts) { try { @@ -2037,16 +2048,16 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if (scope != null) { try { scopeType = Enum.valueOf(ScopeType.class, scope.toUpperCase()); - + } catch (Exception e) { throw new InvalidParameterValueException("invalid scope" + scope); } - + if (scopeType != ScopeType.ZONE) { throw new InvalidParameterValueException("Only zone wide cache storage is supported"); } } - + if (scopeType == ScopeType.ZONE && dcId == null) { throw new InvalidParameterValueException("zone id can't be null, if scope is zone"); } diff --git a/server/src/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/com/cloud/template/HypervisorTemplateAdapter.java index cbffa8ee1ee..3399ca1e92d 100755 --- a/server/src/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/com/cloud/template/HypervisorTemplateAdapter.java @@ -38,6 +38,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ImageDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; @@ -180,7 +181,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase implements Te } for (DataStore imageStore : imageStores) { AsyncCallFuture future = this.imageService - .createTemplateAsync(this.imageFactory.getTemplate(template.getId()), imageStore); + .createTemplateAsync(this.imageFactory.getTemplate(template.getId(), imageStore), imageStore); try { future.get(); } catch (InterruptedException e) { @@ -202,123 +203,60 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase implements Te boolean success = true; VMTemplateVO template = (VMTemplateVO)profile.getTemplate(); - Long zoneId = profile.getZoneId(); - Long templateId = template.getId(); - String zoneName; - List secondaryStorageHosts; - if (!template.isCrossZones() && zoneId != null) { - DataCenterVO zone = _dcDao.findById(zoneId); - zoneName = zone.getName(); - secondaryStorageHosts = _ssvmMgr.listSecondaryStorageHostsInOneZone(zoneId); - } else { - zoneName = "(all zones)"; - secondaryStorageHosts = _ssvmMgr.listSecondaryStorageHostsInAllZones(); - } + // find all eligible image stores for this template + List imageStores = this.templateMgr.getImageStoreByTemplate(template.getId(), profile.getZoneId()); + if ( imageStores == null || imageStores.size() == 0 ){ + throw new CloudRuntimeException("Unable to find image store to delete template "+ profile.getTemplate()); + } - s_logger.debug("Attempting to mark template host refs for template: " + template.getName() + " as destroyed in zone: " + zoneName); + // Make sure the template is downloaded to all found image stores + for (DataStore store : imageStores) { + long storeId = store.getId(); + List templateStores = _tmpltStoreDao.listByTemplateStore(template.getId(), storeId); + for (TemplateDataStoreVO templateStore : templateStores) { + if (templateStore.getDownloadState() == Status.DOWNLOAD_IN_PROGRESS) { + String errorMsg = "Please specify a template that is not currently being downloaded."; + s_logger.debug("Template: " + template.getName() + " is currently being downloaded to secondary storage host: " + store.getName() + "; cant' delete it."); + throw new CloudRuntimeException(errorMsg); + } + } + } - // Make sure the template is downloaded to all the necessary secondary storage hosts - for (HostVO secondaryStorageHost : secondaryStorageHosts) { - long hostId = secondaryStorageHost.getId(); - List templateHostVOs = _tmpltHostDao.listByHostTemplate(hostId, templateId); - for (VMTemplateHostVO templateHostVO : templateHostVOs) { - if (templateHostVO.getDownloadState() == Status.DOWNLOAD_IN_PROGRESS) { - String errorMsg = "Please specify a template that is not currently being downloaded."; - s_logger.debug("Template: " + template.getName() + " is currently being downloaded to secondary storage host: " + secondaryStorageHost.getName() + "; cant' delete it."); - throw new CloudRuntimeException(errorMsg); - } - } - } - Account account = _accountDao.findByIdIncludingRemoved(template.getAccountId()); - String eventType = ""; + for (DataStore imageStore : imageStores) { + s_logger.info("Delete template from image store: " + imageStore.getName()); + AsyncCallFuture future = this.imageService + .deleteTemplateAsync(this.imageFactory.getTemplate(template.getId(), imageStore)); + try { + CommandResult result = future.get(); + success = result.isSuccess(); + if ( !success ) + break; + } catch (InterruptedException e) { + s_logger.debug("delete template Failed", e); + throw new CloudRuntimeException("delete template Failed", e); + } catch (ExecutionException e) { + s_logger.debug("delete template Failed", e); + throw new CloudRuntimeException("delete template Failed", e); + } + } - if (template.getFormat().equals(ImageFormat.ISO)){ - eventType = EventTypes.EVENT_ISO_DELETE; - } else { - eventType = EventTypes.EVENT_TEMPLATE_DELETE; - } + if (success) { + s_logger.info("Delete template from template table"); + // remove template from vm_templates table + if (_tmpltDao.remove(template.getId())) { + // Decrement the number of templates and total secondary storage + // space used by the account + Account account = _accountDao.findByIdIncludingRemoved(template.getAccountId()); + _resourceLimitMgr.decrementResourceCount(template.getAccountId(), ResourceType.template); + _resourceLimitMgr.recalculateResourceCount(template.getAccountId(), account.getDomainId(), + ResourceType.secondary_storage.getOrdinal()); + } + } + return success; - // Iterate through all necessary secondary storage hosts and mark the template on each host as destroyed - for (HostVO secondaryStorageHost : secondaryStorageHosts) { - long hostId = secondaryStorageHost.getId(); - long sZoneId = secondaryStorageHost.getDataCenterId(); - List templateHostVOs = _tmpltHostDao.listByHostTemplate(hostId, templateId); - for (VMTemplateHostVO templateHostVO : templateHostVOs) { - VMTemplateHostVO lock = _tmpltHostDao.acquireInLockTable(templateHostVO.getId()); - try { - if (lock == null) { - s_logger.debug("Failed to acquire lock when deleting templateHostVO with ID: " + templateHostVO.getId()); - success = false; - break; - } - UsageEventUtils.publishUsageEvent(eventType, account.getId(), sZoneId, templateId, null, null, null); - templateHostVO.setDestroyed(true); - _tmpltHostDao.update(templateHostVO.getId(), templateHostVO); - String installPath = templateHostVO.getInstallPath(); - List userVmUsingIso = _userVmDao.listByIsoId(templateId); - //check if there is any VM using this ISO. - if (userVmUsingIso == null || userVmUsingIso.isEmpty()) { - if (installPath != null) { - Answer answer = _agentMgr.sendToSecStorage(secondaryStorageHost, new DeleteTemplateCommand(secondaryStorageHost.getStorageUrl(), installPath)); - if (answer == null || !answer.getResult()) { - s_logger.debug("Failed to delete " + templateHostVO + " due to " + ((answer == null) ? "answer is null" : answer.getDetails())); - } else { - _tmpltHostDao.remove(templateHostVO.getId()); - s_logger.debug("Deleted template at: " + installPath); - } - } else { - _tmpltHostDao.remove(templateHostVO.getId()); - } - } - VMTemplateZoneVO templateZone = _tmpltZoneDao.findByZoneTemplate(sZoneId, templateId); - - if (templateZone != null) { - _tmpltZoneDao.remove(templateZone.getId()); - } - } finally { - if (lock != null) { - _tmpltHostDao.releaseFromLockTable(lock.getId()); - } - } - } - - if (!success) { - break; - } - } - - s_logger.debug("Successfully marked template host refs for template: " + template.getName() + " as destroyed in zone: " + zoneName); - - // If there are no more non-destroyed template host entries for this template, delete it - if (success && (_tmpltHostDao.listByTemplateId(templateId).size() == 0)) { - long accountId = template.getAccountId(); - - VMTemplateVO lock = _tmpltDao.acquireInLockTable(templateId); - - try { - if (lock == null) { - s_logger.debug("Failed to acquire lock when deleting template with ID: " + templateId); - success = false; - } else if (_tmpltDao.remove(templateId)) { - // Decrement the number of templates and total secondary storage space used by the account - _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template); - _resourceLimitMgr.recalculateResourceCount(accountId, account.getDomainId(), - ResourceType.secondary_storage.getOrdinal()); - } - - } finally { - if (lock != null) { - _tmpltDao.releaseFromLockTable(lock.getId()); - } - } - - s_logger.debug("Removed template: " + template.getName() + " because all of its template host refs were marked as destroyed."); - } - - return success; } public TemplateProfile prepareDelete(DeleteTemplateCmd cmd) { diff --git a/server/src/com/cloud/template/TemplateAdapterBase.java b/server/src/com/cloud/template/TemplateAdapterBase.java index 05ba35d1c1a..987e6ed6ab5 100755 --- a/server/src/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/com/cloud/template/TemplateAdapterBase.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; @@ -75,6 +76,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat protected @Inject DataCenterDao _dcDao; protected @Inject VMTemplateDao _tmpltDao; protected @Inject VMTemplateHostDao _tmpltHostDao; + protected @Inject TemplateDataStoreDao _tmpltStoreDao; protected @Inject VMTemplateZoneDao _tmpltZoneDao; protected @Inject UsageEventDao _usageEventDao; protected @Inject HostDao _hostDao; diff --git a/server/src/com/cloud/template/TemplateManager.java b/server/src/com/cloud/template/TemplateManager.java index af5488076e6..a5d94824e30 100755 --- a/server/src/com/cloud/template/TemplateManager.java +++ b/server/src/com/cloud/template/TemplateManager.java @@ -20,6 +20,7 @@ import java.util.List; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import com.cloud.dc.DataCenterVO; import com.cloud.exception.InternalErrorException; @@ -39,7 +40,7 @@ public interface TemplateManager extends TemplateApiService{ /** * Prepares a template for vm creation for a certain storage pool. - * + * * @param template * template to prepare * @param pool @@ -52,7 +53,7 @@ public interface TemplateManager extends TemplateApiService{ /** * Copies a template from its current secondary storage server to the secondary storage server in the specified zone. - * + * * @param template * @param srcSecHost * @param srcZone @@ -66,7 +67,7 @@ public interface TemplateManager extends TemplateApiService{ /** * Deletes a template from secondary storage servers - * + * * @param userId * @param templateId * @param zoneId @@ -77,7 +78,7 @@ public interface TemplateManager extends TemplateApiService{ /** * Lists templates in the specified storage pool that are not being used by any VM. - * + * * @param pool * @return list of VMTemplateStoragePoolVO */ @@ -85,13 +86,15 @@ public interface TemplateManager extends TemplateApiService{ /** * Deletes a template in the specified storage pool. - * + * * @param templatePoolVO */ void evictTemplateFromStoragePool(VMTemplateStoragePoolVO templatePoolVO); boolean templateIsDeleteable(VMTemplateHostVO templateHostRef); + boolean templateIsDeleteable(TemplateDataStoreVO templateStoreRef); + VMTemplateHostVO prepareISOForCreate(VMTemplateVO template, StoragePool pool); @@ -117,4 +120,6 @@ public interface TemplateManager extends TemplateApiService{ String getChecksum(Long hostId, String templatePath); + List getImageStoreByTemplate(long templateId, Long zoneId); + } diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 88a4e7f73f7..a4eb35c9620 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -65,8 +65,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -192,6 +196,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, private final static Logger s_logger = Logger.getLogger(TemplateManagerImpl.class); @Inject VMTemplateDao _tmpltDao; @Inject VMTemplateHostDao _tmpltHostDao; + @Inject TemplateDataStoreDao _tmplStoreDao; @Inject VMTemplatePoolDao _tmpltPoolDao; @Inject VMTemplateZoneDao _tmpltZoneDao; @Inject @@ -253,21 +258,22 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, protected ResourceManager _resourceMgr; @Inject VolumeManager volumeMgr; @Inject VMTemplateHostDao templateHostDao; + @Inject ImageStoreDao _imageStoreDao; + - int _primaryStorageDownloadWait; protected SearchBuilder HostTemplateStatesSearch; - + int _storagePoolMaxWaitSeconds = 3600; boolean _disableExtraction = false; ExecutorService _preloadExecutor; ScheduledExecutorService _swiftTemplateSyncExecutor; - + private ScheduledExecutorService _s3TemplateSyncExecutor = null; @Inject protected List _adapters; - + private TemplateAdapter getAdapter(HypervisorType type) { TemplateAdapter adapter = null; if (type == HypervisorType.BareMetal) { @@ -276,21 +282,21 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, // see HyervisorTemplateAdapter adapter = AdapterBase.getAdapterByName(_adapters, TemplateAdapterType.Hypervisor.getName()); } - + if (adapter == null) { throw new CloudRuntimeException("Cannot find template adapter for " + type.toString()); } - + return adapter; } - + @Override @ActionEvent(eventType = EventTypes.EVENT_ISO_CREATE, eventDescription = "creating iso") public VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws ResourceAllocationException{ TemplateAdapter adapter = getAdapter(HypervisorType.None); - TemplateProfile profile = adapter.prepare(cmd); + TemplateProfile profile = adapter.prepare(cmd); VMTemplateVO template = adapter.create(profile); - + if (template != null){ return template; }else { @@ -307,18 +313,18 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new PermissionDeniedException("Parameter templatetag can only be specified by a Root Admin, permission denied"); } } - + TemplateAdapter adapter = getAdapter(HypervisorType.getType(cmd.getHypervisor())); TemplateProfile profile = adapter.prepare(cmd); VMTemplateVO template = adapter.create(profile); - + if (template != null){ return template; }else { throw new CloudRuntimeException("Failed to create a template"); } } - + @Override public DataStore getImageStore(String storeUuid, Long zoneId) { DataStore imageStore = null; @@ -331,7 +337,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } imageStore = stores.get(0); } - + return imageStore; } @@ -344,7 +350,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, String url = cmd.getUrl(); String mode = cmd.getMode(); Long eventId = cmd.getStartEventId(); - + // FIXME: async job needs fixing Long uploadId = extract(account, templateId, url, zoneId, mode, eventId, true, null, _asyncMgr); if (uploadId != null){ @@ -372,16 +378,16 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new CloudRuntimeException("Failed to extract the teamplate"); } } - + @Override public VirtualMachineTemplate prepareTemplate(long templateId, long zoneId) { - + VMTemplateVO vmTemplate = _tmpltDao.findById(templateId); if(vmTemplate == null) throw new InvalidParameterValueException("Unable to find template id=" + templateId); - + _accountMgr.checkAccess(UserContext.current().getCaller(), AccessType.ModifyEntry, true, vmTemplate); - + prepareTemplateInAllStoragePools(vmTemplate, zoneId); return vmTemplate; } @@ -392,22 +398,22 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, desc = Upload.Type.ISO.toString(); } eventId = eventId == null ? 0:eventId; - + if (!_accountMgr.isRootAdmin(caller.getType()) && _disableExtraction) { throw new PermissionDeniedException("Extraction has been disabled by admin"); } - + VMTemplateVO template = _tmpltDao.findById(templateId); if (template == null || template.getRemoved() != null) { throw new InvalidParameterValueException("Unable to find " +desc+ " with id " + templateId); } - + if (template.getTemplateType() == Storage.TemplateType.SYSTEM){ throw new InvalidParameterValueException("Unable to extract the " + desc + " " + template.getName() + " as it is a default System template"); } else if (template.getTemplateType() == Storage.TemplateType.PERHOST){ throw new InvalidParameterValueException("Unable to extract the " + desc + " " + template.getName() + " as it resides on host and not on SSVM"); } - + if (isISO) { if (template.getFormat() != ImageFormat.ISO ){ throw new InvalidParameterValueException("Unsupported format, could not extract the ISO"); @@ -417,7 +423,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new InvalidParameterValueException("Unsupported format, could not extract the template"); } } - + if (zoneId == null && _swiftMgr.isSwiftEnabled()) { zoneId = _swiftMgr.chooseZoneForTmpltExtract(templateId); } @@ -429,13 +435,13 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (_dcDao.findById(zoneId) == null) { throw new IllegalArgumentException("Please specify a valid zone."); } - + if (!_accountMgr.isRootAdmin(caller.getType()) && !template.isExtractable()) { throw new InvalidParameterValueException("Unable to extract template id=" + templateId + " as it's not extractable"); } - + _accountMgr.checkAccess(caller, AccessType.ModifyEntry, true, template); - + List sservers = getSecondaryStorageHosts(zoneId); VMTemplateHostVO tmpltHostRef = null; @@ -452,7 +458,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } } - + if (tmpltHostRef == null && _swiftMgr.isSwiftEnabled()) { SwiftTO swift = _swiftMgr.getSwiftTO(templateId); if (swift != null && sservers != null) { @@ -468,14 +474,14 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (tmpltHostRef == null) { throw new InvalidParameterValueException("The " + desc + " has not been downloaded "); } - + Upload.Mode extractMode; if (mode == null || (!mode.equalsIgnoreCase(Upload.Mode.FTP_UPLOAD.toString()) && !mode.equalsIgnoreCase(Upload.Mode.HTTP_DOWNLOAD.toString())) ){ throw new InvalidParameterValueException("Please specify a valid extract Mode. Supported modes: "+ Upload.Mode.FTP_UPLOAD + ", " + Upload.Mode.HTTP_DOWNLOAD); } else { extractMode = mode.equalsIgnoreCase(Upload.Mode.FTP_UPLOAD.toString()) ? Upload.Mode.FTP_UPLOAD : Upload.Mode.HTTP_DOWNLOAD; } - + if (extractMode == Upload.Mode.FTP_UPLOAD){ URI uri = null; try { @@ -486,7 +492,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } catch (Exception ex) { throw new InvalidParameterValueException("Invalid url given: " + url); } - + String host = uri.getHost(); try { InetAddress hostAddr = InetAddress.getByName(host); @@ -499,22 +505,22 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } catch (UnknownHostException uhe) { throw new InvalidParameterValueException("Unable to resolve " + host); } - + if (_uploadMonitor.isTypeUploadInProgress(templateId, isISO ? Type.ISO : Type.TEMPLATE) ){ - throw new IllegalArgumentException(template.getName() + " upload is in progress. Please wait for some time to schedule another upload for the same"); + throw new IllegalArgumentException(template.getName() + " upload is in progress. Please wait for some time to schedule another upload for the same"); } - - return _uploadMonitor.extractTemplate(template, url, tmpltHostRef, zoneId, eventId, job.getId(), mgr); + + return _uploadMonitor.extractTemplate(template, url, tmpltHostRef, zoneId, eventId, job.getId(), mgr); } - + UploadVO vo = _uploadMonitor.createEntityDownloadURL(template, tmpltHostRef, zoneId, eventId); - if (vo != null){ + if (vo != null){ return vo.getId(); }else{ return null; } } - + public void prepareTemplateInAllStoragePools(final VMTemplateVO template, long zoneId) { List pools = _poolDao.listByStatus(StoragePoolStatus.Up); for(final StoragePoolVO pool : pools) { @@ -528,7 +534,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, s_logger.warn("Unexpected exception ", e); } } - + private void reallyRun() { s_logger.info("Start to preload template " + template.getId() + " into primary storage " + pool.getId()); StoragePool pol = (StoragePool)dataStoreMgr.getPrimaryDataStore(pool.getId()); @@ -541,7 +547,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } } - + String downloadTemplateFromSwiftToSecondaryStorage(long dcId, long templateId){ VMTemplateVO template = _tmpltDao.findById(templateId); if ( template == null ) { @@ -648,7 +654,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override @DB public VMTemplateStoragePoolVO prepareTemplateForCreate(VMTemplateVO templ, StoragePool pool) { VMTemplateVO template = _tmpltDao.findById(templ.getId(), true); - + long poolId = pool.getId(); long templateId = template.getId(); long dcId = pool.getDataCenterId(); @@ -656,23 +662,23 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, VMTemplateHostVO templateHostRef = null; long templateStoragePoolRefId; String origUrl = null; - + templateStoragePoolRef = _tmpltPoolDao.findByPoolTemplate(poolId, templateId); if (templateStoragePoolRef != null) { templateStoragePoolRef.setMarkedForGC(false); _tmpltPoolDao.update(templateStoragePoolRef.getId(), templateStoragePoolRef); - + if (templateStoragePoolRef.getDownloadState() == Status.DOWNLOADED) { if (s_logger.isDebugEnabled()) { s_logger.debug("Template " + templateId + " has already been downloaded to pool " + poolId); } - + return templateStoragePoolRef; } } - + templateHostRef = findVmTemplateHost(templateId, pool); - + if (templateHostRef == null || templateHostRef.getDownloadState() != Status.DOWNLOADED) { String result = downloadTemplateFromSwiftToSecondaryStorage(dcId, templateId); if (result != null) { @@ -691,13 +697,13 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return null; } } - + HostVO sh = _hostDao.findById(templateHostRef.getHostId()); origUrl = sh.getStorageUrl(); if (origUrl == null) { throw new CloudRuntimeException("Unable to find the orig.url from host " + sh.toString()); } - + if (templateStoragePoolRef == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Downloading template " + templateId + " to pool " + poolId); @@ -706,7 +712,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, try { templateStoragePoolRef = _tmpltPoolDao.persist(templateStoragePoolRef); templateStoragePoolRefId = templateStoragePoolRef.getId(); - + } catch (Exception e) { s_logger.debug("Assuming we're in a race condition: " + e.getMessage()); templateStoragePoolRef = _tmpltPoolDao.findByPoolTemplate(poolId, templateId); @@ -718,12 +724,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } else { templateStoragePoolRefId = templateStoragePoolRef.getId(); } - + List vos = _poolHostDao.listByHostStatus(poolId, com.cloud.host.Status.Up); if (vos == null || vos.isEmpty()){ - throw new CloudRuntimeException("Cannot download " + templateId + " to poolId " + poolId + " since there is no host in the Up state connected to this pool"); - } - + throw new CloudRuntimeException("Cannot download " + templateId + " to poolId " + poolId + " since there is no host in the Up state connected to this pool"); + } + templateStoragePoolRef = _tmpltPoolDao.acquireInLockTable(templateStoragePoolRefId, _storagePoolMaxWaitSeconds); if (templateStoragePoolRef == null) { throw new CloudRuntimeException("Unable to acquire lock on VMTemplateStoragePool: " + templateStoragePoolRefId); @@ -734,13 +740,13 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return templateStoragePoolRef; } String url = origUrl + "/" + templateHostRef.getInstallPath(); - PrimaryStorageDownloadCommand dcmd = new PrimaryStorageDownloadCommand(template.getUniqueName(), url, template.getFormat(), + PrimaryStorageDownloadCommand dcmd = new PrimaryStorageDownloadCommand(template.getUniqueName(), url, template.getFormat(), template.getAccountId(), pool, _primaryStorageDownloadWait); HostVO secondaryStorageHost = _hostDao.findById(templateHostRef.getHostId()); assert(secondaryStorageHost != null); dcmd.setSecondaryStorageUrl(secondaryStorageHost.getStorageUrl()); - - + + for (int retry = 0; retry < 2; retry ++){ Collections.shuffle(vos); // Shuffling to pick a random host in the vm deployment retries StoragePoolHostVO vo = vos.get(0); @@ -749,7 +755,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } dcmd.setLocalPath(vo.getLocalPath()); // set 120 min timeout for this command - + PrimaryStorageDownloadAnswer answer = (PrimaryStorageDownloadAnswer)_agentMgr.easySend( _hvGuruMgr.getGuruProcessedCommandTargetHost(vo.getHostId(), dcmd), dcmd); if (answer != null && answer.getResult() ) { @@ -776,10 +782,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } return null; } - - - - + + + + @Override public VMTemplateHostVO findVmTemplateHost(long templateId, StoragePool pool) { @@ -813,7 +819,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } return null; } - + @Override public String getChecksum(Long hostId, String templatePath) { HostVO ssHost = _hostDao.findById(hostId); @@ -831,7 +837,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } return null; } - + @Override @DB public VMTemplateHostVO prepareISOForCreate(VMTemplateVO template, StoragePool pool) { @@ -873,22 +879,22 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, public boolean resetTemplateDownloadStateOnPool(long templateStoragePoolRefId) { // have to use the same lock that prepareTemplateForCreate use to maintain state consistency VMTemplateStoragePoolVO templateStoragePoolRef = _tmpltPoolDao.acquireInLockTable(templateStoragePoolRefId, 1200); - + if (templateStoragePoolRef == null) { s_logger.warn("resetTemplateDownloadStateOnPool failed - unable to lock TemplateStorgePoolRef " + templateStoragePoolRefId); return false; } - + try { templateStoragePoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED); _tmpltPoolDao.update(templateStoragePoolRefId, templateStoragePoolRef); } finally { _tmpltPoolDao.releaseFromLockTable(templateStoragePoolRefId); } - + return true; } - + @Override @DB public boolean copy(long userId, VMTemplateVO template, HostVO srcSecHost, DataCenterVO srcZone, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException { @@ -930,7 +936,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } else { dstTmpltHost.setDestroyed(false); _tmpltHostDao.update(dstTmpltHost.getId(), dstTmpltHost); - + return true; } } else if (dstTmpltHost != null && dstTmpltHost.getDownloadState() == Status.DOWNLOAD_ERROR){ @@ -951,7 +957,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if(_downloadMonitor.copyTemplate(template, srcSecHost, dstSecHost) ) { _tmpltDao.addTemplateToZone(template, dstZoneId); - + if(account.getId() != Account.ACCOUNT_ID_SYSTEM){ UsageEventUtils.publishUsageEvent(copyEventType, account.getId(), dstZoneId, tmpltId, null, null, null, srcTmpltHost.getSize(), template.getClass().getName(), template.getUuid()); @@ -961,8 +967,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } return false; } - - + + @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_COPY, eventDescription = "copying template", async = true) public VirtualMachineTemplate copyTemplate(CopyTemplateCmd cmd) throws StorageUnavailableException, ResourceAllocationException { @@ -971,7 +977,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, Long sourceZoneId = cmd.getSourceZoneId(); Long destZoneId = cmd.getDestinationZoneId(); Account caller = UserContext.current().getCaller(); - + if (_swiftMgr.isSwiftEnabled()) { throw new CloudRuntimeException("copytemplate API is disabled in Swift setup, templates in Swift can be accessed by all Zones"); } @@ -985,37 +991,37 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (sourceZoneId == destZoneId) { throw new InvalidParameterValueException("Please specify different source and destination zones."); } - + DataCenterVO sourceZone = _dcDao.findById(sourceZoneId); if (sourceZone == null) { throw new InvalidParameterValueException("Please specify a valid source zone."); } - + DataCenterVO dstZone = _dcDao.findById(destZoneId); if (dstZone == null) { throw new InvalidParameterValueException("Please specify a valid destination zone."); } - + VMTemplateVO template = _tmpltDao.findById(templateId); if (template == null || template.getRemoved() != null) { throw new InvalidParameterValueException("Unable to find template with id"); } - + HostVO dstSecHost = getSecondaryStorageHost(destZoneId, templateId); if ( dstSecHost != null ) { s_logger.debug("There is template " + templateId + " in secondary storage " + dstSecHost.getId() + " in zone " + destZoneId + " , don't need to copy"); return template; } - + HostVO srcSecHost = getSecondaryStorageHost(sourceZoneId, templateId); if ( srcSecHost == null ) { throw new InvalidParameterValueException("There is no template " + templateId + " in zone " + sourceZoneId ); } - + _accountMgr.checkAccess(caller, AccessType.ModifyEntry, true, template); - + boolean success = copy(userId, template, srcSecHost, sourceZone, dstZone); - + if (success){ return template; }else { @@ -1029,24 +1035,24 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (template == null || template.getRemoved() != null) { throw new InvalidParameterValueException("Please specify a valid template."); } - + TemplateAdapter adapter = getAdapter(template.getHypervisorType()); return adapter.delete(new TemplateProfile(userId, template, zoneId)); } - + @Override public List getUnusedTemplatesInPool(StoragePoolVO pool) { List unusedTemplatesInPool = new ArrayList(); List allTemplatesInPool = _tmpltPoolDao.listByPoolId(pool.getId()); - + for (VMTemplateStoragePoolVO templatePoolVO : allTemplatesInPool) { - VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId()); - + VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId()); + // If this is a routing template, consider it in use if (template.getTemplateType() == TemplateType.SYSTEM) { continue; } - + // If the template is not yet downloaded to the pool, consider it in use if (templatePoolVO.getDownloadState() != Status.DOWNLOADED) { continue; @@ -1056,24 +1062,24 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, unusedTemplatesInPool.add(templatePoolVO); } } - + return unusedTemplatesInPool; } - + @Override public void evictTemplateFromStoragePool(VMTemplateStoragePoolVO templatePoolVO) { StoragePool pool = (StoragePool)this.dataStoreMgr.getPrimaryDataStore(templatePoolVO.getPoolId()); VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId()); - - + + if (s_logger.isDebugEnabled()) { s_logger.debug("Evicting " + templatePoolVO); } DestroyCommand cmd = new DestroyCommand(pool, templatePoolVO); - + try { Answer answer = _storageMgr.sendToPool(pool, cmd); - + if (answer != null && answer.getResult()) { // Remove the templatePoolVO if (_tmpltPoolDao.remove(templatePoolVO.getId())) { @@ -1087,7 +1093,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } - + void swiftTemplateSync() { GlobalLock swiftTemplateSyncLock = GlobalLock.getInternLock("templatemgr.swiftTemplateSync"); try { @@ -1187,7 +1193,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override public boolean configure(String name, Map params) throws ConfigurationException { - + final Map configs = _configDao.getConfiguration("AgentManager", params); _routerTemplateId = NumbersUtil.parseInt(configs.get("router.template.id"), 1); @@ -1200,14 +1206,14 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, HostTemplateStatesSearch = _tmpltHostDao.createSearchBuilder(); HostTemplateStatesSearch.and("id", HostTemplateStatesSearch.entity().getTemplateId(), SearchCriteria.Op.EQ); HostTemplateStatesSearch.and("state", HostTemplateStatesSearch.entity().getDownloadState(), SearchCriteria.Op.EQ); - + SearchBuilder HostSearch = _hostDao.createSearchBuilder(); HostSearch.and("dcId", HostSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); - + HostTemplateStatesSearch.join("host", HostSearch, HostSearch.entity().getId(), HostTemplateStatesSearch.entity().getHostId(), JoinBuilder.JoinType.INNER); HostSearch.done(); HostTemplateStatesSearch.done(); - + _storagePoolMaxWaitSeconds = NumbersUtil.parseInt(_configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600); _preloadExecutor = Executors.newFixedThreadPool(8, new NamedThreadFactory("Template-Preloader")); _swiftTemplateSyncExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("swift-template-sync-Executor")); @@ -1222,7 +1228,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return true; } - + protected TemplateManagerImpl() { } @@ -1260,23 +1266,57 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return true; } + @Override + public boolean templateIsDeleteable(TemplateDataStoreVO templateStoreRef) { + VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templateStoreRef.getTemplateId()); + long templateId = template.getId(); + ImageStoreVO imageStore = _imageStoreDao.findById(templateStoreRef.getDataStoreId()); + long zoneId = imageStore.getDataCenterId(); + DataCenterVO zone = _dcDao.findById(zoneId); + + // Check if there are VMs running in the template host ref's zone that use the template + List nonExpungedVms = _vmInstanceDao.listNonExpungedByZoneAndTemplate(zoneId, templateId); + + if (!nonExpungedVms.isEmpty()) { + s_logger.debug("Template " + template.getName() + " in zone " + zone.getName() + " is not deleteable because there are non-expunged VMs deployed from this template."); + return false; + } + List userVmUsingIso = _userVmDao.listByIsoId(templateId); + //check if there is any VM using this ISO. + if (!userVmUsingIso.isEmpty()) { + s_logger.debug("ISO " + template.getName() + " in zone " + zone.getName() + " is not deleteable because it is attached to " + userVmUsingIso.size() + " VMs"); + return false; + } + // Check if there are any snapshots for the template in the template host ref's zone + List volumes = _volumeDao.findByTemplateAndZone(templateId, zoneId); + for (VolumeVO volume : volumes) { + List snapshots = _snapshotDao.listByVolumeIdVersion(volume.getId(), "2.1"); + if (!snapshots.isEmpty()) { + s_logger.debug("Template " + template.getName() + " in zone " + zone.getName() + " is not deleteable because there are 2.1 snapshots using this template."); + return false; + } + } + + return true; + } + @Override @ActionEvent(eventType = EventTypes.EVENT_ISO_DETACH, eventDescription = "detaching ISO", async = true) public boolean detachIso(long vmId) { Account caller = UserContext.current().getCaller(); Long userId = UserContext.current().getCallerUserId(); - + // Verify input parameters UserVmVO vmInstanceCheck = _userVmDao.findById(vmId); if (vmInstanceCheck == null) { throw new InvalidParameterValueException ("Unable to find a virtual machine with id " + vmId); } - + UserVm userVM = _userVmDao.findById(vmId); if (userVM == null) { throw new InvalidParameterValueException("Please specify a valid VM."); } - + _accountMgr.checkAccess(caller, null, true, userVM); Long isoId = userVM.getIsoId(); @@ -1284,7 +1324,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new InvalidParameterValueException("The specified VM has no ISO attached to it."); } UserContext.current().setEventDetails("Vm Id: " +vmId+ " ISO Id: "+isoId); - + State vmState = userVM.getState(); if (vmState != State.Running && vmState != State.Stopped) { throw new InvalidParameterValueException("Please specify a VM that is either Stopped or Running."); @@ -1295,44 +1335,44 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return result; }else { throw new CloudRuntimeException("Failed to detach iso"); - } + } } - + @Override @ActionEvent(eventType = EventTypes.EVENT_ISO_ATTACH, eventDescription = "attaching ISO", async = true) public boolean attachIso(long isoId, long vmId) { Account caller = UserContext.current().getCaller(); Long userId = UserContext.current().getCallerUserId(); - + // Verify input parameters UserVmVO vm = _userVmDao.findById(vmId); if (vm == null) { throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId); } - + VMTemplateVO iso = _tmpltDao.findById(isoId); if (iso == null || iso.getRemoved() != null) { throw new InvalidParameterValueException("Unable to find an ISO with id " + isoId); } - + //check permissions - //check if caller has access to VM and ISO + //check if caller has access to VM and ISO //and also check if the VM's owner has access to the ISO. - + _accountMgr.checkAccess(caller, null, false, iso, vm); - + Account vmOwner = _accountDao.findById(vm.getAccountId()); _accountMgr.checkAccess(vmOwner, null, false, iso, vm); - + State vmState = vm.getState(); if (vmState != State.Running && vmState != State.Stopped) { throw new InvalidParameterValueException("Please specify a VM that is either Stopped or Running."); } - + if ("xen-pv-drv-iso".equals(iso.getDisplayText()) && vm.getHypervisorType() != Hypervisor.HypervisorType.XenServer){ throw new InvalidParameterValueException("Cannot attach Xenserver PV drivers to incompatible hypervisor " + vm.getHypervisorType()); } - + if("vmware-tools.iso".equals(iso.getName()) && vm.getHypervisorType() != Hypervisor.HypervisorType.VMware) { throw new InvalidParameterValueException("Cannot attach VMware tools drivers to incompatible hypervisor " + vm.getHypervisorType()); } @@ -1343,7 +1383,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new CloudRuntimeException("Failed to attach iso"); } } - + private boolean attachISOToVM(long vmId, long isoId, boolean attach) { UserVmVO vm = this._userVmDao.findById(vmId); @@ -1397,41 +1437,43 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if ( success && attach) { vm.setIsoId(iso.getId()); _userVmDao.update(vmId, vm); - } + } if ( success && !attach ) { vm.setIsoId(null); _userVmDao.update(vmId, vm); - } + } return success; } - + @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_DELETE, eventDescription = "deleting template", async = true) public boolean deleteTemplate(DeleteTemplateCmd cmd) { Long templateId = cmd.getId(); Account caller = UserContext.current().getCaller(); - + VirtualMachineTemplate template = getTemplate(templateId); if (template == null) { throw new InvalidParameterValueException("unable to find template with id " + templateId); } - + _accountMgr.checkAccess(caller, AccessType.ModifyEntry, true, template); - + if (template.getFormat() == ImageFormat.ISO) { throw new InvalidParameterValueException("Please specify a valid template."); } + /* if (cmd.getZoneId() == null && _swiftMgr.isSwiftEnabled()) { _swiftMgr.deleteTemplate(cmd); } if (cmd.getZoneId() == null && _s3Mgr.isS3Enabled()) { _s3Mgr.deleteTemplate(cmd.getId(), caller.getAccountId()); } + */ TemplateAdapter adapter = getAdapter(template.getHypervisorType()); TemplateProfile profile = adapter.prepareDelete(cmd); boolean result = adapter.delete(profile); - + if (result){ if (cmd.getZoneId() == null && (_swiftMgr.isSwiftEnabled() || _s3Mgr.isS3Enabled())) { @@ -1448,21 +1490,21 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new CloudRuntimeException("Failed to delete template"); } } - + @Override @ActionEvent(eventType = EventTypes.EVENT_ISO_DELETE, eventDescription = "deleting iso", async = true) public boolean deleteIso(DeleteIsoCmd cmd) { Long templateId = cmd.getId(); Account caller = UserContext.current().getCaller(); Long zoneId = cmd.getZoneId(); - + VirtualMachineTemplate template = getTemplate(templateId);; if (template == null) { throw new InvalidParameterValueException("unable to find iso with id " + templateId); } - + _accountMgr.checkAccess(caller, AccessType.ModifyEntry, true, template); - + if (template.getFormat() != ImageFormat.ISO) { throw new InvalidParameterValueException("Please specify a valid iso."); } @@ -1495,17 +1537,17 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new CloudRuntimeException("Failed to delete ISO"); } } - + @Override public VirtualMachineTemplate getTemplate(long templateId) { VMTemplateVO template = _tmpltDao.findById(templateId); if (template != null && template.getRemoved() == null) { return template; } - + return null; } - + @Override public List listTemplatePermissions(BaseListTemplateOrIsoPermissionsCmd cmd) { Account caller = UserContext.current().getCaller(); @@ -1519,7 +1561,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (template == null) { throw new InvalidParameterValueException("unable to find " + cmd.getMediaType() + " with id " + id); } - + if (cmd instanceof ListTemplatePermissionsCmd) { if (template.getFormat().equals(ImageFormat.ISO)) { throw new InvalidParameterValueException("Please provide a valid template"); @@ -1544,7 +1586,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } return accountNames; } - + @DB @Override public boolean updateTemplateOrIsoPermissions(BaseUpdateTemplateOrIsoPermissionsCmd cmd) { @@ -1579,7 +1621,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new InvalidParameterValueException("Please provide a valid iso"); } } - + //convert projectIds to accountNames if (projectIds != null) { for (Long projectId : projectIds) { @@ -1587,7 +1629,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (project == null) { throw new InvalidParameterValueException("Unable to find project by id " + projectId); } - + if (!_projectMgr.canAccessProjectAccount(caller, project.getProjectAccountId())) { throw new InvalidParameterValueException("Account " + caller + " can't access project id=" + projectId); } @@ -1636,7 +1678,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (isFeatured != null) { updatedTemplate.setFeatured(isFeatured.booleanValue()); } - + if (isExtractable != null && caller.getType() == Account.ACCOUNT_TYPE_ADMIN) {//Only ROOT admins allowed to change this powerful attribute updatedTemplate.setExtractable(isExtractable.booleanValue()); }else if (isExtractable != null && caller.getType() != Account.ACCOUNT_TYPE_ADMIN) { @@ -1686,15 +1728,15 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } return true; } - - + + private String getRandomPrivateTemplateName() { return UUID.randomUUID().toString(); } - - - - + + + + @Override @DB @@ -1733,7 +1775,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new CloudRuntimeException( "Creating private Template need to specify snapshotId or volumeId"); } - + CommandResult result = null; try { result = future.get(); @@ -1742,7 +1784,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, s_logger.debug("Failed to create template" + result.getResult()); throw new CloudRuntimeException("Failed to create template" + result.getResult()); } - + privateTemplate = this._tmpltDao.findById(templateId); UsageEventVO usageEvent = new UsageEventVO( EventTypes.EVENT_TEMPLATE_CREATE, @@ -1990,7 +2032,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } - + @Override public Pair getAbsoluteIsoPath(long templateId, long dataCenterId) { @@ -2108,7 +2150,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } return hosts; } - + @Override public Long getTemplateSize(long templateId, long zoneId) { SearchCriteria sc = HostTemplateStatesSearch.create(); @@ -2145,4 +2187,24 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return size; } + // find image store where this template is located + @Override + public List getImageStoreByTemplate(long templateId, Long zoneId) { + // find all eligible image stores for this zone scope + List imageStores = this.dataStoreMgr.getImageStoresByScope(new ZoneScope(zoneId)); + if ( imageStores == null || imageStores.size() == 0 ){ + return null; + } + List stores = new ArrayList(); + for (DataStore store : imageStores){ + // check if the template is stored there + List storeTmpl = this._tmplStoreDao.listByTemplateStore(templateId, store.getId()); + if ( storeTmpl != null && storeTmpl.size() > 0 ){ + stores.add(store); + } + } + return stores; + } + + } diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index e8739fc71c4..65eedee9dbb 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -140,7 +140,7 @@ CREATE TABLE `cloud`.`template_store_ref` ( INDEX `i_template_store_ref__template_id`(`template_id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; -ALTER TABLE `cloud`.`vm_template` ADD COLUMN `image_data_store_id` bigint unsigned; +-- ALTER TABLE `cloud`.`vm_template` ADD COLUMN `image_data_store_id` bigint unsigned; -- Do we still need these columns? TODO, to delete them, remove FK constraints from snapshots table -- ALTER TABLE `cloud`.`snapshots` DROP COLUMN `swift_id`;