diff --git a/api/src/com/cloud/agent/api/storage/DownloadCommand.java b/api/src/com/cloud/agent/api/storage/DownloadCommand.java index 77e7ae32b68..81577b5dec6 100644 --- a/api/src/com/cloud/agent/api/storage/DownloadCommand.java +++ b/api/src/com/cloud/agent/api/storage/DownloadCommand.java @@ -147,9 +147,10 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal this.resourceType = ResourceType.VOLUME; } - public DownloadCommand(String secUrl, String url, VirtualMachineTemplate template, String user, String passwd, Long maxDownloadSizeInBytes) { + public DownloadCommand(DataStoreTO store, String secUrl, String url, VirtualMachineTemplate template, String user, String passwd, Long maxDownloadSizeInBytes) { super(template.getUniqueName(), url, template.getFormat(), template.getAccountId()); - this.hvm = template.isRequiresHvm(); + this._store = store; + this.hvm = template.isRequiresHvm(); this.checksum = template.getChecksum(); this.id = template.getId(); this.description = template.getDisplayText(); diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java b/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java index d1679a6989d..e9485267174 100644 --- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java +++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java @@ -44,6 +44,8 @@ public interface TemplateDataStoreDao extends GenericDao listByTemplate(long templateId); diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 38381082216..c16e47c7ee4 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -388,8 +388,7 @@ public class TemplateServiceImpl implements TemplateService { VMTemplateVO template = _templateDao.findById(tInfo.getId()); DeleteTemplateCommand dtCommand = new DeleteTemplateCommand(store.getTO(), store.getUri(), tInfo.getInstallPath(), template.getId(), template.getAccountId()); try { - HostVO ssAhost = _ssvmMgr.pickSsvmHost(store); - _agentMgr.sendToSecStorage(ssAhost, dtCommand, null); + _agentMgr.sendToSecStorage(store, dtCommand, null); } catch (AgentUnavailableException e) { String err = "Failed to delete " + tInfo.getTemplateName() + " on secondary storage " + storeId + " which isn't in the database"; s_logger.error(err); @@ -430,16 +429,15 @@ public class TemplateServiceImpl implements TemplateService { } - private Map listTemplate(DataStore ssHost) { - ListTemplateCommand cmd = new ListTemplateCommand(ssHost.getUri()); - HostVO ssAhost = _ssvmMgr.pickSsvmHost(ssHost); - Answer answer = _agentMgr.sendToSecStorage(ssAhost, cmd); + private Map listTemplate(DataStore ssStore) { + ListTemplateCommand cmd = new ListTemplateCommand(ssStore.getUri()); + Answer answer = _agentMgr.sendToSecStorage(ssStore, cmd); if (answer != null && answer.getResult()) { ListTemplateAnswer tanswer = (ListTemplateAnswer)answer; return tanswer.getTemplateInfo(); } else { if (s_logger.isDebugEnabled()) { - s_logger.debug("can not list template for secondary storage host " + ssHost.getId()); + s_logger.debug("can not list template for secondary storage host " + ssStore.getId()); } } diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentManagerSimpleImpl.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentManagerSimpleImpl.java index 575bc8e2ba2..0725971ab08 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentManagerSimpleImpl.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentManagerSimpleImpl.java @@ -24,6 +24,7 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; @@ -91,12 +92,12 @@ public class DirectAgentManagerSimpleImpl extends ManagerBase implements AgentMa params.put("password", "password"); params.put("zone", String.valueOf(host.getDataCenterId())); params.put("pod", String.valueOf(host.getPodId())); - + ServerResource resource = null; if (host.getHypervisorType() == HypervisorType.XenServer) { resource = new XcpOssResource(); } - + try { resource.configure(host.getName(), params); hostResourcesMap.put(hostId, resource); @@ -106,7 +107,7 @@ public class DirectAgentManagerSimpleImpl extends ManagerBase implements AgentMa HostEnvironment env = new HostEnvironment(); SetupCommand cmd = new SetupCommand(env); cmd.setNeedSetup(true); - + resource.executeRequest(cmd); } @@ -117,11 +118,11 @@ public class DirectAgentManagerSimpleImpl extends ManagerBase implements AgentMa loadResource(hostId); resource = hostResourcesMap.get(hostId); } - + if (resource == null) { return null; } - + Answer answer = resource.executeRequest(cmd); return answer; } @@ -186,6 +187,19 @@ public class DirectAgentManagerSimpleImpl extends ManagerBase implements AgentMa return null; } + + @Override + public void sendToSecStorage(DataStore ssStore, Command cmd, Listener listener) throws AgentUnavailableException { + // TODO Auto-generated method stub + + } + + @Override + public Answer sendToSecStorage(DataStore ssStore, Command cmd) { + // TODO Auto-generated method stub + return null; + } + @Override public boolean tapLoadingAgents(Long hostId, TapAgentsAction action) { // TODO Auto-generated method stub @@ -243,7 +257,7 @@ public class DirectAgentManagerSimpleImpl extends ManagerBase implements AgentMa @Override public void disconnectWithInvestigation(long hostId, Event event) { // TODO Auto-generated method stub - + } } diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java index dd8e319323f..eccb02aff4e 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java @@ -206,6 +206,19 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase sc = storeTemplateSearch.create(); + sc.setParameters("store_id", storeId); + sc.setParameters("template_id", templateId); + sc.setParameters("destroyed", false); + if (!lock) + return findOneIncludingRemovedBy(sc); + else + return lockOneRandomRow(sc, true); + } + @Override public TemplateDataStoreVO findByTemplate(long templateId) { SearchCriteria sc = templateSearch.create(); diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index d017f4f3600..22320957e63 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -814,8 +814,7 @@ public class VolumeServiceImpl implements VolumeService { TemplateProp vInfo = volumeInfos.get(uniqueName); DeleteVolumeCommand dtCommand = new DeleteVolumeCommand(store.getUri(), vInfo.getInstallPath()); try { - HostVO ssAhost = _ssvmMgr.pickSsvmHost(store); - _agentMgr.sendToSecStorage(ssAhost, dtCommand, null); + _agentMgr.sendToSecStorage(store, dtCommand, null); } catch (AgentUnavailableException e) { String err = "Failed to delete " + vInfo.getTemplateName() + " on image store " + storeId + " which isn't in the database"; s_logger.error(err); @@ -830,8 +829,7 @@ public class VolumeServiceImpl implements VolumeService { private Map listVolume(DataStore store) { ListVolumeCommand cmd = new ListVolumeCommand(store.getUri()); - HostVO ssAhost = _ssvmMgr.pickSsvmHost(store); - Answer answer = _agentMgr.sendToSecStorage(ssAhost, cmd); + Answer answer = _agentMgr.sendToSecStorage(store, cmd); if (answer != null && answer.getResult()) { ListVolumeAnswer tanswer = (ListVolumeAnswer)answer; return tanswer.getTemplateInfo(); diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java index 85ebd230071..fe556a2401d 100644 --- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java +++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java @@ -236,12 +236,11 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver { List userVmUsingIso = _userVmDao.listByIsoId(templateId); // check if there is any VM using this ISO. if (userVmUsingIso == null || userVmUsingIso.isEmpty()) { - HostVO ssAhost = _ssvmMgr.pickSsvmHost(store); - // get installpath of this template on image store + // get installpath of this template on image store TemplateDataStoreVO tmplStore = _templateStoreDao.findByStoreTemplate(storeId, templateId); String installPath = tmplStore.getInstallPath(); if (installPath != null) { - Answer answer = _agentMgr.sendToSecStorage(ssAhost, new DeleteTemplateCommand(store.getTO(), store.getUri(), installPath, template.getId(), template.getAccountId())); + Answer answer = _agentMgr.sendToSecStorage(store, new DeleteTemplateCommand(store.getTO(), store.getUri(), installPath, template.getId(), template.getAccountId())); if (answer == null || !answer.getResult()) { s_logger.debug("Failed to deleted template at store: " + store.getName()); diff --git a/server/src/com/cloud/agent/AgentManager.java b/server/src/com/cloud/agent/AgentManager.java index 6c300ea76fa..8071f28db17 100755 --- a/server/src/com/cloud/agent/AgentManager.java +++ b/server/src/com/cloud/agent/AgentManager.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.agent; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; + import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StartupCommand; @@ -44,10 +46,10 @@ public interface AgentManager extends Manager { Del, Contains, } - + /** * easy send method that returns null if there's any errors. It handles all exceptions. - * + * * @param hostId * host id * @param cmd @@ -58,7 +60,7 @@ public interface AgentManager extends Manager { /** * Synchronous sending a command to the agent. - * + * * @param hostId * id of the agent on host * @param cmd @@ -70,7 +72,7 @@ public interface AgentManager extends Manager { /** * Synchronous sending a list of commands to the agent. - * + * * @param hostId * id of the agent on host * @param cmds @@ -87,7 +89,7 @@ public interface AgentManager extends Manager { /** * Asynchronous sending of a command to the agent. - * + * * @param hostId * id of the agent on the host. * @param cmds @@ -102,7 +104,7 @@ public interface AgentManager extends Manager { /** * Register to listen for host events. These are mostly connection and disconnection events. - * + * * @param listener * @param connections * listen for connections @@ -125,7 +127,7 @@ public interface AgentManager extends Manager { /** * Unregister for listening to host events. - * + * * @param id * returned from registerForHostEvents */ @@ -138,20 +140,24 @@ public interface AgentManager extends Manager { void sendToSecStorage(HostVO ssHost, Command cmd, Listener listener) throws AgentUnavailableException; Answer sendToSecStorage(HostVO ssHost, Command cmd); - + + void sendToSecStorage(DataStore ssStore, Command cmd, Listener listener) throws AgentUnavailableException; + + Answer sendToSecStorage(DataStore ssStore, Command cmd); + /* working as a lock while agent is being loaded */ public boolean tapLoadingAgents(Long hostId, TapAgentsAction action); - + public AgentAttache handleDirectConnectAgent(HostVO host, StartupCommand[] cmds, ServerResource resource, boolean forRebalance) throws ConnectionException; - + public boolean agentStatusTransitTo(HostVO host, Status.Event e, long msId); - + public AgentAttache findAttache(long hostId); - + void disconnectWithoutInvestigation(long hostId, Status.Event event); - + public void pullAgentToMaintenance(long hostId); - + public void pullAgentOutMaintenance(long hostId); boolean reconnect(long hostId); diff --git a/server/src/com/cloud/agent/manager/AgentManagerImpl.java b/server/src/com/cloud/agent/manager/AgentManagerImpl.java index 6baeecf382a..c839b82719b 100755 --- a/server/src/com/cloud/agent/manager/AgentManagerImpl.java +++ b/server/src/com/cloud/agent/manager/AgentManagerImpl.java @@ -39,6 +39,7 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -380,6 +381,18 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl return attache; } + @Override + public Answer sendToSecStorage(DataStore ssStore, Command cmd) { + HostVO ssAhost = _ssvmMgr.pickSsvmHost(ssStore); + return easySend(ssAhost.getId(), cmd); + } + + @Override + public void sendToSecStorage(DataStore ssStore, Command cmd, Listener listener) throws AgentUnavailableException { + HostVO ssAhost = _ssvmMgr.pickSsvmHost(ssStore); + send(ssAhost.getId(), new Commands(cmd), listener); + } + @Override public Answer sendToSecStorage(HostVO ssHost, Command cmd) { if( ssHost.getType() == Host.Type.LocalSecondaryStorage ) { @@ -406,6 +419,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl } } + private void sendToSSVM(final long dcId, final Command cmd, final Listener listener) throws AgentUnavailableException { List ssAHosts = _ssvmMgr.listUpAndConnectingSecondaryStorageVmHost(dcId); if (ssAHosts == null || ssAHosts.isEmpty() ) { diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index cd15167f3fe..d46b2c61369 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -1266,11 +1266,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C String installPath = destroyedTemplateStoreVO .getInstallPath(); - HostVO ssAhost = this._ssvmMgr.pickSsvmHost(store); if (installPath != null) { - Answer answer = _agentMgr.sendToSecStorage( - ssAhost, + Answer answer = _agentMgr.sendToSecStorage(store, new DeleteTemplateCommand( store.getTO(), store.getUri(), diff --git a/server/src/com/cloud/storage/download/DownloadMonitor.java b/server/src/com/cloud/storage/download/DownloadMonitor.java index b5a3f8592b9..e709e90ff70 100644 --- a/server/src/com/cloud/storage/download/DownloadMonitor.java +++ b/server/src/com/cloud/storage/download/DownloadMonitor.java @@ -22,7 +22,6 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import com.cloud.exception.StorageUnavailableException; -import com.cloud.host.HostVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.Storage.ImageFormat; @@ -38,7 +37,7 @@ public interface DownloadMonitor extends Manager{ public void cancelAllDownloads(Long templateId); - public boolean copyTemplate(VMTemplateVO template, HostVO sourceServer, HostVO destServer) + public boolean copyTemplate(VMTemplateVO template, DataStore sourceStore, DataStore Store) throws StorageUnavailableException; //void addSystemVMTemplatesToHost(HostVO host, Map templateInfos); diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java index 58477b43e90..50ded27e349 100755 --- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java @@ -106,10 +106,9 @@ import com.cloud.vm.dao.UserVmDao; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; - @Component -@Local(value={DownloadMonitor.class}) -public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor { +@Local(value = { DownloadMonitor.class }) +public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor { static final Logger s_logger = Logger.getLogger(DownloadMonitorImpl.class); @Inject @@ -119,7 +118,7 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor @Inject VMTemplateZoneDao _vmTemplateZoneDao; @Inject - VMTemplatePoolDao _vmTemplatePoolDao; + VMTemplatePoolDao _vmTemplatePoolDao; @Inject VMTemplateSwiftDao _vmTemplateSwiftlDao; @Inject @@ -139,27 +138,27 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor @Inject SecondaryStorageVmManager _ssvmMgr; @Inject - StorageManager _storageMgr ; + StorageManager _storageMgr; @Inject private final DataCenterDao _dcDao = null; @Inject - VMTemplateDao _templateDao = null; + VMTemplateDao _templateDao = null; @Inject - private AgentManager _agentMgr; - @Inject SecondaryStorageVmManager _secMgr; + private AgentManager _agentMgr; + @Inject + SecondaryStorageVmManager _secMgr; @Inject ConfigurationDao _configDao; @Inject UserVmManager _vmMgr; - @Inject TemplateManager templateMgr; - + @Inject + TemplateManager templateMgr; @Inject private UsageEventDao _usageEventDao; - @Inject private ClusterDao _clusterDao; @Inject @@ -175,34 +174,34 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor @Inject protected AccountManager _accountMgr; - private Boolean _sslCopy = new Boolean(false); - private String _copyAuthPasswd; - private String _proxy = null; + private Boolean _sslCopy = new Boolean(false); + private String _copyAuthPasswd; + private String _proxy = null; protected SearchBuilder ReadyTemplateStatesSearch; - Timer _timer; + Timer _timer; - @Inject DataStoreManager storeMgr; + @Inject + DataStoreManager storeMgr; final Map _listenerTemplateMap = new ConcurrentHashMap(); - final Map _listenerMap = new ConcurrentHashMap(); - final Map _listenerVolumeMap = new ConcurrentHashMap(); - final Map _listenerVolMap = new ConcurrentHashMap(); + final Map _listenerMap = new ConcurrentHashMap(); + final Map _listenerVolumeMap = new ConcurrentHashMap(); + final Map _listenerVolMap = new ConcurrentHashMap(); + public void send(Long hostId, Command cmd, Listener listener) throws AgentUnavailableException { + _agentMgr.send(hostId, new Commands(cmd), listener); + } - public void send(Long hostId, Command cmd, Listener listener) throws AgentUnavailableException { - _agentMgr.send(hostId, new Commands(cmd), listener); - } - - @Override - public boolean configure(String name, Map params) { + @Override + public boolean configure(String name, Map params) { final Map configs = _configDao.getConfiguration("ManagementServer", params); _sslCopy = Boolean.parseBoolean(configs.get("secstorage.encrypt.copy")); _proxy = configs.get(Config.SecStorageProxy.key()); String cert = configs.get("secstorage.ssl.cert.domain"); if (!"realhostip.com".equalsIgnoreCase(cert)) { - s_logger.warn("Only realhostip.com ssl cert is supported, ignoring self-signed and other certs"); + s_logger.warn("Only realhostip.com ssl cert is supported, ignoring self-signed and other certs"); } _copyAuthPasswd = configs.get("secstorage.copy.password"); @@ -222,382 +221,387 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor TemplatesWithNoChecksumSearch.done(); ReadyTemplateStatesSearch.done(); - return true; - } + return true; + } - @Override - public boolean start() { - _timer = new Timer(); - return true; - } + @Override + public boolean start() { + _timer = new Timer(); + return true; + } - @Override - public boolean stop() { - return true; - } + @Override + public boolean stop() { + return true; + } - public boolean isTemplateUpdateable(Long templateId, Long storeId) { - List downloadsInProgress = - _vmTemplateStoreDao.listByTemplateStoreDownloadStatus(templateId, storeId, Status.DOWNLOAD_IN_PROGRESS, Status.DOWNLOADED ); - return (downloadsInProgress.size() == 0); - } + public boolean isTemplateUpdateable(Long templateId, Long storeId) { + List downloadsInProgress = _vmTemplateStoreDao.listByTemplateStoreDownloadStatus(templateId, storeId, + Status.DOWNLOAD_IN_PROGRESS, Status.DOWNLOADED); + return (downloadsInProgress.size() == 0); + } - @Override - public boolean copyTemplate(VMTemplateVO template, HostVO sourceServer, HostVO destServer) throws StorageUnavailableException{ + // TODO: consider using dataMotionStrategy later + @Override + public boolean copyTemplate(VMTemplateVO template, DataStore sourceStore, DataStore destStore) throws StorageUnavailableException { - boolean downloadJobExists = false; - VMTemplateHostVO destTmpltHost = null; - VMTemplateHostVO srcTmpltHost = null; + boolean downloadJobExists = false; + TemplateDataStoreVO destTmpltStore = null; + TemplateDataStoreVO srcTmpltStore = null; - srcTmpltHost = _vmTemplateHostDao.findByHostTemplate(sourceServer.getId(), template.getId()); - if (srcTmpltHost == null) { - throw new InvalidParameterValueException("Template " + template.getName() + " not associated with " + sourceServer.getName()); + srcTmpltStore = this._vmTemplateStoreDao.findByStoreTemplate(sourceStore.getId(), template.getId()); + if (srcTmpltStore == null) { + throw new InvalidParameterValueException("Template " + template.getName() + " not associated with " + sourceStore.getName()); } - String url = generateCopyUrl(sourceServer, srcTmpltHost); - if (url == null) { - s_logger.warn("Unable to start/resume copy of template " + template.getUniqueName() + " to " + destServer.getName() + ", no secondary storage vm in running state in source zone"); - throw new CloudRuntimeException("No secondary VM in running state in zone " + sourceServer.getDataCenterId()); - } - destTmpltHost = _vmTemplateHostDao.findByHostTemplate(destServer.getId(), template.getId()); - if (destTmpltHost == null) { - destTmpltHost = new VMTemplateHostVO(destServer.getId(), template.getId(), new Date(), 0, VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, null, null, "jobid0000", null, url); - destTmpltHost.setCopy(true); - destTmpltHost.setPhysicalSize(srcTmpltHost.getPhysicalSize()); - _vmTemplateHostDao.persist(destTmpltHost); - } else if ((destTmpltHost.getJobId() != null) && (destTmpltHost.getJobId().length() > 2)) { + // generate a storage url on ssvm to copy from + String url = generateCopyUrl(sourceStore, srcTmpltStore); + if (url == null) { + s_logger.warn("Unable to start/resume copy of template " + template.getUniqueName() + " to " + destStore.getName() + + ", no secondary storage vm in running state in source zone"); + throw new CloudRuntimeException("No secondary VM in running state in zone " + sourceStore.getScope().getScopeId()); + } + destTmpltStore = _vmTemplateStoreDao.findByStoreTemplate(destStore.getId(), template.getId()); + if (destTmpltStore == null) { + destTmpltStore = new TemplateDataStoreVO(destStore.getId(), template.getId(), new Date(), 0, + VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, null, null, "jobid0000", null, url); + destTmpltStore.setCopy(true); + destTmpltStore.setPhysicalSize(srcTmpltStore.getPhysicalSize()); + _vmTemplateStoreDao.persist(destTmpltStore); + } else if ((destTmpltStore.getJobId() != null) && (destTmpltStore.getJobId().length() > 2)) { downloadJobExists = true; } Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); - if (srcTmpltHost.getSize() > maxTemplateSizeInBytes){ - throw new CloudRuntimeException("Cant copy the template as the template's size " +srcTmpltHost.getSize()+ - " is greater than max.template.iso.size " + maxTemplateSizeInBytes); + if (srcTmpltStore.getSize() > maxTemplateSizeInBytes) { + throw new CloudRuntimeException("Cant copy the template as the template's size " + srcTmpltStore.getSize() + + " is greater than max.template.iso.size " + maxTemplateSizeInBytes); } - if(destTmpltHost != null) { - start(); - String sourceChecksum = this.templateMgr.getChecksum(srcTmpltHost.getHostId(), srcTmpltHost.getInstallPath()); - DownloadCommand dcmd = - new DownloadCommand(destServer.getStorageUrl(), url, template, TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd, maxTemplateSizeInBytes); - dcmd.setProxy(getHttpProxy()); - if (downloadJobExists) { - dcmd = new DownloadProgressCommand(dcmd, destTmpltHost.getJobId(), RequestType.GET_OR_RESTART); - } - dcmd.setChecksum(sourceChecksum); // We need to set the checksum as the source template might be a compressed url and have cksum for compressed image. Bug #10775 - HostVO ssAhost = _ssvmMgr.pickSsvmHost(destServer); - if( ssAhost == null ) { - s_logger.warn("There is no secondary storage VM for secondary storage host " + destServer.getName()); - return false; - } - DownloadListener dl = new DownloadListener(ssAhost, destServer, template, _timer, _vmTemplateHostDao, destTmpltHost.getId(), this, dcmd, _templateDao, _resourceLimitMgr, _alertMgr, _accountMgr); + if (destTmpltStore != null) { + start(); + String sourceChecksum = this.templateMgr.getChecksum(sourceStore, srcTmpltStore.getInstallPath()); + DownloadCommand dcmd = new DownloadCommand(destStore.getTO(), destStore.getUri(), url, template, + TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd, maxTemplateSizeInBytes); + dcmd.setProxy(getHttpProxy()); if (downloadJobExists) { - dl.setCurrState(destTmpltHost.getDownloadState()); + dcmd = new DownloadProgressCommand(dcmd, destTmpltStore.getJobId(), RequestType.GET_OR_RESTART); + } + dcmd.setChecksum(sourceChecksum); // We need to set the checksum as + // the source template might be a + // compressed url and have cksum + // for compressed image. Bug + // #10775 + HostVO ssAhost = _ssvmMgr.pickSsvmHost(destStore); + if (ssAhost == null) { + s_logger.warn("There is no secondary storage VM for secondary storage host " + destStore.getName()); + return false; + } + DownloadListener dl = new DownloadListener(ssAhost, destStore, template, _timer, _vmTemplateStoreDao, destTmpltStore.getId(), this, dcmd, + _templateDao, _resourceLimitMgr, _alertMgr, _accountMgr, null); + if (downloadJobExists) { + dl.setCurrState(destTmpltStore.getDownloadState()); + } + DownloadListener old = null; + synchronized (_listenerTemplateMap) { + old = _listenerTemplateMap.put(destTmpltStore, dl); + } + if (old != null) { + old.abandon(); } - DownloadListener old = null; - synchronized (_listenerMap) { - old = _listenerMap.put(destTmpltHost, dl); - } - if( old != null ) { - old.abandon(); - } try { - send(ssAhost.getId(), dcmd, dl); - return true; + send(ssAhost.getId(), dcmd, dl); + return true; } catch (AgentUnavailableException e) { - s_logger.warn("Unable to start /resume COPY of template " + template.getUniqueName() + " to " + destServer.getName(), e); - dl.setDisconnected(); - dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); - e.printStackTrace(); + s_logger.warn("Unable to start /resume COPY of template " + template.getUniqueName() + " to " + destStore.getName(), e); + dl.setDisconnected(); + dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); + e.printStackTrace(); } - } + } - return false; - } + return false; + } - private String generateCopyUrl(String ipAddress, String dir, String path){ - String hostname = ipAddress; - String scheme = "http"; - if (_sslCopy) { - hostname = ipAddress.replace(".", "-"); - hostname = hostname + ".realhostip.com"; - scheme = "https"; - } - return scheme + "://" + hostname + "/copy/SecStorage/" + dir + "/" + path; - } + private String generateCopyUrl(String ipAddress, String dir, String path) { + String hostname = ipAddress; + String scheme = "http"; + if (_sslCopy) { + hostname = ipAddress.replace(".", "-"); + hostname = hostname + ".realhostip.com"; + scheme = "https"; + } + return scheme + "://" + hostname + "/copy/SecStorage/" + dir + "/" + path; + } - private String generateCopyUrl(HostVO sourceServer, VMTemplateHostVO srcTmpltHost) { - List ssVms = _secStorageVmDao.getSecStorageVmListInStates(SecondaryStorageVm.Role.templateProcessor, sourceServer.getDataCenterId(), State.Running); - if (ssVms.size() > 0) { - SecondaryStorageVmVO ssVm = ssVms.get(0); - if (ssVm.getPublicIpAddress() == null) { - s_logger.warn("A running secondary storage vm has a null public ip?"); - return null; - } - return generateCopyUrl(ssVm.getPublicIpAddress(), sourceServer.getParent(), srcTmpltHost.getInstallPath()); - } + private String generateCopyUrl(DataStore sourceServer, TemplateDataStoreVO srcTmpltStore) { + List ssVms = _secStorageVmDao.getSecStorageVmListInStates(SecondaryStorageVm.Role.templateProcessor, sourceServer + .getScope().getScopeId(), State.Running); + if (ssVms.size() > 0) { + SecondaryStorageVmVO ssVm = ssVms.get(0); + if (ssVm.getPublicIpAddress() == null) { + s_logger.warn("A running secondary storage vm has a null public ip?"); + return null; + } + //TODO: how to handle parent field from hostVO in image_store? and how we can populate that column? + // return generateCopyUrl(ssVm.getPublicIpAddress(), sourceServer.getParent(), srcTmpltStore.getInstallPath()); + return generateCopyUrl(ssVm.getPublicIpAddress(), null, srcTmpltStore.getInstallPath()); + } - VMTemplateVO tmplt = _templateDao.findById(srcTmpltHost.getTemplateId()); - HypervisorType hyperType = tmplt.getHypervisorType(); - /*No secondary storage vm yet*/ - if (hyperType != null && hyperType == HypervisorType.KVM) { - return "file://" + sourceServer.getParent() + "/" + srcTmpltHost.getInstallPath(); - } - return null; - } + VMTemplateVO tmplt = _templateDao.findById(srcTmpltStore.getTemplateId()); + HypervisorType hyperType = tmplt.getHypervisorType(); + /*No secondary storage vm yet*/ + if (hyperType != null && hyperType == HypervisorType.KVM) { + //return "file://" + sourceServer.getParent() + "/" + srcTmpltStore.getInstallPath(); + return "file://" + "/" + srcTmpltStore.getInstallPath(); + } + return null; + } - private void initiateTemplateDownload(VMTemplateVO template, DataStore store, AsyncCompletionCallback callback) { - boolean downloadJobExists = false; + private void initiateTemplateDownload(VMTemplateVO template, DataStore store, AsyncCompletionCallback callback) { + boolean downloadJobExists = false; TemplateDataStoreVO vmTemplateStore = null; vmTemplateStore = _vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId()); if (vmTemplateStore == null) { - // This method can be invoked other places, for example, handleTemplateSync, in that case, vmTemplateStore may be null - vmTemplateStore = new TemplateDataStoreVO(store.getId(), template.getId(), new Date(), 0, VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, null, null, "jobid0000", null, template.getUrl()); + // This method can be invoked other places, for example, + // handleTemplateSync, in that case, vmTemplateStore may be null + vmTemplateStore = new TemplateDataStoreVO(store.getId(), template.getId(), new Date(), 0, + VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, null, null, "jobid0000", null, template.getUrl()); _vmTemplateStoreDao.persist(vmTemplateStore); - } else - if ((vmTemplateStore.getJobId() != null) && (vmTemplateStore.getJobId().length() > 2)) { + } else if ((vmTemplateStore.getJobId() != null) && (vmTemplateStore.getJobId().length() > 2)) { downloadJobExists = true; } Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); String secUrl = store.getUri(); - if(vmTemplateStore != null) { - start(); - DownloadCommand dcmd = - new DownloadCommand(store.getTO(), secUrl, template, maxTemplateSizeInBytes); - dcmd.setProxy(getHttpProxy()); - if (downloadJobExists) { - dcmd = new DownloadProgressCommand(dcmd, vmTemplateStore.getJobId(), RequestType.GET_OR_RESTART); - } - if (vmTemplateStore.isCopy()) { - dcmd.setCreds(TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd); - } - HostVO ssAhost = _ssvmMgr.pickSsvmHost(store); - if( ssAhost == null ) { - s_logger.warn("There is no secondary storage VM for downloading template to image store " + store.getName()); - return; - } - DownloadListener dl = new DownloadListener(ssAhost, store, template, _timer, _vmTemplateStoreDao, vmTemplateStore.getId(), this, dcmd, _templateDao, _resourceLimitMgr, _alertMgr, _accountMgr, callback); - if (downloadJobExists) { - // due to handling existing download job issues, we still keep downloadState in template_store_ref to avoid big change in DownloadListener to use - // new ObjectInDataStore.State transition. TODO: fix this later to be able to remove downloadState from template_store_ref. - dl.setCurrState(vmTemplateStore.getDownloadState()); - } + if (vmTemplateStore != null) { + start(); + DownloadCommand dcmd = new DownloadCommand(store.getTO(), secUrl, template, maxTemplateSizeInBytes); + dcmd.setProxy(getHttpProxy()); + if (downloadJobExists) { + dcmd = new DownloadProgressCommand(dcmd, vmTemplateStore.getJobId(), RequestType.GET_OR_RESTART); + } + if (vmTemplateStore.isCopy()) { + dcmd.setCreds(TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd); + } + HostVO ssAhost = _ssvmMgr.pickSsvmHost(store); + if (ssAhost == null) { + s_logger.warn("There is no secondary storage VM for downloading template to image store " + store.getName()); + return; + } + DownloadListener dl = new DownloadListener(ssAhost, store, template, _timer, _vmTemplateStoreDao, vmTemplateStore.getId(), this, dcmd, + _templateDao, _resourceLimitMgr, _alertMgr, _accountMgr, callback); + if (downloadJobExists) { + // due to handling existing download job issues, we still keep + // downloadState in template_store_ref to avoid big change in + // DownloadListener to use + // new ObjectInDataStore.State transition. TODO: fix this later + // to be able to remove downloadState from template_store_ref. + dl.setCurrState(vmTemplateStore.getDownloadState()); + } DownloadListener old = null; synchronized (_listenerTemplateMap) { old = _listenerTemplateMap.put(vmTemplateStore, dl); } - if( old != null ) { + if (old != null) { old.abandon(); } - try { - send(ssAhost.getId(), dcmd, dl); + try { + send(ssAhost.getId(), dcmd, dl); } catch (AgentUnavailableException e) { - s_logger.warn("Unable to start /resume download of template " + template.getUniqueName() + " to " + store.getName(), e); - dl.setDisconnected(); - dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); + s_logger.warn("Unable to start /resume download of template " + template.getUniqueName() + " to " + store.getName(), e); + dl.setDisconnected(); + dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); } - } - } + } + } - - - @Override - public void downloadTemplateToStorage(VMTemplateVO template, DataStore store, AsyncCompletionCallback callback) { + @Override + public void downloadTemplateToStorage(VMTemplateVO template, DataStore store, AsyncCompletionCallback callback) { long templateId = template.getId(); if (isTemplateUpdateable(templateId, store.getId())) { - if ( template != null && template.getUrl() != null ){ + if (template != null && template.getUrl() != null) { initiateTemplateDownload(template, store, callback); } } - } + } - - @Override - public void downloadVolumeToStorage(VolumeVO volume, DataStore store, String url, String checkSum, ImageFormat format, AsyncCompletionCallback callback) { - boolean downloadJobExists = false; + @Override + public void downloadVolumeToStorage(VolumeVO volume, DataStore store, String url, String checkSum, ImageFormat format, + AsyncCompletionCallback callback) { + boolean downloadJobExists = false; VolumeDataStoreVO volumeHost = null; volumeHost = _volumeStoreDao.findByStoreVolume(store.getId(), volume.getId()); if (volumeHost == null) { - volumeHost = new VolumeDataStoreVO(store.getId(), volume.getId(), new Date(), 0, VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, null, null, - "jobid0000", null, url, checkSum, format); + volumeHost = new VolumeDataStoreVO(store.getId(), volume.getId(), new Date(), 0, VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, + null, null, "jobid0000", null, url, checkSum, format); _volumeStoreDao.persist(volumeHost); } else if ((volumeHost.getJobId() != null) && (volumeHost.getJobId().length() > 2)) { downloadJobExists = true; } - Long maxVolumeSizeInBytes = getMaxVolumeSizeInBytes(); String secUrl = store.getUri(); - if(volumeHost != null) { - start(); - DownloadCommand dcmd = new DownloadCommand(secUrl, volume, maxVolumeSizeInBytes, checkSum, url, format); - dcmd.setProxy(getHttpProxy()); - if (downloadJobExists) { - dcmd = new DownloadProgressCommand(dcmd, volumeHost.getJobId(), RequestType.GET_OR_RESTART); - dcmd.setResourceType(ResourceType.VOLUME); - } + if (volumeHost != null) { + start(); + DownloadCommand dcmd = new DownloadCommand(secUrl, volume, maxVolumeSizeInBytes, checkSum, url, format); + dcmd.setProxy(getHttpProxy()); + if (downloadJobExists) { + dcmd = new DownloadProgressCommand(dcmd, volumeHost.getJobId(), RequestType.GET_OR_RESTART); + dcmd.setResourceType(ResourceType.VOLUME); + } - HostVO ssAhost = _ssvmMgr.pickSsvmHost(store); - if( ssAhost == null ) { - s_logger.warn("There is no secondary storage VM for image store " + store.getName()); - return; - } - DownloadListener dl = new DownloadListener(ssAhost, store, volume, _timer, _volumeStoreDao, volumeHost.getId(), - this, dcmd, _volumeDao, _storageMgr, _resourceLimitMgr, _alertMgr, _accountMgr, callback); + HostVO ssAhost = _ssvmMgr.pickSsvmHost(store); + if (ssAhost == null) { + s_logger.warn("There is no secondary storage VM for image store " + store.getName()); + return; + } + DownloadListener dl = new DownloadListener(ssAhost, store, volume, _timer, _volumeStoreDao, volumeHost.getId(), this, dcmd, _volumeDao, + _storageMgr, _resourceLimitMgr, _alertMgr, _accountMgr, callback); - if (downloadJobExists) { - dl.setCurrState(volumeHost.getDownloadState()); - } + if (downloadJobExists) { + dl.setCurrState(volumeHost.getDownloadState()); + } DownloadListener old = null; synchronized (_listenerVolMap) { old = _listenerVolMap.put(volumeHost, dl); } - if( old != null ) { + if (old != null) { old.abandon(); } - try { - send(ssAhost.getId(), dcmd, dl); + try { + send(ssAhost.getId(), dcmd, dl); } catch (AgentUnavailableException e) { - s_logger.warn("Unable to start /resume download of volume " + volume.getName() + " to " + store.getName(), e); - dl.setDisconnected(); - dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); + s_logger.warn("Unable to start /resume download of volume " + volume.getName() + " to " + store.getName(), e); + dl.setDisconnected(); + dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); } - } - } + } + } + @DB + public void handleDownloadEvent(HostVO host, VMTemplateVO template, Status dnldStatus) { + if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) || (dnldStatus == Status.ABANDONED)) { + VMTemplateHostVO vmTemplateHost = new VMTemplateHostVO(host.getId(), template.getId()); + synchronized (_listenerMap) { + _listenerMap.remove(vmTemplateHost); + } + } - @DB - public void handleDownloadEvent(HostVO host, VMTemplateVO template, Status dnldStatus) { - if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) || (dnldStatus==Status.ABANDONED)){ - VMTemplateHostVO vmTemplateHost = new VMTemplateHostVO(host.getId(), template.getId()); - synchronized (_listenerMap) { - _listenerMap.remove(vmTemplateHost); - } - } + VMTemplateHostVO vmTemplateHost = _vmTemplateHostDao.findByHostTemplate(host.getId(), template.getId()); - VMTemplateHostVO vmTemplateHost = _vmTemplateHostDao.findByHostTemplate(host.getId(), template.getId()); - - Transaction txn = Transaction.currentTxn(); + Transaction txn = Transaction.currentTxn(); txn.start(); if (dnldStatus == Status.DOWNLOADED) { long size = -1; - if(vmTemplateHost!=null){ + if (vmTemplateHost != null) { size = vmTemplateHost.getPhysicalSize(); template.setSize(size); this._templateDao.update(template.getId(), template); - } - else{ + } else { s_logger.warn("Failed to get size for template" + template.getName()); } String eventType = EventTypes.EVENT_TEMPLATE_CREATE; - if((template.getFormat()).equals(ImageFormat.ISO)){ + if ((template.getFormat()).equals(ImageFormat.ISO)) { eventType = EventTypes.EVENT_ISO_CREATE; } - if(template.getAccountId() != Account.ACCOUNT_ID_SYSTEM){ - UsageEventUtils.publishUsageEvent(eventType, template.getAccountId(), host.getDataCenterId(), - template.getId(), template.getName(), null, template.getSourceTemplateId(), size, - template.getClass().getName(), template.getUuid()); + if (template.getAccountId() != Account.ACCOUNT_ID_SYSTEM) { + UsageEventUtils.publishUsageEvent(eventType, template.getAccountId(), host.getDataCenterId(), template.getId(), template.getName(), + null, template.getSourceTemplateId(), size, template.getClass().getName(), template.getUuid()); } } txn.commit(); - } + } - @DB - public void handleDownloadEvent(HostVO host, VolumeVO volume, Status dnldStatus) { - if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) || (dnldStatus==Status.ABANDONED)){ - VolumeHostVO volumeHost = new VolumeHostVO(host.getId(), volume.getId()); - synchronized (_listenerVolumeMap) { - _listenerVolumeMap.remove(volumeHost); - } - } + @DB + public void handleDownloadEvent(HostVO host, VolumeVO volume, Status dnldStatus) { + if ((dnldStatus == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) || (dnldStatus == Status.ABANDONED)) { + VolumeHostVO volumeHost = new VolumeHostVO(host.getId(), volume.getId()); + synchronized (_listenerVolumeMap) { + _listenerVolumeMap.remove(volumeHost); + } + } - VolumeHostVO volumeHost = _volumeHostDao.findByHostVolume(host.getId(), volume.getId()); + VolumeHostVO volumeHost = _volumeHostDao.findByHostVolume(host.getId(), volume.getId()); - Transaction txn = Transaction.currentTxn(); + Transaction txn = Transaction.currentTxn(); txn.start(); if (dnldStatus == Status.DOWNLOADED) { - //Create usage event + // Create usage event long size = -1; - if(volumeHost!=null){ + if (volumeHost != null) { size = volumeHost.getPhysicalSize(); volume.setSize(size); this._volumeDao.update(volume.getId(), volume); - } - else{ + } else { s_logger.warn("Failed to get size for volume" + volume.getName()); } String eventType = EventTypes.EVENT_VOLUME_UPLOAD; - if(volume.getAccountId() != Account.ACCOUNT_ID_SYSTEM){ - UsageEventUtils.publishUsageEvent(eventType, volume.getAccountId(), host.getDataCenterId(), - volume.getId(), volume.getName(), null, 0l, size, volume.getClass().getName(), volume.getUuid()); + if (volume.getAccountId() != Account.ACCOUNT_ID_SYSTEM) { + UsageEventUtils.publishUsageEvent(eventType, volume.getAccountId(), host.getDataCenterId(), volume.getId(), volume.getName(), null, + 0l, size, volume.getClass().getName(), volume.getUuid()); } - }else if (dnldStatus == Status.DOWNLOAD_ERROR || dnldStatus == Status.ABANDONED || dnldStatus == Status.UNKNOWN){ - //Decrement the volume and secondary storage space count - _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), - com.cloud.configuration.Resource.ResourceType.volume); + } else if (dnldStatus == Status.DOWNLOAD_ERROR || dnldStatus == Status.ABANDONED || dnldStatus == Status.UNKNOWN) { + // Decrement the volume and secondary storage space count + _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), com.cloud.configuration.Resource.ResourceType.volume); _resourceLimitMgr.recalculateResourceCount(volume.getAccountId(), volume.getDomainId(), com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal()); } txn.commit(); - } + } - - - /* + /* @Override - public void addSystemVMTemplatesToHost(HostVO host, Map templateInfos){ - if ( templateInfos == null ) { - return; - } - Long hostId = host.getId(); - List rtngTmplts = _templateDao.listAllSystemVMTemplates(); - for ( VMTemplateVO tmplt : rtngTmplts ) { - TemplateProp tmpltInfo = templateInfos.get(tmplt.getUniqueName()); - if ( tmpltInfo == null ) { - continue; - } - VMTemplateHostVO tmpltHost = _vmTemplateHostDao.findByHostTemplate(hostId, tmplt.getId()); - if ( tmpltHost == null ) { + public void addSystemVMTemplatesToHost(HostVO host, Map templateInfos){ + if ( templateInfos == null ) { + return; + } + Long hostId = host.getId(); + List rtngTmplts = _templateDao.listAllSystemVMTemplates(); + for ( VMTemplateVO tmplt : rtngTmplts ) { + TemplateProp tmpltInfo = templateInfos.get(tmplt.getUniqueName()); + if ( tmpltInfo == null ) { + continue; + } + VMTemplateHostVO tmpltHost = _vmTemplateHostDao.findByHostTemplate(hostId, tmplt.getId()); + if ( tmpltHost == null ) { tmpltHost = new VMTemplateHostVO(hostId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, null, null, null, tmpltInfo.getInstallPath(), tmplt.getUrl()); - tmpltHost.setSize(tmpltInfo.getSize()); - tmpltHost.setPhysicalSize(tmpltInfo.getPhysicalSize()); - _vmTemplateHostDao.persist(tmpltHost); - } - } - } + tmpltHost.setSize(tmpltInfo.getSize()); + tmpltHost.setPhysicalSize(tmpltInfo.getPhysicalSize()); + _vmTemplateHostDao.persist(tmpltHost); + } + } + } */ + @Override + public void cancelAllDownloads(Long templateId) { + List downloadsInProgress = _vmTemplateHostDao.listByTemplateStates(templateId, + VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS, VMTemplateHostVO.Status.NOT_DOWNLOADED); + if (downloadsInProgress.size() > 0) { + for (VMTemplateHostVO vmthvo : downloadsInProgress) { + DownloadListener dl = null; + synchronized (_listenerMap) { + dl = _listenerMap.remove(vmthvo); + } + if (dl != null) { + dl.abandon(); + s_logger.info("Stopping download of template " + templateId + " to storage server " + vmthvo.getHostId()); + } + } + } + } - - @Override - public void cancelAllDownloads(Long templateId) { - List downloadsInProgress = - _vmTemplateHostDao.listByTemplateStates(templateId, VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS, VMTemplateHostVO.Status.NOT_DOWNLOADED); - if (downloadsInProgress.size() > 0){ - for (VMTemplateHostVO vmthvo: downloadsInProgress) { - DownloadListener dl = null; - synchronized (_listenerMap) { - dl = _listenerMap.remove(vmthvo); - } - if (dl != null) { - dl.abandon(); - s_logger.info("Stopping download of template " + templateId + " to storage server " + vmthvo.getHostId()); - } - } - } - } - - /* - private void checksumSync(long hostId){ + /* + private void checksumSync(long hostId){ SearchCriteria sc = ReadyTemplateStatesSearch.create(); sc.setParameters("state", ObjectInDataStoreStateMachine.State.Ready); sc.setParameters("host_id", hostId); @@ -613,38 +617,36 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor _templateDao.update(template.getId(), template); } - } - */ + } + */ - private Long getMaxTemplateSizeInBytes() { - try { - return Long.parseLong(_configDao.getValue("max.template.iso.size")) * 1024L * 1024L * 1024L; - } catch (NumberFormatException e) { - return null; - } - } + private Long getMaxTemplateSizeInBytes() { + try { + return Long.parseLong(_configDao.getValue("max.template.iso.size")) * 1024L * 1024L * 1024L; + } catch (NumberFormatException e) { + return null; + } + } - private Long getMaxVolumeSizeInBytes() { - try { - return Long.parseLong(_configDao.getValue("storage.max.volume.upload.size")) * 1024L * 1024L * 1024L; - } catch (NumberFormatException e) { - return null; - } - } - - private Proxy getHttpProxy() { - if (_proxy == null) { - return null; - } - try { - URI uri = new URI(_proxy); - Proxy prx = new Proxy(uri); - return prx; - } catch (URISyntaxException e) { - return null; - } - } + private Long getMaxVolumeSizeInBytes() { + try { + return Long.parseLong(_configDao.getValue("storage.max.volume.upload.size")) * 1024L * 1024L * 1024L; + } catch (NumberFormatException e) { + return null; + } + } + private Proxy getHttpProxy() { + if (_proxy == null) { + return null; + } + try { + URI uri = new URI(_proxy); + Proxy prx = new Proxy(uri); + return prx; + } catch (URISyntaxException e) { + return null; + } + } } - diff --git a/server/src/com/cloud/template/TemplateManager.java b/server/src/com/cloud/template/TemplateManager.java index a5d94824e30..7c806447bc5 100755 --- a/server/src/com/cloud/template/TemplateManager.java +++ b/server/src/com/cloud/template/TemplateManager.java @@ -55,15 +55,14 @@ public interface TemplateManager extends TemplateApiService{ * Copies a template from its current secondary storage server to the secondary storage server in the specified zone. * * @param template - * @param srcSecHost - * @param srcZone + * @param srcSecStore * @param destZone * @return true if success * @throws InternalErrorException * @throws StorageUnavailableException * @throws ResourceAllocationException */ - boolean copy(long userId, VMTemplateVO template, HostVO srcSecHost, DataCenterVO srcZone, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException; + boolean copy(long userId, VMTemplateVO template, DataStore srcSecStore, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException; /** * Deletes a template from secondary storage servers @@ -107,6 +106,8 @@ public interface TemplateManager extends TemplateApiService{ HostVO getSecondaryStorageHost(long zoneId, long tmpltId); + DataStore getImageStore(long zoneId, long tmpltId); + VMTemplateHostVO getTemplateHostRef(long zoneId, long tmpltId, boolean readyOnly); @@ -120,6 +121,8 @@ public interface TemplateManager extends TemplateApiService{ String getChecksum(Long hostId, String templatePath); + String getChecksum(DataStore store, String templatePath); + List getImageStoreByTemplate(long templateId, Long zoneId); } diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index a4eb35c9620..e9729e4ff31 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -125,6 +125,7 @@ import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; @@ -838,6 +839,20 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return null; } + @Override + public String getChecksum(DataStore store, String templatePath) { + + String secUrl = store.getUri(); + Answer answer; + answer = _agentMgr.sendToSecStorage(store, new ComputeChecksumCommand( + secUrl, templatePath)); + if (answer != null && answer.getResult()) { + return answer.getDetails(); + } + return null; + } + + @Override @DB public VMTemplateHostVO prepareISOForCreate(VMTemplateVO template, StoragePool pool) { @@ -897,18 +912,20 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override @DB - public boolean copy(long userId, VMTemplateVO template, HostVO srcSecHost, DataCenterVO srcZone, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException { - List dstSecHosts = _ssvmMgr.listSecondaryStorageHostsInOneZone(dstZone.getId()); + public boolean copy(long userId, VMTemplateVO template, DataStore srcSecStore, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException { long tmpltId = template.getId(); long dstZoneId = dstZone.getId(); - if (dstSecHosts == null || dstSecHosts.isEmpty() ) { - throw new StorageUnavailableException("Destination zone is not ready", DataCenter.class, dstZone.getId()); + // find all eligible image stores for the destination zone + List dstSecStores = this.dataStoreMgr.getImageStoresByScope(new ZoneScope(dstZoneId)); + if (dstSecStores == null || dstSecStores.isEmpty() ) { + throw new StorageUnavailableException("Destination zone is not ready, no image store associated", DataCenter.class, dstZone.getId()); } AccountVO account = _accountDao.findById(template.getAccountId()); - VMTemplateHostVO srcTmpltHost = _tmpltHostDao.findByHostTemplate(srcSecHost.getId(), tmpltId); + // find the size of the template to be copied + TemplateDataStoreVO srcTmpltStore = this._tmplStoreDao.findByStoreTemplate(srcSecStore.getId(), tmpltId); _resourceLimitMgr.checkResourceLimit(account, ResourceType.template); - _resourceLimitMgr.checkResourceLimit(account, ResourceType.secondary_storage, new Long(srcTmpltHost.getSize())); + _resourceLimitMgr.checkResourceLimit(account, ResourceType.secondary_storage, new Long(srcTmpltStore.getSize())); // Event details String copyEventType; @@ -924,30 +941,31 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, Transaction txn = Transaction.currentTxn(); txn.start(); - for ( HostVO dstSecHost : dstSecHosts ) { - VMTemplateHostVO dstTmpltHost = null; + //Copy will just find one eligible image store for the destination zone and copy template there, not propagate to all image stores + // for that zone + for ( DataStore dstSecStore : dstSecStores ) { + TemplateDataStoreVO dstTmpltStore = null; try { - dstTmpltHost = _tmpltHostDao.findByHostTemplate(dstSecHost.getId(), tmpltId, true); - if (dstTmpltHost != null) { - dstTmpltHost = _tmpltHostDao.lockRow(dstTmpltHost.getId(), true); - if (dstTmpltHost != null && dstTmpltHost.getDownloadState() == Status.DOWNLOADED) { - if (dstTmpltHost.getDestroyed() == false) { - return true; + dstTmpltStore = this._tmplStoreDao.findByStoreTemplate(dstSecStore.getId(), tmpltId, true); + if (dstTmpltStore != null) { + dstTmpltStore = _tmplStoreDao.lockRow(dstTmpltStore.getId(), true); + if (dstTmpltStore != null && dstTmpltStore.getDownloadState() == Status.DOWNLOADED) { + if (dstTmpltStore.getDestroyed() == false) { + return true; // already downloaded on this image store } else { - dstTmpltHost.setDestroyed(false); - _tmpltHostDao.update(dstTmpltHost.getId(), dstTmpltHost); - + dstTmpltStore.setDestroyed(false); + _tmplStoreDao.update(dstTmpltStore.getId(), dstTmpltStore); return true; } - } else if (dstTmpltHost != null && dstTmpltHost.getDownloadState() == Status.DOWNLOAD_ERROR){ - if (dstTmpltHost.getDestroyed() == true) { - dstTmpltHost.setDestroyed(false); - dstTmpltHost.setDownloadState(Status.NOT_DOWNLOADED); - dstTmpltHost.setDownloadPercent(0); - dstTmpltHost.setCopy(true); - dstTmpltHost.setErrorString(""); - dstTmpltHost.setJobId(null); - _tmpltHostDao.update(dstTmpltHost.getId(), dstTmpltHost); + } else if (dstTmpltStore != null && dstTmpltStore.getDownloadState() == Status.DOWNLOAD_ERROR){ + if (dstTmpltStore.getDestroyed() == true) { + dstTmpltStore.setDestroyed(false); + dstTmpltStore.setDownloadState(Status.NOT_DOWNLOADED); + dstTmpltStore.setDownloadPercent(0); + dstTmpltStore.setCopy(true); + dstTmpltStore.setErrorString(""); + dstTmpltStore.setJobId(null); + _tmplStoreDao.update(dstTmpltStore.getId(), dstTmpltStore); } } } @@ -955,11 +973,11 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, txn.commit(); } - if(_downloadMonitor.copyTemplate(template, srcSecHost, dstSecHost) ) { + if(_downloadMonitor.copyTemplate(template, srcSecStore, dstSecStore) ) { _tmpltDao.addTemplateToZone(template, dstZoneId); if(account.getId() != Account.ACCOUNT_ID_SYSTEM){ - UsageEventUtils.publishUsageEvent(copyEventType, account.getId(), dstZoneId, tmpltId, null, null, null, srcTmpltHost.getSize(), + UsageEventUtils.publishUsageEvent(copyEventType, account.getId(), dstZoneId, tmpltId, null, null, null, srcTmpltStore.getSize(), template.getClass().getName(), template.getUuid()); } return true; @@ -969,6 +987,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } + + @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_COPY, eventDescription = "copying template", async = true) public VirtualMachineTemplate copyTemplate(CopyTemplateCmd cmd) throws StorageUnavailableException, ResourceAllocationException { @@ -978,6 +998,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, Long destZoneId = cmd.getDestinationZoneId(); Account caller = UserContext.current().getCaller(); + /* if (_swiftMgr.isSwiftEnabled()) { throw new CloudRuntimeException("copytemplate API is disabled in Swift setup, templates in Swift can be accessed by all Zones"); } @@ -986,6 +1007,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new CloudRuntimeException( "copytemplate API is disabled in S3 setup -- S3 templates are accessible in all zones."); } + */ //Verify parameters if (sourceZoneId == destZoneId) { @@ -1007,20 +1029,24 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new InvalidParameterValueException("Unable to find template with id"); } - HostVO dstSecHost = getSecondaryStorageHost(destZoneId, templateId); - if ( dstSecHost != null ) { - s_logger.debug("There is template " + templateId + " in secondary storage " + dstSecHost.getId() + " in zone " + destZoneId + " , don't need to copy"); + DataStore dstSecStore = getImageStore(destZoneId, templateId); + if ( dstSecStore != null ) { + s_logger.debug("There is template " + templateId + " in secondary storage " + dstSecStore.getName() + " in zone " + destZoneId + " , don't need to copy"); return template; } - HostVO srcSecHost = getSecondaryStorageHost(sourceZoneId, templateId); - if ( srcSecHost == null ) { + DataStore srcSecStore = getImageStore(sourceZoneId, templateId); + if ( srcSecStore == null ) { throw new InvalidParameterValueException("There is no template " + templateId + " in zone " + sourceZoneId ); } + if ( srcSecStore.getScope().getScopeType() == ScopeType.REGION){ + s_logger.debug("Template " + templateId + " is in region-wide secondary storage " + dstSecStore.getName() + " , don't need to copy"); + return template; + } _accountMgr.checkAccess(caller, AccessType.ModifyEntry, true, template); - boolean success = copy(userId, template, srcSecHost, sourceZone, dstZone); + boolean success = copy(userId, template, srcSecStore, dstZone); if (success){ return template; @@ -2073,6 +2099,23 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return secondaryStorageHost.getStorageUrl(); } + // get the image store where a template in a given zone is downloaded to, just pick one is enough. + @Override + public DataStore getImageStore(long zoneId, long tmpltId) { + List stores = this.dataStoreMgr.getImageStoresByScope(new ZoneScope(zoneId)); + if (stores == null || stores.size() == 0) { + return null; + } + for (DataStore host : stores) { + List tmpltStore = this._tmplStoreDao.listByTemplateStoreDownloadStatus( + tmpltId, host.getId(), VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + if ( tmpltStore != null && tmpltStore.size() > 0 ){ + return host; + } + } + return null; + } + @Override public HostVO getSecondaryStorageHost(long zoneId, long tmpltId) { List hosts = _ssvmMgr diff --git a/server/test/com/cloud/agent/MockAgentManagerImpl.java b/server/test/com/cloud/agent/MockAgentManagerImpl.java index 7e3462d8ff8..9e7e163aad9 100755 --- a/server/test/com/cloud/agent/MockAgentManagerImpl.java +++ b/server/test/com/cloud/agent/MockAgentManagerImpl.java @@ -21,6 +21,7 @@ import java.util.Map; import javax.ejb.Local; import javax.naming.ConfigurationException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.springframework.stereotype.Component; import com.cloud.agent.api.Answer; @@ -140,6 +141,19 @@ public class MockAgentManagerImpl extends ManagerBase implements AgentManager { return null; } + + @Override + public void sendToSecStorage(DataStore ssStore, Command cmd, Listener listener) throws AgentUnavailableException { + // TODO Auto-generated method stub + + } + + @Override + public Answer sendToSecStorage(DataStore ssStore, Command cmd) { + // TODO Auto-generated method stub + return null; + } + @Override public boolean tapLoadingAgents(Long hostId, TapAgentsAction action) { // TODO Auto-generated method stub @@ -191,7 +205,7 @@ public class MockAgentManagerImpl extends ManagerBase implements AgentManager { @Override public void disconnectWithInvestigation(long hostId, Event event) { // TODO Auto-generated method stub - + } }