Move data store specific extract template/iso logic from

TemplateManager to data store driver.
This commit is contained in:
Min Chen 2013-06-04 11:50:43 -07:00
parent 2473ceb111
commit ef03d5a122
14 changed files with 156 additions and 138 deletions

View File

@ -75,18 +75,18 @@ public interface TemplateApiService {
*
* @param cmd
* - the command specifying the mode and id of the ISO
* @return extractId.
* @return extractUrl extract url.
*/
Pair<Long, String> extract(ExtractIsoCmd cmd) throws InternalErrorException;
String extract(ExtractIsoCmd cmd) throws InternalErrorException;
/**
* Extracts a Template
*
* @param cmd
* - the command specifying the mode and id of the template
* @return extractId
* @return extractUrl extract url
*/
Pair<Long, String> extract(ExtractTemplateCmd cmd) throws InternalErrorException;
String extract(ExtractTemplateCmd cmd) throws InternalErrorException;
VirtualMachineTemplate getTemplate(long templateId);

View File

@ -215,6 +215,8 @@ public interface ResponseGenerator {
ExtractResponse createExtractResponse(Long uploadId, Long id, Long zoneId, Long accountId, String mode, String url);
ExtractResponse createExtractResponse(Long id, Long zoneId, Long accountId, String mode, String url);
String toSerializedString(CreateCmdResponse response, String responseType);
AsyncJobResponse createAsyncJobResponse(AsyncJob job);
@ -360,7 +362,7 @@ public interface ResponseGenerator {
public NicResponse createNicResponse(Nic result);
ApplicationLoadBalancerResponse createLoadBalancerContainerReponse(ApplicationLoadBalancerRule lb, Map<Ip, UserVm> lbInstances);
AffinityGroupResponse createAffinityGroupResponse(AffinityGroup group);
Long getAffinityGroupId(String name, long entityOwnerId);
@ -370,7 +372,7 @@ public interface ResponseGenerator {
PortableIpResponse createPortableIPResponse(PortableIp portableIp);
InternalLoadBalancerElementResponse createInternalLbElementResponse(VirtualRouterProvider result);
IsolationMethodResponse createIsolationMethodResponse(IsolationType method);
}

View File

@ -124,16 +124,9 @@ public class ExtractIsoCmd extends BaseAsyncCmd {
public void execute(){
try {
UserContext.current().setEventDetails(getEventDescription());
Pair<Long, String> uploadPair = _templateService.extract(this);
if (uploadPair != null){
ExtractResponse response = null;
if (uploadPair.second() != null ) {
// region-wide image store
response = _responseGenerator.createExtractResponse(null, id, zoneId, getEntityOwnerId(), mode, uploadPair.second());
} else {
// nfs image store
response = _responseGenerator.createExtractResponse(uploadPair.first(), id, zoneId, getEntityOwnerId(), mode, null);
}
String uploadUrl = _templateService.extract(this);
if (uploadUrl != null) {
ExtractResponse response = _responseGenerator.createExtractResponse(id, zoneId, getEntityOwnerId(), mode, uploadUrl);
response.setResponseName(getCommandName());
response.setObjectName("iso");
this.setResponseObject(response);

View File

@ -126,16 +126,9 @@ public class ExtractTemplateCmd extends BaseAsyncCmd {
public void execute(){
try {
UserContext.current().setEventDetails(getEventDescription());
Pair<Long, String> uploadPair = _templateService.extract(this);
if (uploadPair != null){
ExtractResponse response = null;
if (uploadPair.second() != null ) {
// region-wide image store
response = _responseGenerator.createExtractResponse(null, id, zoneId, getEntityOwnerId(), mode, uploadPair.second());
} else {
// nfs image store
response = _responseGenerator.createExtractResponse(uploadPair.first(), id, zoneId, getEntityOwnerId(), mode, null);
}
String uploadUrl = _templateService.extract(this);
if (uploadUrl != null) {
ExtractResponse response = _responseGenerator.createExtractResponse(id, zoneId, getEntityOwnerId(), mode, uploadUrl);
response.setResponseName(getCommandName());
this.setResponseObject(response);
} else {

View File

@ -27,6 +27,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import com.cloud.storage.ImageStore;
import com.cloud.storage.Storage.ImageFormat;
public interface ImageStoreEntity extends DataStore, ImageStore {
TemplateInfo getTemplate(long templateId);
@ -40,4 +41,6 @@ public interface ImageStoreEntity extends DataStore, ImageStore {
Set<TemplateInfo> listTemplates();
String getMountPoint(); // get the mount point on ssvm.
String createEntityExtractUrl(String installPath, ImageFormat format); // get the entity download URL
}

View File

@ -42,6 +42,7 @@ import org.apache.log4j.Logger;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.utils.component.ComponentContext;
@ -189,7 +190,13 @@ public class ImageStoreImpl implements ImageStoreEntity {
@Override
public String getMountPoint() {
return this.imageDataStoreVO.getParent();
return imageDataStoreVO.getParent();
}
@Override
public String createEntityExtractUrl(String installPath, ImageFormat format) {
return driver.createEntityExtractUrl(this, installPath, format);
}
}

View File

@ -18,7 +18,11 @@
*/
package org.apache.cloudstack.storage.image;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
import com.cloud.storage.Storage.ImageFormat;
public interface ImageStoreDriver extends DataStoreDriver {
String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format);
}

View File

@ -21,6 +21,7 @@ package org.apache.cloudstack.storage.datastore.driver;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import javax.inject.Inject;
@ -40,6 +41,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import org.apache.cloudstack.storage.image.ImageStoreDriver;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.storage.image.store.ImageStoreImpl;
import org.apache.cloudstack.storage.image.store.TemplateObject;
import org.apache.cloudstack.storage.snapshot.SnapshotObject;
@ -48,6 +50,7 @@ import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.DeleteSnapshotBackupCommand2;
import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
import com.cloud.agent.api.storage.DeleteTemplateCommand;
import com.cloud.agent.api.storage.DeleteVolumeCommand;
import com.cloud.agent.api.storage.DownloadAnswer;
@ -55,12 +58,18 @@ import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.configuration.Config;
import com.cloud.configuration.dao.ConfigurationDao;
import com.cloud.event.EventTypes;
import com.cloud.event.UsageEventUtils;
import com.cloud.host.dao.HostDao;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.SnapshotVO;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Upload.Mode;
import com.cloud.storage.Upload.Status;
import com.cloud.storage.Upload.Type;
import com.cloud.storage.UploadVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VMTemplateZoneVO;
@ -99,6 +108,8 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
@Inject
AccountDao _accountDao;
@Inject
ConfigurationDao _configDao;
@Inject
SecondaryStorageVmManager _ssvmMgr;
@Inject
TemplateDataStoreDao _templateStoreDao;
@ -405,4 +416,39 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
}
@Override
public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format) {
// find an endpoint to send command
EndPoint ep = _epSelector.select(store);
// Create Symlink at ssvm
String path = installPath;
String uuid = UUID.randomUUID().toString() + "." + format.getFileExtension();
CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity) store).getMountPoint(), path, uuid);
Answer ans = ep.sendMessage(cmd);
if (ans == null || !ans.getResult()) {
String errorString = "Unable to create a link for entity at " + installPath + " on ssvm," + ans.getDetails();
s_logger.error(errorString);
throw new CloudRuntimeException(errorString);
}
// Construct actual URL locally now that the symlink exists at SSVM
return generateCopyUrl(ep.getPublicAddr(), uuid);
}
private String generateCopyUrl(String ipAddress, String uuid){
String hostname = ipAddress;
String scheme = "http";
boolean _sslCopy = false;
String sslCfg = _configDao.getValue(Config.SecStorageEncryptCopy.toString());
if ( sslCfg != null ){
_sslCopy = Boolean.parseBoolean(sslCfg);
}
if (_sslCopy) {
hostname = ipAddress.replace(".", "-");
hostname = hostname + ".realhostip.com";
scheme = "https";
}
return scheme + "://" + hostname + "/userdata/" + uuid;
}
}

View File

@ -48,6 +48,7 @@ import org.apache.cloudstack.storage.image.store.TemplateObject;
import org.apache.cloudstack.storage.snapshot.SnapshotObject;
import org.apache.log4j.Logger;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.DeleteSnapshotBackupCommand2;
@ -78,6 +79,7 @@ import com.cloud.storage.secondary.SecondaryStorageVmManager;
import com.cloud.storage.snapshot.SnapshotManager;
import com.cloud.user.Account;
import com.cloud.user.dao.AccountDao;
import com.cloud.utils.S3Utils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.dao.UserVmDao;
@ -422,4 +424,29 @@ public class S3ImageStoreDriverImpl implements ImageStoreDriver {
}
@Override
public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format) {
// for S3, no need to do anything, just return template url for
// extract template. but we need to set object acl as public_read to
// make the url accessible
S3TO s3 = (S3TO)getStoreTO(store);
String key = installPath;
try {
S3Utils.setObjectAcl(s3, s3.getBucketName(), key, CannedAccessControlList.PublicRead);
} catch (Exception ex) {
s_logger.error("Failed to set ACL on S3 object " + key + " to PUBLIC_READ", ex);
throw new CloudRuntimeException("Failed to set ACL on S3 object " + key + " to PUBLIC_READ");
}
// construct the url from s3
StringBuffer s3url = new StringBuffer();
s3url.append(s3.isHttps() ? "https://" : "http://");
s3url.append(s3.getEndPoint());
s3url.append("/");
s3url.append(s3.getBucketName());
s3url.append("/");
s3url.append(key);
return s3url.toString();
}
}

View File

@ -37,6 +37,7 @@ import org.apache.cloudstack.storage.image.ImageStoreDriver;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.dao.VMTemplateDao;
//http-read-only based image store
@ -134,4 +135,12 @@ public class SampleImageStoreDriverImpl implements ImageStoreDriver {
// TODO Auto-generated method stub
}
@Override
public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format) {
// TODO Auto-generated method stub
return null;
}
}

View File

@ -61,6 +61,7 @@ import com.cloud.agent.api.to.SwiftTO;
import com.cloud.api.query.dao.UserVmJoinDao;
import com.cloud.event.EventTypes;
import com.cloud.event.UsageEventUtils;
import com.cloud.exception.UnsupportedServiceException;
import com.cloud.host.dao.HostDao;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.DataStoreRole;
@ -424,4 +425,10 @@ public class SwiftImageStoreDriverImpl implements ImageStoreDriver {
}
@Override
public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format) {
throw new UnsupportedServiceException("Extract entity url is not yet supported for Swift image store provider");
}
}

View File

@ -16,9 +16,6 @@
// under the License.
package com.cloud.api;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Calendar;
@ -40,7 +37,6 @@ import org.apache.cloudstack.affinity.AffinityGroup;
import org.apache.cloudstack.affinity.AffinityGroupResponse;
import org.apache.cloudstack.api.ApiConstants.HostDetails;
import org.apache.cloudstack.api.ApiConstants.VMDetails;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.ResponseGenerator;
import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd;
import org.apache.cloudstack.api.response.AccountResponse;
@ -184,7 +180,6 @@ import com.cloud.configuration.ResourceLimit;
import com.cloud.dao.EntityManager;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.DataCenter;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.HostPodVO;
import com.cloud.dc.Pod;
import com.cloud.dc.StorageNetworkIpRange;
@ -266,16 +261,10 @@ import com.cloud.storage.S3;
import com.cloud.storage.Snapshot;
import com.cloud.storage.SnapshotVO;
import com.cloud.storage.Upload;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.StoragePool;
import com.cloud.storage.Swift;
import com.cloud.storage.UploadVO;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateS3VO;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateSwiftVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeVO;
@ -1601,6 +1590,29 @@ public class ApiResponseHelper implements ResponseGenerator {
return listSgs.get(0);
}
//TODO: we need to deprecate uploadVO, since extract is done in a synchronous fashion
@Override
public ExtractResponse createExtractResponse(Long id, Long zoneId, Long accountId, String mode, String url) {
ExtractResponse response = new ExtractResponse();
response.setObjectName("template");
VMTemplateVO template = ApiDBUtils.findTemplateById(id);
response.setId(template.getUuid());
response.setName(template.getName());
if (zoneId != null) {
DataCenter zone = ApiDBUtils.findZoneById(zoneId);
response.setZoneId(zone.getUuid());
response.setZoneName(zone.getName());
}
response.setMode(mode);
response.setUrl(url);
response.setState(Upload.Status.DOWNLOAD_URL_CREATED.toString());
Account account = ApiDBUtils.findAccountById(accountId);
response.setAccountId(account.getUuid());
return response;
}
@Override
public ExtractResponse createExtractResponse(Long uploadId, Long id, Long zoneId, Long accountId, String mode, String url) {
@ -3480,6 +3492,7 @@ public class ApiResponseHelper implements ResponseGenerator {
return response;
}
@Override
public NicSecondaryIpResponse createSecondaryIPToNicResponse(NicSecondaryIp result) {
NicSecondaryIpResponse response = new NicSecondaryIpResponse();
NicVO nic = _entityMgr.findById(NicVO.class, result.getNicId());
@ -3492,6 +3505,7 @@ public class ApiResponseHelper implements ResponseGenerator {
return response;
}
@Override
public NicResponse createNicResponse(Nic result) {
NicResponse response = new NicResponse();
NetworkVO network = _entityMgr.findById(NetworkVO.class, result.getNetworkId());
@ -3713,6 +3727,7 @@ public class ApiResponseHelper implements ResponseGenerator {
}
@Override
public NetworkACLResponse createNetworkACLResponse(NetworkACL networkACL) {
NetworkACLResponse response = new NetworkACLResponse();
response.setId(networkACL.getUuid());

View File

@ -384,7 +384,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
@Override
@ActionEvent(eventType = EventTypes.EVENT_ISO_EXTRACT, eventDescription = "extracting ISO", async = true)
public Pair<Long, String> extract(ExtractIsoCmd cmd) {
public String extract(ExtractIsoCmd cmd) {
Account account = UserContext.current().getCaller();
Long templateId = cmd.getId();
Long zoneId = cmd.getZoneId();
@ -392,18 +392,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
String mode = cmd.getMode();
Long eventId = cmd.getStartEventId();
// FIXME: async job needs fixing
Pair<Long, String> uploadPair = extract(account, templateId, url, zoneId, mode, eventId, true, null, _asyncMgr);
if (uploadPair != null) {
return uploadPair;
} else {
throw new CloudRuntimeException("Failed to extract the iso");
}
return extract(account, templateId, url, zoneId, mode, eventId, true);
}
@Override
@ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_EXTRACT, eventDescription = "extracting template", async = true)
public Pair<Long, String> extract(ExtractTemplateCmd cmd) {
public String extract(ExtractTemplateCmd cmd) {
Account caller = UserContext.current().getCaller();
Long templateId = cmd.getId();
Long zoneId = cmd.getZoneId();
@ -418,13 +412,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
TemplateAdapter adapter = getAdapter(template.getHypervisorType());
TemplateProfile profile = adapter.prepareExtractTemplate(cmd);
// FIXME: async job needs fixing
Pair<Long, String> uploadPair = extract(caller, templateId, url, zoneId, mode, eventId, false, null, _asyncMgr);
if (uploadPair != null) {
return uploadPair;
} else {
throw new CloudRuntimeException("Failed to extract the teamplate");
}
return extract(caller, templateId, url, zoneId, mode, eventId, false);
}
@Override
@ -440,8 +428,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
return vmTemplate;
}
private Pair<Long, String> extract(Account caller, Long templateId, String url, Long zoneId, String mode, Long eventId, boolean isISO,
AsyncJobVO job, AsyncJobManager mgr) {
private String extract(Account caller, Long templateId, String url, Long zoneId, String mode, Long eventId, boolean isISO) {
String desc = Upload.Type.TEMPLATE.toString();
if (isISO) {
desc = Upload.Type.ISO.toString();
@ -505,82 +492,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
throw new InvalidParameterValueException("The " + desc + " has not been downloaded ");
}
if (tmpltStore.getProviderName().equalsIgnoreCase("Swift")) {
throw new UnsupportedServiceException("ExtractTemplate is not yet supported for Swift image store provider");
}
if (tmpltStore.getProviderName().equalsIgnoreCase("S3")) {
// for S3, no need to do anything, just return template url for
// extract template. but we need to set object acl as public_read to
// make the url accessible
S3TO s3 = (S3TO) tmpltStore.getTO();
String key = tmpltStoreRef.getLocalDownloadPath();
try {
S3Utils.setObjectAcl(s3, s3.getBucketName(), key, CannedAccessControlList.PublicRead);
} catch (Exception ex) {
s_logger.error("Failed to set ACL on S3 object " + key + " to PUBLIC_READ", ex);
throw new CloudRuntimeException("Failed to set ACL on S3 object " + key + " to PUBLIC_READ");
}
// construct the url from s3
StringBuffer s3url = new StringBuffer();
s3url.append(s3.isHttps() ? "https://" : "http://");
s3url.append(s3.getEndPoint());
s3url.append("/");
s3url.append(s3.getBucketName());
s3url.append("/");
s3url.append(key);
return new Pair<Long, String>(null, s3url.toString());
}
// for NFS image store case, control will come here
Upload.Mode extractMode;
if (mode == null
|| (!mode.equalsIgnoreCase(Upload.Mode.FTP_UPLOAD.toString()) && !mode.equalsIgnoreCase(Upload.Mode.HTTP_DOWNLOAD.toString()))) {
throw new InvalidParameterValueException("Please specify a valid extract Mode. Supported modes: " + Upload.Mode.FTP_UPLOAD + ", "
+ Upload.Mode.HTTP_DOWNLOAD);
} else {
extractMode = mode.equalsIgnoreCase(Upload.Mode.FTP_UPLOAD.toString()) ? Upload.Mode.FTP_UPLOAD : Upload.Mode.HTTP_DOWNLOAD;
}
if (extractMode == Upload.Mode.FTP_UPLOAD) {
URI uri = null;
try {
uri = new URI(url);
if ((uri.getScheme() == null) || (!uri.getScheme().equalsIgnoreCase("ftp"))) {
throw new InvalidParameterValueException("Unsupported scheme for url: " + url);
}
} catch (Exception ex) {
throw new InvalidParameterValueException("Invalid url given: " + url);
}
String host = uri.getHost();
try {
InetAddress hostAddr = InetAddress.getByName(host);
if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress()) {
throw new InvalidParameterValueException("Illegal host specified in url");
}
if (hostAddr instanceof Inet6Address) {
throw new InvalidParameterValueException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")");
}
} catch (UnknownHostException uhe) {
throw new InvalidParameterValueException("Unable to resolve " + host);
}
if (_uploadMonitor.isTypeUploadInProgress(templateId, isISO ? Type.ISO : Type.TEMPLATE)) {
throw new IllegalArgumentException(template.getName()
+ " upload is in progress. Please wait for some time to schedule another upload for the same");
}
return new Pair<Long, String>(_uploadMonitor.extractTemplate(template, url, tmpltStoreRef, zoneId, eventId, job.getId(), mgr), null);
}
UploadVO vo = _uploadMonitor.createEntityDownloadURL(template, tmpltStoreRef, zoneId, eventId);
if (vo != null) {
return new Pair<Long, String>(vo.getId(), null);
} else {
return null;
}
return tmpltStore.createEntityExtractUrl(tmpltStoreRef.getInstallPath(), template.getFormat());
}
public void prepareTemplateInAllStoragePools(final VMTemplateVO template, long zoneId) {

View File

@ -305,12 +305,12 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
// status to trigger callback.
td.setStatus(Status.POST_DOWNLOAD_FINISHED);
// set template size for S3
if (td instanceof S3TemplateDownloader){
long size = ((S3TemplateDownloader)td).totalBytes;
DownloadJob dnld = jobs.get(jobId);
dnld.setTemplatesize(size);
dnld.setTemplatePhysicalSize(size);
}
S3TemplateDownloader std = (S3TemplateDownloader) td;
long size = std.totalBytes;
DownloadJob dnld = jobs.get(jobId);
dnld.setTemplatesize(size);
dnld.setTemplatePhysicalSize(size);
dnld.setTmpltPath(std.getDownloadLocalPath()); // update template path to include file name.
}
dj.cleanup();
break;