apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-12-15 18:12:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

Support Multi part upload for S3 using TransferManager.

Browse Source
This commit is contained in:
Min Chen 2013-06-05 09:40:33 -07:00
parent e92cd6d632
commit 4e404953ad
2 changed files with 262 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
core/src/com/cloud/storage/template/S3TemplateDownloader.java
View File

@ -16,20 +16,12 @@
// under the License. // under the License.
package com.cloud.storage.template; package com.cloud.storage.template;
import static com.cloud.utils.StringUtils.join; import static com.cloud.utils.StringUtils.join;
import static java.util.Arrays.asList; import static java.util.Arrays.asList;
import java.io.BufferedInputStream; import java.io.BufferedInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.Date; import java.util.Date;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
@ -51,12 +43,15 @@ import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException; import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.ProgressEvent; import com.amazonaws.services.s3.model.ProgressEvent;
import com.amazonaws.services.s3.model.ProgressListener; import com.amazonaws.services.s3.model.ProgressListener;
import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.StorageClass; import com.amazonaws.services.s3.model.StorageClass;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import com.cloud.agent.api.storage.Proxy; import com.cloud.agent.api.storage.Proxy;
import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.S3TO;
import com.cloud.utils.Pair; import com.cloud.utils.Pair;
@ -75,7 +70,7 @@ public class S3TemplateDownloader implements TemplateDownloader {
private String installPath; private String installPath;
private String s3Key; private String s3Key;
private String fileName; private String fileName;
public TemplateDownloader.Status status= TemplateDownloader.Status.NOT_STARTED; public TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
public String errorString = " "; public String errorString = " ";
private long remoteSize = 0; private long remoteSize = 0;
public long downloadTime = 0; public long downloadTime = 0;
@ -84,32 +79,29 @@ public class S3TemplateDownloader implements TemplateDownloader {
private GetMethod request; private GetMethod request;
private boolean resume = false; private boolean resume = false;
private DownloadCompleteCallback completionCallback; private DownloadCompleteCallback completionCallback;
S3TO s3; private S3TO s3;
boolean inited = true; private boolean inited = true;
private long MAX_TEMPLATE_SIZE_IN_BYTES; private long maxTemplateSizeInByte;
private ResourceType resourceType = ResourceType.TEMPLATE; private ResourceType resourceType = ResourceType.TEMPLATE;
private final HttpMethodRetryHandler myretryhandler; private final HttpMethodRetryHandler myretryhandler;
public S3TemplateDownloader(S3TO storageLayer, String downloadUrl, String installPath,
DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy,
public S3TemplateDownloader (S3TO storageLayer, String downloadUrl, String installPath, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { ResourceType resourceType) {
this.s3 = storageLayer; this.s3 = storageLayer;
this.downloadUrl = downloadUrl; this.downloadUrl = downloadUrl;
this.installPath = installPath; this.installPath = installPath;
this.status = TemplateDownloader.Status.NOT_STARTED; this.status = TemplateDownloader.Status.NOT_STARTED;
this.resourceType = resourceType; this.resourceType = resourceType;
this.MAX_TEMPLATE_SIZE_IN_BYTES = maxTemplateSizeInBytes; this.maxTemplateSizeInByte = maxTemplateSizeInBytes;
this.totalBytes = 0; this.totalBytes = 0;
this.client = new HttpClient(s_httpClientManager); this.client = new HttpClient(s_httpClientManager);
myretryhandler = new HttpMethodRetryHandler() { myretryhandler = new HttpMethodRetryHandler() {
@Override @Override
public boolean retryMethod( public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
final HttpMethod method,
final IOException exception,
int executionCount) {
if (executionCount >= 2) { if (executionCount >= 2) {
// Do not retry if over max retry count // Do not retry if over max retry count
return false; return false;
@ -146,8 +138,10 @@ public class S3TemplateDownloader implements TemplateDownloader {
if ((user != null) && (password != null)) { if ((user != null) && (password != null)) {
client.getParams().setAuthenticationPreemptive(true); client.getParams().setAuthenticationPreemptive(true);
Credentials defaultcreds = new UsernamePasswordCredentials(user, password); Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); client.getState().setCredentials(
s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second()); new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first()
+ ":" + hostAndPort.second());
} else { } else {
s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second()); s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
} }
@ -155,7 +149,7 @@ public class S3TemplateDownloader implements TemplateDownloader {
errorString = iae.getMessage(); errorString = iae.getMessage();
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
inited = false; inited = false;
} catch (Exception ex){ } catch (Exception ex) {
errorString = "Unable to start download -- check url? "; errorString = "Unable to start download -- check url? ";
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
s_logger.warn("Exception in constructor -- " + ex.toString()); s_logger.warn("Exception in constructor -- " + ex.toString());
@ -164,7 +158,6 @@ public class S3TemplateDownloader implements TemplateDownloader {
} }
} }
@Override @Override
public long download(boolean resume, DownloadCompleteCallback callback) { public long download(boolean resume, DownloadCompleteCallback callback) {
switch (status) { switch (status) {
@ -176,15 +169,13 @@ public class S3TemplateDownloader implements TemplateDownloader {
} }
int bytes=0;
try { try {
// execute get method // execute get method
int responseCode = HttpStatus.SC_OK; int responseCode = HttpStatus.SC_OK;
if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) { if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) "; errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
return 0; //FIXME: retry? return 0; // FIXME: retry?
} }
// get the total size of file // get the total size of file
Header contentLengthHeader = request.getResponseHeader("Content-Length"); Header contentLengthHeader = request.getResponseHeader("Content-Length");
@ -194,9 +185,9 @@ public class S3TemplateDownloader implements TemplateDownloader {
Header chunkedHeader = request.getResponseHeader("Transfer-Encoding"); Header chunkedHeader = request.getResponseHeader("Transfer-Encoding");
if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) { if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString=" Failed to receive length of download "; errorString = " Failed to receive length of download ";
return 0; //FIXME: what status do we put here? Do we retry? return 0; // FIXME: what status do we put here? Do we retry?
} else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())){ } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
chunked = true; chunked = true;
} }
} else { } else {
@ -207,38 +198,39 @@ public class S3TemplateDownloader implements TemplateDownloader {
remoteSize = remoteSize2; remoteSize = remoteSize2;
} }
if (remoteSize > MAX_TEMPLATE_SIZE_IN_BYTES) { if (remoteSize > maxTemplateSizeInByte) {
s_logger.info("Remote size is too large: " + remoteSize + " , max=" + MAX_TEMPLATE_SIZE_IN_BYTES); s_logger.info("Remote size is too large: " + remoteSize + " , max=" + maxTemplateSizeInByte);
status = Status.UNRECOVERABLE_ERROR; status = Status.UNRECOVERABLE_ERROR;
errorString = "Download file size is too large"; errorString = "Download file size is too large";
return 0; return 0;
} }
if (remoteSize == 0) { if (remoteSize == 0) {
remoteSize = MAX_TEMPLATE_SIZE_IN_BYTES; remoteSize = maxTemplateSizeInByte;
} }
InputStream in = !chunked?new BufferedInputStream(request.getResponseBodyAsStream()) InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream())
: new ChunkedInputStream(request.getResponseBodyAsStream()); : new ChunkedInputStream(request.getResponseBodyAsStream());
s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3.getBucketName() + " remoteSize=" + remoteSize + " , max size=" + MAX_TEMPLATE_SIZE_IN_BYTES); s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3.getBucketName()
+ " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInByte);
Date start = new Date(); Date start = new Date();
// compute s3 key // compute s3 key
s3Key = join(asList(installPath, fileName), S3Utils.SEPARATOR); s3Key = join(asList(installPath, fileName), S3Utils.SEPARATOR);
// multi-part upload using S3 api to handle > 5G input stream
TransferManager tm = new TransferManager(S3Utils.acquireClient(s3));
// download using S3 API // download using S3 API
ObjectMetadata metadata = new ObjectMetadata(); ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(remoteSize); metadata.setContentLength(remoteSize);
PutObjectRequest putObjectRequest = new PutObjectRequest( PutObjectRequest putObjectRequest = new PutObjectRequest(s3.getBucketName(), s3Key, in, metadata)
s3.getBucketName(), s3Key, in, metadata)
.withStorageClass(StorageClass.ReducedRedundancy); .withStorageClass(StorageClass.ReducedRedundancy);
// register progress listenser // register progress listenser
putObjectRequest putObjectRequest.setProgressListener(new ProgressListener() {
.setProgressListener(new ProgressListener() {
@Override @Override
public void progressChanged( public void progressChanged(ProgressEvent progressEvent) {
ProgressEvent progressEvent) {
// s_logger.debug(progressEvent.getBytesTransfered() // s_logger.debug(progressEvent.getBytesTransfered()
// + " number of byte transferd " // + " number of byte transferd "
// + new Date()); // + new Date());
@ -246,22 +238,22 @@ public class S3TemplateDownloader implements TemplateDownloader {
if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) { if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) {
s_logger.info("download completed"); s_logger.info("download completed");
status = TemplateDownloader.Status.DOWNLOAD_FINISHED; status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
} else if (progressEvent.getEventCode() == ProgressEvent.FAILED_EVENT_CODE){ } else if (progressEvent.getEventCode() == ProgressEvent.FAILED_EVENT_CODE) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
} else if (progressEvent.getEventCode() == ProgressEvent.CANCELED_EVENT_CODE){ } else if (progressEvent.getEventCode() == ProgressEvent.CANCELED_EVENT_CODE) {
status = TemplateDownloader.Status.ABORTED; status = TemplateDownloader.Status.ABORTED;
} else{ } else {
status = TemplateDownloader.Status.IN_PROGRESS; status = TemplateDownloader.Status.IN_PROGRESS;
} }
} }
}); });
S3Utils.putObject(s3, putObjectRequest); // TransferManager processes all transfers asynchronously,
while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED && // so this call will return immediately.
status != TemplateDownloader.Status.UNRECOVERABLE_ERROR && Upload upload = tm.upload(putObjectRequest);
status != TemplateDownloader.Status.ABORTED ){
// wait for completion upload.waitForCompletion();
}
// finished or aborted // finished or aborted
Date finish = new Date(); Date finish = new Date();
String downloaded = "(incomplete download)"; String downloaded = "(incomplete download)";
@ -273,15 +265,21 @@ public class S3TemplateDownloader implements TemplateDownloader {
} }
downloadTime += finish.getTime() - start.getTime(); downloadTime += finish.getTime() - start.getTime();
return totalBytes; return totalBytes;
}catch (HttpException hte) { } catch (HttpException hte) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = hte.getMessage(); errorString = hte.getMessage();
} catch (IOException ioe) { } catch (IOException ioe) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error? // probably a file write error
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = ioe.getMessage(); errorString = ioe.getMessage();
} catch (AmazonClientException ex) { } catch (AmazonClientException ex) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; // S3 api exception // S3 api exception
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = ex.getMessage(); errorString = ex.getMessage();
} catch (InterruptedException e) {
// S3 upload api exception
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = e.getMessage();
} finally { } finally {
// close input stream // close input stream
request.releaseConnection(); request.releaseConnection();
@ -296,19 +294,16 @@ public class S3TemplateDownloader implements TemplateDownloader {
return downloadUrl; return downloadUrl;
} }
@Override @Override
public TemplateDownloader.Status getStatus() { public TemplateDownloader.Status getStatus() {
return status; return status;
} }
@Override @Override
public long getDownloadTime() { public long getDownloadTime() {
return downloadTime; return downloadTime;
} }
@Override @Override
public long getDownloadedBytes() { public long getDownloadedBytes() {
return totalBytes; return totalBytes;
@ -349,7 +344,7 @@ public class S3TemplateDownloader implements TemplateDownloader {
return 0; return 0;
} }
return (int)(100.0*totalBytes/remoteSize); return (int) (100.0 * totalBytes / remoteSize);
} }
@Override @Override
@ -357,7 +352,7 @@ public class S3TemplateDownloader implements TemplateDownloader {
try { try {
download(resume, completionCallback); download(resume, completionCallback);
} catch (Throwable t) { } catch (Throwable t) {
s_logger.warn("Caught exception during download "+ t.getMessage(), t); s_logger.warn("Caught exception during download " + t.getMessage(), t);
errorString = "Failed to install: " + t.getMessage(); errorString = "Failed to install: " + t.getMessage();
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
} }
@ -369,8 +364,6 @@ public class S3TemplateDownloader implements TemplateDownloader {
this.status = status; this.status = status;
} }
public boolean isResume() { public boolean isResume() {
return resume; return resume;
} }
@ -390,28 +383,9 @@ public class S3TemplateDownloader implements TemplateDownloader {
this.resume = resume; this.resume = resume;
} }
@Override @Override
public long getMaxTemplateSizeInBytes() { public long getMaxTemplateSizeInBytes() {
return this.MAX_TEMPLATE_SIZE_IN_BYTES; return this.maxTemplateSizeInByte;
}
public static void main(String[] args) {
String url ="http://dev.mysql.com/get/Downloads/MySQL-5.0/mysql-noinstall-5.0.77-win32.zip/from/http://mirror.services.wisc.edu/mysql/";
try {
URI uri = new java.net.URI(url);
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
TemplateDownloader td = new S3TemplateDownloader(null, url,"/tmp/mysql", null, TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES, null, null, null, null);
long bytes = td.download(true, null);
if (bytes > 0) {
System.out.println("Downloaded (" + bytes + " bytes)" + " in " + td.getDownloadTime()/1000 + " secs");
} else {
System.out.println("Failed download");
}
} }
@Override @Override
@ -419,14 +393,11 @@ public class S3TemplateDownloader implements TemplateDownloader {
errorString = error; errorString = error;
} }
@Override @Override
public boolean isInited() { public boolean isInited() {
return inited; return inited;
} }
public ResourceType getResourceType() { public ResourceType getResourceType() {
return resourceType; return resourceType;
} }

2
utils/src/com/cloud/utils/S3Utils.java
View File

@ -73,7 +73,7 @@ public final class S3Utils {
super(); super();
} }
private static AmazonS3 acquireClient(final ClientOptions clientOptions) { public static AmazonS3 acquireClient(final ClientOptions clientOptions) {
final AWSCredentials credentials = new BasicAWSCredentials( final AWSCredentials credentials = new BasicAWSCredentials(
clientOptions.getAccessKey(), clientOptions.getSecretKey()); clientOptions.getAccessKey(), clientOptions.getSecretKey());