diff --git a/agent/conf/log4j-cloud.xml.in b/agent/conf/log4j-cloud.xml.in
index 9e6c646f585..12228017adb 100644
--- a/agent/conf/log4j-cloud.xml.in
+++ b/agent/conf/log4j-cloud.xml.in
@@ -77,7 +77,17 @@ under the License.
-
+
+
+
+
+
+
+
+
+
+
+
diff --git a/api/src/com/cloud/agent/api/to/S3TO.java b/api/src/com/cloud/agent/api/to/S3TO.java
index a46d609a9d3..ec6bc02cca0 100644
--- a/api/src/com/cloud/agent/api/to/S3TO.java
+++ b/api/src/com/cloud/agent/api/to/S3TO.java
@@ -21,9 +21,9 @@ import java.util.Date;
import com.cloud.agent.api.LogLevel;
import com.cloud.agent.api.LogLevel.Log4jLevel;
import com.cloud.storage.DataStoreRole;
-import com.cloud.utils.S3Utils;
+import com.cloud.utils.storage.S3.ClientOptions;
-public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
+public final class S3TO implements ClientOptions, DataStoreTO {
private Long id;
private String uuid;
@@ -33,6 +33,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
private String secretKey;
private String endPoint;
private String bucketName;
+ private String signer;
private Boolean httpsFlag;
private Boolean useTCPKeepAlive;
private Integer connectionTimeout;
@@ -44,17 +45,9 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
private long maxSingleUploadSizeInBytes;
private static final String pathSeparator = "/";
- public S3TO() {
-
- super();
-
- }
-
public S3TO(final Long id, final String uuid, final String accessKey, final String secretKey, final String endPoint, final String bucketName,
- final Boolean httpsFlag, final Integer connectionTimeout, final Integer maxErrorRetry, final Integer socketTimeout, final Date created,
- final boolean enableRRS, final long maxUploadSize, final Integer connectionTtl, final Boolean useTCPKeepAlive) {
-
- super();
+ final String signer, final Boolean httpsFlag, final Integer connectionTimeout, final Integer maxErrorRetry, final Integer socketTimeout,
+ final Date created, final boolean enableRRS, final long maxUploadSize, final Integer connectionTtl, final Boolean useTCPKeepAlive) {
this.id = id;
this.uuid = uuid;
@@ -62,6 +55,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
this.secretKey = secretKey;
this.endPoint = endPoint;
this.bucketName = bucketName;
+ this.signer = signer;
this.httpsFlag = httpsFlag;
this.connectionTimeout = connectionTimeout;
this.maxErrorRetry = maxErrorRetry;
@@ -74,98 +68,6 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
}
- @Override
- public boolean equals(final Object thatObject) {
-
- if (this == thatObject) {
- return true;
- }
- if (thatObject == null || getClass() != thatObject.getClass()) {
- return false;
- }
-
- final S3TO thatS3TO = (S3TO)thatObject;
-
- if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag) : thatS3TO.httpsFlag != null) {
- return false;
- }
-
- if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey) : thatS3TO.accessKey != null) {
- return false;
- }
-
- if (connectionTimeout != null ? !connectionTimeout.equals(thatS3TO.connectionTimeout) : thatS3TO.connectionTimeout != null) {
- return false;
- }
-
- if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint) : thatS3TO.endPoint != null) {
- return false;
- }
-
- if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) {
- return false;
- }
-
- if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) {
- return false;
- }
-
- if (maxErrorRetry != null ? !maxErrorRetry.equals(thatS3TO.maxErrorRetry) : thatS3TO.maxErrorRetry != null) {
- return false;
- }
-
- if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey) : thatS3TO.secretKey != null) {
- return false;
- }
-
- if (socketTimeout != null ? !socketTimeout.equals(thatS3TO.socketTimeout) : thatS3TO.socketTimeout != null) {
- return false;
- }
-
- if (connectionTtl != null ? !connectionTtl.equals(thatS3TO.connectionTtl) : thatS3TO.connectionTtl != null) {
- return false;
- }
-
- if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatS3TO.useTCPKeepAlive) : thatS3TO.useTCPKeepAlive != null) {
- return false;
- }
-
- if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName) : thatS3TO.bucketName != null) {
- return false;
- }
-
- if (created != null ? !created.equals(thatS3TO.created) : thatS3TO.created != null) {
- return false;
- }
-
- if (enableRRS != thatS3TO.enableRRS) {
- return false;
- }
-
- return true;
-
- }
-
- @Override
- public int hashCode() {
-
- int result = id != null ? id.hashCode() : 0;
-
- result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0);
- result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
- result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
- result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
- result = 31 * result + (httpsFlag ? 1 : 0);
- result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
- result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
- result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
- result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
- result = 31 * result + (useTCPKeepAlive ? 1 : 0);
-
- return result;
-
- }
-
public Long getId() {
return this.id;
}
@@ -223,6 +125,15 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
this.bucketName = bucketName;
}
+ @Override
+ public String getSigner() {
+ return this.signer;
+ }
+
+ public void setSigner(final String signer) {
+ this.signer = signer;
+ }
+
@Override
public Boolean isHttps() {
return this.httpsFlag;
@@ -327,4 +238,100 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
public String getPathSeparator() {
return pathSeparator;
}
+
+ @Override
+ public boolean equals(final Object thatObject) {
+
+ if (this == thatObject) {
+ return true;
+ }
+ if (thatObject == null || getClass() != thatObject.getClass()) {
+ return false;
+ }
+
+ final S3TO thatS3TO = (S3TO)thatObject;
+
+ if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag) : thatS3TO.httpsFlag != null) {
+ return false;
+ }
+
+ if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey) : thatS3TO.accessKey != null) {
+ return false;
+ }
+
+ if (connectionTimeout != null ? !connectionTimeout.equals(thatS3TO.connectionTimeout) : thatS3TO.connectionTimeout != null) {
+ return false;
+ }
+
+ if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint) : thatS3TO.endPoint != null) {
+ return false;
+ }
+
+ if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) {
+ return false;
+ }
+
+ if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) {
+ return false;
+ }
+
+ if (maxErrorRetry != null ? !maxErrorRetry.equals(thatS3TO.maxErrorRetry) : thatS3TO.maxErrorRetry != null) {
+ return false;
+ }
+
+ if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey) : thatS3TO.secretKey != null) {
+ return false;
+ }
+
+ if (socketTimeout != null ? !socketTimeout.equals(thatS3TO.socketTimeout) : thatS3TO.socketTimeout != null) {
+ return false;
+ }
+
+ if (connectionTtl != null ? !connectionTtl.equals(thatS3TO.connectionTtl) : thatS3TO.connectionTtl != null) {
+ return false;
+ }
+
+ if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatS3TO.useTCPKeepAlive) : thatS3TO.useTCPKeepAlive != null) {
+ return false;
+ }
+
+ if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName) : thatS3TO.bucketName != null) {
+ return false;
+ }
+
+ if (signer != null ? !signer.equals(thatS3TO.signer) : thatS3TO.signer != null) {
+ return false;
+ }
+
+ if (created != null ? !created.equals(thatS3TO.created) : thatS3TO.created != null) {
+ return false;
+ }
+
+ if (enableRRS != thatS3TO.enableRRS) {
+ return false;
+ }
+
+ return true;
+
+ }
+
+ @Override
+ public int hashCode() {
+
+ int result = id != null ? id.hashCode() : 0;
+
+ result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0);
+ result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
+ result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
+ result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
+ result = 31 * result + (signer != null ? signer.hashCode() : 0);
+ result = 31 * result + (httpsFlag ? 1 : 0);
+ result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
+ result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
+ result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
+ result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
+ result = 31 * result + (useTCPKeepAlive ? 1 : 0);
+
+ return result;
+ }
}
diff --git a/api/src/com/cloud/storage/StorageService.java b/api/src/com/cloud/storage/StorageService.java
index 7fc1e16ff3a..e40b1e6e14c 100644
--- a/api/src/com/cloud/storage/StorageService.java
+++ b/api/src/com/cloud/storage/StorageService.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package com.cloud.storage;
import java.net.UnknownHostException;
@@ -38,14 +39,14 @@ public interface StorageService {
* Create StoragePool based on uri
*
* @param cmd
- * the command object that specifies the zone, cluster/pod, URI, details, etc. to use to create the
+ * The command object that specifies the zone, cluster/pod, URI, details, etc. to use to create the
* storage pool.
* @return
+ * The StoragePool created.
* @throws ResourceInUseException
* @throws IllegalArgumentException
* @throws UnknownHostException
* @throws ResourceUnavailableException
- * TODO
*/
StoragePool createPool(CreateStoragePoolCmd cmd) throws ResourceInUseException, IllegalArgumentException, UnknownHostException, ResourceUnavailableException;
@@ -63,15 +64,13 @@ public interface StorageService {
/**
* Enable maintenance for primary storage
*
- * @param cmd
- * - the command specifying primaryStorageId
+ * @param primaryStorageId
+ * - the primaryStorageId
* @return the primary storage pool
* @throws ResourceUnavailableException
- * TODO
* @throws InsufficientCapacityException
- * TODO
*/
- public StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException;
+ StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException;
/**
* Complete maintenance for primary storage
@@ -80,19 +79,18 @@ public interface StorageService {
* - the command specifying primaryStorageId
* @return the primary storage pool
* @throws ResourceUnavailableException
- * TODO
*/
- public StoragePool cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException;
+ StoragePool cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException;
- public StoragePool updateStoragePool(UpdateStoragePoolCmd cmd) throws IllegalArgumentException;
+ StoragePool updateStoragePool(UpdateStoragePoolCmd cmd) throws IllegalArgumentException;
- public StoragePool getStoragePool(long id);
+ StoragePool getStoragePool(long id);
boolean deleteImageStore(DeleteImageStoreCmd cmd);
boolean deleteSecondaryStagingStore(DeleteSecondaryStagingStoreCmd cmd);
- public ImageStore discoverImageStore(String name, String url, String providerName, Long dcId, Map details) throws IllegalArgumentException, DiscoveryException,
+ ImageStore discoverImageStore(String name, String url, String providerName, Long zoneId, Map details) throws IllegalArgumentException, DiscoveryException,
InvalidParameterValueException;
@@ -107,7 +105,7 @@ public interface StorageService {
* @throws DiscoveryException
* @throws InvalidParameterValueException
*/
- public ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException,
+ ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException,
InvalidParameterValueException;
diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java
index 2728bcc6fef..5365e14230c 100644
--- a/api/src/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/org/apache/cloudstack/api/ApiConstants.java
@@ -501,6 +501,9 @@ public class ApiConstants {
public static final String S3_SECRET_KEY = "secretkey";
public static final String S3_END_POINT = "endpoint";
public static final String S3_BUCKET_NAME = "bucket";
+ public static final String S3_SIGNER = "s3signer";
+ public static final String S3_V3_SIGNER = "S3SignerType";
+ public static final String S3_V4_SIGNER = "AWSS3V4SignerType";
public static final String S3_HTTPS_FLAG = "usehttps";
public static final String S3_CONNECTION_TIMEOUT = "connectiontimeout";
public static final String S3_CONNECTION_TTL = "connectionttl";
diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
similarity index 81%
rename from api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java
rename to api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
index d54cb7595a4..34ff171b91f 100644
--- a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java
+++ b/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
@@ -26,6 +26,7 @@ import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TTL;
import static org.apache.cloudstack.api.ApiConstants.S3_END_POINT;
import static org.apache.cloudstack.api.ApiConstants.S3_HTTPS_FLAG;
import static org.apache.cloudstack.api.ApiConstants.S3_MAX_ERROR_RETRY;
+import static org.apache.cloudstack.api.ApiConstants.S3_SIGNER;
import static org.apache.cloudstack.api.ApiConstants.S3_SECRET_KEY;
import static org.apache.cloudstack.api.ApiConstants.S3_SOCKET_TIMEOUT;
import static org.apache.cloudstack.api.ApiConstants.S3_USE_TCP_KEEPALIVE;
@@ -36,6 +37,7 @@ import static org.apache.cloudstack.api.BaseCmd.CommandType.STRING;
import java.util.HashMap;
import java.util.Map;
+import com.cloud.utils.storage.S3.ClientOptions;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.APICommand;
@@ -54,12 +56,12 @@ import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.storage.ImageStore;
-@APICommand(name = "addS3", description = "Adds S3", responseObject = ImageStoreResponse.class, since = "4.0.0",
+@APICommand(name = "addImageStoreS3", description = "Adds S3 Image Store", responseObject = ImageStoreResponse.class, since = "4.7.0",
requestHasSensitiveInfo = true, responseHasSensitiveInfo = false)
-public final class AddS3Cmd extends BaseCmd {
- public static final Logger s_logger = Logger.getLogger(AddS3Cmd.class.getName());
+public final class AddImageStoreS3CMD extends BaseCmd implements ClientOptions {
+ public static final Logger s_logger = Logger.getLogger(AddImageStoreS3CMD.class.getName());
- private static final String s_name = "adds3response";
+ private static final String s_name = "addImageStoreS3Response";
@Parameter(name = S3_ACCESS_KEY, type = STRING, required = true, description = "S3 access key")
private String accessKey;
@@ -67,41 +69,49 @@ public final class AddS3Cmd extends BaseCmd {
@Parameter(name = S3_SECRET_KEY, type = STRING, required = true, description = "S3 secret key")
private String secretKey;
- @Parameter(name = S3_END_POINT, type = STRING, required = false, description = "S3 host name")
+ @Parameter(name = S3_END_POINT, type = STRING, required = true, description = "S3 endpoint")
private String endPoint;
- @Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, description = "name of the template storage bucket")
+ @Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, description = "Name of the storage bucket")
private String bucketName;
- @Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, description = "connect to the S3 endpoint via HTTPS?")
+ @Parameter(name = S3_SIGNER, type = STRING, required = false, description = "Signer Algorithm to use, either S3SignerType or AWSS3V4SignerType")
+ private String signer;
+
+ @Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, description = "Use HTTPS instead of HTTP")
private Boolean httpsFlag;
- @Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, description = "connection timeout (milliseconds)")
+ @Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, description = "Connection timeout (milliseconds)")
private Integer connectionTimeout;
- @Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, description = "maximum number of times to retry on error")
+ @Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, description = "Maximum number of times to retry on error")
private Integer maxErrorRetry;
- @Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, description = "socket timeout (milliseconds)")
+ @Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, description = "Socket timeout (milliseconds)")
private Integer socketTimeout;
- @Parameter(name = S3_CONNECTION_TTL, type = INTEGER, required = false, description = "connection ttl (milliseconds)")
+ @Parameter(name = S3_CONNECTION_TTL, type = INTEGER, required = false, description = "Connection TTL (milliseconds)")
private Integer connectionTtl;
- @Parameter(name = S3_USE_TCP_KEEPALIVE, type = BOOLEAN, required = false, description = "whether tcp keepalive is used")
+ @Parameter(name = S3_USE_TCP_KEEPALIVE, type = BOOLEAN, required = false, description = "Whether TCP keep-alive is used")
private Boolean useTCPKeepAlive;
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
ResourceAllocationException, NetworkRuleConflictException {
- Map dm = new HashMap();
+ Map dm = new HashMap();
+
dm.put(ApiConstants.S3_ACCESS_KEY, getAccessKey());
dm.put(ApiConstants.S3_SECRET_KEY, getSecretKey());
dm.put(ApiConstants.S3_END_POINT, getEndPoint());
dm.put(ApiConstants.S3_BUCKET_NAME, getBucketName());
- if (getHttpsFlag() != null) {
- dm.put(ApiConstants.S3_HTTPS_FLAG, getHttpsFlag().toString());
+
+ if (getSigner() != null && (getSigner().equals(ApiConstants.S3_V3_SIGNER) || getSigner().equals(ApiConstants.S3_V4_SIGNER))) {
+ dm.put(ApiConstants.S3_SIGNER, getSigner());
+ }
+ if (isHttps() != null) {
+ dm.put(ApiConstants.S3_HTTPS_FLAG, isHttps().toString());
}
if (getConnectionTimeout() != null) {
dm.put(ApiConstants.S3_CONNECTION_TIMEOUT, getConnectionTimeout().toString());
@@ -119,17 +129,16 @@ public final class AddS3Cmd extends BaseCmd {
dm.put(ApiConstants.S3_USE_TCP_KEEPALIVE, getUseTCPKeepAlive().toString());
}
-
try{
ImageStore result = _storageService.discoverImageStore(null, null, "S3", null, dm);
- ImageStoreResponse storeResponse = null;
+ ImageStoreResponse storeResponse;
if (result != null) {
storeResponse = _responseGenerator.createImageStoreResponse(result);
storeResponse.setResponseName(getCommandName());
- storeResponse.setObjectName("secondarystorage");
+ storeResponse.setObjectName("imagestore");
setResponseObject(storeResponse);
} else {
- throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 secondary storage");
+ throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 Image Store.");
}
} catch (DiscoveryException ex) {
s_logger.warn("Exception: ", ex);
@@ -137,6 +146,60 @@ public final class AddS3Cmd extends BaseCmd {
}
}
+ @Override
+ public String getCommandName() {
+ return s_name;
+ }
+
+ @Override
+ public long getEntityOwnerId() {
+ return ACCOUNT_ID_SYSTEM;
+ }
+
+ public String getAccessKey() {
+ return accessKey;
+ }
+
+ public String getSecretKey() {
+ return secretKey;
+ }
+
+ public String getEndPoint() {
+ return endPoint;
+ }
+
+ public String getBucketName() {
+ return bucketName;
+ }
+
+ public String getSigner() {
+ return signer;
+ }
+
+ public Boolean isHttps() {
+ return httpsFlag;
+ }
+
+ public Integer getConnectionTimeout() {
+ return connectionTimeout;
+ }
+
+ public Integer getMaxErrorRetry() {
+ return maxErrorRetry;
+ }
+
+ public Integer getSocketTimeout() {
+ return socketTimeout;
+ }
+
+ public Integer getConnectionTtl() {
+ return connectionTtl;
+ }
+
+ public Boolean getUseTCPKeepAlive() {
+ return useTCPKeepAlive;
+ }
+
@Override
public boolean equals(final Object thatObject) {
@@ -148,7 +211,7 @@ public final class AddS3Cmd extends BaseCmd {
return false;
}
- final AddS3Cmd thatAddS3Cmd = (AddS3Cmd)thatObject;
+ final AddImageStoreS3CMD thatAddS3Cmd = (AddImageStoreS3CMD)thatObject;
if (httpsFlag != null ? !httpsFlag.equals(thatAddS3Cmd.httpsFlag) : thatAddS3Cmd.httpsFlag != null) {
return false;
@@ -191,7 +254,6 @@ public final class AddS3Cmd extends BaseCmd {
}
return true;
-
}
@Override
@@ -201,64 +263,14 @@ public final class AddS3Cmd extends BaseCmd {
result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
- result = 31 * result + (httpsFlag != null && httpsFlag == true ? 1 : 0);
+ result = 31 * result + (signer != null ? signer.hashCode() : 0);
+ result = 31 * result + (httpsFlag != null && httpsFlag ? 1 : 0);
result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
- result = 31 * result + (useTCPKeepAlive != null && useTCPKeepAlive == true ? 1 : 0);
+ result = 31 * result + (useTCPKeepAlive != null && useTCPKeepAlive ? 1 : 0);
return result;
-
- }
-
- @Override
- public String getCommandName() {
- return s_name;
- }
-
- @Override
- public long getEntityOwnerId() {
- return ACCOUNT_ID_SYSTEM;
- }
-
- public String getAccessKey() {
- return accessKey;
- }
-
- public String getSecretKey() {
- return secretKey;
- }
-
- public String getEndPoint() {
- return endPoint;
- }
-
- public String getBucketName() {
- return bucketName;
- }
-
- public Boolean getHttpsFlag() {
- return httpsFlag;
- }
-
- public Integer getConnectionTimeout() {
- return connectionTimeout;
- }
-
- public Integer getMaxErrorRetry() {
- return maxErrorRetry;
- }
-
- public Integer getSocketTimeout() {
- return socketTimeout;
- }
-
- public Integer getConnectionTtl() {
- return connectionTtl;
- }
-
- public Boolean getUseTCPKeepAlive() {
- return useTCPKeepAlive;
}
}
diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java
deleted file mode 100644
index c1889e72a0b..00000000000
--- a/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cloudstack.api.command.admin.storage;
-
-import org.apache.cloudstack.api.APICommand;
-import org.apache.cloudstack.api.BaseListCmd;
-import org.apache.cloudstack.api.ServerApiException;
-import org.apache.cloudstack.api.response.ImageStoreResponse;
-import org.apache.cloudstack.api.response.ListResponse;
-
-import com.cloud.exception.ConcurrentOperationException;
-import com.cloud.exception.InsufficientCapacityException;
-import com.cloud.exception.NetworkRuleConflictException;
-import com.cloud.exception.ResourceAllocationException;
-import com.cloud.exception.ResourceUnavailableException;
-
-@APICommand(name = "listS3s", description = "Lists S3s", responseObject = ImageStoreResponse.class, since = "4.0.0",
- requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
-public class ListS3sCmd extends BaseListCmd {
-
- private static final String COMMAND_NAME = "lists3sresponse";
-
- @Override
- public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
- ResourceAllocationException, NetworkRuleConflictException {
-
- ListImageStoresCmd cmd = new ListImageStoresCmd();
- cmd.setProvider("S3");
- ListResponse response = _queryService.searchForImageStores(cmd);
- response.setResponseName(getCommandName());
- this.setResponseObject(response);
- }
-
- @Override
- public String getCommandName() {
- return COMMAND_NAME;
- }
-
-}
diff --git a/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java b/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
index 659d1c5c4d6..68d53f9df6f 100644
--- a/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
+++ b/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
@@ -32,7 +32,7 @@ import org.apache.cloudstack.api.response.TemplateResponse;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.template.VirtualMachineTemplate;
-@APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud. ", responseObject = TemplateResponse.class, responseView = ResponseView.Full,
+@APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud.", responseObject = TemplateResponse.class, responseView = ResponseView.Full,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
public class RegisterTemplateCmdByAdmin extends RegisterTemplateCmd {
public static final Logger s_logger = Logger.getLogger(RegisterTemplateCmdByAdmin.class.getName());
diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in
index 4f93b977733..1c788a85153 100644
--- a/client/tomcatconf/commands.properties.in
+++ b/client/tomcatconf/commands.properties.in
@@ -282,12 +282,9 @@ listCapacity=3
addSwift=1
listSwifts=1
-#### s3 commands
-addS3=1
-listS3s=1
-
#### image store commands
addImageStore=1
+addImageStoreS3=1
listImageStores=1
deleteImageStore=1
createSecondaryStagingStore=1
diff --git a/client/tomcatconf/log4j-cloud.xml.in b/client/tomcatconf/log4j-cloud.xml.in
index 587aa86aff2..0a12e7d06e4 100755
--- a/client/tomcatconf/log4j-cloud.xml.in
+++ b/client/tomcatconf/log4j-cloud.xml.in
@@ -162,6 +162,16 @@ under the License.
+
+
+
+
+
+
+
+
+
+
diff --git a/core/src/com/cloud/resource/ServerResource.java b/core/src/com/cloud/resource/ServerResource.java
index 1bbcbff7d63..9030db72f00 100644
--- a/core/src/com/cloud/resource/ServerResource.java
+++ b/core/src/com/cloud/resource/ServerResource.java
@@ -28,7 +28,6 @@ import com.cloud.host.Host;
import com.cloud.utils.component.Manager;
/**
- *
* ServerResource is a generic container to execute commands sent
*/
public interface ServerResource extends Manager {
@@ -70,4 +69,5 @@ public interface ServerResource extends Manager {
IAgentControl getAgentControl();
void setAgentControl(IAgentControl agentControl);
+
}
diff --git a/core/src/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
index 632a80963e9..76d1ac9f7b2 100644
--- a/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
@@ -47,10 +47,10 @@ import org.apache.log4j.Logger;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
-import com.cloud.agent.api.storage.Proxy;
import com.cloud.storage.StorageLayer;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
+import com.cloud.utils.net.Proxy;
/**
* Download a template file using HTTP
diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
index ac47decefc6..d584cdffb10 100644
--- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
@@ -19,303 +19,235 @@
package com.cloud.storage.template;
-import static com.cloud.utils.StringUtils.join;
-import static java.util.Arrays.asList;
+import com.amazonaws.event.ProgressEvent;
+import com.amazonaws.event.ProgressEventType;
+import com.amazonaws.event.ProgressListener;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.StorageClass;
+import com.amazonaws.services.s3.transfer.Upload;
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.utils.net.HTTPUtils;
+import com.cloud.utils.net.Proxy;
+import com.cloud.utils.storage.S3.S3Utils;
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
+import org.apache.commons.httpclient.Header;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.URIException;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.params.HttpMethodParams;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
-import org.apache.cloudstack.managed.context.ManagedContextRunnable;
-import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
-import org.apache.commons.httpclient.ChunkedInputStream;
-import org.apache.commons.httpclient.Credentials;
-import org.apache.commons.httpclient.Header;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpMethodRetryHandler;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
-import org.apache.commons.httpclient.NoHttpResponseException;
-import org.apache.commons.httpclient.UsernamePasswordCredentials;
-import org.apache.commons.httpclient.auth.AuthScope;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.params.HttpMethodParams;
-import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.ProgressEvent;
-import com.amazonaws.services.s3.model.ProgressListener;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.StorageClass;
-import com.cloud.agent.api.storage.Proxy;
-import com.cloud.agent.api.to.S3TO;
-import com.cloud.utils.Pair;
-import com.cloud.utils.S3Utils;
-import com.cloud.utils.UriUtils;
+import static com.cloud.utils.StringUtils.join;
+import static java.util.Arrays.asList;
/**
* Download a template file using HTTP(S)
+ *
+ * This class, once instantiated, has the purpose to download a single Template to an S3 Image Store.
+ *
+ * Execution of the instance is started when runInContext() is called.
*/
public class S3TemplateDownloader extends ManagedContextRunnable implements TemplateDownloader {
- private static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
- private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
-
- private String downloadUrl;
- private String installPath;
- private String s3Key;
- private String fileName;
- private String fileExtension;
- private String errorString = " ";
+ private static final Logger LOGGER = Logger.getLogger(S3TemplateDownloader.class.getName());
+ private final String downloadUrl;
+ private final String s3Key;
+ private final String fileExtension;
+ private final HttpClient httpClient;
+ private final GetMethod getMethod;
+ private final DownloadCompleteCallback downloadCompleteCallback;
+ private final S3TO s3TO;
+ private String errorString = "";
private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
private ResourceType resourceType = ResourceType.TEMPLATE;
- private final HttpClient client;
- private final HttpMethodRetryHandler myretryhandler;
- private GetMethod request;
- private DownloadCompleteCallback completionCallback;
- private S3TO s3to;
-
- private long remoteSize = 0;
- private long downloadTime = 0;
+ private long remoteSize;
+ private long downloadTime;
private long totalBytes;
private long maxTemplateSizeInByte;
private boolean resume = false;
- private boolean inited = true;
- public S3TemplateDownloader(S3TO s3to, String downloadUrl, String installPath, DownloadCompleteCallback callback,
- long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) {
- this.s3to = s3to;
+ public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
+ long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
this.downloadUrl = downloadUrl;
- this.installPath = installPath;
- this.status = TemplateDownloader.Status.NOT_STARTED;
+ this.s3TO = s3TO;
this.resourceType = resourceType;
this.maxTemplateSizeInByte = maxTemplateSizeInBytes;
+ this.httpClient = HTTPUtils.getHTTPClient();
+ this.downloadCompleteCallback = downloadCompleteCallback;
- this.totalBytes = 0;
- this.client = new HttpClient(s_httpClientManager);
+ // Create a GET method for the download url.
+ this.getMethod = new GetMethod(downloadUrl);
- this.myretryhandler = new HttpMethodRetryHandler() {
- @Override
- public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
- if (executionCount >= 2) {
- // Do not retry if over max retry count
- return false;
- }
- if (exception instanceof NoHttpResponseException) {
- // Retry if the server dropped connection on us
- return true;
- }
- if (!method.isRequestSent()) {
- // Retry if the request has not been sent fully or
- // if it's OK to retry methods that have been sent
- return true;
- }
- // otherwise do not retry
- return false;
- }
- };
+ // Set the retry handler, default to retry 5 times.
+ this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
- try {
- request = new GetMethod(downloadUrl);
- request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
- completionCallback = callback;
+ // Follow redirects
+ this.getMethod.setFollowRedirects(true);
- Pair hostAndPort = UriUtils.validateUrl(downloadUrl);
- fileName = StringUtils.substringAfterLast(downloadUrl, "/");
- fileExtension = StringUtils.substringAfterLast(fileName, ".");
+ // Set file extension.
+ this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
- if (proxy != null) {
- client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
- if (proxy.getUserName() != null) {
- Credentials proxyCreds = new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword());
- client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds);
- }
- }
- if ((user != null) && (password != null)) {
- client.getParams().setAuthenticationPreemptive(true);
- Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
- client.getState().setCredentials(
- new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
- s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first()
- + ":" + hostAndPort.second());
- } else {
- s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
- }
- } catch (IllegalArgumentException iae) {
- errorString = iae.getMessage();
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- inited = false;
- } catch (Exception ex) {
- errorString = "Unable to start download -- check url? ";
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- s_logger.warn("Exception in constructor -- " + ex.toString());
- } catch (Throwable th) {
- s_logger.warn("throwable caught ", th);
- }
+ // Calculate and set S3 Key.
+ this.s3Key = join(asList(installPath, StringUtils.substringAfterLast(downloadUrl, "/")), S3Utils.SEPARATOR);
+
+ // Set proxy if available.
+ HTTPUtils.setProxy(proxy, this.httpClient);
+
+ // Set credentials if available.
+ HTTPUtils.setCredentials(username, password, this.httpClient);
}
@Override
public long download(boolean resume, DownloadCompleteCallback callback) {
- switch (status) {
- case ABORTED:
- case UNRECOVERABLE_ERROR:
- case DOWNLOAD_FINISHED:
- return 0;
- default:
+ if (!status.equals(Status.NOT_STARTED)) {
+ // Only start downloading if we haven't started yet.
+ LOGGER.debug("Template download is already started, not starting again. Template: " + downloadUrl);
+ return 0;
}
+ int responseCode;
+ if ((responseCode = HTTPUtils.executeMethod(httpClient, getMethod)) == -1) {
+ errorString = "Exception while executing HttpMethod " + getMethod.getName() + " on URL " + downloadUrl;
+ LOGGER.warn(errorString);
+
+ status = Status.UNRECOVERABLE_ERROR;
+ return 0;
+ }
+
+ if (!HTTPUtils.verifyResponseCode(responseCode)) {
+ errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
+ LOGGER.warn(errorString);
+
+ status = Status.UNRECOVERABLE_ERROR;
+ return 0;
+ }
+
+ // Headers
+ Header contentLengthHeader = getMethod.getResponseHeader("Content-Length");
+ Header contentTypeHeader = getMethod.getResponseHeader("Content-Type");
+
+ // Check the contentLengthHeader and transferEncodingHeader.
+ if (contentLengthHeader == null) {
+ errorString = "The ContentLengthHeader of " + downloadUrl + " isn't supplied";
+ LOGGER.warn(errorString);
+
+ status = Status.UNRECOVERABLE_ERROR;
+ return 0;
+ } else {
+ // The ContentLengthHeader is supplied, parse it's value.
+ remoteSize = Long.parseLong(contentLengthHeader.getValue());
+ }
+
+ if (remoteSize > maxTemplateSizeInByte) {
+ errorString = "Remote size is too large for template " + downloadUrl + " remote size is " + remoteSize + " max allowed is " + maxTemplateSizeInByte;
+ LOGGER.warn(errorString);
+
+ status = Status.UNRECOVERABLE_ERROR;
+ return 0;
+ }
+
+ InputStream inputStream;
+
try {
- // execute get method
- int responseCode = HttpStatus.SC_OK;
- if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
- return 0; // FIXME: retry?
- }
- // get the total size of file
- Header contentLengthHeader = request.getResponseHeader("Content-Length");
- boolean chunked = false;
- long remoteSize2 = 0;
- if (contentLengthHeader == null) {
- Header chunkedHeader = request.getResponseHeader("Transfer-Encoding");
- if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = " Failed to receive length of download ";
- return 0; // FIXME: what status do we put here? Do we retry?
- } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
- chunked = true;
- }
- } else {
- remoteSize2 = Long.parseLong(contentLengthHeader.getValue());
- }
+ inputStream = new BufferedInputStream(getMethod.getResponseBodyAsStream());
+ } catch (IOException e) {
+ errorString = "Exception occurred while opening InputStream for template " + downloadUrl;
+ LOGGER.warn(errorString);
- if (remoteSize == 0) {
- remoteSize = remoteSize2;
- }
-
- if (remoteSize > maxTemplateSizeInByte) {
- s_logger.info("Remote size is too large: " + remoteSize + " , max=" + maxTemplateSizeInByte);
- status = Status.UNRECOVERABLE_ERROR;
- errorString = "Download file size is too large";
- return 0;
- }
-
- if (remoteSize == 0) {
- remoteSize = maxTemplateSizeInByte;
- }
-
- // get content type
- String contentType = null;
- Header contentTypeHeader = request.getResponseHeader("Content-Type");
- if (contentTypeHeader != null) {
- contentType = contentTypeHeader.getValue();
- }
-
- InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream())
- : new ChunkedInputStream(request.getResponseBodyAsStream());
-
- s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3to.getBucketName()
- + " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInByte);
-
- Date start = new Date();
- // compute s3 key
- s3Key = join(asList(installPath, fileName), S3Utils.SEPARATOR);
-
- // download using S3 API
- ObjectMetadata metadata = new ObjectMetadata();
- metadata.setContentLength(remoteSize);
- if (contentType != null) {
- metadata.setContentType(contentType);
- }
- PutObjectRequest putObjectRequest = new PutObjectRequest(s3to.getBucketName(), s3Key, in, metadata);
- // check if RRS is enabled
- if (s3to.getEnableRRS()) {
- putObjectRequest = putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
- }
- // register progress listenser
- putObjectRequest.setProgressListener(new ProgressListener() {
- @Override
- public void progressChanged(ProgressEvent progressEvent) {
- // s_logger.debug(progressEvent.getBytesTransfered()
- // + " number of byte transferd "
- // + new Date());
- totalBytes += progressEvent.getBytesTransfered();
- if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) {
- s_logger.info("download completed");
- status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
- } else if (progressEvent.getEventCode() == ProgressEvent.FAILED_EVENT_CODE) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- } else if (progressEvent.getEventCode() == ProgressEvent.CANCELED_EVENT_CODE) {
- status = TemplateDownloader.Status.ABORTED;
- } else {
- status = TemplateDownloader.Status.IN_PROGRESS;
- }
- }
-
- });
-
- if (!s3to.getSingleUpload(remoteSize)) {
- // use TransferManager to do multipart upload
- S3Utils.mputObject(s3to, putObjectRequest);
- } else {
- // single part upload, with 5GB limit in Amazon
- S3Utils.putObject(s3to, putObjectRequest);
- while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED
- && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR
- && status != TemplateDownloader.Status.ABORTED) {
- // wait for completion
- }
- }
-
- // finished or aborted
- Date finish = new Date();
- String downloaded = "(incomplete download)";
- if (totalBytes >= remoteSize) {
- status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
- downloaded = "(download complete remote=" + remoteSize + "bytes)";
- } else {
- errorString = "Downloaded " + totalBytes + " bytes " + downloaded;
- }
- downloadTime += finish.getTime() - start.getTime();
- return totalBytes;
- } catch (HttpException hte) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = hte.getMessage();
- } catch (IOException ioe) {
- // probably a file write error
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = ioe.getMessage();
- } catch (AmazonClientException ex) {
- // S3 api exception
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = ex.getMessage();
- } catch (InterruptedException e) {
- // S3 upload api exception
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = e.getMessage();
- } finally {
- // close input stream
- request.releaseConnection();
- if (callback != null) {
- callback.downloadComplete(status);
- }
+ status = Status.UNRECOVERABLE_ERROR;
+ return 0;
}
- return 0;
+
+ LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + remoteSize + " bytes");
+
+ // Time the upload starts.
+ final Date start = new Date();
+
+ ObjectMetadata objectMetadata = new ObjectMetadata();
+ objectMetadata.setContentLength(remoteSize);
+
+ if (contentTypeHeader.getValue() != null) {
+ objectMetadata.setContentType(contentTypeHeader.getValue());
+ }
+
+ // Create the PutObjectRequest.
+ PutObjectRequest putObjectRequest = new PutObjectRequest(s3TO.getBucketName(), s3Key, inputStream, objectMetadata);
+
+ // If reduced redundancy is enabled, set it.
+ if (s3TO.getEnableRRS()) {
+ putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
+ }
+
+ Upload upload = S3Utils.putObject(s3TO, putObjectRequest);
+
+ upload.addProgressListener(new ProgressListener() {
+ @Override
+ public void progressChanged(ProgressEvent progressEvent) {
+
+ // Record the amount of bytes transferred.
+ totalBytes += progressEvent.getBytesTransferred();
+
+ LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds");
+
+ if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) {
+ status = Status.IN_PROGRESS;
+ } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
+ status = Status.DOWNLOAD_FINISHED;
+ } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_CANCELED_EVENT) {
+ status = Status.ABORTED;
+ } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_FAILED_EVENT) {
+ status = Status.UNRECOVERABLE_ERROR;
+ }
+ }
+ });
+
+ try {
+ // Wait for the upload to complete.
+ upload.waitForCompletion();
+ } catch (InterruptedException e) {
+ // Interruption while waiting for the upload to complete.
+ LOGGER.warn("Interruption occurred while waiting for upload of " + downloadUrl + " to complete");
+ }
+
+ downloadTime = new Date().getTime() - start.getTime();
+
+ if (status == Status.DOWNLOAD_FINISHED) {
+ LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!");
+ } else {
+ LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString());
+ }
+
+ // Close input stream
+ getMethod.releaseConnection();
+
+ // Call the callback!
+ if (callback != null) {
+ callback.downloadComplete(status);
+ }
+
+ return totalBytes;
}
public String getDownloadUrl() {
- return downloadUrl;
+ try {
+ return getMethod.getURI().toString();
+ } catch (URIException e) {
+ return null;
+ }
}
@Override
- public TemplateDownloader.Status getStatus() {
+ public Status getStatus() {
return status;
}
@@ -342,47 +274,38 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
return null;
}
- return S3Utils.getObjectStream(s3to, s3to.getBucketName(), s3Key);
+ return S3Utils.getObjectStream(s3TO, s3TO.getBucketName(), s3Key);
}
public void cleanupAfterError() {
- if (status != Status.UNRECOVERABLE_ERROR) {
- s_logger.debug("S3Template downloader does not have state UNRECOVERABLE_ERROR, no cleanup neccesarry.");
- return;
- }
+ LOGGER.warn("Cleanup after error, trying to remove object: " + s3Key);
- s_logger.info("Cleanup after UNRECOVERABLE_ERROR, trying to remove object: " + s3Key);
-
- S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
+ S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key);
}
@Override
- @SuppressWarnings("fallthrough")
public boolean stopDownload() {
- switch (getStatus()) {
- case IN_PROGRESS:
- if (request != null) {
- request.abort();
- }
- status = TemplateDownloader.Status.ABORTED;
- return true;
- case UNKNOWN:
- case NOT_STARTED:
- case RECOVERABLE_ERROR:
- case UNRECOVERABLE_ERROR:
- case ABORTED:
- status = TemplateDownloader.Status.ABORTED;
- case DOWNLOAD_FINISHED:
- try {
- S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
- } catch (Exception ex) {
- // ignore delete exception if it is not there
- }
- return true;
-
- default:
- return true;
+ switch (status) {
+ case IN_PROGRESS:
+ if (getMethod != null) {
+ getMethod.abort();
+ }
+ break;
+ case UNKNOWN:
+ case NOT_STARTED:
+ case RECOVERABLE_ERROR:
+ case UNRECOVERABLE_ERROR:
+ case ABORTED:
+ case DOWNLOAD_FINISHED:
+ // Remove the object if it already has been uploaded.
+ S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key);
+ break;
+ default:
+ break;
}
+
+ status = TemplateDownloader.Status.ABORTED;
+ return true;
}
@Override
@@ -396,18 +319,12 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
@Override
protected void runInContext() {
- try {
- download(resume, completionCallback);
- } catch (Throwable t) {
- s_logger.warn("Caught exception during download " + t.getMessage(), t);
- errorString = "Failed to install: " + t.getMessage();
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- }
-
+ // Start the download!
+ download(resume, downloadCompleteCallback);
}
@Override
- public void setStatus(TemplateDownloader.Status status) {
+ public void setStatus(Status status) {
this.status = status;
}
@@ -442,7 +359,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
@Override
public boolean isInited() {
- return inited;
+ return true;
}
public ResourceType getResourceType() {
diff --git a/core/src/com/cloud/storage/template/TemplateDownloader.java b/core/src/com/cloud/storage/template/TemplateDownloader.java
index da9787d6d99..5db3d2425a5 100644
--- a/core/src/com/cloud/storage/template/TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/TemplateDownloader.java
@@ -25,16 +25,16 @@ public interface TemplateDownloader extends Runnable {
* Callback used to notify completion of download
*
*/
- public interface DownloadCompleteCallback {
+ interface DownloadCompleteCallback {
void downloadComplete(Status status);
}
- public static enum Status {
+ enum Status {
UNKNOWN, NOT_STARTED, IN_PROGRESS, ABORTED, UNRECOVERABLE_ERROR, RECOVERABLE_ERROR, DOWNLOAD_FINISHED, POST_DOWNLOAD_FINISHED
}
- public static long DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES = 50L * 1024L * 1024L * 1024L;
+ long DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES = 50L * 1024L * 1024L * 1024L;
/**
* Initiate download, resuming a previous one if required
@@ -42,55 +42,54 @@ public interface TemplateDownloader extends Runnable {
* @param callback completion callback to be called after download is complete
* @return bytes downloaded
*/
- public long download(boolean resume, DownloadCompleteCallback callback);
+ long download(boolean resume, DownloadCompleteCallback callback);
/**
* @return
*/
- public boolean stopDownload();
+ boolean stopDownload();
/**
* @return percent of file downloaded
*/
- public int getDownloadPercent();
+ int getDownloadPercent();
/**
* Get the status of the download
* @return status of download
*/
- public TemplateDownloader.Status getStatus();
+ TemplateDownloader.Status getStatus();
/**
* Get time taken to download so far
* @return time in seconds taken to download
*/
- public long getDownloadTime();
+ long getDownloadTime();
/**
* Get bytes downloaded
* @return bytes downloaded so far
*/
- public long getDownloadedBytes();
+ long getDownloadedBytes();
/**
* Get the error if any
* @return error string if any
*/
- public String getDownloadError();
+ String getDownloadError();
/** Get local path of the downloaded file
* @return local path of the file downloaded
*/
- public String getDownloadLocalPath();
+ String getDownloadLocalPath();
- public void setStatus(TemplateDownloader.Status status);
+ void setStatus(TemplateDownloader.Status status);
- public void setDownloadError(String string);
+ void setDownloadError(String string);
- public void setResume(boolean resume);
+ void setResume(boolean resume);
- public boolean isInited();
-
- public long getMaxTemplateSizeInBytes();
+ boolean isInited();
+ long getMaxTemplateSizeInBytes();
}
diff --git a/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
index 5dfc22e785b..29d737fcce9 100644
--- a/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
+++ b/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
@@ -25,7 +25,7 @@ import org.apache.cloudstack.storage.to.VolumeObjectTO;
import com.cloud.agent.api.storage.AbstractDownloadCommand;
import com.cloud.agent.api.storage.PasswordAuth;
-import com.cloud.agent.api.storage.Proxy;
+import com.cloud.utils.net.Proxy;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.storage.Storage.ImageFormat;
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
index 7b5f8d967ee..3e5761ef37f 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
@@ -23,14 +23,14 @@ import java.util.Set;
public interface DataStoreProvider {
// constants for provider names
- static final String NFS_IMAGE = "NFS";
- static final String S3_IMAGE = "S3";
- static final String SWIFT_IMAGE = "Swift";
- static final String SAMPLE_IMAGE = "Sample";
- static final String SMB = "NFS";
- static final String DEFAULT_PRIMARY = "DefaultPrimary";
+ String NFS_IMAGE = "NFS";
+ String S3_IMAGE = "S3";
+ String SWIFT_IMAGE = "Swift";
+ String SAMPLE_IMAGE = "Sample";
+ String SMB = "NFS";
+ String DEFAULT_PRIMARY = "DefaultPrimary";
- static enum DataStoreProviderType {
+ enum DataStoreProviderType {
PRIMARY, IMAGE, ImageCache
}
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index 69dc7c71294..9ab35953711 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -196,7 +196,7 @@ public class TemplateServiceImpl implements TemplateService {
@Override
public void downloadBootstrapSysTemplate(DataStore store) {
- Set toBeDownloaded = new HashSet();
+ Set toBeDownloaded = new HashSet();
List rtngTmplts = _templateDao.listAllSystemVMTemplates();
diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
index 3dc6ac497a9..8f081d3af39 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
@@ -27,15 +27,15 @@ import com.cloud.storage.DataStoreRole;
import com.cloud.utils.fsm.NoTransitionException;
public interface ObjectInDataStoreManager {
- public DataObject create(DataObject dataObj, DataStore dataStore);
+ DataObject create(DataObject dataObj, DataStore dataStore);
- public boolean delete(DataObject dataObj);
+ boolean delete(DataObject dataObj);
- public boolean deleteIfNotReady(DataObject dataObj);
+ boolean deleteIfNotReady(DataObject dataObj);
- public DataObject get(DataObject dataObj, DataStore store);
+ DataObject get(DataObject dataObj, DataStore store);
- public boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException;
+ boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException;
DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role);
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
index 380040b230e..0068c2d4f7b 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
@@ -47,7 +47,6 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.DownloadAnswer;
-import com.cloud.agent.api.storage.Proxy;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataTO;
import com.cloud.alert.AlertManager;
@@ -58,6 +57,7 @@ import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.download.DownloadMonitor;
+import com.cloud.utils.net.Proxy;
public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
private static final Logger s_logger = Logger.getLogger(BaseImageStoreDriverImpl.class);
diff --git a/framework/rest/pom.xml b/framework/rest/pom.xml
index 3b88e405088..372770f48cd 100644
--- a/framework/rest/pom.xml
+++ b/framework/rest/pom.xml
@@ -33,27 +33,27 @@
com.fasterxml.jackson.module
jackson-module-jaxb-annotations
- 2.4.4
+ 2.6.3
com.fasterxml.jackson.core
jackson-annotations
- 2.4.4
+ 2.6.3
com.fasterxml.jackson.core
jackson-core
- 2.4.4
+ 2.6.3
com.fasterxml.jackson.core
jackson-databind
- 2.4.4
+ 2.6.3
com.fasterxml.jackson.jaxrs
jackson-jaxrs-json-provider
- 2.4.4
+ 2.6.3
org.apache.cxf
diff --git a/packaging/centos7/tomcat7/log4j-cloud.xml b/packaging/centos7/tomcat7/log4j-cloud.xml
index d03775cc41e..1ebcbf8dc71 100644
--- a/packaging/centos7/tomcat7/log4j-cloud.xml
+++ b/packaging/centos7/tomcat7/log4j-cloud.xml
@@ -162,6 +162,16 @@ under the License.
+
+
+
+
+
+
+
+
+
+
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
index 08ce05ad3ea..ee7448d6c6d 100644
--- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
@@ -18,8 +18,7 @@
*/
package com.cloud.hypervisor.kvm.storage;
-import static com.cloud.utils.S3Utils.mputFile;
-import static com.cloud.utils.S3Utils.putFile;
+import static com.cloud.utils.storage.S3.S3Utils.putFile;
import java.io.File;
import java.io.FileNotFoundException;
@@ -95,7 +94,7 @@ import com.cloud.storage.template.Processor.FormatInfo;
import com.cloud.storage.template.QCOW2Processor;
import com.cloud.storage.template.TemplateLocation;
import com.cloud.utils.NumbersUtil;
-import com.cloud.utils.S3Utils;
+import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
@@ -594,15 +593,10 @@ public class KVMStorageProcessor implements StorageProcessor {
}
protected String copyToS3(final File srcFile, final S3TO destStore, final String destPath) throws InterruptedException {
- final String bucket = destStore.getBucketName();
-
- final long srcSize = srcFile.length();
final String key = destPath + S3Utils.SEPARATOR + srcFile.getName();
- if (!destStore.getSingleUpload(srcSize)) {
- mputFile(destStore, srcFile, bucket, key);
- } else {
- putFile(destStore, srcFile, bucket, key);
- }
+
+ putFile(destStore, srcFile, destStore.getBucketName(), key).waitForCompletion();
+
return key;
}
@@ -668,7 +662,7 @@ public class KVMStorageProcessor implements StorageProcessor {
final SnapshotObjectTO snapshotOnCacheStore = (SnapshotObjectTO)answer.getNewData();
snapshotOnCacheStore.setDataStore(cacheStore);
((SnapshotObjectTO)destData).setDataStore(imageStore);
- final CopyCommand newCpyCmd = new CopyCommand(snapshotOnCacheStore, destData, cmd.getWaitInMillSeconds(), cmd.executeInSequence());
+ final CopyCommand newCpyCmd = new CopyCommand(snapshotOnCacheStore, destData, cmd.getWaitInMillSeconds(), cmd.executeInSequence());
return copyToObjectStore(newCpyCmd);
}
diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
index 94cf5df7ee9..38b45d02466 100644
--- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
+++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
@@ -70,11 +70,11 @@ import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.resource.StorageProcessor;
-import com.cloud.utils.S3Utils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.storage.encoding.DecodedDataObject;
import com.cloud.utils.storage.encoding.DecodedDataStore;
import com.cloud.utils.storage.encoding.Decoder;
+import com.cloud.utils.storage.S3.ClientOptions;
import com.xensource.xenapi.Connection;
import com.xensource.xenapi.Host;
import com.xensource.xenapi.PBD;
@@ -1061,7 +1061,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
try {
- final List parameters = newArrayList(flattenProperties(s3, S3Utils.ClientOptions.class));
+ final List parameters = newArrayList(flattenProperties(s3, ClientOptions.class));
// https workaround for Introspector bug that does not
// recognize Boolean accessor methods ...
diff --git a/plugins/storage/image/s3/pom.xml b/plugins/storage/image/s3/pom.xml
index bb12cf78d6c..6fb4d98260b 100644
--- a/plugins/storage/image/s3/pom.xml
+++ b/plugins/storage/image/s3/pom.xml
@@ -19,7 +19,7 @@
4.0.0
cloud-plugin-storage-image-s3
- Apache CloudStack Plugin - Storage Image S3
+ Apache CloudStack Plugin - Storage Image S3 provider
org.apache.cloudstack
cloudstack-plugins
diff --git a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
index 2d723f4d7d0..3c2bc953d97 100644
--- a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
+++ b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
@@ -39,7 +39,7 @@ import com.cloud.agent.api.to.S3TO;
import com.cloud.configuration.Config;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.utils.NumbersUtil;
-import com.cloud.utils.S3Utils;
+import com.cloud.utils.storage.S3.S3Utils;
public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
private static final Logger s_logger = Logger.getLogger(S3ImageStoreDriverImpl.class);
@@ -58,7 +58,9 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
imgStore.getUuid(),
details.get(ApiConstants.S3_ACCESS_KEY),
details.get(ApiConstants.S3_SECRET_KEY),
- details.get(ApiConstants.S3_END_POINT), details.get(ApiConstants.S3_BUCKET_NAME),
+ details.get(ApiConstants.S3_END_POINT),
+ details.get(ApiConstants.S3_BUCKET_NAME),
+ details.get(ApiConstants.S3_SIGNER),
details.get(ApiConstants.S3_HTTPS_FLAG) == null ? false : Boolean.parseBoolean(details.get(ApiConstants.S3_HTTPS_FLAG)),
details.get(ApiConstants.S3_CONNECTION_TIMEOUT) == null ? null : Integer.valueOf(details.get(ApiConstants.S3_CONNECTION_TIMEOUT)),
details.get(ApiConstants.S3_MAX_ERROR_RETRY) == null ? null : Integer.valueOf(details.get(ApiConstants.S3_MAX_ERROR_RETRY)),
@@ -74,27 +76,29 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
try {
return Long.parseLong(_configDao.getValue(Config.S3MaxSingleUploadSize.toString())) * 1024L * 1024L * 1024L;
} catch (NumberFormatException e) {
- // use default 5GB
- return 5L * 1024L * 1024L * 1024L;
+ // use default 1TB
+ return 1024L * 1024L * 1024L * 1024L;
}
}
@Override
- public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject) {
- // for S3, no need to do anything, just return template url for
- // extract template. but we need to set object acl as public_read to
- // make the url accessible
+ public String createEntityExtractUrl(DataStore store, String key, ImageFormat format, DataObject dataObject) {
+ /**
+ * Generate a pre-signed URL for the given object.
+ */
S3TO s3 = (S3TO)getStoreTO(store);
- String key = installPath;
- s_logger.info("Generating pre-signed s3 entity extraction URL.");
+ if(s_logger.isDebugEnabled()) {
+ s_logger.debug("Generating pre-signed s3 entity extraction URL for object: " + key);
+ }
Date expiration = new Date();
long milliSeconds = expiration.getTime();
- // get extract url expiration interval set in global configuration (in seconds)
+ // Get extract url expiration interval set in global configuration (in seconds)
String urlExpirationInterval = _configDao.getValue(Config.ExtractURLExpirationInterval.toString());
- int expirationInterval = NumbersUtil.parseInt(urlExpirationInterval, 14400);
- milliSeconds += 1000 * expirationInterval; // expired after configured interval (in milliseconds)
+
+ // Expired after configured interval (in milliseconds), default 14400 seconds
+ milliSeconds += 1000 * NumbersUtil.parseInt(urlExpirationInterval, 14400);
expiration.setTime(milliSeconds);
URL s3url = S3Utils.generatePresignedUrl(s3, s3.getBucketName(), key, expiration);
@@ -103,5 +107,4 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
return s3url.toString();
}
-
}
diff --git a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java
index 718c591b7fb..062fb70ae63 100644
--- a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java
+++ b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java
@@ -71,7 +71,6 @@ public class S3ImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
@Override
public DataStore initialize(Map dsInfos) {
- Long dcId = (Long)dsInfos.get("zoneId");
String url = (String)dsInfos.get("url");
String name = (String)dsInfos.get("name");
String providerName = (String)dsInfos.get("providerName");
@@ -79,11 +78,10 @@ public class S3ImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
DataStoreRole role = (DataStoreRole)dsInfos.get("role");
Map details = (Map)dsInfos.get("details");
- s_logger.info("Trying to add a S3 store in data center " + dcId);
+ s_logger.info("Trying to add a S3 store with endpoint: " + details.get(ApiConstants.S3_END_POINT));
- Map imageStoreParameters = new HashMap();
+ Map imageStoreParameters = new HashMap();
imageStoreParameters.put("name", name);
- imageStoreParameters.put("zoneId", dcId);
imageStoreParameters.put("url", url);
String protocol = "http";
String useHttps = details.get(ApiConstants.S3_HTTPS_FLAG);
diff --git a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/provider/S3ImageStoreProviderImpl.java b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/provider/S3ImageStoreProviderImpl.java
index fff55a06f04..18947949831 100644
--- a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/provider/S3ImageStoreProviderImpl.java
+++ b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/provider/S3ImageStoreProviderImpl.java
@@ -44,14 +44,15 @@ import com.cloud.utils.component.ComponentContext;
@Component
public class S3ImageStoreProviderImpl implements ImageStoreProvider {
- private final String providerName = DataStoreProvider.S3_IMAGE;
- protected ImageStoreLifeCycle lifeCycle;
- protected ImageStoreDriver driver;
@Inject
ImageStoreProviderManager storeMgr;
@Inject
ImageStoreHelper helper;
+ private final String providerName = DataStoreProvider.S3_IMAGE;
+ protected ImageStoreLifeCycle lifeCycle;
+ protected ImageStoreDriver driver;
+
@Override
public DataStoreLifeCycle getDataStoreLifeCycle() {
return lifeCycle;
diff --git a/pom.xml b/pom.xml
index d9a131c2a7f..9fce1fc1f65 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,7 +96,8 @@
3.2.12.RELEASE
1.9.5
1.5.3
- 1.3.22
+ 1.10.34
+ 2.6.3
2.6
3.4
2.4
diff --git a/scripts/vm/hypervisor/xenserver/s3xenserver b/scripts/vm/hypervisor/xenserver/s3xenserver
index d0cea6c693b..7a05e0559df 100644
--- a/scripts/vm/hypervisor/xenserver/s3xenserver
+++ b/scripts/vm/hypervisor/xenserver/s3xenserver
@@ -377,7 +377,7 @@ class S3Client(object):
def parseArguments(args):
# The keys in the args map will correspond to the properties defined on
- # the com.cloud.utils.S3Utils#ClientOptions interface
+ # the com.cloud.utils.storage.S3.S3Utils#ClientOptions interface
client = S3Client(
args['accessKey'], args['secretKey'], args['endPoint'],
args['https'], args['connectionTimeout'], args['socketTimeout'])
diff --git a/server/conf/log4j-cloud.xml.in b/server/conf/log4j-cloud.xml.in
index 3b4bff106d1..9dc81e57f63 100755
--- a/server/conf/log4j-cloud.xml.in
+++ b/server/conf/log4j-cloud.xml.in
@@ -109,6 +109,16 @@ under the License.
+
+
+
+
+
+
+
+
+
+
diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java
index 25071a2b28f..a4e14f1aa09 100644
--- a/server/src/com/cloud/server/ManagementServerImpl.java
+++ b/server/src/com/cloud/server/ManagementServerImpl.java
@@ -162,7 +162,7 @@ import org.apache.cloudstack.api.command.admin.router.StopRouterCmd;
import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd;
-import org.apache.cloudstack.api.command.admin.storage.AddS3Cmd;
+import org.apache.cloudstack.api.command.admin.storage.AddImageStoreS3CMD;
import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd;
@@ -171,7 +171,6 @@ import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.FindStoragePoolsForMigrationCmd;
import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd;
-import org.apache.cloudstack.api.command.admin.storage.ListS3sCmd;
import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd;
import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd;
import org.apache.cloudstack.api.command.admin.storage.ListStorageProvidersCmd;
@@ -2640,12 +2639,10 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
cmdList.add(StartRouterCmd.class);
cmdList.add(StopRouterCmd.class);
cmdList.add(UpgradeRouterCmd.class);
- cmdList.add(AddS3Cmd.class);
cmdList.add(AddSwiftCmd.class);
cmdList.add(CancelPrimaryStorageMaintenanceCmd.class);
cmdList.add(CreateStoragePoolCmd.class);
cmdList.add(DeletePoolCmd.class);
- cmdList.add(ListS3sCmd.class);
cmdList.add(ListSwiftsCmd.class);
cmdList.add(ListStoragePoolsCmd.class);
cmdList.add(ListStorageTagsCmd.class);
@@ -2911,6 +2908,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
cmdList.add(RemoveFromGlobalLoadBalancerRuleCmd.class);
cmdList.add(ListStorageProvidersCmd.class);
cmdList.add(AddImageStoreCmd.class);
+ cmdList.add(AddImageStoreS3CMD.class);
cmdList.add(ListImageStoresCmd.class);
cmdList.add(DeleteImageStoreCmd.class);
cmdList.add(CreateSecondaryStagingStoreCmd.class);
diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java
index ba39e1f0fa8..41f00197ad2 100644
--- a/server/src/com/cloud/storage/StorageManagerImpl.java
+++ b/server/src/com/cloud/storage/StorageManagerImpl.java
@@ -1844,7 +1844,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
@Override
- public ImageStore discoverImageStore(String name, String url, String providerName, Long dcId, Map details) throws IllegalArgumentException, DiscoveryException,
+ public ImageStore discoverImageStore(String name, String url, String providerName, Long zoneId, Map details) throws IllegalArgumentException, DiscoveryException,
InvalidParameterValueException {
DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName);
@@ -1857,13 +1857,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
ScopeType scopeType = ScopeType.ZONE;
- if (dcId == null) {
+ if (zoneId == null) {
scopeType = ScopeType.REGION;
}
if (name == null) {
name = url;
}
+
ImageStoreVO imageStore = _imageStoreDao.findByName(name);
if (imageStore != null) {
throw new InvalidParameterValueException("The image store with name " + name + " already exists, try creating with another name");
@@ -1884,11 +1885,11 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
}
- if (dcId != null) {
+ if (zoneId != null) {
// Check if the zone exists in the system
- DataCenterVO zone = _dcDao.findById(dcId);
+ DataCenterVO zone = _dcDao.findById(zoneId);
if (zone == null) {
- throw new InvalidParameterValueException("Can't find zone by id " + dcId);
+ throw new InvalidParameterValueException("Can't find zone by id " + zoneId);
}
Account account = CallContext.current().getCallingAccount();
@@ -1901,8 +1902,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
}
- Map params = new HashMap();
- params.put("zoneId", dcId);
+ Map params = new HashMap();
+ params.put("zoneId", zoneId);
params.put("url", url);
params.put("name", name);
params.put("details", details);
@@ -1911,11 +1912,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
params.put("role", DataStoreRole.Image);
DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle();
+
DataStore store;
try {
store = lifeCycle.initialize(params);
} catch (Exception e) {
- s_logger.debug("Failed to add data store: " + e.getMessage(), e);
+ if(s_logger.isDebugEnabled()) {
+ s_logger.debug("Failed to add data store: " + e.getMessage(), e);
+ }
throw new CloudRuntimeException("Failed to add data store: " + e.getMessage(), e);
}
@@ -1927,9 +1931,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
_imageSrv.addSystemVMTemplatesToSecondary(store);
}
- // associate builtin template with zones associated with this image
- // store
- associateCrosszoneTemplatesToZone(dcId);
+ // associate builtin template with zones associated with this image store
+ associateCrosszoneTemplatesToZone(zoneId);
// duplicate cache store records to region wide storage
if (scopeType == ScopeType.REGION) {
diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
index f1937f85232..65471dd624c 100644
--- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
+++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
@@ -49,7 +49,7 @@ import org.apache.cloudstack.storage.to.VolumeObjectTO;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.storage.DownloadAnswer;
-import com.cloud.agent.api.storage.Proxy;
+import com.cloud.utils.net.Proxy;
import com.cloud.configuration.Config;
import com.cloud.storage.RegisterVolumePayload;
import com.cloud.storage.Storage.ImageFormat;
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index be5969104fd..b5c15767d6b 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@ -16,8 +16,7 @@
// under the License.
package org.apache.cloudstack.storage.resource;
-import static com.cloud.utils.S3Utils.mputFile;
-import static com.cloud.utils.S3Utils.putFile;
+import static com.cloud.utils.storage.S3.S3Utils.putFile;
import static com.cloud.utils.StringUtils.join;
import static java.lang.String.format;
import static java.util.Arrays.asList;
@@ -155,8 +154,7 @@ import com.cloud.storage.template.TemplateProp;
import com.cloud.storage.template.VhdProcessor;
import com.cloud.storage.template.VmdkProcessor;
import com.cloud.utils.NumbersUtil;
-import com.cloud.utils.S3Utils;
-import com.cloud.utils.S3Utils.FileNamingStrategy;
+import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.utils.SwiftUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.net.NetUtils;
@@ -386,12 +384,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
}
- File destFile = S3Utils.getFile(s3, s3.getBucketName(), srcData.getPath(), downloadDirectory, new FileNamingStrategy() {
- @Override
- public String determineFileName(final String key) {
- return substringAfterLast(key, S3Utils.SEPARATOR);
- }
- });
+ File destFile = new File(downloadDirectory, substringAfterLast(srcData.getPath(), S3Utils.SEPARATOR));
+
+ S3Utils.getFile(s3, s3.getBucketName(), srcData.getPath(), destFile).waitForCompletion();
+
if (destFile == null) {
return new CopyCmdAnswer("Can't find template");
@@ -400,7 +396,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return postProcessing(destFile, downloadPath, destPath, srcData, destData);
} catch (Exception e) {
- final String errMsg = format("Failed to download" + "due to $2%s", e.getMessage());
+ final String errMsg = format("Failed to download" + "due to $1%s", e.getMessage());
s_logger.error(errMsg, e);
return new CopyCmdAnswer(errMsg);
}
@@ -907,14 +903,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
}
- long srcSize = srcFile.length();
ImageFormat format = getTemplateFormat(srcFile.getName());
String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName();
- if (!s3.getSingleUpload(srcSize)) {
- mputFile(s3, srcFile, bucket, key);
- } else {
- putFile(s3, srcFile, bucket, key);
- }
+
+ putFile(s3, srcFile, bucket, key).waitForCompletion();
DataTO retObj = null;
if (destData.getObjectType() == DataObjectType.TEMPLATE) {
@@ -1509,7 +1501,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
Map s3ListTemplate(S3TO s3) {
String bucket = s3.getBucketName();
// List the objects in the source directory on S3
- final List objectSummaries = S3Utils.getDirectory(s3, bucket, TEMPLATE_ROOT_DIR);
+ final List objectSummaries = S3Utils.listDirectory(s3, bucket, TEMPLATE_ROOT_DIR);
if (objectSummaries == null) {
return null;
}
@@ -1530,7 +1522,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
Map s3ListVolume(S3TO s3) {
String bucket = s3.getBucketName();
// List the objects in the source directory on S3
- final List objectSummaries = S3Utils.getDirectory(s3, bucket, VOLUME_ROOT_DIR);
+ final List objectSummaries = S3Utils.listDirectory(s3, bucket, VOLUME_ROOT_DIR);
if (objectSummaries == null) {
return null;
}
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResource.java
index 93fd8eae201..4d3f04819e8 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResource.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResource.java
@@ -19,11 +19,10 @@ package org.apache.cloudstack.storage.resource;
import com.cloud.resource.ServerResource;
/**
- *
* SecondaryStorageServerResource is a generic container to execute commands sent
*/
public interface SecondaryStorageResource extends ServerResource {
- public String getRootDir(String cmd);
+ String getRootDir(String cmd);
}
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResourceHandler.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResourceHandler.java
index 14ebc7101e8..5c97e48e5e9 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResourceHandler.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResourceHandler.java
@@ -20,5 +20,7 @@ import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command;
public interface SecondaryStorageResourceHandler {
+
Answer executeRequest(Command cmd);
+
}
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
index 0b6d47d351b..78190af0598 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
@@ -24,7 +24,7 @@ import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
import com.cloud.agent.api.storage.DownloadAnswer;
-import com.cloud.agent.api.storage.Proxy;
+import com.cloud.utils.net.Proxy;
import com.cloud.agent.api.to.S3TO;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.VMTemplateHostVO;
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
index 431a20425a2..f1706e22966 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -49,7 +49,7 @@ import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
import org.apache.log4j.Logger;
import com.cloud.agent.api.storage.DownloadAnswer;
-import com.cloud.agent.api.storage.Proxy;
+import com.cloud.utils.net.Proxy;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
diff --git a/systemvm/conf.dom0/log4j-cloud.xml.in b/systemvm/conf.dom0/log4j-cloud.xml.in
index dde844fabe4..bc9f35e1286 100644
--- a/systemvm/conf.dom0/log4j-cloud.xml.in
+++ b/systemvm/conf.dom0/log4j-cloud.xml.in
@@ -88,6 +88,16 @@ under the License.
+
+
+
+
+
+
+
+
+
+
diff --git a/systemvm/conf/log4j-cloud.xml b/systemvm/conf/log4j-cloud.xml
index 2d1d361c939..9c26bf4dd7c 100644
--- a/systemvm/conf/log4j-cloud.xml
+++ b/systemvm/conf/log4j-cloud.xml
@@ -89,6 +89,16 @@ under the License.
+
+
+
+
+
+
+
+
+
+
diff --git a/utils/pom.xml b/utils/pom.xml
index f52014aff2a..5502c58d0d2 100755
--- a/utils/pom.xml
+++ b/utils/pom.xml
@@ -168,6 +168,11 @@
guava-testlib
${cs.guava-testlib.version}
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ ${cs.jackson.version}
+
src/main/java
diff --git a/utils/src/main/java/com/cloud/utils/S3Utils.java b/utils/src/main/java/com/cloud/utils/S3Utils.java
deleted file mode 100644
index c07db332337..00000000000
--- a/utils/src/main/java/com/cloud/utils/S3Utils.java
+++ /dev/null
@@ -1,619 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-package com.cloud.utils;
-
-import static com.amazonaws.Protocol.HTTP;
-import static com.amazonaws.Protocol.HTTPS;
-import static com.cloud.utils.StringUtils.join;
-import static java.io.File.createTempFile;
-import static java.lang.String.format;
-import static java.lang.System.currentTimeMillis;
-import static java.util.Collections.emptyList;
-import static java.util.Collections.singletonList;
-import static java.util.Collections.unmodifiableList;
-import static org.apache.commons.lang.ArrayUtils.isEmpty;
-import static org.apache.commons.lang.StringUtils.isBlank;
-import static org.apache.commons.lang.StringUtils.isNotBlank;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.log4j.Logger;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.HttpMethod;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.Bucket;
-import com.amazonaws.services.s3.model.CannedAccessControlList;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.Upload;
-import com.cloud.utils.exception.CloudRuntimeException;
-
-public final class S3Utils {
-
- private static final Logger LOGGER = Logger.getLogger(S3Utils.class);
-
- public static final String SEPARATOR = "/";
-
- private static final int MIN_BUCKET_NAME_LENGTH = 3;
- private static final int MAX_BUCKET_NAME_LENGTH = 63;
-
- private S3Utils() {
- super();
- }
-
- public static AmazonS3 acquireClient(final ClientOptions clientOptions) {
-
- final AWSCredentials credentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());
-
- final ClientConfiguration configuration = new ClientConfiguration();
-
- if (clientOptions.isHttps() != null) {
- configuration.setProtocol(clientOptions.isHttps() == true ? HTTPS : HTTP);
- }
-
- if (clientOptions.getConnectionTimeout() != null) {
- configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
- }
-
- if (clientOptions.getMaxErrorRetry() != null) {
- configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
- }
-
- if (clientOptions.getSocketTimeout() != null) {
- configuration.setSocketTimeout(clientOptions.getSocketTimeout());
- }
-
- if (clientOptions.getUseTCPKeepAlive() != null) {
- //configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
- LOGGER.debug("useTCPKeepAlive not supported by old AWS SDK");
- }
-
- if (clientOptions.getConnectionTtl() != null) {
- //configuration.setConnectionTTL(clientOptions.getConnectionTtl());
- LOGGER.debug("connectionTtl not supported by old AWS SDK");
- }
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, connectionTimeOut: " + "%2$s, maxErrorRetry: %3$s, socketTimeout: %4$s, useTCPKeepAlive: %5$s, connectionTtl: %6$s]",
- configuration.getProtocol(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(),
- -1, -1));
- }
-
- final AmazonS3Client client = new AmazonS3Client(credentials, configuration);
-
- if (isNotBlank(clientOptions.getEndPoint())) {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Setting the end point for S3 client %1$s to %2$s.", client, clientOptions.getEndPoint()));
- }
- client.setEndpoint(clientOptions.getEndPoint());
- }
-
- return client;
-
- }
-
- public static void putFile(final ClientOptions clientOptions, final File sourceFile, final String bucketName, final String key) {
-
- assert clientOptions != null;
- assert sourceFile != null;
- assert !isBlank(bucketName);
- assert !isBlank(key);
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Sending file %1$s as S3 object %2$s in " + "bucket %3$s", sourceFile.getName(), key, bucketName));
- }
-
- acquireClient(clientOptions).putObject(bucketName, key, sourceFile);
-
- }
-
- public static void putObject(final ClientOptions clientOptions, final InputStream sourceStream, final String bucketName, final String key) {
-
- assert clientOptions != null;
- assert sourceStream != null;
- assert !isBlank(bucketName);
- assert !isBlank(key);
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Sending stream as S3 object %1$s in " + "bucket %2$s", key, bucketName));
- }
-
- acquireClient(clientOptions).putObject(bucketName, key, sourceStream, null);
-
- }
-
- public static void putObject(final ClientOptions clientOptions, final PutObjectRequest req) {
-
- assert clientOptions != null;
- assert req != null;
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Sending stream as S3 object using PutObjectRequest"));
- }
-
- acquireClient(clientOptions).putObject(req);
-
- }
-
- // multi-part upload file
- public static void mputFile(final ClientOptions clientOptions, final File sourceFile, final String bucketName, final String key) throws InterruptedException {
-
- assert clientOptions != null;
- assert sourceFile != null;
- assert !isBlank(bucketName);
- assert !isBlank(key);
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Multipart sending file %1$s as S3 object %2$s in " + "bucket %3$s", sourceFile.getName(), key, bucketName));
- }
- TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
- Upload upload = tm.upload(bucketName, key, sourceFile);
- upload.waitForCompletion();
- }
-
- // multi-part upload object
- public static void mputObject(final ClientOptions clientOptions, final InputStream sourceStream, final String bucketName, final String key)
- throws InterruptedException {
-
- assert clientOptions != null;
- assert sourceStream != null;
- assert !isBlank(bucketName);
- assert !isBlank(key);
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Multipart sending stream as S3 object %1$s in " + "bucket %2$s", key, bucketName));
- }
- TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
- Upload upload = tm.upload(bucketName, key, sourceStream, null);
- upload.waitForCompletion();
- }
-
- // multi-part upload object
- public static void mputObject(final ClientOptions clientOptions, final PutObjectRequest req) throws InterruptedException {
-
- assert clientOptions != null;
- assert req != null;
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Multipart sending object to S3 using PutObjectRequest");
- }
- TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
- Upload upload = tm.upload(req);
- upload.waitForCompletion();
-
- }
-
- public static void setObjectAcl(final ClientOptions clientOptions, final String bucketName, final String key, final CannedAccessControlList acl) {
-
- assert clientOptions != null;
- assert acl != null;
-
- acquireClient(clientOptions).setObjectAcl(bucketName, key, acl);
-
- }
-
- public static URL generatePresignedUrl(final ClientOptions clientOptions, final String bucketName, final String key, final Date expiration) {
-
- assert clientOptions != null;
- assert !isBlank(bucketName);
- assert !isBlank(key);
-
- return acquireClient(clientOptions).generatePresignedUrl(bucketName, key, expiration, HttpMethod.GET);
-
- }
-
- // Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak.
- public static S3Object getObject(final ClientOptions clientOptions, final String bucketName, final String key) {
-
- assert clientOptions != null;
- assert !isBlank(bucketName);
- assert !isBlank(key);
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Get S3 object %1$s in " + "bucket %2$s", key, bucketName));
- }
-
- return acquireClient(clientOptions).getObject(bucketName, key);
-
- }
-
- // Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak.
- public static S3ObjectInputStream getObjectStream(final ClientOptions clientOptions, final String bucketName, final String key) {
-
- assert clientOptions != null;
- assert !isBlank(bucketName);
- assert !isBlank(key);
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Get S3 object %1$s in " + "bucket %2$s", key, bucketName));
- }
-
- return acquireClient(clientOptions).getObject(bucketName, key).getObjectContent();
-
- }
-
- @SuppressWarnings("unchecked")
- public static File getFile(final ClientOptions clientOptions, final String bucketName, final String key, final File targetDirectory,
- final FileNamingStrategy namingStrategy) {
-
- assert clientOptions != null;
- assert isNotBlank(bucketName);
- assert isNotBlank(key);
- assert targetDirectory != null && targetDirectory.isDirectory();
- assert namingStrategy != null;
-
- final AmazonS3 connection = acquireClient(clientOptions);
-
- File tempFile = null;
- try {
-
- tempFile = createTempFile(join("-", targetDirectory.getName(), currentTimeMillis(), "part"), "tmp", targetDirectory);
- tempFile.deleteOnExit();
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Downloading object %1$s from bucket %2$s to temp file %3$s", key, bucketName, tempFile.getName()));
- }
-
- try {
- connection.getObject(new GetObjectRequest(bucketName, key), tempFile);
- } catch (AmazonClientException ex) {
- // hack to handle different ETAG format generated from RiakCS for multi-part uploaded object
- String msg = ex.getMessage();
- if (!msg.contains("verify integrity")) {
- throw ex;
- }
- }
-
- final File targetFile = new File(targetDirectory, namingStrategy.determineFileName(key));
- tempFile.renameTo(targetFile);
-
- return targetFile;
-
- } catch (FileNotFoundException e) {
-
- throw new CloudRuntimeException(format("Failed open file %1$s in order to get object %2$s from bucket %3$s.", targetDirectory.getAbsoluteFile(), bucketName,
- key), e);
-
- } catch (IOException e) {
-
- throw new CloudRuntimeException(format("Unable to allocate temporary file in directory %1$s to download %2$s:%3$s from S3",
- targetDirectory.getAbsolutePath(), bucketName, key), e);
-
- } finally {
-
- if (tempFile != null) {
- tempFile.delete();
- }
-
- }
-
- }
-
- public static List getDirectory(final ClientOptions clientOptions, final String bucketName, final String sourcePath, final File targetDirectory,
- final FileNamingStrategy namingStrategy) {
-
- assert clientOptions != null;
- assert isNotBlank(bucketName);
- assert isNotBlank(sourcePath);
- assert targetDirectory != null;
-
- final AmazonS3 connection = acquireClient(clientOptions);
-
- // List the objects in the source directory on S3
- final List objectSummaries = listDirectory(bucketName, sourcePath, connection);
- final List files = new ArrayList();
-
- for (final S3ObjectSummary objectSummary : objectSummaries) {
-
- files.add(getFile(clientOptions, bucketName, objectSummary.getKey(), targetDirectory, namingStrategy));
-
- }
-
- return unmodifiableList(files);
-
- }
-
- public static List getDirectory(final ClientOptions clientOptions, final String bucketName, final String sourcePath) {
- assert clientOptions != null;
- assert isNotBlank(bucketName);
- assert isNotBlank(sourcePath);
-
- final AmazonS3 connection = acquireClient(clientOptions);
-
- // List the objects in the source directory on S3
- return listDirectory(bucketName, sourcePath, connection);
- }
-
- private static List listDirectory(final String bucketName, final String directory, final AmazonS3 client) {
-
- List objects = new ArrayList();
- ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(directory + SEPARATOR);
-
- ObjectListing ol = client.listObjects(listObjectsRequest);
- if(ol.isTruncated()) {
- do {
- objects.addAll(ol.getObjectSummaries());
- listObjectsRequest.setMarker(ol.getNextMarker());
- ol = client.listObjects(listObjectsRequest);
- } while (ol.isTruncated());
- }
- else {
- objects.addAll(ol.getObjectSummaries());
- }
-
- if (objects.isEmpty()) {
- return emptyList();
- }
-
- return unmodifiableList(objects);
- }
-
- public static void putDirectory(final ClientOptions clientOptions, final String bucketName, final File directory, final FilenameFilter fileNameFilter,
- final ObjectNamingStrategy namingStrategy) {
-
- assert clientOptions != null;
- assert isNotBlank(bucketName);
- assert directory != null && directory.isDirectory();
- assert fileNameFilter != null;
- assert namingStrategy != null;
-
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Putting directory %1$s in S3 bucket %2$s.", directory.getAbsolutePath(), bucketName));
- }
-
- // Determine the list of files to be sent using the passed filter ...
- final File[] files = directory.listFiles(fileNameFilter);
-
- if (LOGGER.isTraceEnabled()) {
- LOGGER.trace(format("Putting files (%1$s) in S3 bucket %2$s.", ArrayUtils.toString(files, "no files found"), bucketName));
- }
-
- // Skip spinning up an S3 connection when no files will be sent ...
- if (isEmpty(files)) {
- return;
- }
-
- final AmazonS3 client = acquireClient(clientOptions);
-
- // Send the files to S3 using the passed ObjectNaming strategy to
- // determine the key ...
- for (final File file : files) {
- final String key = namingStrategy.determineKey(file);
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(format("Putting file %1$s into bucket %2$s with key %3$s.", file.getAbsolutePath(), bucketName, key));
- }
- client.putObject(bucketName, key, file);
- }
-
- }
-
- public static void deleteObject(final ClientOptions clientOptions, final String bucketName, final String key) {
-
- assert clientOptions != null;
- assert isNotBlank(bucketName);
- assert isNotBlank(key);
-
- final AmazonS3 client = acquireClient(clientOptions);
-
- client.deleteObject(bucketName, key);
-
- }
-
- public static void deleteDirectory(final ClientOptions clientOptions, final String bucketName, final String directoryName) {
-
- assert clientOptions != null;
- assert isNotBlank(bucketName);
- assert isNotBlank(directoryName);
-
- final AmazonS3 client = acquireClient(clientOptions);
-
- final List objects = listDirectory(bucketName, directoryName, client);
-
- for (final S3ObjectSummary object : objects) {
-
- client.deleteObject(bucketName, object.getKey());
-
- }
-
- client.deleteObject(bucketName, directoryName);
-
- }
-
- public static boolean canConnect(final ClientOptions clientOptions) {
-
- try {
-
- acquireClient(clientOptions);
- return true;
-
- } catch (AmazonClientException e) {
-
- LOGGER.warn("Ignored Exception while checking connection options", e);
- return false;
-
- }
-
- }
-
- public static boolean doesBucketExist(final ClientOptions clientOptions, final String bucketName) {
-
- assert clientOptions != null;
- assert !isBlank(bucketName);
-
- try {
-
- final List buckets = acquireClient(clientOptions).listBuckets();
-
- for (Bucket bucket : buckets) {
- if (bucket.getName().equals(bucketName)) {
- return true;
- }
- }
-
- return false;
-
- } catch (AmazonClientException e) {
-
- LOGGER.warn("Ignored Exception while checking bucket existence", e);
- return false;
-
- }
-
- }
-
- public static boolean canReadWriteBucket(final ClientOptions clientOptions, final String bucketName) {
-
- assert clientOptions != null;
- assert isNotBlank(bucketName);
-
- try {
-
- final AmazonS3 client = acquireClient(clientOptions);
-
- final String fileContent = "testing put and delete";
- final InputStream inputStream = new ByteArrayInputStream(fileContent.getBytes());
- final String key = UUID.randomUUID().toString() + ".txt";
-
- final ObjectMetadata metadata = new ObjectMetadata();
- metadata.setContentLength(fileContent.length());
-
- client.putObject(bucketName, key, inputStream, metadata);
- client.deleteObject(bucketName, key);
-
- return true;
-
- } catch (AmazonClientException e) {
-
- return false;
-
- }
-
- }
-
- public static List checkClientOptions(ClientOptions clientOptions) {
-
- assert clientOptions != null;
-
- List errorMessages = new ArrayList();
-
- errorMessages.addAll(checkRequiredField("access key", clientOptions.getAccessKey()));
- errorMessages.addAll(checkRequiredField("secret key", clientOptions.getSecretKey()));
-
- errorMessages.addAll(checkOptionalField("connection timeout", clientOptions.getConnectionTimeout()));
- errorMessages.addAll(checkOptionalField("socket timeout", clientOptions.getSocketTimeout()));
- errorMessages.addAll(checkOptionalField("max error retries", clientOptions.getMaxErrorRetry()));
- errorMessages.addAll(checkOptionalField("connection ttl", clientOptions.getConnectionTtl()));
-
- return unmodifiableList(errorMessages);
-
- }
-
- public static List checkBucketName(final String bucketLabel, final String bucket) {
-
- assert isNotBlank(bucketLabel);
- assert isNotBlank(bucket);
-
- final List errorMessages = new ArrayList();
-
- if (bucket.length() < MIN_BUCKET_NAME_LENGTH) {
- errorMessages.add(format("The length of %1$s " + "for the %2$s must have a length of at least %3$s " + "characters", bucket, bucketLabel,
- MIN_BUCKET_NAME_LENGTH));
- }
-
- if (bucket.length() > MAX_BUCKET_NAME_LENGTH) {
- errorMessages.add(format("The length of %1$s " + "for the %2$s must not have a length of at greater" + " than %3$s characters", bucket, bucketLabel,
- MAX_BUCKET_NAME_LENGTH));
- }
-
- return unmodifiableList(errorMessages);
-
- }
-
- private static List checkOptionalField(final String fieldName, final Integer fieldValue) {
- if (fieldValue != null && fieldValue < 0) {
- return singletonList(format("The value of %1$s must " + "be greater than zero.", fieldName));
- }
- return emptyList();
- }
-
- private static List checkRequiredField(String fieldName, String fieldValue) {
- if (isBlank(fieldValue)) {
- return singletonList(format("A %1$s must be specified.", fieldName));
- }
- return emptyList();
- }
-
- public interface ClientOptions {
-
- String getAccessKey();
-
- String getSecretKey();
-
- String getEndPoint();
-
- Boolean isHttps();
-
- Integer getConnectionTimeout();
-
- Integer getMaxErrorRetry();
-
- Integer getSocketTimeout();
-
- Boolean getUseTCPKeepAlive();
-
- Integer getConnectionTtl();
- }
-
- public interface ObjectNamingStrategy {
-
- String determineKey(File file);
-
- }
-
- public interface FileNamingStrategy {
-
- String determineFileName(String key);
-
- }
-
-}
diff --git a/utils/src/main/java/com/cloud/utils/net/HTTPUtils.java b/utils/src/main/java/com/cloud/utils/net/HTTPUtils.java
new file mode 100644
index 00000000000..95aee6e5aff
--- /dev/null
+++ b/utils/src/main/java/com/cloud/utils/net/HTTPUtils.java
@@ -0,0 +1,143 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils.net;
+
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.HttpMethodRetryHandler;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.NoHttpResponseException;
+import org.apache.commons.httpclient.UsernamePasswordCredentials;
+import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+
+public final class HTTPUtils {
+
+ private static final Logger LOGGER = Logger.getLogger(HTTPUtils.class);
+
+ // The connection manager.
+ private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
+
+ private HTTPUtils() {}
+
+ public static HttpClient getHTTPClient() {
+ return new HttpClient(s_httpClientManager);
+ }
+
+ /**
+ * @return A HttpMethodRetryHandler with given number of retries.
+ */
+ public static HttpMethodRetryHandler getHttpMethodRetryHandler(final int retryCount) {
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Initializing new HttpMethodRetryHandler with retry count " + retryCount);
+ }
+
+ return new HttpMethodRetryHandler() {
+ @Override
+ public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
+ if (executionCount >= retryCount) {
+ // Do not retry if over max retry count
+ return false;
+ }
+ if (exception instanceof NoHttpResponseException) {
+ // Retry if the server dropped connection on us
+ return true;
+ }
+ if (!method.isRequestSent()) {
+ // Retry if the request has not been sent fully or
+ // if it's OK to retry methods that have been sent
+ return true;
+ }
+ // otherwise do not retry
+ return false;
+ }
+ };
+ }
+
+ /**
+ * @param proxy
+ * @param httpClient
+ */
+ public static void setProxy(Proxy proxy, HttpClient httpClient) {
+ if (proxy != null && httpClient != null) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Setting proxy with host " + proxy.getHost() + " and port " + proxy.getPort() + " for host " + httpClient.getHostConfiguration().getHost() + ":" + httpClient.getHostConfiguration().getPort());
+ }
+
+ httpClient.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
+ if (proxy.getUserName() != null && proxy.getPassword() != null) {
+ httpClient.getState().setProxyCredentials(AuthScope.ANY, new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword()));
+ }
+ }
+ }
+
+ /**
+ * @param username
+ * @param password
+ * @param httpClient
+ */
+ public static void setCredentials(String username, String password, HttpClient httpClient) {
+ if (username != null && password != null && httpClient != null) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Setting credentials with username " + username + " for host " + httpClient.getHostConfiguration().getHost() + ":" + httpClient.getHostConfiguration().getPort());
+ }
+
+ httpClient.getParams().setAuthenticationPreemptive(true);
+ httpClient.getState().setCredentials(
+ new AuthScope(httpClient.getHostConfiguration().getHost(), httpClient.getHostConfiguration().getPort(), AuthScope.ANY_REALM), new UsernamePasswordCredentials(username, password));
+ }
+ }
+
+ /**
+ * @param httpClient
+ * @param httpMethod
+ * @return
+ * Returns the HTTP Status Code or -1 if an exception occurred.
+ */
+ public static int executeMethod(HttpClient httpClient, HttpMethod httpMethod) {
+ // Execute GetMethod
+ try {
+ return httpClient.executeMethod(httpMethod);
+ } catch (IOException e) {
+ LOGGER.warn("Exception while executing HttpMethod " + httpMethod.getName() + " on URL " + httpMethod.getPath());
+ return -1;
+ }
+ }
+
+ /**
+ * @param responseCode
+ * @return
+ */
+ public static boolean verifyResponseCode(int responseCode) {
+ switch (responseCode) {
+ case HttpStatus.SC_OK:
+ case HttpStatus.SC_MOVED_PERMANENTLY:
+ case HttpStatus.SC_MOVED_TEMPORARILY:
+ return true;
+ default:
+ return false;
+
+ }
+ }
+}
diff --git a/api/src/com/cloud/agent/api/storage/Proxy.java b/utils/src/main/java/com/cloud/utils/net/Proxy.java
similarity index 96%
rename from api/src/com/cloud/agent/api/storage/Proxy.java
rename to utils/src/main/java/com/cloud/utils/net/Proxy.java
index 5adc1146bed..a4475c2cc10 100644
--- a/api/src/com/cloud/agent/api/storage/Proxy.java
+++ b/utils/src/main/java/com/cloud/utils/net/Proxy.java
@@ -14,7 +14,8 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package com.cloud.agent.api.storage;
+
+package com.cloud.utils.net;
import java.net.URI;
@@ -27,10 +28,6 @@ public class Proxy {
private String _userName;
private String _password;
- public Proxy() {
-
- }
-
public Proxy(String host, int port, String userName, String password) {
this._host = host;
this._port = port;
diff --git a/utils/src/main/java/com/cloud/utils/storage/S3/ClientOptions.java b/utils/src/main/java/com/cloud/utils/storage/S3/ClientOptions.java
new file mode 100644
index 00000000000..9c9e0aaae12
--- /dev/null
+++ b/utils/src/main/java/com/cloud/utils/storage/S3/ClientOptions.java
@@ -0,0 +1,42 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils.storage.S3;
+
+public interface ClientOptions {
+ String getAccessKey();
+
+ String getSecretKey();
+
+ String getEndPoint();
+
+ String getSigner();
+
+ Boolean isHttps();
+
+ Integer getConnectionTimeout();
+
+ Integer getMaxErrorRetry();
+
+ Integer getSocketTimeout();
+
+ Boolean getUseTCPKeepAlive();
+
+ Integer getConnectionTtl();
+}
\ No newline at end of file
diff --git a/utils/src/main/java/com/cloud/utils/storage/S3/FileNamingStrategy.java b/utils/src/main/java/com/cloud/utils/storage/S3/FileNamingStrategy.java
new file mode 100644
index 00000000000..5c80e520cbd
--- /dev/null
+++ b/utils/src/main/java/com/cloud/utils/storage/S3/FileNamingStrategy.java
@@ -0,0 +1,25 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils.storage.S3;
+
+public interface FileNamingStrategy {
+
+ String determineFileName(String key);
+}
diff --git a/utils/src/main/java/com/cloud/utils/storage/S3/ObjectNamingStrategy.java b/utils/src/main/java/com/cloud/utils/storage/S3/ObjectNamingStrategy.java
new file mode 100644
index 00000000000..04f3e87f936
--- /dev/null
+++ b/utils/src/main/java/com/cloud/utils/storage/S3/ObjectNamingStrategy.java
@@ -0,0 +1,27 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils.storage.S3;
+
+import java.io.File;
+
+public interface ObjectNamingStrategy {
+
+ String determineKey(File file);
+}
\ No newline at end of file
diff --git a/utils/src/main/java/com/cloud/utils/storage/S3/S3Utils.java b/utils/src/main/java/com/cloud/utils/storage/S3/S3Utils.java
new file mode 100644
index 00000000000..274ff9bc994
--- /dev/null
+++ b/utils/src/main/java/com/cloud/utils/storage/S3/S3Utils.java
@@ -0,0 +1,216 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils.storage.S3;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.HttpMethod;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.transfer.Download;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.Upload;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static com.amazonaws.Protocol.HTTP;
+import static com.amazonaws.Protocol.HTTPS;
+import static java.lang.String.format;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.unmodifiableList;
+import static org.apache.commons.lang.StringUtils.isNotBlank;
+
+
+public final class S3Utils {
+
+ private static final Logger LOGGER = Logger.getLogger(S3Utils.class);
+
+ public static final String SEPARATOR = "/";
+
+ private static final Map TRANSFERMANAGER_ACCESSKEY_MAP = new HashMap<>();
+
+ private S3Utils() {}
+
+ public static TransferManager getTransferManager(final ClientOptions clientOptions) {
+
+ if(TRANSFERMANAGER_ACCESSKEY_MAP.containsKey(clientOptions.getAccessKey())) {
+ return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
+ }
+
+ final AWSCredentials basicAWSCredentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());
+
+ final ClientConfiguration configuration = new ClientConfiguration();
+
+ if (clientOptions.isHttps() != null) {
+ configuration.setProtocol(clientOptions.isHttps() ? HTTPS : HTTP);
+ }
+
+ if (clientOptions.getConnectionTimeout() != null) {
+ configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
+ }
+
+ if (clientOptions.getMaxErrorRetry() != null) {
+ configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
+ }
+
+ if (clientOptions.getSocketTimeout() != null) {
+ configuration.setSocketTimeout(clientOptions.getSocketTimeout());
+ }
+
+ if (clientOptions.getUseTCPKeepAlive() != null) {
+ configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
+ }
+
+ if (clientOptions.getConnectionTtl() != null) {
+ configuration.setConnectionTTL(clientOptions.getConnectionTtl());
+ }
+
+ if (clientOptions.getSigner() != null) {
+
+ configuration.setSignerOverride(clientOptions.getSigner());
+ }
+
+ LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, signer: %2$s, connectionTimeOut: %3$s, maxErrorRetry: %4$s, socketTimeout: %5$s, useTCPKeepAlive: %6$s, connectionTtl: %7$s]",
+ configuration.getProtocol(), configuration.getSignerOverride(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(),
+ clientOptions.getUseTCPKeepAlive(), clientOptions.getConnectionTtl()));
+
+ final AmazonS3Client client = new AmazonS3Client(basicAWSCredentials, configuration);
+
+ if (isNotBlank(clientOptions.getEndPoint())) {
+ LOGGER.debug(format("Setting the end point for S3 client with access key %1$s to %2$s.", clientOptions.getAccessKey(), clientOptions.getEndPoint()));
+
+ client.setEndpoint(clientOptions.getEndPoint());
+ }
+
+ TRANSFERMANAGER_ACCESSKEY_MAP.put(clientOptions.getAccessKey(), new TransferManager(client));
+
+ return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
+ }
+
+ public static AmazonS3 getAmazonS3Client(final ClientOptions clientOptions) {
+
+ return getTransferManager(clientOptions).getAmazonS3Client();
+ }
+
+ public static Upload putFile(final ClientOptions clientOptions, final File sourceFile, final String bucketName, final String key) {
+ LOGGER.debug(format("Sending file %1$s as S3 object %2$s in bucket %3$s", sourceFile.getName(), key, bucketName));
+
+ return getTransferManager(clientOptions).upload(bucketName, key, sourceFile);
+ }
+
+ public static Upload putObject(final ClientOptions clientOptions, final InputStream sourceStream, final String bucketName, final String key) {
+ LOGGER.debug(format("Sending stream as S3 object %1$s in bucket %2$s", key, bucketName));
+
+ return getTransferManager(clientOptions).upload(bucketName, key, sourceStream, null);
+ }
+
+ public static Upload putObject(final ClientOptions clientOptions, final PutObjectRequest req) {
+ LOGGER.debug(format("Sending stream as S3 object %1$s in bucket %2$s using PutObjectRequest", req.getKey(), req.getBucketName()));
+
+ return getTransferManager(clientOptions).upload(req);
+ }
+
+ public static Download getFile(final ClientOptions clientOptions, final String bucketName, final String key, final File file) {
+ LOGGER.debug(format("Receiving object %1$s as file %2$s from bucket %3$s", key, file.getAbsolutePath(), bucketName));
+
+ return getTransferManager(clientOptions).download(bucketName, key, file);
+ }
+
+ public static Download getFile(final ClientOptions clientOptions, final GetObjectRequest getObjectRequest, final File file) {
+ LOGGER.debug(format("Receiving object %1$s as file %2$s from bucket %3$s using GetObjectRequest", getObjectRequest.getKey(), file.getAbsolutePath(), getObjectRequest.getBucketName()));
+
+ return getTransferManager(clientOptions).download(getObjectRequest, file);
+ }
+
+ public static URL generatePresignedUrl(final ClientOptions clientOptions, final String bucketName, final String key, final Date expiration) {
+ LOGGER.debug(format("Generating presigned url for key %1s in bucket %2s with expiration date %3s", key, bucketName, expiration.toString()));
+
+ return getTransferManager(clientOptions).getAmazonS3Client().generatePresignedUrl(bucketName, key, expiration, HttpMethod.GET);
+ }
+
+ // Note that whenever S3ObjectInputStream is returned, client code needs to close the internal stream to avoid resource leak.
+ public static S3ObjectInputStream getObjectStream(final ClientOptions clientOptions, final String bucketName, final String key) {
+ LOGGER.debug(format("Get S3ObjectInputStream from S3 Object %1$s in bucket %2$s", key, bucketName));
+
+ return getTransferManager(clientOptions).getAmazonS3Client().getObject(bucketName, key).getObjectContent();
+ }
+
+ public static List listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) {
+ LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName));
+
+ List objects = new ArrayList<>();
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
+
+ listObjectsRequest.withBucketName(bucketName);
+ listObjectsRequest.withPrefix(directory);
+
+ ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
+ if(ol.isTruncated()) {
+ do {
+ objects.addAll(ol.getObjectSummaries());
+ listObjectsRequest.setMarker(ol.getNextMarker());
+ ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
+ } while (ol.isTruncated());
+ }
+ else {
+ objects.addAll(ol.getObjectSummaries());
+ }
+
+ if (objects.isEmpty()) {
+ return emptyList();
+ }
+
+ return unmodifiableList(objects);
+ }
+
+ public static void deleteObject(final ClientOptions clientOptions, final String bucketName, final String key) {
+ LOGGER.debug(format("Deleting S3 Object %1$s in bucket %2$s", key, bucketName));
+
+ getAmazonS3Client(clientOptions).deleteObject(bucketName,key);
+ }
+
+ public static void deleteDirectory(final ClientOptions clientOptions, final String bucketName, final String directoryName) {
+ LOGGER.debug(format("Deleting S3 Directory %1$s in bucket %2$s", directoryName, bucketName));
+
+ final List objects = listDirectory(clientOptions, bucketName, directoryName);
+
+ for (final S3ObjectSummary object : objects) {
+
+ deleteObject(clientOptions, bucketName, object.getKey());
+ }
+
+ deleteObject(clientOptions, bucketName, directoryName);
+ }
+}
diff --git a/utils/src/test/java/com/cloud/utils/rest/RESTServiceConnectorTest.java b/utils/src/test/java/com/cloud/utils/rest/RESTServiceConnectorTest.java
index e26ee6acd57..63add63181a 100644
--- a/utils/src/test/java/com/cloud/utils/rest/RESTServiceConnectorTest.java
+++ b/utils/src/test/java/com/cloud/utils/rest/RESTServiceConnectorTest.java
@@ -32,6 +32,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.CollectionType;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.http.HttpEntity;
@@ -43,8 +45,6 @@ import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicStatusLine;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.type.CollectionType;
import org.junit.Test;
import com.google.gson.FieldNamingPolicy;