apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-9062: Improve S3 implementation.

Browse Source 
The S3 implementation is far from finished, this commit focusses on the bases.

 - Upgrade AWS SDK to latest version.
 - Rewrite S3 Template downloader.
 - Rewrite S3Utils utility class.
 - Improve addImageStoreS3 API command.
 - Split various classes for convenience.
 - Various minor improvements and code optimalisations.

A side effect of the new AWS SDK is that it, by default, uses the V4 signature. Therefore I added an option to specify the Signer, so it stays compatible with previous versions.
This commit is contained in:
Boris Schrijver 2015-11-13 02:19:24 +01:00
parent fe2917e91b
commit 5c0366c99e
48 changed files with 1055 additions and 1290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
agent/conf/log4j-cloud.xml.in
View File

@ -77,7 +77,17 @@ under the License.
</category>
<category name="net">
<priority value="INFO"/>
<priority value="INFO"/>
</category>
<!-- Limit the com.amazonaws category to INFO as its DEBUG is verbose -->
<category name="com.amazonaws">
<priority value="INFO"/>
</category>
<!-- Limit the httpclient.wire category to INFO as its DEBUG is verbose -->
<category name="httpclient.wire">
<priority value="INFO"/>
</category>
<!-- ======================= -->

215
api/src/com/cloud/agent/api/to/S3TO.java
View File

@ -21,9 +21,9 @@ import java.util.Date;
import com.cloud.agent.api.LogLevel;
import com.cloud.agent.api.LogLevel.Log4jLevel;
import com.cloud.storage.DataStoreRole;
import com.cloud.utils.S3Utils;
import com.cloud.utils.storage.S3.ClientOptions;
public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
public final class S3TO implements ClientOptions, DataStoreTO {
private Long id;
private String uuid;
@ -33,6 +33,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
private String secretKey;
private String endPoint;
private String bucketName;
private String signer;
private Boolean httpsFlag;
private Boolean useTCPKeepAlive;
private Integer connectionTimeout;
@ -44,17 +45,9 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
private long maxSingleUploadSizeInBytes;
private static final String pathSeparator = "/";
public S3TO() {
super();
}
public S3TO(final Long id, final String uuid, final String accessKey, final String secretKey, final String endPoint, final String bucketName,
final Boolean httpsFlag, final Integer connectionTimeout, final Integer maxErrorRetry, final Integer socketTimeout, final Date created,
final boolean enableRRS, final long maxUploadSize, final Integer connectionTtl, final Boolean useTCPKeepAlive) {
super();
final String signer, final Boolean httpsFlag, final Integer connectionTimeout, final Integer maxErrorRetry, final Integer socketTimeout,
final Date created, final boolean enableRRS, final long maxUploadSize, final Integer connectionTtl, final Boolean useTCPKeepAlive) {
this.id = id;
this.uuid = uuid;
@ -62,6 +55,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
this.secretKey = secretKey;
this.endPoint = endPoint;
this.bucketName = bucketName;
this.signer = signer;
this.httpsFlag = httpsFlag;
this.connectionTimeout = connectionTimeout;
this.maxErrorRetry = maxErrorRetry;
@ -74,98 +68,6 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
}
@Override
public boolean equals(final Object thatObject) {
if (this == thatObject) {
return true;
}
if (thatObject == null || getClass() != thatObject.getClass()) {
return false;
}
final S3TO thatS3TO = (S3TO)thatObject;
if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag) : thatS3TO.httpsFlag != null) {
return false;
}
if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey) : thatS3TO.accessKey != null) {
return false;
}
if (connectionTimeout != null ? !connectionTimeout.equals(thatS3TO.connectionTimeout) : thatS3TO.connectionTimeout != null) {
return false;
}
if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint) : thatS3TO.endPoint != null) {
return false;
}
if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) {
return false;
}
if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) {
return false;
}
if (maxErrorRetry != null ? !maxErrorRetry.equals(thatS3TO.maxErrorRetry) : thatS3TO.maxErrorRetry != null) {
return false;
}
if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey) : thatS3TO.secretKey != null) {
return false;
}
if (socketTimeout != null ? !socketTimeout.equals(thatS3TO.socketTimeout) : thatS3TO.socketTimeout != null) {
return false;
}
if (connectionTtl != null ? !connectionTtl.equals(thatS3TO.connectionTtl) : thatS3TO.connectionTtl != null) {
return false;
}
if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatS3TO.useTCPKeepAlive) : thatS3TO.useTCPKeepAlive != null) {
return false;
}
if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName) : thatS3TO.bucketName != null) {
return false;
}
if (created != null ? !created.equals(thatS3TO.created) : thatS3TO.created != null) {
return false;
}
if (enableRRS != thatS3TO.enableRRS) {
return false;
}
return true;
}
@Override
public int hashCode() {
int result = id != null ? id.hashCode() : 0;
result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0);
result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
result = 31 * result + (httpsFlag ? 1 : 0);
result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
result = 31 * result + (useTCPKeepAlive ? 1 : 0);
return result;
}
public Long getId() {
return this.id;
}
@ -223,6 +125,15 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
this.bucketName = bucketName;
}
@Override
public String getSigner() {
return this.signer;
}
public void setSigner(final String signer) {
this.signer = signer;
}
@Override
public Boolean isHttps() {
return this.httpsFlag;
@ -327,4 +238,100 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
public String getPathSeparator() {
return pathSeparator;
}
@Override
public boolean equals(final Object thatObject) {
if (this == thatObject) {
return true;
}
if (thatObject == null || getClass() != thatObject.getClass()) {
return false;
}
final S3TO thatS3TO = (S3TO)thatObject;
if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag) : thatS3TO.httpsFlag != null) {
return false;
}
if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey) : thatS3TO.accessKey != null) {
return false;
}
if (connectionTimeout != null ? !connectionTimeout.equals(thatS3TO.connectionTimeout) : thatS3TO.connectionTimeout != null) {
return false;
}
if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint) : thatS3TO.endPoint != null) {
return false;
}
if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) {
return false;
}
if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) {
return false;
}
if (maxErrorRetry != null ? !maxErrorRetry.equals(thatS3TO.maxErrorRetry) : thatS3TO.maxErrorRetry != null) {
return false;
}
if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey) : thatS3TO.secretKey != null) {
return false;
}
if (socketTimeout != null ? !socketTimeout.equals(thatS3TO.socketTimeout) : thatS3TO.socketTimeout != null) {
return false;
}
if (connectionTtl != null ? !connectionTtl.equals(thatS3TO.connectionTtl) : thatS3TO.connectionTtl != null) {
return false;
}
if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatS3TO.useTCPKeepAlive) : thatS3TO.useTCPKeepAlive != null) {
return false;
}
if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName) : thatS3TO.bucketName != null) {
return false;
}
if (signer != null ? !signer.equals(thatS3TO.signer) : thatS3TO.signer != null) {
return false;
}
if (created != null ? !created.equals(thatS3TO.created) : thatS3TO.created != null) {
return false;
}
if (enableRRS != thatS3TO.enableRRS) {
return false;
}
return true;
}
@Override
public int hashCode() {
int result = id != null ? id.hashCode() : 0;
result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0);
result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
result = 31 * result + (signer != null ? signer.hashCode() : 0);
result = 31 * result + (httpsFlag ? 1 : 0);
result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
result = 31 * result + (useTCPKeepAlive ? 1 : 0);
return result;
}
}

24
api/src/com/cloud/storage/StorageService.java
View File

@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.storage;
import java.net.UnknownHostException;
@ -38,14 +39,14 @@ public interface StorageService {
* Create StoragePool based on uri
*
* @param cmd
* the command object that specifies the zone, cluster/pod, URI, details, etc. to use to create the
* The command object that specifies the zone, cluster/pod, URI, details, etc. to use to create the
* storage pool.
* @return
* The StoragePool created.
* @throws ResourceInUseException
* @throws IllegalArgumentException
* @throws UnknownHostException
* @throws ResourceUnavailableException
* TODO
*/
StoragePool createPool(CreateStoragePoolCmd cmd) throws ResourceInUseException, IllegalArgumentException, UnknownHostException, ResourceUnavailableException;
@ -63,15 +64,13 @@ public interface StorageService {
/**
* Enable maintenance for primary storage
*
* @param cmd
* - the command specifying primaryStorageId
* @param primaryStorageId
* - the primaryStorageId
* @return the primary storage pool
* @throws ResourceUnavailableException
* TODO
* @throws InsufficientCapacityException
* TODO
*/
public StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException;
StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException;
/**
* Complete maintenance for primary storage
@ -80,19 +79,18 @@ public interface StorageService {
* - the command specifying primaryStorageId
* @return the primary storage pool
* @throws ResourceUnavailableException
* TODO
*/
public StoragePool cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException;
StoragePool cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException;
public StoragePool updateStoragePool(UpdateStoragePoolCmd cmd) throws IllegalArgumentException;
StoragePool updateStoragePool(UpdateStoragePoolCmd cmd) throws IllegalArgumentException;
public StoragePool getStoragePool(long id);
StoragePool getStoragePool(long id);
boolean deleteImageStore(DeleteImageStoreCmd cmd);
boolean deleteSecondaryStagingStore(DeleteSecondaryStagingStoreCmd cmd);
public ImageStore discoverImageStore(String name, String url, String providerName, Long dcId, Map details) throws IllegalArgumentException, DiscoveryException,
ImageStore discoverImageStore(String name, String url, String providerName, Long zoneId, Map details) throws IllegalArgumentException, DiscoveryException,
InvalidParameterValueException;
@ -107,7 +105,7 @@ public interface StorageService {
* @throws DiscoveryException
* @throws InvalidParameterValueException
*/
public ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException,
ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException,
InvalidParameterValueException;

3
api/src/org/apache/cloudstack/api/ApiConstants.java
View File

@ -501,6 +501,9 @@ public class ApiConstants {
public static final String S3_SECRET_KEY = "secretkey";
public static final String S3_END_POINT = "endpoint";
public static final String S3_BUCKET_NAME = "bucket";
public static final String S3_SIGNER = "s3signer";
public static final String S3_V3_SIGNER = "S3SignerType";
public static final String S3_V4_SIGNER = "AWSS3V4SignerType";
public static final String S3_HTTPS_FLAG = "usehttps";
public static final String S3_CONNECTION_TIMEOUT = "connectiontimeout";
public static final String S3_CONNECTION_TTL = "connectionttl";

160
api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java → api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
View File

@ -26,6 +26,7 @@ import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TTL;
import static org.apache.cloudstack.api.ApiConstants.S3_END_POINT;
import static org.apache.cloudstack.api.ApiConstants.S3_HTTPS_FLAG;
import static org.apache.cloudstack.api.ApiConstants.S3_MAX_ERROR_RETRY;
import static org.apache.cloudstack.api.ApiConstants.S3_SIGNER;
import static org.apache.cloudstack.api.ApiConstants.S3_SECRET_KEY;
import static org.apache.cloudstack.api.ApiConstants.S3_SOCKET_TIMEOUT;
import static org.apache.cloudstack.api.ApiConstants.S3_USE_TCP_KEEPALIVE;
@ -36,6 +37,7 @@ import static org.apache.cloudstack.api.BaseCmd.CommandType.STRING;
import java.util.HashMap;
import java.util.Map;
import com.cloud.utils.storage.S3.ClientOptions;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.APICommand;
@ -54,12 +56,12 @@ import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.storage.ImageStore;
@APICommand(name = "addS3", description = "Adds S3", responseObject = ImageStoreResponse.class, since = "4.0.0",
@APICommand(name = "addImageStoreS3", description = "Adds S3 Image Store", responseObject = ImageStoreResponse.class, since = "4.7.0",
requestHasSensitiveInfo = true, responseHasSensitiveInfo = false)
public final class AddS3Cmd extends BaseCmd {
public static final Logger s_logger = Logger.getLogger(AddS3Cmd.class.getName());
public final class AddImageStoreS3CMD extends BaseCmd implements ClientOptions {
public static final Logger s_logger = Logger.getLogger(AddImageStoreS3CMD.class.getName());
private static final String s_name = "adds3response";
private static final String s_name = "addImageStoreS3Response";
@Parameter(name = S3_ACCESS_KEY, type = STRING, required = true, description = "S3 access key")
private String accessKey;
@ -67,41 +69,49 @@ public final class AddS3Cmd extends BaseCmd {
@Parameter(name = S3_SECRET_KEY, type = STRING, required = true, description = "S3 secret key")
private String secretKey;
@Parameter(name = S3_END_POINT, type = STRING, required = false, description = "S3 host name")
@Parameter(name = S3_END_POINT, type = STRING, required = true, description = "S3 endpoint")
private String endPoint;
@Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, description = "name of the template storage bucket")
@Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, description = "Name of the storage bucket")
private String bucketName;
@Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, description = "connect to the S3 endpoint via HTTPS?")
@Parameter(name = S3_SIGNER, type = STRING, required = false, description = "Signer Algorithm to use, either S3SignerType or AWSS3V4SignerType")
private String signer;
@Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, description = "Use HTTPS instead of HTTP")
private Boolean httpsFlag;
@Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, description = "connection timeout (milliseconds)")
@Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, description = "Connection timeout (milliseconds)")
private Integer connectionTimeout;
@Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, description = "maximum number of times to retry on error")
@Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, description = "Maximum number of times to retry on error")
private Integer maxErrorRetry;
@Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, description = "socket timeout (milliseconds)")
@Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, description = "Socket timeout (milliseconds)")
private Integer socketTimeout;
@Parameter(name = S3_CONNECTION_TTL, type = INTEGER, required = false, description = "connection ttl (milliseconds)")
@Parameter(name = S3_CONNECTION_TTL, type = INTEGER, required = false, description = "Connection TTL (milliseconds)")
private Integer connectionTtl;
@Parameter(name = S3_USE_TCP_KEEPALIVE, type = BOOLEAN, required = false, description = "whether tcp keepalive is used")
@Parameter(name = S3_USE_TCP_KEEPALIVE, type = BOOLEAN, required = false, description = "Whether TCP keep-alive is used")
private Boolean useTCPKeepAlive;
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
ResourceAllocationException, NetworkRuleConflictException {
Map<String, String> dm = new HashMap<String, String>();
Map<String, String> dm = new HashMap();
dm.put(ApiConstants.S3_ACCESS_KEY, getAccessKey());
dm.put(ApiConstants.S3_SECRET_KEY, getSecretKey());
dm.put(ApiConstants.S3_END_POINT, getEndPoint());
dm.put(ApiConstants.S3_BUCKET_NAME, getBucketName());
if (getHttpsFlag() != null) {
dm.put(ApiConstants.S3_HTTPS_FLAG, getHttpsFlag().toString());
if (getSigner() != null && (getSigner().equals(ApiConstants.S3_V3_SIGNER) || getSigner().equals(ApiConstants.S3_V4_SIGNER))) {
dm.put(ApiConstants.S3_SIGNER, getSigner());
}
if (isHttps() != null) {
dm.put(ApiConstants.S3_HTTPS_FLAG, isHttps().toString());
}
if (getConnectionTimeout() != null) {
dm.put(ApiConstants.S3_CONNECTION_TIMEOUT, getConnectionTimeout().toString());
@ -119,17 +129,16 @@ public final class AddS3Cmd extends BaseCmd {
dm.put(ApiConstants.S3_USE_TCP_KEEPALIVE, getUseTCPKeepAlive().toString());
}
try{
ImageStore result = _storageService.discoverImageStore(null, null, "S3", null, dm);
ImageStoreResponse storeResponse = null;
ImageStoreResponse storeResponse;
if (result != null) {
storeResponse = _responseGenerator.createImageStoreResponse(result);
storeResponse.setResponseName(getCommandName());
storeResponse.setObjectName("secondarystorage");
storeResponse.setObjectName("imagestore");
setResponseObject(storeResponse);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 secondary storage");
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 Image Store.");
}
} catch (DiscoveryException ex) {
s_logger.warn("Exception: ", ex);
@ -137,6 +146,60 @@ public final class AddS3Cmd extends BaseCmd {
}
}
@Override
public String getCommandName() {
return s_name;
}
@Override
public long getEntityOwnerId() {
return ACCOUNT_ID_SYSTEM;
}
public String getAccessKey() {
return accessKey;
}
public String getSecretKey() {
return secretKey;
}
public String getEndPoint() {
return endPoint;
}
public String getBucketName() {
return bucketName;
}
public String getSigner() {
return signer;
}
public Boolean isHttps() {
return httpsFlag;
}
public Integer getConnectionTimeout() {
return connectionTimeout;
}
public Integer getMaxErrorRetry() {
return maxErrorRetry;
}
public Integer getSocketTimeout() {
return socketTimeout;
}
public Integer getConnectionTtl() {
return connectionTtl;
}
public Boolean getUseTCPKeepAlive() {
return useTCPKeepAlive;
}
@Override
public boolean equals(final Object thatObject) {
@ -148,7 +211,7 @@ public final class AddS3Cmd extends BaseCmd {
return false;
}
final AddS3Cmd thatAddS3Cmd = (AddS3Cmd)thatObject;
final AddImageStoreS3CMD thatAddS3Cmd = (AddImageStoreS3CMD)thatObject;
if (httpsFlag != null ? !httpsFlag.equals(thatAddS3Cmd.httpsFlag) : thatAddS3Cmd.httpsFlag != null) {
return false;
@ -191,7 +254,6 @@ public final class AddS3Cmd extends BaseCmd {
}
return true;
}
@Override
@ -201,64 +263,14 @@ public final class AddS3Cmd extends BaseCmd {
result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
result = 31 * result + (httpsFlag != null && httpsFlag == true ? 1 : 0);
result = 31 * result + (signer != null ? signer.hashCode() : 0);
result = 31 * result + (httpsFlag != null && httpsFlag ? 1 : 0);
result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
result = 31 * result + (useTCPKeepAlive != null && useTCPKeepAlive == true ? 1 : 0);
result = 31 * result + (useTCPKeepAlive != null && useTCPKeepAlive ? 1 : 0);
return result;
}
@Override
public String getCommandName() {
return s_name;
}
@Override
public long getEntityOwnerId() {
return ACCOUNT_ID_SYSTEM;
}
public String getAccessKey() {
return accessKey;
}
public String getSecretKey() {
return secretKey;
}
public String getEndPoint() {
return endPoint;
}
public String getBucketName() {
return bucketName;
}
public Boolean getHttpsFlag() {
return httpsFlag;
}
public Integer getConnectionTimeout() {
return connectionTimeout;
}
public Integer getMaxErrorRetry() {
return maxErrorRetry;
}
public Integer getSocketTimeout() {
return socketTimeout;
}
public Integer getConnectionTtl() {
return connectionTtl;
}
public Boolean getUseTCPKeepAlive() {
return useTCPKeepAlive;
}
}

55
api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java
View File

@ -1,55 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cloudstack.api.command.admin.storage;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.BaseListCmd;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.ImageStoreResponse;
import org.apache.cloudstack.api.response.ListResponse;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
@APICommand(name = "listS3s", description = "Lists S3s", responseObject = ImageStoreResponse.class, since = "4.0.0",
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
public class ListS3sCmd extends BaseListCmd {
private static final String COMMAND_NAME = "lists3sresponse";
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
ResourceAllocationException, NetworkRuleConflictException {
ListImageStoresCmd cmd = new ListImageStoresCmd();
cmd.setProvider("S3");
ListResponse<ImageStoreResponse> response = _queryService.searchForImageStores(cmd);
response.setResponseName(getCommandName());
this.setResponseObject(response);
}
@Override
public String getCommandName() {
return COMMAND_NAME;
}
}

2
api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
View File

@ -32,7 +32,7 @@ import org.apache.cloudstack.api.response.TemplateResponse;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.template.VirtualMachineTemplate;
@APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud. ", responseObject = TemplateResponse.class, responseView = ResponseView.Full,
@APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud.", responseObject = TemplateResponse.class, responseView = ResponseView.Full,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
public class RegisterTemplateCmdByAdmin extends RegisterTemplateCmd {
public static final Logger s_logger = Logger.getLogger(RegisterTemplateCmdByAdmin.class.getName());

5
client/tomcatconf/commands.properties.in
View File

@ -282,12 +282,9 @@ listCapacity=3
addSwift=1
listSwifts=1
#### s3 commands
addS3=1
listS3s=1
#### image store commands
addImageStore=1
addImageStoreS3=1
listImageStores=1
deleteImageStore=1
createSecondaryStagingStore=1

10
client/tomcatconf/log4j-cloud.xml.in
View File

@ -162,6 +162,16 @@ under the License.
<appender-ref ref="APISERVER"/>
</logger>
<!-- Limit the com.amazonaws category to INFO as its DEBUG is verbose -->
<category name="com.amazonaws">
<priority value="INFO"/>
</category>
<!-- Limit the httpclient.wire category to INFO as its DEBUG is verbose -->
<category name="httpclient.wire">
<priority value="INFO"/>
</category>
<!-- ============================== -->
<!-- Add or remove these logger for SNMP, this logger is for SNMP alerts plugin -->
<!-- ============================== -->

2
core/src/com/cloud/resource/ServerResource.java
View File

@ -28,7 +28,6 @@ import com.cloud.host.Host;
import com.cloud.utils.component.Manager;
/**
*
* ServerResource is a generic container to execute commands sent
*/
public interface ServerResource extends Manager {
@ -70,4 +69,5 @@ public interface ServerResource extends Manager {
IAgentControl getAgentControl();
void setAgentControl(IAgentControl agentControl);
}

2
core/src/com/cloud/storage/template/HttpTemplateDownloader.java
View File

@ -47,10 +47,10 @@ import org.apache.log4j.Logger;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import com.cloud.agent.api.storage.Proxy;
import com.cloud.storage.StorageLayer;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.net.Proxy;
/**
* Download a template file using HTTP

509
core/src/com/cloud/storage/template/S3TemplateDownloader.java
View File

@ -19,303 +19,235 @@
package com.cloud.storage.template;
import static com.cloud.utils.StringUtils.join;
import static java.util.Arrays.asList;
import com.amazonaws.event.ProgressEvent;
import com.amazonaws.event.ProgressEventType;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.StorageClass;
import com.amazonaws.services.s3.transfer.Upload;
import com.cloud.agent.api.to.S3TO;
import com.cloud.utils.net.HTTPUtils;
import com.cloud.utils.net.Proxy;
import com.cloud.utils.storage.S3.S3Utils;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.commons.httpclient.ChunkedInputStream;
import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.NoHttpResponseException;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.ProgressEvent;
import com.amazonaws.services.s3.model.ProgressListener;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.StorageClass;
import com.cloud.agent.api.storage.Proxy;
import com.cloud.agent.api.to.S3TO;
import com.cloud.utils.Pair;
import com.cloud.utils.S3Utils;
import com.cloud.utils.UriUtils;
import static com.cloud.utils.StringUtils.join;
import static java.util.Arrays.asList;
/**
* Download a template file using HTTP(S)
*
* This class, once instantiated, has the purpose to download a single Template to an S3 Image Store.
*
* Execution of the instance is started when runInContext() is called.
*/
public class S3TemplateDownloader extends ManagedContextRunnable implements TemplateDownloader {
private static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
private String downloadUrl;
private String installPath;
private String s3Key;
private String fileName;
private String fileExtension;
private String errorString = " ";
private static final Logger LOGGER = Logger.getLogger(S3TemplateDownloader.class.getName());
private final String downloadUrl;
private final String s3Key;
private final String fileExtension;
private final HttpClient httpClient;
private final GetMethod getMethod;
private final DownloadCompleteCallback downloadCompleteCallback;
private final S3TO s3TO;
private String errorString = "";
private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
private ResourceType resourceType = ResourceType.TEMPLATE;
private final HttpClient client;
private final HttpMethodRetryHandler myretryhandler;
private GetMethod request;
private DownloadCompleteCallback completionCallback;
private S3TO s3to;
private long remoteSize = 0;
private long downloadTime = 0;
private long remoteSize;
private long downloadTime;
private long totalBytes;
private long maxTemplateSizeInByte;
private boolean resume = false;
private boolean inited = true;
public S3TemplateDownloader(S3TO s3to, String downloadUrl, String installPath, DownloadCompleteCallback callback,
long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) {
this.s3to = s3to;
public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
this.downloadUrl = downloadUrl;
this.installPath = installPath;
this.status = TemplateDownloader.Status.NOT_STARTED;
this.s3TO = s3TO;
this.resourceType = resourceType;
this.maxTemplateSizeInByte = maxTemplateSizeInBytes;
this.httpClient = HTTPUtils.getHTTPClient();
this.downloadCompleteCallback = downloadCompleteCallback;
this.totalBytes = 0;
this.client = new HttpClient(s_httpClientManager);
// Create a GET method for the download url.
this.getMethod = new GetMethod(downloadUrl);
this.myretryhandler = new HttpMethodRetryHandler() {
@Override
public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
if (executionCount >= 2) {
// Do not retry if over max retry count
return false;
}
if (exception instanceof NoHttpResponseException) {
// Retry if the server dropped connection on us
return true;
}
if (!method.isRequestSent()) {
// Retry if the request has not been sent fully or
// if it's OK to retry methods that have been sent
return true;
}
// otherwise do not retry
return false;
}
};
// Set the retry handler, default to retry 5 times.
this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
try {
request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
completionCallback = callback;
// Follow redirects
this.getMethod.setFollowRedirects(true);
Pair<String, Integer> hostAndPort = UriUtils.validateUrl(downloadUrl);
fileName = StringUtils.substringAfterLast(downloadUrl, "/");
fileExtension = StringUtils.substringAfterLast(fileName, ".");
// Set file extension.
this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
if (proxy != null) {
client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
if (proxy.getUserName() != null) {
Credentials proxyCreds = new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword());
client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds);
}
}
if ((user != null) && (password != null)) {
client.getParams().setAuthenticationPreemptive(true);
Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
client.getState().setCredentials(
new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first()
+ ":" + hostAndPort.second());
} else {
s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
}
} catch (IllegalArgumentException iae) {
errorString = iae.getMessage();
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
inited = false;
} catch (Exception ex) {
errorString = "Unable to start download -- check url? ";
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
s_logger.warn("Exception in constructor -- " + ex.toString());
} catch (Throwable th) {
s_logger.warn("throwable caught ", th);
}
// Calculate and set S3 Key.
this.s3Key = join(asList(installPath, StringUtils.substringAfterLast(downloadUrl, "/")), S3Utils.SEPARATOR);
// Set proxy if available.
HTTPUtils.setProxy(proxy, this.httpClient);
// Set credentials if available.
HTTPUtils.setCredentials(username, password, this.httpClient);
}
@Override
public long download(boolean resume, DownloadCompleteCallback callback) {
switch (status) {
case ABORTED:
case UNRECOVERABLE_ERROR:
case DOWNLOAD_FINISHED:
return 0;
default:
if (!status.equals(Status.NOT_STARTED)) {
// Only start downloading if we haven't started yet.
LOGGER.debug("Template download is already started, not starting again. Template: " + downloadUrl);
return 0;
}
int responseCode;
if ((responseCode = HTTPUtils.executeMethod(httpClient, getMethod)) == -1) {
errorString = "Exception while executing HttpMethod " + getMethod.getName() + " on URL " + downloadUrl;
LOGGER.warn(errorString);
status = Status.UNRECOVERABLE_ERROR;
return 0;
}
if (!HTTPUtils.verifyResponseCode(responseCode)) {
errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
LOGGER.warn(errorString);
status = Status.UNRECOVERABLE_ERROR;
return 0;
}
// Headers
Header contentLengthHeader = getMethod.getResponseHeader("Content-Length");
Header contentTypeHeader = getMethod.getResponseHeader("Content-Type");
// Check the contentLengthHeader and transferEncodingHeader.
if (contentLengthHeader == null) {
errorString = "The ContentLengthHeader of " + downloadUrl + " isn't supplied";
LOGGER.warn(errorString);
status = Status.UNRECOVERABLE_ERROR;
return 0;
} else {
// The ContentLengthHeader is supplied, parse it's value.
remoteSize = Long.parseLong(contentLengthHeader.getValue());
}
if (remoteSize > maxTemplateSizeInByte) {
errorString = "Remote size is too large for template " + downloadUrl + " remote size is " + remoteSize + " max allowed is " + maxTemplateSizeInByte;
LOGGER.warn(errorString);
status = Status.UNRECOVERABLE_ERROR;
return 0;
}
InputStream inputStream;
try {
// execute get method
int responseCode = HttpStatus.SC_OK;
if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
return 0; // FIXME: retry?
}
// get the total size of file
Header contentLengthHeader = request.getResponseHeader("Content-Length");
boolean chunked = false;
long remoteSize2 = 0;
if (contentLengthHeader == null) {
Header chunkedHeader = request.getResponseHeader("Transfer-Encoding");
if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = " Failed to receive length of download ";
return 0; // FIXME: what status do we put here? Do we retry?
} else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
chunked = true;
}
} else {
remoteSize2 = Long.parseLong(contentLengthHeader.getValue());
}
inputStream = new BufferedInputStream(getMethod.getResponseBodyAsStream());
} catch (IOException e) {
errorString = "Exception occurred while opening InputStream for template " + downloadUrl;
LOGGER.warn(errorString);
if (remoteSize == 0) {
remoteSize = remoteSize2;
}
if (remoteSize > maxTemplateSizeInByte) {
s_logger.info("Remote size is too large: " + remoteSize + " , max=" + maxTemplateSizeInByte);
status = Status.UNRECOVERABLE_ERROR;
errorString = "Download file size is too large";
return 0;
}
if (remoteSize == 0) {
remoteSize = maxTemplateSizeInByte;
}
// get content type
String contentType = null;
Header contentTypeHeader = request.getResponseHeader("Content-Type");
if (contentTypeHeader != null) {
contentType = contentTypeHeader.getValue();
}
InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream())
: new ChunkedInputStream(request.getResponseBodyAsStream());
s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3to.getBucketName()
+ " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInByte);
Date start = new Date();
// compute s3 key
s3Key = join(asList(installPath, fileName), S3Utils.SEPARATOR);
// download using S3 API
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(remoteSize);
if (contentType != null) {
metadata.setContentType(contentType);
}
PutObjectRequest putObjectRequest = new PutObjectRequest(s3to.getBucketName(), s3Key, in, metadata);
// check if RRS is enabled
if (s3to.getEnableRRS()) {
putObjectRequest = putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
}
// register progress listenser
putObjectRequest.setProgressListener(new ProgressListener() {
@Override
public void progressChanged(ProgressEvent progressEvent) {
// s_logger.debug(progressEvent.getBytesTransfered()
// + " number of byte transferd "
// + new Date());
totalBytes += progressEvent.getBytesTransfered();
if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) {
s_logger.info("download completed");
status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
} else if (progressEvent.getEventCode() == ProgressEvent.FAILED_EVENT_CODE) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
} else if (progressEvent.getEventCode() == ProgressEvent.CANCELED_EVENT_CODE) {
status = TemplateDownloader.Status.ABORTED;
} else {
status = TemplateDownloader.Status.IN_PROGRESS;
}
}
});
if (!s3to.getSingleUpload(remoteSize)) {
// use TransferManager to do multipart upload
S3Utils.mputObject(s3to, putObjectRequest);
} else {
// single part upload, with 5GB limit in Amazon
S3Utils.putObject(s3to, putObjectRequest);
while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED
&& status != TemplateDownloader.Status.UNRECOVERABLE_ERROR
&& status != TemplateDownloader.Status.ABORTED) {
// wait for completion
}
}
// finished or aborted
Date finish = new Date();
String downloaded = "(incomplete download)";
if (totalBytes >= remoteSize) {
status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
downloaded = "(download complete remote=" + remoteSize + "bytes)";
} else {
errorString = "Downloaded " + totalBytes + " bytes " + downloaded;
}
downloadTime += finish.getTime() - start.getTime();
return totalBytes;
} catch (HttpException hte) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = hte.getMessage();
} catch (IOException ioe) {
// probably a file write error
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = ioe.getMessage();
} catch (AmazonClientException ex) {
// S3 api exception
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = ex.getMessage();
} catch (InterruptedException e) {
// S3 upload api exception
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = e.getMessage();
} finally {
// close input stream
request.releaseConnection();
if (callback != null) {
callback.downloadComplete(status);
}
status = Status.UNRECOVERABLE_ERROR;
return 0;
}
return 0;
LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + remoteSize + " bytes");
// Time the upload starts.
final Date start = new Date();
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(remoteSize);
if (contentTypeHeader.getValue() != null) {
objectMetadata.setContentType(contentTypeHeader.getValue());
}
// Create the PutObjectRequest.
PutObjectRequest putObjectRequest = new PutObjectRequest(s3TO.getBucketName(), s3Key, inputStream, objectMetadata);
// If reduced redundancy is enabled, set it.
if (s3TO.getEnableRRS()) {
putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
}
Upload upload = S3Utils.putObject(s3TO, putObjectRequest);
upload.addProgressListener(new ProgressListener() {
@Override
public void progressChanged(ProgressEvent progressEvent) {
// Record the amount of bytes transferred.
totalBytes += progressEvent.getBytesTransferred();
LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds");
if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) {
status = Status.IN_PROGRESS;
} else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
status = Status.DOWNLOAD_FINISHED;
} else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_CANCELED_EVENT) {
status = Status.ABORTED;
} else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_FAILED_EVENT) {
status = Status.UNRECOVERABLE_ERROR;
}
}
});
try {
// Wait for the upload to complete.
upload.waitForCompletion();
} catch (InterruptedException e) {
// Interruption while waiting for the upload to complete.
LOGGER.warn("Interruption occurred while waiting for upload of " + downloadUrl + " to complete");
}
downloadTime = new Date().getTime() - start.getTime();
if (status == Status.DOWNLOAD_FINISHED) {
LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!");
} else {
LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString());
}
// Close input stream
getMethod.releaseConnection();
// Call the callback!
if (callback != null) {
callback.downloadComplete(status);
}
return totalBytes;
}
public String getDownloadUrl() {
return downloadUrl;
try {
return getMethod.getURI().toString();
} catch (URIException e) {
return null;
}
}
@Override
public TemplateDownloader.Status getStatus() {
public Status getStatus() {
return status;
}
@ -342,47 +274,38 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
return null;
}
return S3Utils.getObjectStream(s3to, s3to.getBucketName(), s3Key);
return S3Utils.getObjectStream(s3TO, s3TO.getBucketName(), s3Key);
}
public void cleanupAfterError() {
if (status != Status.UNRECOVERABLE_ERROR) {
s_logger.debug("S3Template downloader does not have state UNRECOVERABLE_ERROR, no cleanup neccesarry.");
return;
}
LOGGER.warn("Cleanup after error, trying to remove object: " + s3Key);
s_logger.info("Cleanup after UNRECOVERABLE_ERROR, trying to remove object: " + s3Key);
S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key);
}
@Override
@SuppressWarnings("fallthrough")
public boolean stopDownload() {
switch (getStatus()) {
case IN_PROGRESS:
if (request != null) {
request.abort();
}
status = TemplateDownloader.Status.ABORTED;
return true;
case UNKNOWN:
case NOT_STARTED:
case RECOVERABLE_ERROR:
case UNRECOVERABLE_ERROR:
case ABORTED:
status = TemplateDownloader.Status.ABORTED;
case DOWNLOAD_FINISHED:
try {
S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
} catch (Exception ex) {
// ignore delete exception if it is not there
}
return true;
default:
return true;
switch (status) {
case IN_PROGRESS:
if (getMethod != null) {
getMethod.abort();
}
break;
case UNKNOWN:
case NOT_STARTED:
case RECOVERABLE_ERROR:
case UNRECOVERABLE_ERROR:
case ABORTED:
case DOWNLOAD_FINISHED:
// Remove the object if it already has been uploaded.
S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key);
break;
default:
break;
}
status = TemplateDownloader.Status.ABORTED;
return true;
}
@Override
@ -396,18 +319,12 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
@Override
protected void runInContext() {
try {
download(resume, completionCallback);
} catch (Throwable t) {
s_logger.warn("Caught exception during download " + t.getMessage(), t);
errorString = "Failed to install: " + t.getMessage();
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
}
// Start the download!
download(resume, downloadCompleteCallback);
}
@Override
public void setStatus(TemplateDownloader.Status status) {
public void setStatus(Status status) {
this.status = status;
}
@ -442,7 +359,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
@Override
public boolean isInited() {
return inited;
return true;
}
public ResourceType getResourceType() {

33
core/src/com/cloud/storage/template/TemplateDownloader.java
View File

@ -25,16 +25,16 @@ public interface TemplateDownloader extends Runnable {
* Callback used to notify completion of download
*
*/
public interface DownloadCompleteCallback {
interface DownloadCompleteCallback {
void downloadComplete(Status status);
}
public static enum Status {
enum Status {
UNKNOWN, NOT_STARTED, IN_PROGRESS, ABORTED, UNRECOVERABLE_ERROR, RECOVERABLE_ERROR, DOWNLOAD_FINISHED, POST_DOWNLOAD_FINISHED
}
public static long DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES = 50L * 1024L * 1024L * 1024L;
long DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES = 50L * 1024L * 1024L * 1024L;
/**
* Initiate download, resuming a previous one if required
@ -42,55 +42,54 @@ public interface TemplateDownloader extends Runnable {
* @param callback completion callback to be called after download is complete
* @return bytes downloaded
*/
public long download(boolean resume, DownloadCompleteCallback callback);
long download(boolean resume, DownloadCompleteCallback callback);
/**
* @return
*/
public boolean stopDownload();
boolean stopDownload();
/**
* @return percent of file downloaded
*/
public int getDownloadPercent();
int getDownloadPercent();
/**
* Get the status of the download
* @return status of download
*/
public TemplateDownloader.Status getStatus();
TemplateDownloader.Status getStatus();
/**
* Get time taken to download so far
* @return time in seconds taken to download
*/
public long getDownloadTime();
long getDownloadTime();
/**
* Get bytes downloaded
* @return bytes downloaded so far
*/
public long getDownloadedBytes();
long getDownloadedBytes();
/**
* Get the error if any
* @return error string if any
*/
public String getDownloadError();
String getDownloadError();
/** Get local path of the downloaded file
* @return local path of the file downloaded
*/
public String getDownloadLocalPath();
String getDownloadLocalPath();
public void setStatus(TemplateDownloader.Status status);
void setStatus(TemplateDownloader.Status status);
public void setDownloadError(String string);
void setDownloadError(String string);
public void setResume(boolean resume);
void setResume(boolean resume);
public boolean isInited();
public long getMaxTemplateSizeInBytes();
boolean isInited();
long getMaxTemplateSizeInBytes();
}

2
core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
View File

@ -25,7 +25,7 @@ import org.apache.cloudstack.storage.to.VolumeObjectTO;
import com.cloud.agent.api.storage.AbstractDownloadCommand;
import com.cloud.agent.api.storage.PasswordAuth;
import com.cloud.agent.api.storage.Proxy;
import com.cloud.utils.net.Proxy;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.storage.Storage.ImageFormat;

14
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
View File

@ -23,14 +23,14 @@ import java.util.Set;
public interface DataStoreProvider {
// constants for provider names
static final String NFS_IMAGE = "NFS";
static final String S3_IMAGE = "S3";
static final String SWIFT_IMAGE = "Swift";
static final String SAMPLE_IMAGE = "Sample";
static final String SMB = "NFS";
static final String DEFAULT_PRIMARY = "DefaultPrimary";
String NFS_IMAGE = "NFS";
String S3_IMAGE = "S3";
String SWIFT_IMAGE = "Swift";
String SAMPLE_IMAGE = "Sample";
String SMB = "NFS";
String DEFAULT_PRIMARY = "DefaultPrimary";
static enum DataStoreProviderType {
enum DataStoreProviderType {
PRIMARY, IMAGE, ImageCache
}

2
engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
View File

@ -196,7 +196,7 @@ public class TemplateServiceImpl implements TemplateService {
@Override
public void downloadBootstrapSysTemplate(DataStore store) {
Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
Set<VMTemplateVO> toBeDownloaded = new HashSet();
List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();

10
engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
View File

@ -27,15 +27,15 @@ import com.cloud.storage.DataStoreRole;
import com.cloud.utils.fsm.NoTransitionException;
public interface ObjectInDataStoreManager {
public DataObject create(DataObject dataObj, DataStore dataStore);
DataObject create(DataObject dataObj, DataStore dataStore);
public boolean delete(DataObject dataObj);
boolean delete(DataObject dataObj);
public boolean deleteIfNotReady(DataObject dataObj);
boolean deleteIfNotReady(DataObject dataObj);
public DataObject get(DataObject dataObj, DataStore store);
DataObject get(DataObject dataObj, DataStore store);
public boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException;
boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException;
DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role);

2
engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
View File

@ -47,7 +47,6 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.storage.Proxy;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataTO;
import com.cloud.alert.AlertManager;
@ -58,6 +57,7 @@ import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.download.DownloadMonitor;
import com.cloud.utils.net.Proxy;
public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
private static final Logger s_logger = Logger.getLogger(BaseImageStoreDriverImpl.class);

10
framework/rest/pom.xml
View File

@ -33,27 +33,27 @@
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-jaxb-annotations</artifactId>
<version>2.4.4</version>
<version>2.6.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.4.4</version>
<version>2.6.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.4.4</version>
<version>2.6.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.4.4</version>
<version>2.6.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>2.4.4</version>
<version>2.6.3</version>
</dependency>
<dependency>
<groupId>org.apache.cxf</groupId>

10
packaging/centos7/tomcat7/log4j-cloud.xml
View File

@ -162,6 +162,16 @@ under the License.
<appender-ref ref="APISERVER"/>
</logger>
<!-- Limit the com.amazonaws category to INFO as its DEBUG is verbose -->
<category name="com.amazonaws">
<priority value="INFO"/>
</category>
<!-- Limit the httpclient.wire category to INFO as its DEBUG is verbose -->
<category name="httpclient.wire">
<priority value="INFO"/>
</category>
<!-- ============================== -->
<!-- Add or remove these logger for SNMP, this logger is for SNMP alerts plugin -->
<!-- ============================== -->

18
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -18,8 +18,7 @@
*/
package com.cloud.hypervisor.kvm.storage;
import static com.cloud.utils.S3Utils.mputFile;
import static com.cloud.utils.S3Utils.putFile;
import static com.cloud.utils.storage.S3.S3Utils.putFile;
import java.io.File;
import java.io.FileNotFoundException;
@ -95,7 +94,7 @@ import com.cloud.storage.template.Processor.FormatInfo;
import com.cloud.storage.template.QCOW2Processor;
import com.cloud.storage.template.TemplateLocation;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.S3Utils;
import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
@ -594,15 +593,10 @@ public class KVMStorageProcessor implements StorageProcessor {
}
protected String copyToS3(final File srcFile, final S3TO destStore, final String destPath) throws InterruptedException {
final String bucket = destStore.getBucketName();
final long srcSize = srcFile.length();
final String key = destPath + S3Utils.SEPARATOR + srcFile.getName();
if (!destStore.getSingleUpload(srcSize)) {
mputFile(destStore, srcFile, bucket, key);
} else {
putFile(destStore, srcFile, bucket, key);
}
putFile(destStore, srcFile, destStore.getBucketName(), key).waitForCompletion();
return key;
}
@ -668,7 +662,7 @@ public class KVMStorageProcessor implements StorageProcessor {
final SnapshotObjectTO snapshotOnCacheStore = (SnapshotObjectTO)answer.getNewData();
snapshotOnCacheStore.setDataStore(cacheStore);
((SnapshotObjectTO)destData).setDataStore(imageStore);
final CopyCommand newCpyCmd = new CopyCommand(snapshotOnCacheStore, destData, cmd.getWaitInMillSeconds(), cmd.executeInSequence());
final CopyCommand newCpyCmd = new CopyCommand(snapshotOnCacheStore, destData, cmd.getWaitInMillSeconds(), cmd.executeInSequence());
return copyToObjectStore(newCpyCmd);
}

4
plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
View File

@ -70,11 +70,11 @@ import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.resource.StorageProcessor;
import com.cloud.utils.S3Utils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.storage.encoding.DecodedDataObject;
import com.cloud.utils.storage.encoding.DecodedDataStore;
import com.cloud.utils.storage.encoding.Decoder;
import com.cloud.utils.storage.S3.ClientOptions;
import com.xensource.xenapi.Connection;
import com.xensource.xenapi.Host;
import com.xensource.xenapi.PBD;
@ -1061,7 +1061,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
try {
final List<String> parameters = newArrayList(flattenProperties(s3, S3Utils.ClientOptions.class));
final List<String> parameters = newArrayList(flattenProperties(s3, ClientOptions.class));
// https workaround for Introspector bug that does not
// recognize Boolean accessor methods ...

2
plugins/storage/image/s3/pom.xml
View File

@ -19,7 +19,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>cloud-plugin-storage-image-s3</artifactId>
<name>Apache CloudStack Plugin - Storage Image S3</name>
<name>Apache CloudStack Plugin - Storage Image S3 provider</name>
<parent>
<groupId>org.apache.cloudstack</groupId>
<artifactId>cloudstack-plugins</artifactId>

31
plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
View File

@ -39,7 +39,7 @@ import com.cloud.agent.api.to.S3TO;
import com.cloud.configuration.Config;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.S3Utils;
import com.cloud.utils.storage.S3.S3Utils;
public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
private static final Logger s_logger = Logger.getLogger(S3ImageStoreDriverImpl.class);
@ -58,7 +58,9 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
imgStore.getUuid(),
details.get(ApiConstants.S3_ACCESS_KEY),
details.get(ApiConstants.S3_SECRET_KEY),
details.get(ApiConstants.S3_END_POINT), details.get(ApiConstants.S3_BUCKET_NAME),
details.get(ApiConstants.S3_END_POINT),
details.get(ApiConstants.S3_BUCKET_NAME),
details.get(ApiConstants.S3_SIGNER),
details.get(ApiConstants.S3_HTTPS_FLAG) == null ? false : Boolean.parseBoolean(details.get(ApiConstants.S3_HTTPS_FLAG)),
details.get(ApiConstants.S3_CONNECTION_TIMEOUT) == null ? null : Integer.valueOf(details.get(ApiConstants.S3_CONNECTION_TIMEOUT)),
details.get(ApiConstants.S3_MAX_ERROR_RETRY) == null ? null : Integer.valueOf(details.get(ApiConstants.S3_MAX_ERROR_RETRY)),
@ -74,27 +76,29 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
try {
return Long.parseLong(_configDao.getValue(Config.S3MaxSingleUploadSize.toString())) * 1024L * 1024L * 1024L;
} catch (NumberFormatException e) {
// use default 5GB
return 5L * 1024L * 1024L * 1024L;
// use default 1TB
return 1024L * 1024L * 1024L * 1024L;
}
}
@Override
public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject) {
// for S3, no need to do anything, just return template url for
// extract template. but we need to set object acl as public_read to
// make the url accessible
public String createEntityExtractUrl(DataStore store, String key, ImageFormat format, DataObject dataObject) {
/**
* Generate a pre-signed URL for the given object.
*/
S3TO s3 = (S3TO)getStoreTO(store);
String key = installPath;
s_logger.info("Generating pre-signed s3 entity extraction URL.");
if(s_logger.isDebugEnabled()) {
s_logger.debug("Generating pre-signed s3 entity extraction URL for object: " + key);
}
Date expiration = new Date();
long milliSeconds = expiration.getTime();
// get extract url expiration interval set in global configuration (in seconds)
// Get extract url expiration interval set in global configuration (in seconds)
String urlExpirationInterval = _configDao.getValue(Config.ExtractURLExpirationInterval.toString());
int expirationInterval = NumbersUtil.parseInt(urlExpirationInterval, 14400);
milliSeconds += 1000 * expirationInterval; // expired after configured interval (in milliseconds)
// Expired after configured interval (in milliseconds), default 14400 seconds
milliSeconds += 1000 * NumbersUtil.parseInt(urlExpirationInterval, 14400);
expiration.setTime(milliSeconds);
URL s3url = S3Utils.generatePresignedUrl(s3, s3.getBucketName(), key, expiration);
@ -103,5 +107,4 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
return s3url.toString();
}
}

6
plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java
View File

@ -71,7 +71,6 @@ public class S3ImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
@Override
public DataStore initialize(Map<String, Object> dsInfos) {
Long dcId = (Long)dsInfos.get("zoneId");
String url = (String)dsInfos.get("url");
String name = (String)dsInfos.get("name");
String providerName = (String)dsInfos.get("providerName");
@ -79,11 +78,10 @@ public class S3ImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
DataStoreRole role = (DataStoreRole)dsInfos.get("role");
Map<String, String> details = (Map<String, String>)dsInfos.get("details");
s_logger.info("Trying to add a S3 store in data center " + dcId);
s_logger.info("Trying to add a S3 store with endpoint: " + details.get(ApiConstants.S3_END_POINT));
Map<String, Object> imageStoreParameters = new HashMap<String, Object>();
Map<String, Object> imageStoreParameters = new HashMap();
imageStoreParameters.put("name", name);
imageStoreParameters.put("zoneId", dcId);
imageStoreParameters.put("url", url);
String protocol = "http";
String useHttps = details.get(ApiConstants.S3_HTTPS_FLAG);

7
plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/provider/S3ImageStoreProviderImpl.java
View File

@ -44,14 +44,15 @@ import com.cloud.utils.component.ComponentContext;
@Component
public class S3ImageStoreProviderImpl implements ImageStoreProvider {
private final String providerName = DataStoreProvider.S3_IMAGE;
protected ImageStoreLifeCycle lifeCycle;
protected ImageStoreDriver driver;
@Inject
ImageStoreProviderManager storeMgr;
@Inject
ImageStoreHelper helper;
private final String providerName = DataStoreProvider.S3_IMAGE;
protected ImageStoreLifeCycle lifeCycle;
protected ImageStoreDriver driver;
@Override
public DataStoreLifeCycle getDataStoreLifeCycle() {
return lifeCycle;

3
pom.xml
View File

@ -96,7 +96,8 @@
<org.springframework.version>3.2.12.RELEASE</org.springframework.version>
<cs.mockito.version>1.9.5</cs.mockito.version>
<cs.powermock.version>1.5.3</cs.powermock.version>
<cs.aws.sdk.version>1.3.22</cs.aws.sdk.version>
<cs.aws.sdk.version>1.10.34</cs.aws.sdk.version>
<cs.jackson.version>2.6.3</cs.jackson.version>
<cs.lang.version>2.6</cs.lang.version>
<cs.lang3.version>3.4</cs.lang3.version>
<cs.commons-io.version>2.4</cs.commons-io.version>

2
scripts/vm/hypervisor/xenserver/s3xenserver
View File

@ -377,7 +377,7 @@ class S3Client(object):
def parseArguments(args):
# The keys in the args map will correspond to the properties defined on
# the com.cloud.utils.S3Utils#ClientOptions interface
# the com.cloud.utils.storage.S3.S3Utils#ClientOptions interface
client = S3Client(
args['accessKey'], args['secretKey'], args['endPoint'],
args['https'], args['connectionTimeout'], args['socketTimeout'])

10
server/conf/log4j-cloud.xml.in
View File

@ -109,6 +109,16 @@ under the License.
<appender-ref ref="APISERVER"/>
</logger>
<!-- Limit the com.amazonaws category to INFO as its DEBUG is verbose -->
<category name="com.amazonaws">
<priority value="INFO"/>
</category>
<!-- Limit the httpclient.wire category to INFO as its DEBUG is verbose -->
<category name="httpclient.wire">
<priority value="INFO"/>
</category>
<!-- ======================= -->
<!-- Setup the Root category -->
<!-- ======================= -->

6
server/src/com/cloud/server/ManagementServerImpl.java
View File

@ -162,7 +162,7 @@ import org.apache.cloudstack.api.command.admin.router.StopRouterCmd;
import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.AddS3Cmd;
import org.apache.cloudstack.api.command.admin.storage.AddImageStoreS3CMD;
import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd;
@ -171,7 +171,6 @@ import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.FindStoragePoolsForMigrationCmd;
import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd;
import org.apache.cloudstack.api.command.admin.storage.ListS3sCmd;
import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd;
import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd;
import org.apache.cloudstack.api.command.admin.storage.ListStorageProvidersCmd;
@ -2640,12 +2639,10 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
cmdList.add(StartRouterCmd.class);
cmdList.add(StopRouterCmd.class);
cmdList.add(UpgradeRouterCmd.class);
cmdList.add(AddS3Cmd.class);
cmdList.add(AddSwiftCmd.class);
cmdList.add(CancelPrimaryStorageMaintenanceCmd.class);
cmdList.add(CreateStoragePoolCmd.class);
cmdList.add(DeletePoolCmd.class);
cmdList.add(ListS3sCmd.class);
cmdList.add(ListSwiftsCmd.class);
cmdList.add(ListStoragePoolsCmd.class);
cmdList.add(ListStorageTagsCmd.class);
@ -2911,6 +2908,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
cmdList.add(RemoveFromGlobalLoadBalancerRuleCmd.class);
cmdList.add(ListStorageProvidersCmd.class);
cmdList.add(AddImageStoreCmd.class);
cmdList.add(AddImageStoreS3CMD.class);
cmdList.add(ListImageStoresCmd.class);
cmdList.add(DeleteImageStoreCmd.class);
cmdList.add(CreateSecondaryStagingStoreCmd.class);

25
server/src/com/cloud/storage/StorageManagerImpl.java
View File

@ -1844,7 +1844,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
@Override
public ImageStore discoverImageStore(String name, String url, String providerName, Long dcId, Map details) throws IllegalArgumentException, DiscoveryException,
public ImageStore discoverImageStore(String name, String url, String providerName, Long zoneId, Map details) throws IllegalArgumentException, DiscoveryException,
InvalidParameterValueException {
DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName);
@ -1857,13 +1857,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
ScopeType scopeType = ScopeType.ZONE;
if (dcId == null) {
if (zoneId == null) {
scopeType = ScopeType.REGION;
}
if (name == null) {
name = url;
}
ImageStoreVO imageStore = _imageStoreDao.findByName(name);
if (imageStore != null) {
throw new InvalidParameterValueException("The image store with name " + name + " already exists, try creating with another name");
@ -1884,11 +1885,11 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
}
if (dcId != null) {
if (zoneId != null) {
// Check if the zone exists in the system
DataCenterVO zone = _dcDao.findById(dcId);
DataCenterVO zone = _dcDao.findById(zoneId);
if (zone == null) {
throw new InvalidParameterValueException("Can't find zone by id " + dcId);
throw new InvalidParameterValueException("Can't find zone by id " + zoneId);
}
Account account = CallContext.current().getCallingAccount();
@ -1901,8 +1902,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
}
Map<String, Object> params = new HashMap<String, Object>();
params.put("zoneId", dcId);
Map<String, Object> params = new HashMap();
params.put("zoneId", zoneId);
params.put("url", url);
params.put("name", name);
params.put("details", details);
@ -1911,11 +1912,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
params.put("role", DataStoreRole.Image);
DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle();
DataStore store;
try {
store = lifeCycle.initialize(params);
} catch (Exception e) {
s_logger.debug("Failed to add data store: " + e.getMessage(), e);
if(s_logger.isDebugEnabled()) {
s_logger.debug("Failed to add data store: " + e.getMessage(), e);
}
throw new CloudRuntimeException("Failed to add data store: " + e.getMessage(), e);
}
@ -1927,9 +1931,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
_imageSrv.addSystemVMTemplatesToSecondary(store);
}
// associate builtin template with zones associated with this image
// store
associateCrosszoneTemplatesToZone(dcId);
// associate builtin template with zones associated with this image store
associateCrosszoneTemplatesToZone(zoneId);
// duplicate cache store records to region wide storage
if (scopeType == ScopeType.REGION) {

2
server/src/com/cloud/storage/download/DownloadMonitorImpl.java
View File

@ -49,7 +49,7 @@ import org.apache.cloudstack.storage.to.VolumeObjectTO;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.storage.Proxy;
import com.cloud.utils.net.Proxy;
import com.cloud.configuration.Config;
import com.cloud.storage.RegisterVolumePayload;
import com.cloud.storage.Storage.ImageFormat;

30
services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
View File

@ -16,8 +16,7 @@
// under the License.
package org.apache.cloudstack.storage.resource;
import static com.cloud.utils.S3Utils.mputFile;
import static com.cloud.utils.S3Utils.putFile;
import static com.cloud.utils.storage.S3.S3Utils.putFile;
import static com.cloud.utils.StringUtils.join;
import static java.lang.String.format;
import static java.util.Arrays.asList;
@ -155,8 +154,7 @@ import com.cloud.storage.template.TemplateProp;
import com.cloud.storage.template.VhdProcessor;
import com.cloud.storage.template.VmdkProcessor;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.S3Utils;
import com.cloud.utils.S3Utils.FileNamingStrategy;
import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.utils.SwiftUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.net.NetUtils;
@ -386,12 +384,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
}
File destFile = S3Utils.getFile(s3, s3.getBucketName(), srcData.getPath(), downloadDirectory, new FileNamingStrategy() {
@Override
public String determineFileName(final String key) {
return substringAfterLast(key, S3Utils.SEPARATOR);
}
});
File destFile = new File(downloadDirectory, substringAfterLast(srcData.getPath(), S3Utils.SEPARATOR));
S3Utils.getFile(s3, s3.getBucketName(), srcData.getPath(), destFile).waitForCompletion();
if (destFile == null) {
return new CopyCmdAnswer("Can't find template");
@ -400,7 +396,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return postProcessing(destFile, downloadPath, destPath, srcData, destData);
} catch (Exception e) {
final String errMsg = format("Failed to download" + "due to $2%s", e.getMessage());
final String errMsg = format("Failed to download" + "due to $1%s", e.getMessage());
s_logger.error(errMsg, e);
return new CopyCmdAnswer(errMsg);
}
@ -907,14 +903,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
}
long srcSize = srcFile.length();
ImageFormat format = getTemplateFormat(srcFile.getName());
String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName();
if (!s3.getSingleUpload(srcSize)) {
mputFile(s3, srcFile, bucket, key);
} else {
putFile(s3, srcFile, bucket, key);
}
putFile(s3, srcFile, bucket, key).waitForCompletion();
DataTO retObj = null;
if (destData.getObjectType() == DataObjectType.TEMPLATE) {
@ -1509,7 +1501,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
Map<String, TemplateProp> s3ListTemplate(S3TO s3) {
String bucket = s3.getBucketName();
// List the objects in the source directory on S3
final List<S3ObjectSummary> objectSummaries = S3Utils.getDirectory(s3, bucket, TEMPLATE_ROOT_DIR);
final List<S3ObjectSummary> objectSummaries = S3Utils.listDirectory(s3, bucket, TEMPLATE_ROOT_DIR);
if (objectSummaries == null) {
return null;
}
@ -1530,7 +1522,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
Map<Long, TemplateProp> s3ListVolume(S3TO s3) {
String bucket = s3.getBucketName();
// List the objects in the source directory on S3
final List<S3ObjectSummary> objectSummaries = S3Utils.getDirectory(s3, bucket, VOLUME_ROOT_DIR);
final List<S3ObjectSummary> objectSummaries = S3Utils.listDirectory(s3, bucket, VOLUME_ROOT_DIR);
if (objectSummaries == null) {
return null;
}

3
services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResource.java
View File

@ -19,11 +19,10 @@ package org.apache.cloudstack.storage.resource;
import com.cloud.resource.ServerResource;
/**
*
* SecondaryStorageServerResource is a generic container to execute commands sent
*/
public interface SecondaryStorageResource extends ServerResource {
public String getRootDir(String cmd);
String getRootDir(String cmd);
}

2
services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResourceHandler.java
View File

@ -20,5 +20,7 @@ import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command;
public interface SecondaryStorageResourceHandler {
Answer executeRequest(Command cmd);
}

2
services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
View File

@ -24,7 +24,7 @@ import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.storage.Proxy;
import com.cloud.utils.net.Proxy;
import com.cloud.agent.api.to.S3TO;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.VMTemplateHostVO;

2
services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
View File

@ -49,7 +49,7 @@ import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
import org.apache.log4j.Logger;
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.storage.Proxy;
import com.cloud.utils.net.Proxy;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;

10
systemvm/conf.dom0/log4j-cloud.xml.in
View File

@ -88,6 +88,16 @@ under the License.
<priority value="INFO"/>
</category>
<!-- Limit the com.amazonaws category to INFO as its DEBUG is verbose -->
<category name="com.amazonaws">
<priority value="INFO"/>
</category>
<!-- Limit the httpclient.wire category to INFO as its DEBUG is verbose -->
<category name="httpclient.wire">
<priority value="INFO"/>
</category>
<!-- ======================= -->
<!-- Setup the Root category -->
<!-- ======================= -->

10
systemvm/conf/log4j-cloud.xml
View File

@ -89,6 +89,16 @@ under the License.
<priority value="INFO"/>
</category>
<!-- Limit the com.amazonaws category to INFO as its DEBUG is verbose -->
<category name="com.amazonaws">
<priority value="INFO"/>
</category>
<!-- Limit the httpclient.wire category to INFO as its DEBUG is verbose -->
<category name="httpclient.wire">
<priority value="INFO"/>
</category>
<!-- ======================= -->
<!-- Setup the Root category -->
<!-- ======================= -->

5
utils/pom.xml
View File

@ -168,6 +168,11 @@
<artifactId>guava-testlib</artifactId>
<version>${cs.guava-testlib.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${cs.jackson.version}</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>

619
utils/src/main/java/com/cloud/utils/S3Utils.java
View File

@ -1,619 +0,0 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.utils;
import static com.amazonaws.Protocol.HTTP;
import static com.amazonaws.Protocol.HTTPS;
import static com.cloud.utils.StringUtils.join;
import static java.io.File.createTempFile;
import static java.lang.String.format;
import static java.lang.System.currentTimeMillis;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static java.util.Collections.unmodifiableList;
import static org.apache.commons.lang.ArrayUtils.isEmpty;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.commons.lang.StringUtils.isNotBlank;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import org.apache.commons.lang.ArrayUtils;
import org.apache.log4j.Logger;
import com.amazonaws.AmazonClientException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.HttpMethod;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import com.cloud.utils.exception.CloudRuntimeException;
public final class S3Utils {
private static final Logger LOGGER = Logger.getLogger(S3Utils.class);
public static final String SEPARATOR = "/";
private static final int MIN_BUCKET_NAME_LENGTH = 3;
private static final int MAX_BUCKET_NAME_LENGTH = 63;
private S3Utils() {
super();
}
public static AmazonS3 acquireClient(final ClientOptions clientOptions) {
final AWSCredentials credentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());
final ClientConfiguration configuration = new ClientConfiguration();
if (clientOptions.isHttps() != null) {
configuration.setProtocol(clientOptions.isHttps() == true ? HTTPS : HTTP);
}
if (clientOptions.getConnectionTimeout() != null) {
configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
}
if (clientOptions.getMaxErrorRetry() != null) {
configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
}
if (clientOptions.getSocketTimeout() != null) {
configuration.setSocketTimeout(clientOptions.getSocketTimeout());
}
if (clientOptions.getUseTCPKeepAlive() != null) {
//configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
LOGGER.debug("useTCPKeepAlive not supported by old AWS SDK");
}
if (clientOptions.getConnectionTtl() != null) {
//configuration.setConnectionTTL(clientOptions.getConnectionTtl());
LOGGER.debug("connectionTtl not supported by old AWS SDK");
}
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, connectionTimeOut: " + "%2$s, maxErrorRetry: %3$s, socketTimeout: %4$s, useTCPKeepAlive: %5$s, connectionTtl: %6$s]",
configuration.getProtocol(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(),
-1, -1));
}
final AmazonS3Client client = new AmazonS3Client(credentials, configuration);
if (isNotBlank(clientOptions.getEndPoint())) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Setting the end point for S3 client %1$s to %2$s.", client, clientOptions.getEndPoint()));
}
client.setEndpoint(clientOptions.getEndPoint());
}
return client;
}
public static void putFile(final ClientOptions clientOptions, final File sourceFile, final String bucketName, final String key) {
assert clientOptions != null;
assert sourceFile != null;
assert !isBlank(bucketName);
assert !isBlank(key);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Sending file %1$s as S3 object %2$s in " + "bucket %3$s", sourceFile.getName(), key, bucketName));
}
acquireClient(clientOptions).putObject(bucketName, key, sourceFile);
}
public static void putObject(final ClientOptions clientOptions, final InputStream sourceStream, final String bucketName, final String key) {
assert clientOptions != null;
assert sourceStream != null;
assert !isBlank(bucketName);
assert !isBlank(key);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Sending stream as S3 object %1$s in " + "bucket %2$s", key, bucketName));
}
acquireClient(clientOptions).putObject(bucketName, key, sourceStream, null);
}
public static void putObject(final ClientOptions clientOptions, final PutObjectRequest req) {
assert clientOptions != null;
assert req != null;
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Sending stream as S3 object using PutObjectRequest"));
}
acquireClient(clientOptions).putObject(req);
}
// multi-part upload file
public static void mputFile(final ClientOptions clientOptions, final File sourceFile, final String bucketName, final String key) throws InterruptedException {
assert clientOptions != null;
assert sourceFile != null;
assert !isBlank(bucketName);
assert !isBlank(key);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Multipart sending file %1$s as S3 object %2$s in " + "bucket %3$s", sourceFile.getName(), key, bucketName));
}
TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
Upload upload = tm.upload(bucketName, key, sourceFile);
upload.waitForCompletion();
}
// multi-part upload object
public static void mputObject(final ClientOptions clientOptions, final InputStream sourceStream, final String bucketName, final String key)
throws InterruptedException {
assert clientOptions != null;
assert sourceStream != null;
assert !isBlank(bucketName);
assert !isBlank(key);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Multipart sending stream as S3 object %1$s in " + "bucket %2$s", key, bucketName));
}
TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
Upload upload = tm.upload(bucketName, key, sourceStream, null);
upload.waitForCompletion();
}
// multi-part upload object
public static void mputObject(final ClientOptions clientOptions, final PutObjectRequest req) throws InterruptedException {
assert clientOptions != null;
assert req != null;
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Multipart sending object to S3 using PutObjectRequest");
}
TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
Upload upload = tm.upload(req);
upload.waitForCompletion();
}
public static void setObjectAcl(final ClientOptions clientOptions, final String bucketName, final String key, final CannedAccessControlList acl) {
assert clientOptions != null;
assert acl != null;
acquireClient(clientOptions).setObjectAcl(bucketName, key, acl);
}
public static URL generatePresignedUrl(final ClientOptions clientOptions, final String bucketName, final String key, final Date expiration) {
assert clientOptions != null;
assert !isBlank(bucketName);
assert !isBlank(key);
return acquireClient(clientOptions).generatePresignedUrl(bucketName, key, expiration, HttpMethod.GET);
}
// Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak.
public static S3Object getObject(final ClientOptions clientOptions, final String bucketName, final String key) {
assert clientOptions != null;
assert !isBlank(bucketName);
assert !isBlank(key);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Get S3 object %1$s in " + "bucket %2$s", key, bucketName));
}
return acquireClient(clientOptions).getObject(bucketName, key);
}
// Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak.
public static S3ObjectInputStream getObjectStream(final ClientOptions clientOptions, final String bucketName, final String key) {
assert clientOptions != null;
assert !isBlank(bucketName);
assert !isBlank(key);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Get S3 object %1$s in " + "bucket %2$s", key, bucketName));
}
return acquireClient(clientOptions).getObject(bucketName, key).getObjectContent();
}
@SuppressWarnings("unchecked")
public static File getFile(final ClientOptions clientOptions, final String bucketName, final String key, final File targetDirectory,
final FileNamingStrategy namingStrategy) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert isNotBlank(key);
assert targetDirectory != null && targetDirectory.isDirectory();
assert namingStrategy != null;
final AmazonS3 connection = acquireClient(clientOptions);
File tempFile = null;
try {
tempFile = createTempFile(join("-", targetDirectory.getName(), currentTimeMillis(), "part"), "tmp", targetDirectory);
tempFile.deleteOnExit();
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Downloading object %1$s from bucket %2$s to temp file %3$s", key, bucketName, tempFile.getName()));
}
try {
connection.getObject(new GetObjectRequest(bucketName, key), tempFile);
} catch (AmazonClientException ex) {
// hack to handle different ETAG format generated from RiakCS for multi-part uploaded object
String msg = ex.getMessage();
if (!msg.contains("verify integrity")) {
throw ex;
}
}
final File targetFile = new File(targetDirectory, namingStrategy.determineFileName(key));
tempFile.renameTo(targetFile);
return targetFile;
} catch (FileNotFoundException e) {
throw new CloudRuntimeException(format("Failed open file %1$s in order to get object %2$s from bucket %3$s.", targetDirectory.getAbsoluteFile(), bucketName,
key), e);
} catch (IOException e) {
throw new CloudRuntimeException(format("Unable to allocate temporary file in directory %1$s to download %2$s:%3$s from S3",
targetDirectory.getAbsolutePath(), bucketName, key), e);
} finally {
if (tempFile != null) {
tempFile.delete();
}
}
}
public static List<File> getDirectory(final ClientOptions clientOptions, final String bucketName, final String sourcePath, final File targetDirectory,
final FileNamingStrategy namingStrategy) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert isNotBlank(sourcePath);
assert targetDirectory != null;
final AmazonS3 connection = acquireClient(clientOptions);
// List the objects in the source directory on S3
final List<S3ObjectSummary> objectSummaries = listDirectory(bucketName, sourcePath, connection);
final List<File> files = new ArrayList<File>();
for (final S3ObjectSummary objectSummary : objectSummaries) {
files.add(getFile(clientOptions, bucketName, objectSummary.getKey(), targetDirectory, namingStrategy));
}
return unmodifiableList(files);
}
public static List<S3ObjectSummary> getDirectory(final ClientOptions clientOptions, final String bucketName, final String sourcePath) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert isNotBlank(sourcePath);
final AmazonS3 connection = acquireClient(clientOptions);
// List the objects in the source directory on S3
return listDirectory(bucketName, sourcePath, connection);
}
private static List<S3ObjectSummary> listDirectory(final String bucketName, final String directory, final AmazonS3 client) {
List<S3ObjectSummary> objects = new ArrayList<S3ObjectSummary>();
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(directory + SEPARATOR);
ObjectListing ol = client.listObjects(listObjectsRequest);
if(ol.isTruncated()) {
do {
objects.addAll(ol.getObjectSummaries());
listObjectsRequest.setMarker(ol.getNextMarker());
ol = client.listObjects(listObjectsRequest);
} while (ol.isTruncated());
}
else {
objects.addAll(ol.getObjectSummaries());
}
if (objects.isEmpty()) {
return emptyList();
}
return unmodifiableList(objects);
}
public static void putDirectory(final ClientOptions clientOptions, final String bucketName, final File directory, final FilenameFilter fileNameFilter,
final ObjectNamingStrategy namingStrategy) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert directory != null && directory.isDirectory();
assert fileNameFilter != null;
assert namingStrategy != null;
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Putting directory %1$s in S3 bucket %2$s.", directory.getAbsolutePath(), bucketName));
}
// Determine the list of files to be sent using the passed filter ...
final File[] files = directory.listFiles(fileNameFilter);
if (LOGGER.isTraceEnabled()) {
LOGGER.trace(format("Putting files (%1$s) in S3 bucket %2$s.", ArrayUtils.toString(files, "no files found"), bucketName));
}
// Skip spinning up an S3 connection when no files will be sent ...
if (isEmpty(files)) {
return;
}
final AmazonS3 client = acquireClient(clientOptions);
// Send the files to S3 using the passed ObjectNaming strategy to
// determine the key ...
for (final File file : files) {
final String key = namingStrategy.determineKey(file);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Putting file %1$s into bucket %2$s with key %3$s.", file.getAbsolutePath(), bucketName, key));
}
client.putObject(bucketName, key, file);
}
}
public static void deleteObject(final ClientOptions clientOptions, final String bucketName, final String key) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert isNotBlank(key);
final AmazonS3 client = acquireClient(clientOptions);
client.deleteObject(bucketName, key);
}
public static void deleteDirectory(final ClientOptions clientOptions, final String bucketName, final String directoryName) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert isNotBlank(directoryName);
final AmazonS3 client = acquireClient(clientOptions);
final List<S3ObjectSummary> objects = listDirectory(bucketName, directoryName, client);
for (final S3ObjectSummary object : objects) {
client.deleteObject(bucketName, object.getKey());
}
client.deleteObject(bucketName, directoryName);
}
public static boolean canConnect(final ClientOptions clientOptions) {
try {
acquireClient(clientOptions);
return true;
} catch (AmazonClientException e) {
LOGGER.warn("Ignored Exception while checking connection options", e);
return false;
}
}
public static boolean doesBucketExist(final ClientOptions clientOptions, final String bucketName) {
assert clientOptions != null;
assert !isBlank(bucketName);
try {
final List<Bucket> buckets = acquireClient(clientOptions).listBuckets();
for (Bucket bucket : buckets) {
if (bucket.getName().equals(bucketName)) {
return true;
}
}
return false;
} catch (AmazonClientException e) {
LOGGER.warn("Ignored Exception while checking bucket existence", e);
return false;
}
}
public static boolean canReadWriteBucket(final ClientOptions clientOptions, final String bucketName) {
assert clientOptions != null;
assert isNotBlank(bucketName);
try {
final AmazonS3 client = acquireClient(clientOptions);
final String fileContent = "testing put and delete";
final InputStream inputStream = new ByteArrayInputStream(fileContent.getBytes());
final String key = UUID.randomUUID().toString() + ".txt";
final ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(fileContent.length());
client.putObject(bucketName, key, inputStream, metadata);
client.deleteObject(bucketName, key);
return true;
} catch (AmazonClientException e) {
return false;
}
}
public static List<String> checkClientOptions(ClientOptions clientOptions) {
assert clientOptions != null;
List<String> errorMessages = new ArrayList<String>();
errorMessages.addAll(checkRequiredField("access key", clientOptions.getAccessKey()));
errorMessages.addAll(checkRequiredField("secret key", clientOptions.getSecretKey()));
errorMessages.addAll(checkOptionalField("connection timeout", clientOptions.getConnectionTimeout()));
errorMessages.addAll(checkOptionalField("socket timeout", clientOptions.getSocketTimeout()));
errorMessages.addAll(checkOptionalField("max error retries", clientOptions.getMaxErrorRetry()));
errorMessages.addAll(checkOptionalField("connection ttl", clientOptions.getConnectionTtl()));
return unmodifiableList(errorMessages);
}
public static List<String> checkBucketName(final String bucketLabel, final String bucket) {
assert isNotBlank(bucketLabel);
assert isNotBlank(bucket);
final List<String> errorMessages = new ArrayList<String>();
if (bucket.length() < MIN_BUCKET_NAME_LENGTH) {
errorMessages.add(format("The length of %1$s " + "for the %2$s must have a length of at least %3$s " + "characters", bucket, bucketLabel,
MIN_BUCKET_NAME_LENGTH));
}
if (bucket.length() > MAX_BUCKET_NAME_LENGTH) {
errorMessages.add(format("The length of %1$s " + "for the %2$s must not have a length of at greater" + " than %3$s characters", bucket, bucketLabel,
MAX_BUCKET_NAME_LENGTH));
}
return unmodifiableList(errorMessages);
}
private static List<String> checkOptionalField(final String fieldName, final Integer fieldValue) {
if (fieldValue != null && fieldValue < 0) {
return singletonList(format("The value of %1$s must " + "be greater than zero.", fieldName));
}
return emptyList();
}
private static List<String> checkRequiredField(String fieldName, String fieldValue) {
if (isBlank(fieldValue)) {
return singletonList(format("A %1$s must be specified.", fieldName));
}
return emptyList();
}
public interface ClientOptions {
String getAccessKey();
String getSecretKey();
String getEndPoint();
Boolean isHttps();
Integer getConnectionTimeout();
Integer getMaxErrorRetry();
Integer getSocketTimeout();
Boolean getUseTCPKeepAlive();
Integer getConnectionTtl();
}
public interface ObjectNamingStrategy {
String determineKey(File file);
}
public interface FileNamingStrategy {
String determineFileName(String key);
}
}

143
utils/src/main/java/com/cloud/utils/net/HTTPUtils.java Normal file
View File

@ -0,0 +1,143 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.utils.net;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.NoHttpResponseException;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.log4j.Logger;
import java.io.IOException;
public final class HTTPUtils {
private static final Logger LOGGER = Logger.getLogger(HTTPUtils.class);
// The connection manager.
private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
private HTTPUtils() {}
public static HttpClient getHTTPClient() {
return new HttpClient(s_httpClientManager);
}
/**
* @return A HttpMethodRetryHandler with given number of retries.
*/
public static HttpMethodRetryHandler getHttpMethodRetryHandler(final int retryCount) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Initializing new HttpMethodRetryHandler with retry count " + retryCount);
}
return new HttpMethodRetryHandler() {
@Override
public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
if (executionCount >= retryCount) {
// Do not retry if over max retry count
return false;
}
if (exception instanceof NoHttpResponseException) {
// Retry if the server dropped connection on us
return true;
}
if (!method.isRequestSent()) {
// Retry if the request has not been sent fully or
// if it's OK to retry methods that have been sent
return true;
}
// otherwise do not retry
return false;
}
};
}
/**
* @param proxy
* @param httpClient
*/
public static void setProxy(Proxy proxy, HttpClient httpClient) {
if (proxy != null && httpClient != null) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Setting proxy with host " + proxy.getHost() + " and port " + proxy.getPort() + " for host " + httpClient.getHostConfiguration().getHost() + ":" + httpClient.getHostConfiguration().getPort());
}
httpClient.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
if (proxy.getUserName() != null && proxy.getPassword() != null) {
httpClient.getState().setProxyCredentials(AuthScope.ANY, new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword()));
}
}
}
/**
* @param username
* @param password
* @param httpClient
*/
public static void setCredentials(String username, String password, HttpClient httpClient) {
if (username != null && password != null && httpClient != null) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Setting credentials with username " + username + " for host " + httpClient.getHostConfiguration().getHost() + ":" + httpClient.getHostConfiguration().getPort());
}
httpClient.getParams().setAuthenticationPreemptive(true);
httpClient.getState().setCredentials(
new AuthScope(httpClient.getHostConfiguration().getHost(), httpClient.getHostConfiguration().getPort(), AuthScope.ANY_REALM), new UsernamePasswordCredentials(username, password));
}
}
/**
* @param httpClient
* @param httpMethod
* @return
* Returns the HTTP Status Code or -1 if an exception occurred.
*/
public static int executeMethod(HttpClient httpClient, HttpMethod httpMethod) {
// Execute GetMethod
try {
return httpClient.executeMethod(httpMethod);
} catch (IOException e) {
LOGGER.warn("Exception while executing HttpMethod " + httpMethod.getName() + " on URL " + httpMethod.getPath());
return -1;
}
}
/**
* @param responseCode
* @return
*/
public static boolean verifyResponseCode(int responseCode) {
switch (responseCode) {
case HttpStatus.SC_OK:
case HttpStatus.SC_MOVED_PERMANENTLY:
case HttpStatus.SC_MOVED_TEMPORARILY:
return true;
default:
return false;
}
}
}

7
api/src/com/cloud/agent/api/storage/Proxy.java → utils/src/main/java/com/cloud/utils/net/Proxy.java
View File

@ -14,7 +14,8 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.agent.api.storage;
package com.cloud.utils.net;
import java.net.URI;
@ -27,10 +28,6 @@ public class Proxy {
private String _userName;
private String _password;
public Proxy() {
}
public Proxy(String host, int port, String userName, String password) {
this._host = host;
this._port = port;

42
utils/src/main/java/com/cloud/utils/storage/S3/ClientOptions.java Normal file
View File

@ -0,0 +1,42 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.utils.storage.S3;
public interface ClientOptions {
String getAccessKey();
String getSecretKey();
String getEndPoint();
String getSigner();
Boolean isHttps();
Integer getConnectionTimeout();
Integer getMaxErrorRetry();
Integer getSocketTimeout();
Boolean getUseTCPKeepAlive();
Integer getConnectionTtl();
}

25
utils/src/main/java/com/cloud/utils/storage/S3/FileNamingStrategy.java Normal file
View File

@ -0,0 +1,25 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.utils.storage.S3;
public interface FileNamingStrategy {
String determineFileName(String key);
}

27
utils/src/main/java/com/cloud/utils/storage/S3/ObjectNamingStrategy.java Normal file
View File

@ -0,0 +1,27 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.utils.storage.S3;
import java.io.File;
public interface ObjectNamingStrategy {
String determineKey(File file);
}

216
utils/src/main/java/com/cloud/utils/storage/S3/S3Utils.java Normal file
View File

@ -0,0 +1,216 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.utils.storage.S3;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.HttpMethod;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.Download;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import org.apache.log4j.Logger;
import java.io.File;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.amazonaws.Protocol.HTTP;
import static com.amazonaws.Protocol.HTTPS;
import static java.lang.String.format;
import static java.util.Collections.emptyList;
import static java.util.Collections.unmodifiableList;
import static org.apache.commons.lang.StringUtils.isNotBlank;
public final class S3Utils {
private static final Logger LOGGER = Logger.getLogger(S3Utils.class);
public static final String SEPARATOR = "/";
private static final Map<String, TransferManager> TRANSFERMANAGER_ACCESSKEY_MAP = new HashMap<>();
private S3Utils() {}
public static TransferManager getTransferManager(final ClientOptions clientOptions) {
if(TRANSFERMANAGER_ACCESSKEY_MAP.containsKey(clientOptions.getAccessKey())) {
return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
}
final AWSCredentials basicAWSCredentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());
final ClientConfiguration configuration = new ClientConfiguration();
if (clientOptions.isHttps() != null) {
configuration.setProtocol(clientOptions.isHttps() ? HTTPS : HTTP);
}
if (clientOptions.getConnectionTimeout() != null) {
configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
}
if (clientOptions.getMaxErrorRetry() != null) {
configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
}
if (clientOptions.getSocketTimeout() != null) {
configuration.setSocketTimeout(clientOptions.getSocketTimeout());
}
if (clientOptions.getUseTCPKeepAlive() != null) {
configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
}
if (clientOptions.getConnectionTtl() != null) {
configuration.setConnectionTTL(clientOptions.getConnectionTtl());
}
if (clientOptions.getSigner() != null) {
configuration.setSignerOverride(clientOptions.getSigner());
}
LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, signer: %2$s, connectionTimeOut: %3$s, maxErrorRetry: %4$s, socketTimeout: %5$s, useTCPKeepAlive: %6$s, connectionTtl: %7$s]",
configuration.getProtocol(), configuration.getSignerOverride(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(),
clientOptions.getUseTCPKeepAlive(), clientOptions.getConnectionTtl()));
final AmazonS3Client client = new AmazonS3Client(basicAWSCredentials, configuration);
if (isNotBlank(clientOptions.getEndPoint())) {
LOGGER.debug(format("Setting the end point for S3 client with access key %1$s to %2$s.", clientOptions.getAccessKey(), clientOptions.getEndPoint()));
client.setEndpoint(clientOptions.getEndPoint());
}
TRANSFERMANAGER_ACCESSKEY_MAP.put(clientOptions.getAccessKey(), new TransferManager(client));
return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
}
public static AmazonS3 getAmazonS3Client(final ClientOptions clientOptions) {
return getTransferManager(clientOptions).getAmazonS3Client();
}
public static Upload putFile(final ClientOptions clientOptions, final File sourceFile, final String bucketName, final String key) {
LOGGER.debug(format("Sending file %1$s as S3 object %2$s in bucket %3$s", sourceFile.getName(), key, bucketName));
return getTransferManager(clientOptions).upload(bucketName, key, sourceFile);
}
public static Upload putObject(final ClientOptions clientOptions, final InputStream sourceStream, final String bucketName, final String key) {
LOGGER.debug(format("Sending stream as S3 object %1$s in bucket %2$s", key, bucketName));
return getTransferManager(clientOptions).upload(bucketName, key, sourceStream, null);
}
public static Upload putObject(final ClientOptions clientOptions, final PutObjectRequest req) {
LOGGER.debug(format("Sending stream as S3 object %1$s in bucket %2$s using PutObjectRequest", req.getKey(), req.getBucketName()));
return getTransferManager(clientOptions).upload(req);
}
public static Download getFile(final ClientOptions clientOptions, final String bucketName, final String key, final File file) {
LOGGER.debug(format("Receiving object %1$s as file %2$s from bucket %3$s", key, file.getAbsolutePath(), bucketName));
return getTransferManager(clientOptions).download(bucketName, key, file);
}
public static Download getFile(final ClientOptions clientOptions, final GetObjectRequest getObjectRequest, final File file) {
LOGGER.debug(format("Receiving object %1$s as file %2$s from bucket %3$s using GetObjectRequest", getObjectRequest.getKey(), file.getAbsolutePath(), getObjectRequest.getBucketName()));
return getTransferManager(clientOptions).download(getObjectRequest, file);
}
public static URL generatePresignedUrl(final ClientOptions clientOptions, final String bucketName, final String key, final Date expiration) {
LOGGER.debug(format("Generating presigned url for key %1s in bucket %2s with expiration date %3s", key, bucketName, expiration.toString()));
return getTransferManager(clientOptions).getAmazonS3Client().generatePresignedUrl(bucketName, key, expiration, HttpMethod.GET);
}
// Note that whenever S3ObjectInputStream is returned, client code needs to close the internal stream to avoid resource leak.
public static S3ObjectInputStream getObjectStream(final ClientOptions clientOptions, final String bucketName, final String key) {
LOGGER.debug(format("Get S3ObjectInputStream from S3 Object %1$s in bucket %2$s", key, bucketName));
return getTransferManager(clientOptions).getAmazonS3Client().getObject(bucketName, key).getObjectContent();
}
public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) {
LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName));
List<S3ObjectSummary> objects = new ArrayList<>();
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.withBucketName(bucketName);
listObjectsRequest.withPrefix(directory);
ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
if(ol.isTruncated()) {
do {
objects.addAll(ol.getObjectSummaries());
listObjectsRequest.setMarker(ol.getNextMarker());
ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
} while (ol.isTruncated());
}
else {
objects.addAll(ol.getObjectSummaries());
}
if (objects.isEmpty()) {
return emptyList();
}
return unmodifiableList(objects);
}
public static void deleteObject(final ClientOptions clientOptions, final String bucketName, final String key) {
LOGGER.debug(format("Deleting S3 Object %1$s in bucket %2$s", key, bucketName));
getAmazonS3Client(clientOptions).deleteObject(bucketName,key);
}
public static void deleteDirectory(final ClientOptions clientOptions, final String bucketName, final String directoryName) {
LOGGER.debug(format("Deleting S3 Directory %1$s in bucket %2$s", directoryName, bucketName));
final List<S3ObjectSummary> objects = listDirectory(clientOptions, bucketName, directoryName);
for (final S3ObjectSummary object : objects) {
deleteObject(clientOptions, bucketName, object.getKey());
}
deleteObject(clientOptions, bucketName, directoryName);
}
}

4
utils/src/test/java/com/cloud/utils/rest/RESTServiceConnectorTest.java
View File

@ -32,6 +32,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.CollectionType;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.http.HttpEntity;
@ -43,8 +45,6 @@ import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicStatusLine;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.type.CollectionType;
import org.junit.Test;
import com.google.gson.FieldNamingPolicy;