apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

engine-storage: control download redirection

Browse Source 
Add a global setting to control whether redirection is allowed while
downloading templates and volumes

core: some changes on SimpleHttpMultiFileDownloader
similar as HttpTemplateDownloader

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
(cherry picked from commit b1642bc3bf58ccde9f56f632b5a9fe46a3eb5356)
Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
This commit is contained in:
Wei Zhou 2024-03-27 10:07:47 +01:00 committed by Rohit Yadav
parent 67e2061f4b
commit 939d0b9011
46 changed files with 457 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java
View File

@ -28,6 +28,7 @@ import java.io.RandomAccessFile;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Date;
import java.util.List;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
@ -80,6 +81,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
private long maxTemplateSizeInBytes;
private ResourceType resourceType = ResourceType.TEMPLATE;
private final HttpMethodRetryHandler myretryhandler;
private boolean followRedirects = false;
public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
String user, String password, Proxy proxy, ResourceType resourceType) {
@ -111,7 +113,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
private GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
request.setFollowRedirects(true);
request.setFollowRedirects(followRedirects);
return request;
}
@ -337,6 +339,12 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
} else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
status = Status.UNRECOVERABLE_ERROR;
errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
&& !followRedirects) {
errorString = String.format("Failed to download %s due to redirection, response code: %d",
downloadUrl, responseCode);
s_logger.error(errorString);
}
return true; //FIXME: retry?
}
return false;
@ -538,4 +546,12 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
return this;
}
}
@Override
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
if (this.request != null) {
this.request.setFollowRedirects(followRedirects);
}
}
}

10
core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java
View File

@ -60,7 +60,7 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement
protected GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
request.setFollowRedirects(true);
request.setFollowRedirects(followRedirects);
if (!toFileSet) {
String[] parts = downloadUrl.split("/");
String filename = parts[parts.length - 1];
@ -173,4 +173,12 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement
public void setStatus(Status status) {
this.status = status;
}
@Override
public void setFollowRedirects(boolean followRedirects) {
super.setFollowRedirects(followRedirects);
if (this.request != null) {
this.request.setFollowRedirects(followRedirects);
}
}
}

19
core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java
View File

@ -34,6 +34,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
@ -44,6 +45,7 @@ import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import java.util.List;
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
import static java.util.Arrays.asList;
@ -72,8 +74,8 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
private long downloadTime;
private long totalBytes;
private long maxTemplateSizeInByte;
private boolean resume = false;
private boolean followRedirects = false;
public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
@ -91,7 +93,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
// Follow redirects
this.getMethod.setFollowRedirects(true);
this.getMethod.setFollowRedirects(followRedirects);
// Set file extension.
this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
@ -124,10 +126,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
return 0;
}
if (!HTTPUtils.verifyResponseCode(responseCode)) {
boolean failedDueToRedirection = List.of(HttpStatus.SC_MOVED_PERMANENTLY,
HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) && !followRedirects;
if (!HTTPUtils.verifyResponseCode(responseCode) || failedDueToRedirection) {
errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
LOGGER.warn(errorString);
status = Status.UNRECOVERABLE_ERROR;
return 0;
}
@ -373,4 +376,12 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
public String getFileExtension() {
return fileExtension;
}
@Override
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
if (this.getMethod != null) {
this.getMethod.setFollowRedirects(followRedirects);
}
}
}

22
core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java
View File

@ -25,6 +25,7 @@ import java.io.InputStream;
import java.io.RandomAccessFile;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
@ -73,6 +74,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
private final HttpMethodRetryHandler retryHandler;
private HashMap<String, String> urlFileMap;
private boolean followRedirects = false;
public SimpleHttpMultiFileDownloader(StorageLayer storageLayer, String[] downloadUrls, String toDir,
DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
@ -94,7 +96,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
private GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler);
request.setFollowRedirects(true);
request.setFollowRedirects(followRedirects);
return request;
}
@ -170,7 +172,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
urlFileMap.put(downloadUrl, currentToFile);
file = new File(currentToFile);
long localFileSize = checkLocalFileSizeForResume(resume, file);
if (checkServerResponse(localFileSize)) return 0;
if (checkServerResponse(localFileSize, downloadUrl)) return 0;
if (!tryAndGetRemoteSize()) return 0;
if (!canHandleDownloadSize()) return 0;
checkAndSetDownloadSize();
@ -317,7 +319,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
return true;
}
private boolean checkServerResponse(long localFileSize) throws IOException {
private boolean checkServerResponse(long localFileSize, String downloadUrl) throws IOException {
int responseCode = 0;
if (localFileSize > 0) {
@ -331,6 +333,12 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
} else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
currentStatus = Status.UNRECOVERABLE_ERROR;
errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
&& !followRedirects) {
errorString = String.format("Failed to download %s due to redirection, response code: %d",
downloadUrl, responseCode);
s_logger.error(errorString);
}
return true; //FIXME: retry?
}
return false;
@ -478,4 +486,12 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
public Map<String, String> getDownloadedFilesMap() {
return urlFileMap;
}
@Override
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
if (this.request != null) {
this.request.setFollowRedirects(followRedirects);
}
}
}

2
core/src/main/java/com/cloud/storage/template/TemplateDownloader.java
View File

@ -92,4 +92,6 @@ public interface TemplateDownloader extends Runnable {
boolean isInited();
long getMaxTemplateSizeInBytes();
void setFollowRedirects(boolean followRedirects);
}

6
core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java
View File

@ -43,6 +43,7 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl
protected long _start;
protected StorageLayer _storage;
protected boolean _inited = false;
protected boolean followRedirects = false;
private long maxTemplateSizeInBytes;
public TemplateDownloaderBase(StorageLayer storage, String downloadUrl, String toDir, long maxTemplateSizeInBytes, DownloadCompleteCallback callback) {
@ -149,4 +150,9 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl
public boolean isInited() {
return _inited;
}
@Override
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
}
}

11
core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
View File

@ -28,6 +28,7 @@ public class CheckUrlCommand extends Command {
private Integer connectTimeout;
private Integer connectionRequestTimeout;
private Integer socketTimeout;
private boolean followRedirects;
public String getFormat() {
return format;
@ -43,19 +44,25 @@ public class CheckUrlCommand extends Command {
public Integer getSocketTimeout() { return socketTimeout; }
public CheckUrlCommand(final String format,final String url) {
public boolean isFollowRedirects() {
return followRedirects;
}
public CheckUrlCommand(final String format, final String url, final boolean followRedirects) {
super();
this.format = format;
this.url = url;
this.followRedirects = followRedirects;
}
public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, final boolean followRedirects) {
super();
this.format = format;
this.url = url;
this.connectTimeout = connectTimeout;
this.socketTimeout = socketTimeout;
this.connectionRequestTimeout = connectionRequestTimeout;
this.followRedirects = followRedirects;
}
@Override

15
core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
View File

@ -45,7 +45,11 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
private Long templateSize;
private Storage.ImageFormat format;
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) {
private boolean followRedirects;
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool,
final String checksum, final Map<String, String> headers, final Integer connectTimeout,
final Integer soTimeout, final Integer connectionRequestTimeout, final boolean followRedirects) {
this.url = url;
this.templateId = templateId;
this.destData = destData;
@ -55,6 +59,7 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;
this.connectionRequestTimeout = connectionRequestTimeout;
this.followRedirects = followRedirects;
}
public String getUrl() {
@ -137,4 +142,12 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
public int getWaitInMillSeconds() {
return getWait() * 1000;
}
public boolean isFollowRedirects() {
return followRedirects;
}
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
}
}

6
core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
View File

@ -24,8 +24,10 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class HttpDirectDownloadCommand extends DirectDownloadCommand {
public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
null, followRedirects);
}
}

7
core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
View File

@ -25,7 +25,10 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, connectionRequestTimeout);
public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout,
boolean followRedirects) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
connectionRequestTimeout, followRedirects);
}
}

5
core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
View File

@ -24,8 +24,9 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null, followRedirects);
}
}

5
core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
View File

@ -24,8 +24,9 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class NfsDirectDownloadCommand extends DirectDownloadCommand {
public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
super(url, templateId, destPool, checksum, headers, null, null, null);
public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool,
final String checksum, final Map<String, String> headers) {
super(url, templateId, destPool, checksum, headers, null, null, null, false);
}
}

45
core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
View File

@ -34,27 +34,30 @@ public class DirectDownloadHelper {
* Get direct template downloader from direct download command and destination pool
*/
public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
String destPoolLocalPath,
String temporaryDownloadPath) {
String destPoolLocalPath, String temporaryDownloadPath) {
if (cmd instanceof HttpDirectDownloadCommand) {
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
temporaryDownloadPath, cmd.isFollowRedirects());
} else if (cmd instanceof HttpsDirectDownloadCommand) {
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
cmd.getConnectionRequestTimeout(), temporaryDownloadPath, cmd.isFollowRedirects());
} else if (cmd instanceof NfsDirectDownloadCommand) {
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
cmd.getChecksum(), temporaryDownloadPath);
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
temporaryDownloadPath, cmd.isFollowRedirects());
} else {
throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}
}
public static boolean checkUrlExistence(String url) {
public static boolean checkUrlExistence(String url, boolean followRedirects) {
try {
DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects);
return checker.checkUrl(url);
} catch (CloudRuntimeException e) {
LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@ -62,9 +65,9 @@ public class DirectDownloadHelper {
}
}
public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
try {
DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
return checker.checkUrl(url);
} catch (CloudRuntimeException e) {
LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@ -72,27 +75,27 @@ public class DirectDownloadHelper {
}
}
private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
if (url.toLowerCase().startsWith("https:")) {
return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
} else if (url.toLowerCase().startsWith("http:")) {
return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout);
return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects);
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else if (url.toLowerCase().endsWith(".metalink")) {
return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout);
return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects);
} else {
throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url));
}
}
public static Long getFileSize(String url, String format) {
DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
public static Long getFileSize(String url, String format, boolean followRedirects) {
DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects);
return checker.getRemoteFileSize(url, format);
}
public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
return checker.getRemoteFileSize(url, format);
}
}

13
core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java
View File

@ -42,16 +42,19 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
private String checksum;
private boolean redownload = false;
protected String temporaryDownloadPath;
private boolean followRedirects;
public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName());
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId,
final String checksum, final String temporaryDownloadPath) {
final String checksum, final String temporaryDownloadPath,
final boolean followRedirects) {
this.url = url;
this.destPoolPath = destPoolPath;
this.templateId = templateId;
this.checksum = checksum;
this.temporaryDownloadPath = temporaryDownloadPath;
this.followRedirects = followRedirects;
}
private static String directDownloadDir = "template";
@ -111,6 +114,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
return redownload;
}
public boolean isFollowRedirects() {
return followRedirects;
}
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
}
/**
* Create download directory (if it does not exist)
*/

21
core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
View File

@ -50,13 +50,14 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected GetMethod request;
protected Map<String, String> reqHeaders = new HashMap<>();
protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
this(url, null, null, null, null, connectTimeout, socketTimeout, null);
protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout, boolean followRedirects) {
this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects);
}
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, destPoolPath, templateId, checksum, downloadPath);
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
boolean followRedirects) {
super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout);
s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout);
client = new HttpClient(s_httpClientManager);
@ -68,7 +69,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
GetMethod request = new GetMethod(downloadUrl);
request.setFollowRedirects(true);
request.setFollowRedirects(this.isFollowRedirects());
if (MapUtils.isNotEmpty(headers)) {
for (String key : headers.keySet()) {
request.setRequestHeader(key, headers.get(key));
@ -111,9 +112,11 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
@Override
public boolean checkUrl(String url) {
HeadMethod httpHead = new HeadMethod(url);
httpHead.setFollowRedirects(this.isFollowRedirects());
try {
if (client.executeMethod(httpHead) != HttpStatus.SC_OK) {
s_logger.error(String.format("Invalid URL: %s", url));
int responseCode = client.executeMethod(httpHead);
if (responseCode != HttpStatus.SC_OK) {
s_logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
return false;
}
return true;
@ -128,9 +131,9 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
@Override
public Long getRemoteFileSize(String url, String format) {
if ("qcow2".equalsIgnoreCase(format)) {
return QCOW2Utils.getVirtualSize(url);
return QCOW2Utils.getVirtualSizeFromUrl(url, this.isFollowRedirects());
} else {
return UriUtils.getRemoteSize(url);
return UriUtils.getRemoteSize(url, this.isFollowRedirects());
}
}

26
core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
View File

@ -68,20 +68,26 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
protected CloseableHttpClient httpsClient;
private HttpUriRequest req;
protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null);
protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null, followRedirects);
}
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
super(url, destPoolPath, templateId, checksum, temporaryDownloadPath);
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout,
Integer connectionRequestTimeout, String temporaryDownloadPath, boolean followRedirects) {
super(url, destPoolPath, templateId, checksum, temporaryDownloadPath, followRedirects);
SSLContext sslcontext = getSSLContext();
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
.setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
.setSocketTimeout(soTimeout == null ? 5000 : soTimeout)
.setRedirectsEnabled(followRedirects)
.build();
httpsClient = HttpClients.custom()
.setSSLSocketFactory(factory)
.setDefaultRequestConfig(config)
.build();
createUriRequest(url, headers);
String downloadDir = getDirectDownloadTempPath(templateId);
File tempFile = createTemporaryDirectoryAndFile(downloadDir);
@ -90,6 +96,7 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
req = new HttpGet(downloadUrl);
setFollowRedirects(this.isFollowRedirects());
if (MapUtils.isNotEmpty(headers)) {
for (String headerKey: headers.keySet()) {
req.setHeader(headerKey, headers.get(headerKey));
@ -164,8 +171,9 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
HttpHead httpHead = new HttpHead(url);
try {
CloseableHttpResponse response = httpsClient.execute(httpHead);
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
s_logger.error(String.format("Invalid URL: %s", url));
int responseCode = response.getStatusLine().getStatusCode();
if (responseCode != HttpStatus.SC_OK) {
s_logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
return false;
}
return true;

20
core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
View File

@ -42,16 +42,15 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm
private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId,
String destPoolPath, String checksum,
Map<String, String> headers,
Integer connectTimeout, Integer soTimeout,
Integer connectionRequestTimeout, String temporaryDownloadPath) {
String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout,
Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
if (url.toLowerCase().startsWith("https:")) {
return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath);
connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath,
this.isFollowRedirects());
} else if (url.toLowerCase().startsWith("http:")) {
return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
connectTimeout, soTimeout, temporaryDownloadPath);
connectTimeout, soTimeout, temporaryDownloadPath, this.isFollowRedirects());
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else {
@ -60,13 +59,14 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm
}
}
protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
this(url, null, null, null, null, connectTimeout, socketTimeout, null);
protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout, boolean followRedirects) {
this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects);
}
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, destPoolPath, templateId, checksum, downloadPath);
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
boolean followRedirects) {
super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
this.headers = headers;
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;

5
core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java
View File

@ -57,8 +57,9 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
this(url, null, null, null, null);
}
public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) {
super(url, destPool, templateId, checksum, downloadPath);
public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum,
String downloadPath) {
super(url, destPool, templateId, checksum, downloadPath, false);
parseUrl();
}

13
core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java
View File

@ -49,6 +49,8 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
private DataStoreTO _store;
private DataStoreTO cacheStore;
private boolean followRedirects = false;
protected DownloadCommand() {
}
@ -65,6 +67,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
installPath = that.installPath;
_store = that._store;
_proxy = that._proxy;
followRedirects = that.followRedirects;
}
public DownloadCommand(TemplateObjectTO template, Long maxDownloadSizeInBytes) {
@ -80,6 +83,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
setSecUrl(((NfsTO)_store).getUrl());
}
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
this.followRedirects = template.isFollowRedirects();
}
public DownloadCommand(TemplateObjectTO template, String user, String passwd, Long maxDownloadSizeInBytes) {
@ -95,6 +99,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
_store = volume.getDataStore();
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
resourceType = ResourceType.VOLUME;
this.followRedirects = volume.isFollowRedirects();
}
public DownloadCommand(SnapshotObjectTO snapshot, Long maxDownloadSizeInBytes, String url) {
@ -194,4 +199,12 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
public DataStoreTO getCacheStore() {
return cacheStore;
}
public boolean isFollowRedirects() {
return followRedirects;
}
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
}
}

30
core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java Normal file
View File

@ -0,0 +1,30 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.to;
public class DownloadableObjectTO {
protected boolean followRedirects = false;
public boolean isFollowRedirects() {
return followRedirects;
}
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
}
}

2
core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java
View File

@ -30,7 +30,7 @@ import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
public class SnapshotObjectTO implements DataTO {
public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO {
private String path;
private VolumeObjectTO volume;
private String parentSnapshotPath;

3
core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java
View File

@ -28,7 +28,7 @@ import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.template.VirtualMachineTemplate;
public class TemplateObjectTO implements DataTO {
public class TemplateObjectTO extends DownloadableObjectTO implements DataTO {
private String path;
private String origUrl;
private String uuid;
@ -87,6 +87,7 @@ public class TemplateObjectTO implements DataTO {
this.deployAsIs = template.isDeployAsIs();
this.deployAsIsConfiguration = template.getDeployAsIsConfiguration();
this.directDownload = template.isDirectDownload();
this.followRedirects = template.isFollowRedirects();
}
@Override

3
core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
View File

@ -33,7 +33,7 @@ import com.cloud.storage.Volume;
import java.util.Arrays;
public class VolumeObjectTO implements DataTO {
public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
private String uuid;
private Volume.Type volumeType;
private DataStoreTO dataStore;
@ -119,6 +119,7 @@ public class VolumeObjectTO implements DataTO {
this.vSphereStoragePolicyId = volume.getvSphereStoragePolicyId();
this.passphrase = volume.getPassphrase();
this.encryptFormat = volume.getEncryptFormat();
this.followRedirects = volume.isFollowRedirects();
}
public String getUuid() {

2
core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
View File

@ -56,7 +56,7 @@ public class BaseDirectTemplateDownloaderTest {
private HttpEntity httpEntity;
@InjectMocks
protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000);
protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000, false);
@Before
public void init() throws IOException {

3
core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
View File

@ -25,8 +25,7 @@ import org.mockito.InjectMocks;
public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
@InjectMocks
protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000);
protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000, false);
@Test
public void testCheckUrlMetalink() {
metalinkDownloader.downloader = httpsDownloader;

24
engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java Normal file
View File

@ -0,0 +1,24 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.engine.subsystem.api.storage;
public interface DownloadableDataInfo extends DataObject {
default public boolean isFollowRedirects() {
return true;
}
}

2
engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
View File

@ -21,7 +21,7 @@ package org.apache.cloudstack.engine.subsystem.api.storage;
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.user.UserData;
public interface TemplateInfo extends DataObject, VirtualMachineTemplate {
public interface TemplateInfo extends DownloadableDataInfo, VirtualMachineTemplate {
@Override
String getUniqueName();

2
engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java
View File

@ -26,7 +26,7 @@ import com.cloud.storage.Storage;
import com.cloud.storage.Volume;
import com.cloud.vm.VirtualMachine;
public interface VolumeInfo extends DataObject, Volume {
public interface VolumeInfo extends DownloadableDataInfo, Volume {
boolean isAttachedVM();

4
engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
View File

@ -199,6 +199,10 @@ public interface StorageManager extends StorageService {
true,
ConfigKey.Scope.Global,
null);
static final ConfigKey<Boolean> DataStoreDownloadFollowRedirects = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED,
Boolean.class, "store.download.follow.redirects", "false",
"Whether HTTP redirect is followed during store downloads for objects such as template, volume etc.",
true, ConfigKey.Scope.Global);
ConfigKey<Long> HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000",
"The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true);

5
engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
View File

@ -91,6 +91,7 @@ import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
@ -366,6 +367,7 @@ public class TemplateServiceImpl implements TemplateService {
toBeDownloaded.addAll(allTemplates);
final StateMachine2<VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate> stateMachine = VirtualMachineTemplate.State.getStateMachine();
Boolean followRedirect = StorageManager.DataStoreDownloadFollowRedirects.value();
for (VMTemplateVO tmplt : allTemplates) {
String uniqueName = tmplt.getUniqueName();
TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
@ -446,7 +448,8 @@ public class TemplateServiceImpl implements TemplateService {
try {
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId),
com.cloud.configuration.Resource.ResourceType.secondary_storage,
tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl(),
followRedirect));
} catch (ResourceAllocationException e) {
s_logger.warn(e.getMessage());
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null, e.getMessage(), e.getMessage());

8
engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java
View File

@ -23,6 +23,7 @@ import java.util.Map;
import javax.inject.Inject;
import com.cloud.storage.StorageManager;
import com.cloud.user.UserData;
import org.apache.log4j.Logger;
@ -73,8 +74,10 @@ public class TemplateObject implements TemplateInfo {
VMTemplatePoolDao templatePoolDao;
@Inject
TemplateDataStoreDao templateStoreDao;
final private boolean followRedirects;
public TemplateObject() {
this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
}
protected void configure(VMTemplateVO template, DataStore dataStore) {
@ -573,4 +576,9 @@ public class TemplateObject implements TemplateInfo {
// TODO Auto-generated method stub
return null;
}
@Override
public boolean isFollowRedirects() {
return followRedirects;
}
}

8
engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
View File

@ -23,6 +23,7 @@ import javax.inject.Inject;
import com.cloud.configuration.Resource.ResourceType;
import com.cloud.dc.VsphereStoragePolicyVO;
import com.cloud.dc.dao.VsphereStoragePolicyDao;
import com.cloud.storage.StorageManager;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;
@ -117,6 +118,7 @@ public class VolumeObject implements VolumeInfo {
private MigrationOptions migrationOptions;
private boolean directDownload;
private String vSphereStoragePolicyId;
private boolean followRedirects;
private final List<Volume.State> volumeStatesThatShouldNotTransitWhenDataStoreRoleIsImage = Arrays.asList(Volume.State.Migrating, Volume.State.Uploaded, Volume.State.Copying,
Volume.State.Expunged);
@ -127,6 +129,7 @@ public class VolumeObject implements VolumeInfo {
public VolumeObject() {
_volStateMachine = Volume.State.getStateMachine();
this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
}
protected void configure(DataStore dataStore, VolumeVO volumeVO) {
@ -930,4 +933,9 @@ public class VolumeObject implements VolumeInfo {
public void setEncryptFormat(String encryptFormat) {
volumeVO.setEncryptFormat(encryptFormat);
}
@Override
public boolean isFollowRedirects() {
return followRedirects;
}
}

1
framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java
View File

@ -31,4 +31,5 @@ public interface ConfigDepot {
<T> void set(ConfigKey<T> key, T value);
<T> void createOrUpdateConfigObject(String componentName, ConfigKey<T> key, String value);
boolean isNewConfig(ConfigKey<?> configKey);
}

7
framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java
View File

@ -81,6 +81,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
List<Configurable> _configurables;
List<ScopedConfigStorage> _scopedStorages;
Set<Configurable> _configured = Collections.synchronizedSet(new HashSet<Configurable>());
Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
private HashMap<String, Pair<String, ConfigKey<?>>> _allKeys = new HashMap<String, Pair<String, ConfigKey<?>>>(1007);
@ -193,6 +194,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
}
_configDao.persist(vo);
newConfigs.add(vo.getName());
} else {
boolean configUpdated = false;
if (vo.isDynamic() != key.isDynamic() || !ObjectUtils.equals(vo.getDescription(), key.description()) || !ObjectUtils.equals(vo.getDefaultValue(), key.defaultValue()) ||
@ -343,4 +345,9 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
return new Pair<>(groupId, subGroupId);
}
@Override
public boolean isNewConfig(ConfigKey<?> configKey) {
return newConfigs.contains(configKey.key());
}
}

17
framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java
View File

@ -18,9 +18,14 @@
//
package org.apache.cloudstack.framework.config.impl;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.test.util.ReflectionTestUtils;
public class ConfigDepotImplTest {
@ -40,4 +45,16 @@ public class ConfigDepotImplTest {
}
}
@Test
public void testIsNewConfig() {
String validNewConfigKey = "CONFIG";
ConfigKey<Boolean> validNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG", "true", "", true);
ConfigKey<Boolean> invalidNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG1", "true", "", true);
Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
newConfigs.add(validNewConfigKey);
ReflectionTestUtils.setField(configDepotImpl, "newConfigs", newConfigs);
Assert.assertTrue(configDepotImpl.isNewConfig(validNewConfig));
Assert.assertFalse(configDepotImpl.isNewConfig(invalidNewConfig));
}
}

4
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
View File

@ -42,9 +42,9 @@ public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, Chec
s_logger.info(String.format("Checking URL: %s, with connect timeout: %d, connect request timeout: %d, socket timeout: %d", url, connectTimeout, connectionRequestTimeout, socketTimeout));
Long remoteSize = null;
boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout);
boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout, cmd.isFollowRedirects());
if (checkResult) {
remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout);
remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout, cmd.isFollowRedirects());
if (remoteSize == null || remoteSize < 0) {
s_logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
"url %s, obtained size = %s", url, remoteSize));

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -2378,7 +2378,7 @@ public class KVMStorageProcessor implements StorageProcessor {
Long templateSize = null;
if (StringUtils.isNotBlank(cmd.getUrl())) {
String url = cmd.getUrl();
templateSize = UriUtils.getRemoteSize(url);
templateSize = UriUtils.getRemoteSize(url, cmd.isFollowRedirects());
}
s_logger.debug("Checking for free space on the host for downloading the template with physical size: " + templateSize + " and virtual size: " + cmd.getTemplateSize());

29
server/src/main/java/com/cloud/storage/StorageManagerImpl.java
View File

@ -97,6 +97,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
@ -239,6 +240,7 @@ import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.EntityManager;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.db.JoinBuilder;
@ -382,6 +384,11 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
@Inject
protected BucketDao _bucketDao;
@Inject
ConfigDepot configDepot;
@Inject
ConfigurationDao configurationDao;
protected List<StoragePoolDiscoverer> _discoverers;
public List<StoragePoolDiscoverer> getDiscoverers() {
@ -442,6 +449,23 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
}
protected void enableDefaultDatastoreDownloadRedirectionForExistingInstallations() {
if (!configDepot.isNewConfig(DataStoreDownloadFollowRedirects)) {
if (s_logger.isTraceEnabled()) {
s_logger.trace(String.format("%s is not a new configuration, skipping updating its value",
DataStoreDownloadFollowRedirects.key()));
}
return;
}
List<DataCenterVO> zones =
_dcDao.listAll(new Filter(1));
if (CollectionUtils.isNotEmpty(zones)) {
s_logger.debug(String.format("Updating value for configuration: %s to true",
DataStoreDownloadFollowRedirects.key()));
configurationDao.update(DataStoreDownloadFollowRedirects.key(), "true");
}
}
@Override
public List<StoragePoolVO> ListByDataCenterHypervisor(long datacenterId, HypervisorType type) {
List<StoragePoolVO> pools = _storagePoolDao.listByDataCenterId(datacenterId);
@ -673,7 +697,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
_executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS);
enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
return true;
}
@ -3719,7 +3743,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
MountDisabledStoragePool,
VmwareCreateCloneFull,
VmwareAllowParallelExecution,
ConvertVmwareInstanceToKvmTimeout
ConvertVmwareInstanceToKvmTimeout,
DataStoreDownloadFollowRedirects
};
}

12
server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
View File

@ -552,12 +552,14 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
throw new InvalidParameterValueException("File:// type urls are currently unsupported");
}
UriUtils.validateUrl(format, url);
boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access
s_logger.debug("Checking url: " + url);
DirectDownloadHelper.checkUrlExistence(url);
DirectDownloadHelper.checkUrlExistence(url, followRedirects);
}
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage,
UriUtils.getRemoteSize(url, followRedirects));
} else {
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage);
}
@ -659,8 +661,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume);
//url can be null incase of postupload
if (url != null) {
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
volume.setSize(UriUtils.getRemoteSize(url));
long remoteSize = UriUtils.getRemoteSize(url, StorageManager.DataStoreDownloadFollowRedirects.value());
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage,
remoteSize);
volume.setSize(remoteSize);
}
return volume;

19
server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
View File

@ -89,6 +89,7 @@ import com.cloud.server.StatsCollector;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.TemplateProfile;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateVO;
@ -161,7 +162,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
* @param url url
*/
private Long performDirectDownloadUrlValidation(final String format, final Hypervisor.HypervisorType hypervisor,
final String url, final List<Long> zoneIds) {
final String url, final List<Long> zoneIds, final boolean followRedirects) {
HostVO host = null;
if (zoneIds != null && !zoneIds.isEmpty()) {
for (Long zoneId : zoneIds) {
@ -180,7 +181,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
Integer socketTimeout = DirectDownloadManager.DirectDownloadSocketTimeout.value();
Integer connectRequestTimeout = DirectDownloadManager.DirectDownloadConnectionRequestTimeout.value();
Integer connectTimeout = DirectDownloadManager.DirectDownloadConnectTimeout.value();
CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout);
CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout, followRedirects);
s_logger.debug("Performing URL " + url + " validation on host " + host.getId());
Answer answer = _agentMgr.easySend(host.getId(), cmd);
if (answer == null || !answer.getResult()) {
@ -204,6 +205,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url);
boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (cmd.isDirectDownload()) {
DigestHelper.validateChecksumString(cmd.getChecksum());
List<Long> zoneIds = null;
@ -212,12 +214,14 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
zoneIds.add(cmd.getZoneId());
}
Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(),
Hypervisor.HypervisorType.KVM, url, zoneIds);
Hypervisor.HypervisorType.KVM, url, zoneIds, followRedirects);
profile.setSize(templateSize);
}
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
ResourceType.secondary_storage,
UriUtils.getRemoteSize(url, followRedirects));
return profile;
}
@ -236,15 +240,18 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
String url = profile.getUrl();
UriUtils.validateUrl(cmd.getFormat(), url, cmd.isDirectDownload());
Hypervisor.HypervisorType hypervisor = Hypervisor.HypervisorType.getType(cmd.getHypervisor());
boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (cmd.isDirectDownload()) {
DigestHelper.validateChecksumString(cmd.getChecksum());
Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(),
hypervisor, url, cmd.getZoneIds());
hypervisor, url, cmd.getZoneIds(), followRedirects);
profile.setSize(templateSize);
}
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
ResourceType.secondary_storage,
UriUtils.getRemoteSize(url, followRedirects));
return profile;
}

17
server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
View File

@ -272,7 +272,8 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO();
DownloadProtocol protocol = getProtocolFromUrl(url);
DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum,
headers);
cmd.setTemplateSize(template.getSize());
cmd.setFormat(template.getFormat());
@ -393,19 +394,23 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
/**
* Return DirectDownloadCommand according to the protocol
*/
private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
String checksum, Map<String, String> httpHeaders) {
private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url,
Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> httpHeaders) {
int connectTimeout = DirectDownloadConnectTimeout.value();
int soTimeout = DirectDownloadSocketTimeout.value();
int connectionRequestTimeout = DirectDownloadConnectionRequestTimeout.value();
boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (protocol.equals(DownloadProtocol.HTTP)) {
return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
soTimeout, followRedirects);
} else if (protocol.equals(DownloadProtocol.HTTPS)) {
return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout, connectionRequestTimeout);
return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
soTimeout, connectionRequestTimeout, followRedirects);
} else if (protocol.equals(DownloadProtocol.NFS)) {
return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
} else if (protocol.equals(DownloadProtocol.METALINK)) {
return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
soTimeout, followRedirects);
} else {
return null;
}

43
server/src/test/java/com/cloud/storage/StorageManagerImplTest.java
View File

@ -17,12 +17,17 @@
package com.cloud.storage;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.exception.ConnectionException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.host.Host;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.dao.VMInstanceDao;
import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.commons.collections.MapUtils;
@ -48,6 +53,12 @@ public class StorageManagerImplTest {
@Mock
VMInstanceDao vmInstanceDao;
@Mock
ConfigDepot configDepot;
@Mock
ConfigurationDao configurationDao;
@Mock
DataCenterDao dataCenterDao;
@Spy
@InjectMocks
@ -206,4 +217,36 @@ public class StorageManagerImplTest {
throw new RuntimeException(e);
}
}
@Test
public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNoChange() {
Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
.thenReturn(false);
storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
Mockito.verify(configurationDao, Mockito.never()).update(Mockito.anyString(), Mockito.anyString());
}
@Test
public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsOldInstall() {
Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
.thenReturn(true);
Mockito.when(dataCenterDao.listAll(Mockito.any()))
.thenReturn(List.of(Mockito.mock(DataCenterVO.class)));
Mockito.doReturn(true).when(configurationDao).update(Mockito.anyString(), Mockito.anyString());
storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
Mockito.verify(configurationDao, Mockito.times(1))
.update(StorageManager.DataStoreDownloadFollowRedirects.key(), "true");
}
@Test
public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNewInstall() {
Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
.thenReturn(true);
Mockito.when(dataCenterDao.listAll(Mockito.any()))
.thenReturn(new ArrayList<>()); //new installation
storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
Mockito.verify(configurationDao, Mockito.never())
.update(StorageManager.DataStoreDownloadFollowRedirects.key(),StorageManager.DataStoreDownloadFollowRedirects.defaultValue());
}
}

16
services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java
View File

@ -41,17 +41,21 @@ public interface DownloadManager extends Manager {
* @param hvm whether the template is a hardware virtual machine
* @param accountId the accountId of the iso owner (null if public iso)
* @param descr description of the template
* @param user username used for authentication to the server
* @param password password used for authentication to the server
* @param userName username used for authentication to the server
* @param passwd password used for authentication to the server
* @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes.
* @param resourceType signifying the type of resource like template, volume etc.
* @param followRedirects whether downloader follows redirections
* @return job-id that can be used to interrogate the status of the download.
*/
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
String installPathPrefix, String templatePath, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType);
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
Long accountId, String descr, String cksum, String installPathPrefix, String templatePath,
String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType,
boolean followRedirects);
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType);
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects);
Map<String, Processor> getProcessors();

25
services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
View File

@ -661,8 +661,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
@Override
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects) {
UUID uuid = UUID.randomUUID();
String jobId = uuid.toString();
@ -682,6 +683,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
} else {
throw new CloudRuntimeException("Unable to download from URL: " + url);
}
td.setFollowRedirects(followRedirects);
DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
dj.setTmpltPath(installPathPrefix);
jobs.put(jobId, dj);
@ -717,8 +719,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
@Override
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user,
String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType,
boolean followRedirects) {
UUID uuid = UUID.randomUUID();
String jobId = uuid.toString();
String tmpDir = installPathPrefix;
@ -765,6 +769,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
throw new CloudRuntimeException("Unable to download from URL: " + url);
}
}
td.setFollowRedirects(followRedirects);
// NOTE the difference between installPathPrefix and templatePath
// here. instalPathPrefix is the absolute path for template
// including mount directory
@ -901,12 +906,16 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
String jobId = null;
if (dstore instanceof S3TO) {
jobId =
downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(),
cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(),
installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
cmd.isFollowRedirects());
} else {
jobId =
downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(),
cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix,
cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
cmd.isFollowRedirects());
}
sleep();
if (jobId == null) {

3
utils/src/main/java/com/cloud/utils/UriUtils.java
View File

@ -213,7 +213,7 @@ public class UriUtils {
}
// Get the size of a file from URL response header.
public static long getRemoteSize(String url) {
public static long getRemoteSize(String url, Boolean followRedirect) {
long remoteSize = 0L;
final String[] methods = new String[]{"HEAD", "GET"};
IllegalArgumentException exception = null;
@ -228,6 +228,7 @@ public class UriUtils {
httpConn.setRequestMethod(method);
httpConn.setConnectTimeout(2000);
httpConn.setReadTimeout(5000);
httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect));
String contentLength = httpConn.getHeaderField("content-length");
if (contentLength != null) {
remoteSize = Long.parseLong(contentLength);

20
utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
View File

@ -22,8 +22,9 @@ package com.cloud.utils.storage;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@ -112,16 +113,23 @@ public final class QCOW2Utils {
}
}
public static long getVirtualSize(String urlStr) {
public static long getVirtualSizeFromUrl(String urlStr, boolean followRedirects) {
HttpURLConnection httpConn = null;
try {
URL url = new URL(urlStr);
return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr));
} catch (MalformedURLException e) {
URI url = new URI(urlStr);
httpConn = (HttpURLConnection)url.toURL().openConnection();
httpConn.setInstanceFollowRedirects(followRedirects);
return getVirtualSize(httpConn.getInputStream(), UriUtils.isUrlForCompressedFile(urlStr));
} catch (URISyntaxException e) {
LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
throw new IllegalArgumentException("Invalid URL: " + urlStr);
} catch (IOException e) {
LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
} finally {
if (httpConn != null) {
httpConn.disconnect();
}
}
}
}