engine-storage: control download redirection

Add a global setting to control whether redirection is allowed while
downloading templates and volumes

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java

@@ -26,6 +26,7 @@ import java.io.RandomAccessFile;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Date;
+import java.util.List;
 
 import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
@@ -81,6 +82,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
     private long maxTemplateSizeInBytes;
     private ResourceType resourceType = ResourceType.TEMPLATE;
     private final HttpMethodRetryHandler myretryhandler;
+    private boolean followRedirects = false;
 
     public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
             String user, String password, Proxy proxy, ResourceType resourceType) {
@@ -112,7 +114,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
     private GetMethod createRequest(String downloadUrl) {
         GetMethod request = new GetMethod(downloadUrl);
         request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
-        request.setFollowRedirects(true);
+        request.setFollowRedirects(followRedirects);
         return request;
     }
 
@@ -336,6 +338,12 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
         } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
             status = Status.UNRECOVERABLE_ERROR;
             errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
+            if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
+                    && !followRedirects) {
+                errorString = String.format("Failed to download %s due to redirection, response code: %d",
+                        downloadUrl, responseCode);
+                s_logger.error(errorString);
+            }
             return true; //FIXME: retry?
         }
         return false;
@@ -537,4 +545,12 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
             return this;
         }
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+        if (this.request != null) {
+            this.request.setFollowRedirects(followRedirects);
+        }
+    }
 }

core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java

@@ -60,7 +60,7 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement
     protected GetMethod createRequest(String downloadUrl) {
         GetMethod request = new GetMethod(downloadUrl);
         request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
-        request.setFollowRedirects(true);
+        request.setFollowRedirects(followRedirects);
         if (!toFileSet) {
             String[] parts = downloadUrl.split("/");
             String filename = parts[parts.length - 1];
@@ -173,4 +173,12 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement
     public void setStatus(Status status) {
         this.status = status;
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        super.setFollowRedirects(followRedirects);
+        if (this.request != null) {
+            this.request.setFollowRedirects(followRedirects);
+        }
+    }
 }

core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java

@@ -34,6 +34,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
 import org.apache.commons.httpclient.Header;
 import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.params.HttpMethodParams;
@@ -44,6 +45,7 @@ import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Date;
+import java.util.List;
 
 import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
 import static java.util.Arrays.asList;
@@ -72,8 +74,8 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
     private long downloadTime;
     private long totalBytes;
     private long maxTemplateSizeInByte;
-
     private boolean resume = false;
+    private boolean followRedirects = false;
 
     public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
             long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
@@ -91,7 +93,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
         this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
 
         // Follow redirects
-        this.getMethod.setFollowRedirects(true);
+        this.getMethod.setFollowRedirects(followRedirects);
 
         // Set file extension.
         this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
@@ -124,10 +126,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
             return 0;
         }
 
-        if (!HTTPUtils.verifyResponseCode(responseCode)) {
+        boolean failedDueToRedirection = List.of(HttpStatus.SC_MOVED_PERMANENTLY,
+                HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) && !followRedirects;
+        if (!HTTPUtils.verifyResponseCode(responseCode) || failedDueToRedirection) {
             errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
             LOGGER.warn(errorString);
-
             status = Status.UNRECOVERABLE_ERROR;
             return 0;
         }
@@ -373,4 +376,12 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
     public String getFileExtension() {
         return fileExtension;
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+        if (this.getMethod != null) {
+            this.getMethod.setFollowRedirects(followRedirects);
+        }
+    }
 }

core/src/main/java/com/cloud/storage/template/TemplateDownloader.java

@@ -92,4 +92,6 @@ public interface TemplateDownloader extends Runnable {
     boolean isInited();
 
     long getMaxTemplateSizeInBytes();
+
+    void setFollowRedirects(boolean followRedirects);
 }

core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java

@@ -43,6 +43,7 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl
     protected long _start;
     protected StorageLayer _storage;
     protected boolean _inited = false;
+    protected boolean followRedirects = false;
     private long maxTemplateSizeInBytes;
 
     public TemplateDownloaderBase(StorageLayer storage, String downloadUrl, String toDir, long maxTemplateSizeInBytes, DownloadCompleteCallback callback) {
@@ -149,4 +150,9 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl
     public boolean isInited() {
         return _inited;
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java

@@ -28,6 +28,7 @@ public class CheckUrlCommand extends Command {
     private Integer connectTimeout;
     private Integer connectionRequestTimeout;
     private Integer socketTimeout;
+    private boolean followRedirects;
 
     public String getFormat() {
         return format;
@@ -43,13 +44,19 @@ public class CheckUrlCommand extends Command {
 
     public Integer getSocketTimeout() { return socketTimeout; }
 
-    public CheckUrlCommand(final String format,final String url) {
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public CheckUrlCommand(final String format, final String url, final boolean followRedirects) {
         super();
         this.format = format;
         this.url = url;
+        this.followRedirects = followRedirects;
     }
 
-    public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+    public CheckUrlCommand(final String format,final String url, Integer connectTimeout,
+               Integer connectionRequestTimeout, Integer socketTimeout, final boolean followRedirects) {
         super();
         this.format = format;
         this.url = url;

core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java

@@ -45,7 +45,11 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
     private Long templateSize;
     private Storage.ImageFormat format;
 
-    protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) {
+    private boolean followRedirects;
+
+    protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool,
+             final String checksum, final Map<String, String> headers, final Integer connectTimeout,
+             final Integer soTimeout, final Integer connectionRequestTimeout, final boolean followRedirects) {
         this.url = url;
         this.templateId = templateId;
         this.destData = destData;
@@ -55,6 +59,7 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
         this.connectTimeout = connectTimeout;
         this.soTimeout = soTimeout;
         this.connectionRequestTimeout = connectionRequestTimeout;
+        this.followRedirects = followRedirects;
     }
 
     public String getUrl() {
@@ -137,4 +142,12 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
     public int getWaitInMillSeconds() {
         return getWait() * 1000;
     }
+
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java

@@ -24,8 +24,10 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
 public class HttpDirectDownloadCommand extends DirectDownloadCommand {
 
-    public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
+    public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
-        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
+             Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
+        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
+                null, followRedirects);
     }
 
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java

@@ -25,7 +25,10 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
 public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
 
-    public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout) {
+    public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
-        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, connectionRequestTimeout);
+              Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout,
+              boolean followRedirects) {
+        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
+                connectionRequestTimeout, followRedirects);
     }
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java

@@ -24,8 +24,9 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
 public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
 
-    public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
+    public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
-        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
+                 Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
+        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null, followRedirects);
     }
 
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java

@@ -24,8 +24,9 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
 public class NfsDirectDownloadCommand extends DirectDownloadCommand {
 
-    public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
+    public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool,
-        super(url, templateId, destPool, checksum, headers, null, null, null);
+                final String checksum, final Map<String, String> headers) {
+        super(url, templateId, destPool, checksum, headers, null, null, null, false);
     }
 
 }

core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java

@@ -34,27 +34,30 @@ public class DirectDownloadHelper {
      * Get direct template downloader from direct download command and destination pool
      */
     public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
-                                                                                  String destPoolLocalPath,
+                  String destPoolLocalPath,  String temporaryDownloadPath) {
-                                                                                  String temporaryDownloadPath) {
         if (cmd instanceof HttpDirectDownloadCommand) {
-            return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
+            return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
-                    cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
+                    cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+                    temporaryDownloadPath, cmd.isFollowRedirects());
         } else if (cmd instanceof HttpsDirectDownloadCommand) {
-            return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
+            return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
-                    cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
+                    cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+                    cmd.getConnectionRequestTimeout(), temporaryDownloadPath, cmd.isFollowRedirects());
         } else if (cmd instanceof NfsDirectDownloadCommand) {
-            return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
+            return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
+                    cmd.getChecksum(), temporaryDownloadPath);
         } else if (cmd instanceof MetalinkDirectDownloadCommand) {
-            return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
+            return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
-                    cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
+                    cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+                    temporaryDownloadPath, cmd.isFollowRedirects());
         } else {
             throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
         }
     }
 
-    public static boolean checkUrlExistence(String url) {
+    public static boolean checkUrlExistence(String url, boolean followRedirects) {
         try {
-            DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
+            DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects);
             return checker.checkUrl(url);
         } catch (CloudRuntimeException e) {
             LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@@ -62,9 +65,11 @@ public class DirectDownloadHelper {
         }
     }
 
-    public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+    public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout,
+            Integer socketTimeout, boolean followRedirects) {
         try {
-            DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+            DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout,
+                    socketTimeout, followRedirects);
             return checker.checkUrl(url);
         } catch (CloudRuntimeException e) {
             LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@@ -72,27 +77,29 @@ public class DirectDownloadHelper {
         }
     }
 
-    private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+    private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout,
+             Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
         if (url.toLowerCase().startsWith("https:")) {
-            return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+            return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
         } else if (url.toLowerCase().startsWith("http:")) {
-            return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout);
+            return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects);
         } else if (url.toLowerCase().startsWith("nfs:")) {
             return new NfsDirectTemplateDownloader(url);
         } else if (url.toLowerCase().endsWith(".metalink")) {
-            return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout);
+            return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects);
         } else {
             throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url));
         }
     }
 
-    public static Long getFileSize(String url, String format) {
+    public static Long getFileSize(String url, String format, boolean followRedirects) {
-        DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
+        DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects);
         return checker.getRemoteFileSize(url, format);
     }
 
-    public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+    public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout,
-        DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+           Integer socketTimeout, boolean followRedirects) {
+        DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
         return checker.getRemoteFileSize(url, format);
     }
 }

core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java

@@ -42,16 +42,19 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
     private String checksum;
     private boolean redownload = false;
     protected String temporaryDownloadPath;
+    private boolean followRedirects;
 
     public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName());
 
     protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId,
-                                           final String checksum, final String temporaryDownloadPath) {
+                                           final String checksum, final String temporaryDownloadPath,
+                                           final boolean followRedirects) {
         this.url = url;
         this.destPoolPath = destPoolPath;
         this.templateId = templateId;
         this.checksum = checksum;
         this.temporaryDownloadPath = temporaryDownloadPath;
+        this.followRedirects = followRedirects;
     }
 
     private static String directDownloadDir = "template";
@@ -111,6 +114,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
         return redownload;
     }
 
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
+
     /**
      * Create download directory (if it does not exist)
      */

core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java

@@ -50,13 +50,15 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
     protected GetMethod request;
     protected Map<String, String> reqHeaders = new HashMap<>();
 
-    protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
+    protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout,
-        this(url, null, null, null, null, connectTimeout, socketTimeout, null);
+           boolean followRedirects) {
+        this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects);
     }
 
     public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
-                                        Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
+                Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
-        super(url, destPoolPath, templateId, checksum, downloadPath);
+                boolean followRedirects) {
+        super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
         s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout);
         s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout);
         client = new HttpClient(s_httpClientManager);
@@ -68,7 +70,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
 
     protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
         GetMethod request = new GetMethod(downloadUrl);
-        request.setFollowRedirects(true);
+        request.setFollowRedirects(this.isFollowRedirects());
         if (MapUtils.isNotEmpty(headers)) {
             for (String key : headers.keySet()) {
                 request.setRequestHeader(key, headers.get(key));
@@ -111,9 +113,11 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
     @Override
     public boolean checkUrl(String url) {
         HeadMethod httpHead = new HeadMethod(url);
+        httpHead.setFollowRedirects(this.isFollowRedirects());
         try {
-            if (client.executeMethod(httpHead) != HttpStatus.SC_OK) {
+            int responseCode = client.executeMethod(httpHead);
-                s_logger.error(String.format("Invalid URL: %s", url));
+            if (responseCode != HttpStatus.SC_OK) {
+                s_logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
                 return false;
             }
             return true;
@@ -128,9 +132,9 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
     @Override
     public Long getRemoteFileSize(String url, String format) {
         if ("qcow2".equalsIgnoreCase(format)) {
-            return QCOW2Utils.getVirtualSize(url);
+            return QCOW2Utils.getVirtualSizeFromUrl(url, this.isFollowRedirects());
         } else {
-            return UriUtils.getRemoteSize(url);
+            return UriUtils.getRemoteSize(url, this.isFollowRedirects());
         }
     }
 

core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java

@@ -68,20 +68,28 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
     protected CloseableHttpClient httpsClient;
     private HttpUriRequest req;
 
-    protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+    protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout,
-        this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null);
+            Integer socketTimeout, boolean followRedirects) {
+        this(url, null, null, null, null, connectTimeout, socketTimeout,
+                connectionRequestTimeout, null, followRedirects);
     }
 
-    public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
+    public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
-                                         Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
+                 Map<String, String> headers, Integer connectTimeout, Integer soTimeout,
-        super(url, destPoolPath, templateId, checksum, temporaryDownloadPath);
+                 Integer connectionRequestTimeout, String temporaryDownloadPath, boolean followRedirects) {
+        super(url, destPoolPath, templateId, checksum, temporaryDownloadPath, followRedirects);
         SSLContext sslcontext = getSSLContext();
         SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
         RequestConfig config = RequestConfig.custom()
                 .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
                 .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
-                .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
+                .setSocketTimeout(soTimeout == null ? 5000 : soTimeout)
-        httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
+                .setRedirectsEnabled(followRedirects)
+                .build();
+        httpsClient = HttpClients.custom()
+                .setSSLSocketFactory(factory)
+                .setDefaultRequestConfig(config)
+                .build();
         createUriRequest(url, headers);
         String downloadDir = getDirectDownloadTempPath(templateId);
         File tempFile = createTemporaryDirectoryAndFile(downloadDir);
@@ -90,6 +98,7 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
 
     protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
         req = new HttpGet(downloadUrl);
+        setFollowRedirects(this.isFollowRedirects());
         if (MapUtils.isNotEmpty(headers)) {
             for (String headerKey: headers.keySet()) {
                 req.setHeader(headerKey, headers.get(headerKey));
@@ -164,8 +173,9 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
         HttpHead httpHead = new HttpHead(url);
         try {
             CloseableHttpResponse response = httpsClient.execute(httpHead);
-            if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+            int responseCode = response.getStatusLine().getStatusCode();
-                s_logger.error(String.format("Invalid URL: %s", url));
+            if (responseCode != HttpStatus.SC_OK) {
+                s_logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
                 return false;
             }
             return true;

core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java

@@ -42,16 +42,15 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm
     private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
 
     protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId,
-                                                                    String destPoolPath, String checksum,
+                String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout,
-                                                                    Map<String, String> headers,
+                Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
-                                                                    Integer connectTimeout, Integer soTimeout,
-                                                                    Integer connectionRequestTimeout, String temporaryDownloadPath) {
         if (url.toLowerCase().startsWith("https:")) {
             return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
-                    connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath);
+                    connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath,
+                    this.isFollowRedirects());
         } else if (url.toLowerCase().startsWith("http:")) {
             return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
-                    connectTimeout, soTimeout, temporaryDownloadPath);
+                    connectTimeout, soTimeout, temporaryDownloadPath, this.isFollowRedirects());
         } else if (url.toLowerCase().startsWith("nfs:")) {
             return new NfsDirectTemplateDownloader(url);
         } else {
@@ -60,13 +59,15 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm
         }
     }
 
-    protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
+    protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout,
-        this(url, null, null, null, null, connectTimeout, socketTimeout, null);
+           boolean followRedirects) {
+        this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects);
     }
 
     public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
-                                            Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
+                Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
-        super(url, destPoolPath, templateId, checksum, downloadPath);
+                boolean followRedirects) {
+        super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
         this.headers = headers;
         this.connectTimeout = connectTimeout;
         this.soTimeout = soTimeout;

core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java

@@ -57,8 +57,9 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
         this(url, null, null, null, null);
     }
 
-    public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) {
+    public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum,
-        super(url, destPool, templateId, checksum, downloadPath);
+               String downloadPath) {
+        super(url, destPool, templateId, checksum, downloadPath, false);
         parseUrl();
     }
 

core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java

@@ -48,6 +48,8 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
     private DataStoreTO _store;
     private DataStoreTO cacheStore;
 
+    private boolean followRedirects = false;
+
     protected DownloadCommand() {
     }
 
@@ -64,6 +66,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
         installPath = that.installPath;
         _store = that._store;
         _proxy = that._proxy;
+        followRedirects = that.followRedirects;
     }
 
     public DownloadCommand(TemplateObjectTO template, Long maxDownloadSizeInBytes) {
@@ -79,6 +82,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
             setSecUrl(((NfsTO)_store).getUrl());
         }
         this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
+        this.followRedirects = template.isFollowRedirects();
     }
 
     public DownloadCommand(TemplateObjectTO template, String user, String passwd, Long maxDownloadSizeInBytes) {
@@ -94,6 +98,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
         _store = volume.getDataStore();
         this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
         resourceType = ResourceType.VOLUME;
+        this.followRedirects = volume.isFollowRedirects();
     }
 
     @Override
@@ -181,4 +186,12 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
     public DataStoreTO getCacheStore() {
         return cacheStore;
     }
+
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
 }

core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.storage.to;
+
+public class DownloadableObjectTO {
+    protected boolean followRedirects = false;
+
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
+}

core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java

@@ -30,7 +30,7 @@ import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 
-public class SnapshotObjectTO implements DataTO {
+public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO {
     private String path;
     private VolumeObjectTO volume;
     private String parentSnapshotPath;

core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java

@@ -28,7 +28,7 @@ import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.template.VirtualMachineTemplate;
 
-public class TemplateObjectTO implements DataTO {
+public class TemplateObjectTO extends DownloadableObjectTO implements DataTO {
     private String path;
     private String origUrl;
     private String uuid;
@@ -87,6 +87,7 @@ public class TemplateObjectTO implements DataTO {
         this.deployAsIs = template.isDeployAsIs();
         this.deployAsIsConfiguration = template.getDeployAsIsConfiguration();
         this.directDownload = template.isDirectDownload();
+        this.followRedirects = template.isFollowRedirects();
     }
 
     @Override

core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java

@@ -33,7 +33,7 @@ import com.cloud.storage.Volume;
 
 import java.util.Arrays;
 
-public class VolumeObjectTO implements DataTO {
+public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
     private String uuid;
     private Volume.Type volumeType;
     private DataStoreTO dataStore;
@@ -119,6 +119,7 @@ public class VolumeObjectTO implements DataTO {
         this.vSphereStoragePolicyId = volume.getvSphereStoragePolicyId();
         this.passphrase = volume.getPassphrase();
         this.encryptFormat = volume.getEncryptFormat();
+        this.followRedirects = volume.isFollowRedirects();
     }
 
     public String getUuid() {

core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java

@@ -56,7 +56,7 @@ public class BaseDirectTemplateDownloaderTest {
     private HttpEntity httpEntity;
 
     @InjectMocks
-    protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000);
+    protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000, false);
 
     @Before
     public void init() throws IOException {

core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java

@@ -25,7 +25,7 @@ import org.mockito.InjectMocks;
 public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
 
     @InjectMocks
-    protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000);
+    protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000, false);
 
     @Test
     public void testCheckUrlMetalink() {

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.engine.subsystem.api.storage;
+
+public interface DownloadableDataInfo extends DataObject {
+    default public boolean isFollowRedirects() {
+        return true;
+    }
+}

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java

@@ -21,7 +21,7 @@ package org.apache.cloudstack.engine.subsystem.api.storage;
 import com.cloud.template.VirtualMachineTemplate;
 import com.cloud.user.UserData;
 
-public interface TemplateInfo extends DataObject, VirtualMachineTemplate {
+public interface TemplateInfo extends DownloadableDataInfo, VirtualMachineTemplate {
     @Override
     String getUniqueName();
 

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java

@@ -26,7 +26,7 @@ import com.cloud.storage.Storage;
 import com.cloud.storage.Volume;
 import com.cloud.vm.VirtualMachine;
 
-public interface VolumeInfo extends DataObject, Volume {
+public interface VolumeInfo extends DownloadableDataInfo, Volume {
 
     boolean isAttachedVM();
 

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -191,6 +191,10 @@ public interface StorageManager extends StorageService {
             true,
             ConfigKey.Scope.Global,
             null);
+    static final ConfigKey<Boolean> DataStoreDownloadFollowRedirects = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED,
+            Boolean.class, "store.download.follow.redirects", "false",
+            "Whether HTTP redirect is followed during store downloads for objects such as template, volume etc.",
+            true, ConfigKey.Scope.Global);
 
     /**
      * should we execute in sequence not involving any storages?

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -91,6 +91,7 @@ import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.TemplateType;
+import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
@@ -366,6 +367,7 @@ public class TemplateServiceImpl implements TemplateService {
                     toBeDownloaded.addAll(allTemplates);
 
                     final StateMachine2<VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate> stateMachine = VirtualMachineTemplate.State.getStateMachine();
+                    Boolean followRedirect = StorageManager.DataStoreDownloadFollowRedirects.value();
                     for (VMTemplateVO tmplt : allTemplates) {
                         String uniqueName = tmplt.getUniqueName();
                         TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
@@ -446,7 +448,8 @@ public class TemplateServiceImpl implements TemplateService {
                                         try {
                                             _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId),
                                                     com.cloud.configuration.Resource.ResourceType.secondary_storage,
-                                                    tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
+                                                    tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl(),
+                                                            followRedirect));
                                         } catch (ResourceAllocationException e) {
                                             s_logger.warn(e.getMessage());
                                             _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null, e.getMessage(), e.getMessage());

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java

@@ -23,6 +23,7 @@ import java.util.Map;
 
 import javax.inject.Inject;
 
+import com.cloud.storage.StorageManager;
 import com.cloud.user.UserData;
 import org.apache.log4j.Logger;
 
@@ -73,8 +74,10 @@ public class TemplateObject implements TemplateInfo {
     VMTemplatePoolDao templatePoolDao;
     @Inject
     TemplateDataStoreDao templateStoreDao;
+    final private boolean followRedirects;
 
     public TemplateObject() {
+        this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
     }
 
     protected void configure(VMTemplateVO template, DataStore dataStore) {
@@ -573,4 +576,9 @@ public class TemplateObject implements TemplateInfo {
         // TODO Auto-generated method stub
         return null;
     }
+
+    @Override
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
 }

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java

@@ -23,6 +23,7 @@ import javax.inject.Inject;
 import com.cloud.configuration.Resource.ResourceType;
 import com.cloud.dc.VsphereStoragePolicyVO;
 import com.cloud.dc.dao.VsphereStoragePolicyDao;
+import com.cloud.storage.StorageManager;
 import com.cloud.utils.db.Transaction;
 import com.cloud.utils.db.TransactionCallbackNoReturn;
 import com.cloud.utils.db.TransactionStatus;
@@ -117,6 +118,7 @@ public class VolumeObject implements VolumeInfo {
     private MigrationOptions migrationOptions;
     private boolean directDownload;
     private String vSphereStoragePolicyId;
+    private boolean followRedirects;
 
     private final List<Volume.State> volumeStatesThatShouldNotTransitWhenDataStoreRoleIsImage = Arrays.asList(Volume.State.Migrating, Volume.State.Uploaded, Volume.State.Copying,
       Volume.State.Expunged);
@@ -127,6 +129,7 @@ public class VolumeObject implements VolumeInfo {
 
     public VolumeObject() {
         _volStateMachine = Volume.State.getStateMachine();
+        this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
     }
 
     protected void configure(DataStore dataStore, VolumeVO volumeVO) {
@@ -930,4 +933,9 @@ public class VolumeObject implements VolumeInfo {
     public void setEncryptFormat(String encryptFormat) {
         volumeVO.setEncryptFormat(encryptFormat);
     }
+
+    @Override
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
 }

framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java

@@ -31,4 +31,5 @@ public interface ConfigDepot {
     <T> void set(ConfigKey<T> key, T value);
 
     <T> void createOrUpdateConfigObject(String componentName, ConfigKey<T> key, String value);
+    boolean isNewConfig(ConfigKey<?> configKey);
 }

framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java

@@ -81,6 +81,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
     List<Configurable> _configurables;
     List<ScopedConfigStorage> _scopedStorages;
     Set<Configurable> _configured = Collections.synchronizedSet(new HashSet<Configurable>());
+    Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
 
     private HashMap<String, Pair<String, ConfigKey<?>>> _allKeys = new HashMap<String, Pair<String, ConfigKey<?>>>(1007);
 
@@ -193,6 +194,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
             }
 
             _configDao.persist(vo);
+            newConfigs.add(vo.getName());
         } else {
             boolean configUpdated = false;
             if (vo.isDynamic() != key.isDynamic() || !ObjectUtils.equals(vo.getDescription(), key.description()) || !ObjectUtils.equals(vo.getDefaultValue(), key.defaultValue()) ||
@@ -343,4 +345,9 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
 
         return new Pair<>(groupId, subGroupId);
     }
+
+    @Override
+    public boolean isNewConfig(ConfigKey<?> configKey) {
+        return newConfigs.contains(configKey.key());
+    }
 }

framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java

@@ -18,9 +18,14 @@
 //
 package org.apache.cloudstack.framework.config.impl;
 
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.junit.Assert;
 import org.junit.Test;
+import org.springframework.test.util.ReflectionTestUtils;
 
 public class ConfigDepotImplTest {
 
@@ -40,4 +45,16 @@ public class ConfigDepotImplTest {
         }
     }
 
+    @Test
+    public void testIsNewConfig() {
+        String validNewConfigKey = "CONFIG";
+        ConfigKey<Boolean> validNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG", "true", "", true);
+        ConfigKey<Boolean> invalidNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG1", "true", "", true);
+        Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
+        newConfigs.add(validNewConfigKey);
+        ReflectionTestUtils.setField(configDepotImpl, "newConfigs", newConfigs);
+        Assert.assertTrue(configDepotImpl.isNewConfig(validNewConfig));
+        Assert.assertFalse(configDepotImpl.isNewConfig(invalidNewConfig));
+    }
+
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java

@@ -38,13 +38,14 @@ public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, Chec
         final Integer connectTimeout = cmd.getConnectTimeout();
         final Integer connectionRequestTimeout = cmd.getConnectionRequestTimeout();
         final Integer socketTimeout = cmd.getSocketTimeout();
+        final boolean followRedirects = cmd.isFollowRedirects();
 
         s_logger.info(String.format("Checking URL: %s, with connect timeout: %d, connect request timeout: %d, socket timeout: %d", url, connectTimeout, connectionRequestTimeout, socketTimeout));
         Long remoteSize = null;
 
-        boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+        boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
         if (checkResult) {
-            remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout);
+            remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
             if (remoteSize == null || remoteSize < 0) {
                 s_logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
                         "url %s, obtained size = %s", url, remoteSize));

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -2361,7 +2361,7 @@ public class KVMStorageProcessor implements StorageProcessor {
             Long templateSize = null;
             if (StringUtils.isNotBlank(cmd.getUrl())) {
                 String url = cmd.getUrl();
-                templateSize = UriUtils.getRemoteSize(url);
+                templateSize = UriUtils.getRemoteSize(url, cmd.isFollowRedirects());
             }
 
             s_logger.debug("Checking for free space on the host for downloading the template with physical size: " + templateSize + " and virtual size: " + cmd.getTemplateSize());

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -89,6 +89,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
+import org.apache.cloudstack.framework.config.ConfigDepot;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
@@ -216,6 +217,7 @@ import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.EntityManager;
+import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericSearchBuilder;
 import com.cloud.utils.db.GlobalLock;
 import com.cloud.utils.db.JoinBuilder;
@@ -347,6 +349,11 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
     @Inject
     protected UserVmManager userVmManager;
+    @Inject
+    ConfigDepot configDepot;
+    @Inject
+    ConfigurationDao configurationDao;
+
     protected List<StoragePoolDiscoverer> _discoverers;
 
     public List<StoragePoolDiscoverer> getDiscoverers() {
@@ -407,6 +414,23 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         }
     }
 
+    protected void enableDefaultDatastoreDownloadRedirectionForExistingInstallations() {
+        if (!configDepot.isNewConfig(DataStoreDownloadFollowRedirects)) {
+            if (s_logger.isTraceEnabled()) {
+                s_logger.trace(String.format("%s is not a new configuration, skipping updating its value",
+                        DataStoreDownloadFollowRedirects.key()));
+            }
+            return;
+        }
+        List<DataCenterVO> zones =
+                _dcDao.listAll(new Filter(1));
+        if (CollectionUtils.isNotEmpty(zones)) {
+            s_logger.debug(String.format("Updating value for configuration: %s to true",
+                DataStoreDownloadFollowRedirects.key()));
+            configurationDao.update(DataStoreDownloadFollowRedirects.key(), "true");
+        }
+    }
+
     @Override
     public List<StoragePoolVO> ListByDataCenterHypervisor(long datacenterId, HypervisorType type) {
         List<StoragePoolVO> pools = _storagePoolDao.listByDataCenterId(datacenterId);
@@ -638,7 +662,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         }
 
         _executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS);
-
+        enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
         return true;
     }
 
@@ -3417,7 +3441,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
                 SecStorageVMAutoScaleDown,
                 MountDisabledStoragePool,
                 VmwareCreateCloneFull,
-                VmwareAllowParallelExecution
+                VmwareAllowParallelExecution,
+                DataStoreDownloadFollowRedirects
         };
     }
 

server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java

@@ -535,12 +535,14 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
                 throw new InvalidParameterValueException("File:// type urls are currently unsupported");
             }
             UriUtils.validateUrl(format, url);
+            boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
             if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access
                 s_logger.debug("Checking url: " + url);
-                DirectDownloadHelper.checkUrlExistence(url);
+                DirectDownloadHelper.checkUrlExistence(url, followRedirects);
             }
             // Check that the resource limit for secondary storage won't be exceeded
-            _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+            _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage,
+                    UriUtils.getRemoteSize(url, followRedirects));
         } else {
             _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage);
         }
@@ -642,7 +644,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
                 _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume);
                 //url can be null incase of postupload
                 if (url != null) {
-                    _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+                    _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage,
+                            UriUtils.getRemoteSize(url, StorageManager.DataStoreDownloadFollowRedirects.value()));
                 }
 
                 return volume;

server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java

@@ -87,6 +87,7 @@ import com.cloud.server.StatsCollector;
 import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.TemplateType;
+import com.cloud.storage.StorageManager;
 import com.cloud.storage.TemplateProfile;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.storage.VMTemplateVO;
@@ -156,7 +157,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
      * Validate on random running KVM host that URL is reachable
      * @param url url
      */
-    private Long performDirectDownloadUrlValidation(final String format, final String url, final List<Long> zoneIds) {
+    private Long performDirectDownloadUrlValidation(final String format, final String url, final List<Long> zoneIds,
+                boolean followRedirects) {
         HostVO host = null;
         if (zoneIds != null && !zoneIds.isEmpty()) {
             for (Long zoneId : zoneIds) {
@@ -175,7 +177,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
         Integer socketTimeout = DirectDownloadManager.DirectDownloadSocketTimeout.value();
         Integer connectRequestTimeout = DirectDownloadManager.DirectDownloadConnectionRequestTimeout.value();
         Integer connectTimeout = DirectDownloadManager.DirectDownloadConnectTimeout.value();
-        CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout);
+        CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout,
+                followRedirects);
         s_logger.debug("Performing URL " + url + " validation on host " + host.getId());
         Answer answer = _agentMgr.easySend(host.getId(), cmd);
         if (answer == null || !answer.getResult()) {
@@ -199,6 +202,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
         TemplateProfile profile = super.prepare(cmd);
         String url = profile.getUrl();
         UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url);
+        boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
         if (cmd.isDirectDownload()) {
             DigestHelper.validateChecksumString(cmd.getChecksum());
             List<Long> zoneIds = null;
@@ -206,12 +210,15 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
                 zoneIds =  new ArrayList<>();
                 zoneIds.add(cmd.getZoneId());
             }
-            Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(), url, zoneIds);
+            Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(), url, zoneIds,
+                    followRedirects);
             profile.setSize(templateSize);
         }
         profile.setUrl(url);
         // Check that the resource limit for secondary storage won't be exceeded
-        _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+        _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
+                ResourceType.secondary_storage,
+                UriUtils.getRemoteSize(url, followRedirects));
         return profile;
     }
 
@@ -229,14 +236,18 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
         TemplateProfile profile = super.prepare(cmd);
         String url = profile.getUrl();
         UriUtils.validateUrl(cmd.getFormat(), url, cmd.isDirectDownload());
+        boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
         if (cmd.isDirectDownload()) {
             DigestHelper.validateChecksumString(cmd.getChecksum());
-            Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(), url, cmd.getZoneIds());
+            Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(), url, cmd.getZoneIds(),
+                    followRedirects);
             profile.setSize(templateSize);
         }
         profile.setUrl(url);
         // Check that the resource limit for secondary storage won't be exceeded
-        _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+        _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
+                ResourceType.secondary_storage,
+                UriUtils.getRemoteSize(url, followRedirects));
         return profile;
     }
 

server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java

@@ -272,7 +272,8 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
         PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO();
 
         DownloadProtocol protocol = getProtocolFromUrl(url);
-        DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
+        DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum,
+                headers);
         cmd.setTemplateSize(template.getSize());
         cmd.setFormat(template.getFormat());
 
@@ -393,19 +394,23 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
     /**
      * Return DirectDownloadCommand according to the protocol
      */
-    private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
+    private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url,
-                                                                       String checksum, Map<String, String> httpHeaders) {
+               Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> httpHeaders) {
         int connectTimeout = DirectDownloadConnectTimeout.value();
         int soTimeout = DirectDownloadSocketTimeout.value();
         int connectionRequestTimeout = DirectDownloadConnectionRequestTimeout.value();
+        boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
         if (protocol.equals(DownloadProtocol.HTTP)) {
-            return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
+            return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+                    soTimeout, followRedirects);
         } else if (protocol.equals(DownloadProtocol.HTTPS)) {
-            return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout, connectionRequestTimeout);
+            return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+                    soTimeout, connectionRequestTimeout, followRedirects);
         } else if (protocol.equals(DownloadProtocol.NFS)) {
             return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
         } else if (protocol.equals(DownloadProtocol.METALINK)) {
-            return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
+            return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+                    soTimeout, followRedirects);
         } else {
             return null;
         }

server/src/test/java/com/cloud/storage/StorageManagerImplTest.java

@@ -19,6 +19,8 @@ package com.cloud.storage;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.cloudstack.framework.config.ConfigDepot;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.junit.Assert;
@@ -31,6 +33,8 @@ import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
 import com.cloud.agent.api.StoragePoolInfo;
+import com.cloud.dc.DataCenterVO;
+import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.host.Host;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.vm.VMInstanceVO;
@@ -44,6 +48,12 @@ public class StorageManagerImplTest {
 
     @Mock
     VMInstanceDao vmInstanceDao;
+    @Mock
+    ConfigDepot configDepot;
+    @Mock
+    ConfigurationDao configurationDao;
+    @Mock
+    DataCenterDao dataCenterDao;
 
     @Spy
     @InjectMocks
@@ -155,7 +165,38 @@ public class StorageManagerImplTest {
         PrimaryDataStoreDao storagePoolDao = Mockito.mock(PrimaryDataStoreDao.class);
         storageManagerImpl._storagePoolDao = storagePoolDao;
         Assert.assertTrue(storageManagerImpl.storagePoolCompatibleWithVolumePool(storagePool, volume));
+    }
 
+    @Test
+    public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNoChange() {
+        Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+                .thenReturn(false);
+        storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+        Mockito.verify(configurationDao, Mockito.never()).update(Mockito.anyString(), Mockito.anyString());
+    }
+
+    @Test
+    public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsOldInstall() {
+        Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+                .thenReturn(true);
+        Mockito.when(dataCenterDao.listAll(Mockito.any()))
+                .thenReturn(List.of(Mockito.mock(DataCenterVO.class)));
+        Mockito.doReturn(true).when(configurationDao).update(Mockito.anyString(), Mockito.anyString());
+        storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+        Mockito.verify(configurationDao, Mockito.times(1))
+                .update(StorageManager.DataStoreDownloadFollowRedirects.key(), "true");
+    }
+
+    @Test
+    public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNewInstall() {
+        Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+                .thenReturn(true);
+        Mockito.when(dataCenterDao.listAll(Mockito.any()))
+                .thenReturn(new ArrayList<>()); //new installation
+        storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+        Mockito.verify(configurationDao, Mockito.never())
+                .update(StorageManager.DataStoreDownloadFollowRedirects.key(),
+                        StorageManager.DataStoreDownloadFollowRedirects.defaultValue());
     }
 
 }

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java

@@ -41,17 +41,21 @@ public interface DownloadManager extends Manager {
      * @param hvm  whether the template is a hardware virtual machine
      * @param accountId the accountId of the iso owner (null if public iso)
      * @param descr    description of the template
-     * @param user username used for authentication to the server
+     * @param userName username used for authentication to the server
-     * @param password password used for authentication to the server
+     * @param passwd password used for authentication to the server
      * @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes.
      * @param resourceType signifying the type of resource like template, volume etc.
+     * @param followRedirects whether downloader follows redirections
      * @return job-id that can be used to interrogate the status of the download.
      */
-    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
-        String installPathPrefix, String templatePath, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType);
+             Long accountId, String descr, String cksum, String installPathPrefix, String templatePath,
+             String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType,
+             boolean followRedirects);
 
-    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
-        String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType);
+             Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
+             long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects);
 
     Map<String, Processor> getProcessors();
 

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java

@@ -549,8 +549,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
     }
 
     @Override
-    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
-            String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+            Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
+            long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects) {
         UUID uuid = UUID.randomUUID();
         String jobId = uuid.toString();
 
@@ -570,6 +571,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         } else {
             throw new CloudRuntimeException("Unable to download from URL: " + url);
         }
+        td.setFollowRedirects(followRedirects);
         DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
         dj.setTmpltPath(installPathPrefix);
         jobs.put(jobId, dj);
@@ -579,8 +581,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
     }
 
     @Override
-    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
-            String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+            Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user,
+            String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType,
+            boolean followRedirects) {
         UUID uuid = UUID.randomUUID();
         String jobId = uuid.toString();
         String tmpDir = installPathPrefix;
@@ -632,6 +636,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
                     } else {
                         throw new CloudRuntimeException("Unable to download from URL: " + url);
                     }
+                    td.setFollowRedirects(followRedirects);
                     // NOTE the difference between installPathPrefix and templatePath
                     // here. instalPathPrefix is the absolute path for template
                     // including mount directory
@@ -768,12 +773,16 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         String jobId = null;
         if (dstore instanceof S3TO) {
             jobId =
-                    downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
+                    downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(),
-                            cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+                            cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(),
+                            installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
+                            cmd.isFollowRedirects());
         } else {
             jobId =
-                    downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
+                    downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(),
-                            cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+                            cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix,
+                            cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
+                            cmd.isFollowRedirects());
         }
         sleep();
         if (jobId == null) {

utils/src/main/java/com/cloud/utils/UriUtils.java

@@ -213,7 +213,7 @@ public class UriUtils {
     }
 
     // Get the size of a file from URL response header.
-    public static long getRemoteSize(String url) {
+    public static long getRemoteSize(String url, Boolean followRedirect) {
         long remoteSize = 0L;
         final String[] methods = new String[]{"HEAD", "GET"};
         IllegalArgumentException exception = null;
@@ -228,6 +228,7 @@ public class UriUtils {
                 httpConn.setRequestMethod(method);
                 httpConn.setConnectTimeout(2000);
                 httpConn.setReadTimeout(5000);
+                httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect));
                 String contentLength = httpConn.getHeaderField("content-length");
                 if (contentLength != null) {
                     remoteSize = Long.parseLong(contentLength);

utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java

@@ -22,8 +22,9 @@ package com.cloud.utils.storage;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.MalformedURLException;
+import java.net.HttpURLConnection;
-import java.net.URL;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 
@@ -32,7 +33,6 @@ import org.apache.commons.compress.compressors.CompressorStreamFactory;
 import org.apache.log4j.Logger;
 
 import com.cloud.utils.NumbersUtil;
-import com.cloud.utils.UriUtils;
 
 public final class QCOW2Utils {
     public static final Logger LOGGER = Logger.getLogger(QCOW2Utils.class.getName());
@@ -112,16 +112,23 @@ public final class QCOW2Utils {
         }
     }
 
-    public static long getVirtualSize(String urlStr) {
+    public static long getVirtualSizeFromUrl(String urlStr, boolean followRedirects) {
+        HttpURLConnection httpConn = null;
         try {
-            URL url = new URL(urlStr);
+            URI url = new URI(urlStr);
-            return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr));
+            httpConn = (HttpURLConnection)url.toURL().openConnection();
-        } catch (MalformedURLException e) {
+            httpConn.setInstanceFollowRedirects(followRedirects);
+            return getVirtualSizeFromInputStream(httpConn.getInputStream());
+        } catch (URISyntaxException e) {
             LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
             throw new IllegalArgumentException("Invalid URL: " + urlStr);
         }  catch (IOException e) {
             LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
             throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
+        } finally {
+            if (httpConn != null) {
+                httpConn.disconnect();
+            }
         }
     }
 }