Merge remote-tracking branch 'apache/4.19'

api/src/main/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmd.java

@@ -29,6 +29,7 @@ import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.response.VolumeResponse;
 import org.apache.cloudstack.context.CallContext;
+import org.apache.commons.lang3.EnumUtils;
 
 import com.cloud.event.EventTypes;
 import com.cloud.exception.InvalidParameterValueException;
@@ -69,9 +70,9 @@ public class CheckAndRepairVolumeCmd extends BaseAsyncCmd {
 
     public String getRepair() {
         if (org.apache.commons.lang3.StringUtils.isNotEmpty(repair)) {
-            RepairValues repairType = Enum.valueOf(RepairValues.class, repair.toUpperCase());
+            RepairValues repairType = EnumUtils.getEnumIgnoreCase(RepairValues.class, repair);
             if (repairType == null) {
-                throw new InvalidParameterValueException(String.format("Repair parameter can only take the following values: %s" + Arrays.toString(RepairValues.values())));
+                throw new InvalidParameterValueException(String.format("Repair parameter can only take the following values: %s", Arrays.toString(RepairValues.values())));
             }
             return repair.toLowerCase();
         }

api/src/main/java/org/apache/cloudstack/api/response/UserDataResponse.java

@@ -24,7 +24,7 @@ import org.apache.cloudstack.api.BaseResponseWithAnnotations;
 import org.apache.cloudstack.api.EntityReference;
 
 @EntityReference(value = UserData.class)
-public class UserDataResponse extends BaseResponseWithAnnotations {
+public class UserDataResponse extends BaseResponseWithAnnotations implements ControlledEntityResponse {
 
     @SerializedName(ApiConstants.ID)
     @Param(description = "ID of the ssh keypair")
@@ -40,6 +40,14 @@ public class UserDataResponse extends BaseResponseWithAnnotations {
     @SerializedName(ApiConstants.ACCOUNT) @Param(description="the owner of the userdata")
     private String accountName;
 
+    @SerializedName(ApiConstants.PROJECT_ID)
+    @Param(description = "the project id of the userdata", since = "4.19.1")
+    private String projectId;
+
+    @SerializedName(ApiConstants.PROJECT)
+    @Param(description = "the project name of the userdata", since = "4.19.1")
+    private String projectName;
+
     @SerializedName(ApiConstants.DOMAIN_ID) @Param(description="the domain id of the userdata owner")
     private String domainId;
 
@@ -118,6 +126,16 @@ public class UserDataResponse extends BaseResponseWithAnnotations {
         this.accountName = accountName;
     }
 
+    @Override
+    public void setProjectId(String projectId) {
+        this.projectId = projectId;
+    }
+
+    @Override
+    public void setProjectName(String projectName) {
+        this.projectName = projectName;
+    }
+
     public String getDomainName() {
         return domain;
     }

api/src/test/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmdTest.java

@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.api.command.user.volume;
+
+import com.cloud.exception.InvalidParameterValueException;
+import junit.framework.TestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.MockitoAnnotations;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
+@RunWith(MockitoJUnitRunner.class)
+public class CheckAndRepairVolumeCmdTest extends TestCase {
+    private CheckAndRepairVolumeCmd checkAndRepairVolumeCmd;
+    private AutoCloseable closeable;
+
+    @Before
+    public void setup() {
+        closeable = MockitoAnnotations.openMocks(this);
+        checkAndRepairVolumeCmd = new CheckAndRepairVolumeCmd();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        closeable.close();
+    }
+
+    @Test
+    public void testGetRepair() {
+        ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", "all");
+        assertEquals("all", checkAndRepairVolumeCmd.getRepair());
+
+        ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", "LEAKS");
+        assertEquals("leaks", checkAndRepairVolumeCmd.getRepair());
+
+        ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", null);
+        assertNull(checkAndRepairVolumeCmd.getRepair());
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testGetRepairInvalid() {
+        ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", "RANDOM STRING");
+        checkAndRepairVolumeCmd.getRepair();
+    }
+}

client/src/main/java/org/apache/cloudstack/ServerDaemon.java

@@ -30,7 +30,6 @@ import org.apache.commons.daemon.Daemon;
 import org.apache.commons.daemon.DaemonContext;
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.jetty.jmx.MBeanContainer;
-import org.eclipse.jetty.server.ForwardedRequestCustomizer;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.NCSARequestLog;
@@ -174,7 +173,7 @@ public class ServerDaemon implements Daemon {
 
         // HTTP config
         final HttpConfiguration httpConfig = new HttpConfiguration();
-        httpConfig.addCustomizer( new ForwardedRequestCustomizer() );
+// it would be nice to make this dynamic but we take care of this ourselves for now: httpConfig.addCustomizer( new ForwardedRequestCustomizer() );
         httpConfig.setSecureScheme("https");
         httpConfig.setSecurePort(httpsPort);
         httpConfig.setOutputBufferSize(32768);

core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java

@@ -28,6 +28,7 @@ import java.io.RandomAccessFile;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Date;
+import java.util.List;
 
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
@@ -78,6 +79,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
     private long maxTemplateSizeInBytes;
     private ResourceType resourceType = ResourceType.TEMPLATE;
     private final HttpMethodRetryHandler myretryhandler;
+    private boolean followRedirects = false;
 
     public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
             String user, String password, Proxy proxy, ResourceType resourceType) {
@@ -109,7 +111,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
     private GetMethod createRequest(String downloadUrl) {
         GetMethod request = new GetMethod(downloadUrl);
         request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
-        request.setFollowRedirects(true);
+        request.setFollowRedirects(followRedirects);
         return request;
     }
 
@@ -335,6 +337,12 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
         } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
             status = Status.UNRECOVERABLE_ERROR;
             errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
+            if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
+                    && !followRedirects) {
+                errorString = String.format("Failed to download %s due to redirection, response code: %d",
+                        downloadUrl, responseCode);
+                logger.error(errorString);
+            }
             return true; //FIXME: retry?
         }
         return false;
@@ -536,4 +544,12 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
             return this;
         }
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+        if (this.request != null) {
+            this.request.setFollowRedirects(followRedirects);
+        }
+    }
 }

core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java

@@ -58,7 +58,7 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement
     protected GetMethod createRequest(String downloadUrl) {
         GetMethod request = new GetMethod(downloadUrl);
         request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
-        request.setFollowRedirects(true);
+        request.setFollowRedirects(followRedirects);
         if (!toFileSet) {
             String[] parts = downloadUrl.split("/");
             String filename = parts[parts.length - 1];
@@ -171,4 +171,12 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement
     public void setStatus(Status status) {
         this.status = status;
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        super.setFollowRedirects(followRedirects);
+        if (this.request != null) {
+            this.request.setFollowRedirects(followRedirects);
+        }
+    }
 }

core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java

@@ -34,6 +34,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
 import org.apache.commons.httpclient.Header;
 import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.params.HttpMethodParams;
@@ -43,6 +44,7 @@ import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Date;
+import java.util.List;
 
 import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
 import static java.util.Arrays.asList;
@@ -70,8 +72,8 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
     private long downloadTime;
     private long totalBytes;
     private long maxTemplateSizeInByte;
-
     private boolean resume = false;
+    private boolean followRedirects = false;
 
     public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
             long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
@@ -89,7 +91,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
         this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
 
         // Follow redirects
-        this.getMethod.setFollowRedirects(true);
+        this.getMethod.setFollowRedirects(followRedirects);
 
         // Set file extension.
         this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
@@ -122,10 +124,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
             return 0;
         }
 
-        if (!HTTPUtils.verifyResponseCode(responseCode)) {
+        boolean failedDueToRedirection = List.of(HttpStatus.SC_MOVED_PERMANENTLY,
+                HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) && !followRedirects;
+        if (!HTTPUtils.verifyResponseCode(responseCode) || failedDueToRedirection) {
             errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
             logger.warn(errorString);
-
             status = Status.UNRECOVERABLE_ERROR;
             return 0;
         }
@@ -371,4 +374,12 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
     public String getFileExtension() {
         return fileExtension;
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+        if (this.getMethod != null) {
+            this.getMethod.setFollowRedirects(followRedirects);
+        }
+    }
 }

core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java

@@ -25,6 +25,7 @@ import java.io.InputStream;
 import java.io.RandomAccessFile;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
@@ -71,6 +72,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
     private final HttpMethodRetryHandler retryHandler;
 
     private HashMap<String, String> urlFileMap;
+    private boolean followRedirects = false;
 
     public SimpleHttpMultiFileDownloader(StorageLayer storageLayer, String[] downloadUrls, String toDir,
                                          DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
@@ -92,7 +94,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
     private GetMethod createRequest(String downloadUrl) {
         GetMethod request = new GetMethod(downloadUrl);
         request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler);
-        request.setFollowRedirects(true);
+        request.setFollowRedirects(followRedirects);
         return request;
     }
 
@@ -168,7 +170,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
             urlFileMap.put(downloadUrl, currentToFile);
             file = new File(currentToFile);
             long localFileSize = checkLocalFileSizeForResume(resume, file);
-            if (checkServerResponse(localFileSize)) return 0;
+            if (checkServerResponse(localFileSize, downloadUrl)) return 0;
             if (!tryAndGetRemoteSize()) return 0;
             if (!canHandleDownloadSize()) return 0;
             checkAndSetDownloadSize();
@@ -315,7 +317,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
         return true;
     }
 
-    private boolean checkServerResponse(long localFileSize) throws IOException {
+    private boolean checkServerResponse(long localFileSize, String downloadUrl) throws IOException {
         int responseCode = 0;
 
         if (localFileSize > 0) {
@@ -329,6 +331,12 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
         } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
             currentStatus = Status.UNRECOVERABLE_ERROR;
             errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
+            if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
+                    && !followRedirects) {
+                errorString = String.format("Failed to download %s due to redirection, response code: %d",
+                        downloadUrl, responseCode);
+                logger.error(errorString);
+            }
             return true; //FIXME: retry?
         }
         return false;
@@ -476,4 +484,12 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
     public Map<String, String> getDownloadedFilesMap() {
         return urlFileMap;
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+        if (this.request != null) {
+            this.request.setFollowRedirects(followRedirects);
+        }
+    }
 }

core/src/main/java/com/cloud/storage/template/TemplateDownloader.java

@@ -92,4 +92,6 @@ public interface TemplateDownloader extends Runnable {
     boolean isInited();
 
     long getMaxTemplateSizeInBytes();
+
+    void setFollowRedirects(boolean followRedirects);
 }

core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java

@@ -41,6 +41,7 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl
     protected long _start;
     protected StorageLayer _storage;
     protected boolean _inited = false;
+    protected boolean followRedirects = false;
     private long maxTemplateSizeInBytes;
 
     public TemplateDownloaderBase(StorageLayer storage, String downloadUrl, String toDir, long maxTemplateSizeInBytes, DownloadCompleteCallback callback) {
@@ -147,4 +148,9 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl
     public boolean isInited() {
         return _inited;
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java

@@ -28,6 +28,7 @@ public class CheckUrlCommand extends Command {
     private Integer connectTimeout;
     private Integer connectionRequestTimeout;
     private Integer socketTimeout;
+    private boolean followRedirects;
 
     public String getFormat() {
         return format;
@@ -43,19 +44,25 @@ public class CheckUrlCommand extends Command {
 
     public Integer getSocketTimeout() { return socketTimeout; }
 
-    public CheckUrlCommand(final String format,final String url) {
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public CheckUrlCommand(final String format, final String url, final boolean followRedirects) {
         super();
         this.format = format;
         this.url = url;
+        this.followRedirects = followRedirects;
     }
 
-    public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+    public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, final boolean followRedirects) {
         super();
         this.format = format;
         this.url = url;
         this.connectTimeout = connectTimeout;
         this.socketTimeout = socketTimeout;
         this.connectionRequestTimeout = connectionRequestTimeout;
+        this.followRedirects = followRedirects;
     }
 
     @Override

core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java

@@ -45,7 +45,11 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
     private Long templateSize;
     private Storage.ImageFormat format;
 
-    protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) {
+    private boolean followRedirects;
+
+    protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool,
+             final String checksum, final Map<String, String> headers, final Integer connectTimeout,
+             final Integer soTimeout, final Integer connectionRequestTimeout, final boolean followRedirects) {
         this.url = url;
         this.templateId = templateId;
         this.destData = destData;
@@ -55,6 +59,7 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
         this.connectTimeout = connectTimeout;
         this.soTimeout = soTimeout;
         this.connectionRequestTimeout = connectionRequestTimeout;
+        this.followRedirects = followRedirects;
     }
 
     public String getUrl() {
@@ -137,4 +142,12 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
     public int getWaitInMillSeconds() {
         return getWait() * 1000;
     }
+
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java

@@ -24,8 +24,10 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
 public class HttpDirectDownloadCommand extends DirectDownloadCommand {
 
-    public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
+    public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
-        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
+             Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
+        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
+                null, followRedirects);
     }
 
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java

@@ -25,7 +25,10 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
 public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
 
-    public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout) {
+    public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
-        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, connectionRequestTimeout);
+              Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout,
+              boolean followRedirects) {
+        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
+                connectionRequestTimeout, followRedirects);
     }
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java

@@ -24,8 +24,9 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
 public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
 
-    public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
+    public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
-        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
+                 Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
+        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null, followRedirects);
     }
 
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java

@@ -24,8 +24,9 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
 public class NfsDirectDownloadCommand extends DirectDownloadCommand {
 
-    public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
+    public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool,
-        super(url, templateId, destPool, checksum, headers, null, null, null);
+                final String checksum, final Map<String, String> headers) {
+        super(url, templateId, destPool, checksum, headers, null, null, null, false);
     }
 
 }

core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java

@@ -35,27 +35,30 @@ public class DirectDownloadHelper {
      * Get direct template downloader from direct download command and destination pool
      */
     public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
-                                                                                  String destPoolLocalPath,
+                  String destPoolLocalPath,  String temporaryDownloadPath) {
-                                                                                  String temporaryDownloadPath) {
         if (cmd instanceof HttpDirectDownloadCommand) {
-            return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
+            return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
-                    cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
+                    cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+                    temporaryDownloadPath, cmd.isFollowRedirects());
         } else if (cmd instanceof HttpsDirectDownloadCommand) {
-            return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
+            return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
-                    cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
+                    cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+                    cmd.getConnectionRequestTimeout(), temporaryDownloadPath, cmd.isFollowRedirects());
         } else if (cmd instanceof NfsDirectDownloadCommand) {
-            return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
+            return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
+                    cmd.getChecksum(), temporaryDownloadPath);
         } else if (cmd instanceof MetalinkDirectDownloadCommand) {
-            return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
+            return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
-                    cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
+                    cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+                    temporaryDownloadPath, cmd.isFollowRedirects());
         } else {
             throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
         }
     }
 
-    public static boolean checkUrlExistence(String url) {
+    public static boolean checkUrlExistence(String url, boolean followRedirects) {
         try {
-            DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
+            DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects);
             return checker.checkUrl(url);
         } catch (CloudRuntimeException e) {
             LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@@ -63,9 +66,9 @@ public class DirectDownloadHelper {
         }
     }
 
-    public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+    public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
         try {
-            DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+            DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
             return checker.checkUrl(url);
         } catch (CloudRuntimeException e) {
             LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@@ -73,27 +76,27 @@ public class DirectDownloadHelper {
         }
     }
 
-    private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+    private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
         if (url.toLowerCase().startsWith("https:")) {
-            return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+            return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
         } else if (url.toLowerCase().startsWith("http:")) {
-            return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout);
+            return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects);
         } else if (url.toLowerCase().startsWith("nfs:")) {
             return new NfsDirectTemplateDownloader(url);
         } else if (url.toLowerCase().endsWith(".metalink")) {
-            return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout);
+            return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects);
         } else {
             throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url));
         }
     }
 
-    public static Long getFileSize(String url, String format) {
+    public static Long getFileSize(String url, String format, boolean followRedirects) {
-        DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
+        DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects);
         return checker.getRemoteFileSize(url, format);
     }
 
-    public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+    public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
-        DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+        DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
         return checker.getRemoteFileSize(url, format);
     }
 }

core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java

@@ -43,16 +43,19 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
     private String checksum;
     private boolean redownload = false;
     protected String temporaryDownloadPath;
+    private boolean followRedirects;
 
     protected Logger logger = LogManager.getLogger(getClass());
 
     protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId,
-                                           final String checksum, final String temporaryDownloadPath) {
+                                           final String checksum, final String temporaryDownloadPath,
+                                           final boolean followRedirects) {
         this.url = url;
         this.destPoolPath = destPoolPath;
         this.templateId = templateId;
         this.checksum = checksum;
         this.temporaryDownloadPath = temporaryDownloadPath;
+        this.followRedirects = followRedirects;
     }
 
     private static String directDownloadDir = "template";
@@ -112,6 +115,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
         return redownload;
     }
 
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
+
     /**
      * Create download directory (if it does not exist)
      */

core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java

@@ -48,13 +48,14 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
     protected GetMethod request;
     protected Map<String, String> reqHeaders = new HashMap<>();
 
-    protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
+    protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout, boolean followRedirects) {
-        this(url, null, null, null, null, connectTimeout, socketTimeout, null);
+        this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects);
     }
 
     public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
-                                        Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
+                Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
-        super(url, destPoolPath, templateId, checksum, downloadPath);
+                boolean followRedirects) {
+        super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
         s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout);
         s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout);
         client = new HttpClient(s_httpClientManager);
@@ -66,7 +67,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
 
     protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
         GetMethod request = new GetMethod(downloadUrl);
-        request.setFollowRedirects(true);
+        request.setFollowRedirects(this.isFollowRedirects());
         if (MapUtils.isNotEmpty(headers)) {
             for (String key : headers.keySet()) {
                 request.setRequestHeader(key, headers.get(key));
@@ -109,9 +110,11 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
     @Override
     public boolean checkUrl(String url) {
         HeadMethod httpHead = new HeadMethod(url);
+        httpHead.setFollowRedirects(this.isFollowRedirects());
         try {
-            if (client.executeMethod(httpHead) != HttpStatus.SC_OK) {
+            int responseCode = client.executeMethod(httpHead);
-                logger.error(String.format("Invalid URL: %s", url));
+            if (responseCode != HttpStatus.SC_OK) {
+                logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
                 return false;
             }
             return true;
@@ -126,9 +129,9 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
     @Override
     public Long getRemoteFileSize(String url, String format) {
         if ("qcow2".equalsIgnoreCase(format)) {
-            return QCOW2Utils.getVirtualSize(url);
+            return QCOW2Utils.getVirtualSizeFromUrl(url, this.isFollowRedirects());
         } else {
-            return UriUtils.getRemoteSize(url);
+            return UriUtils.getRemoteSize(url, this.isFollowRedirects());
         }
     }
 

core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java

@@ -68,20 +68,26 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
     protected CloseableHttpClient httpsClient;
     private HttpUriRequest req;
 
-    protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+    protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
-        this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null);
+        this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null, followRedirects);
     }
 
-    public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
+    public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
-                                         Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
+                 Map<String, String> headers, Integer connectTimeout, Integer soTimeout,
-        super(url, destPoolPath, templateId, checksum, temporaryDownloadPath);
+                 Integer connectionRequestTimeout, String temporaryDownloadPath, boolean followRedirects) {
+        super(url, destPoolPath, templateId, checksum, temporaryDownloadPath, followRedirects);
         SSLContext sslcontext = getSSLContext();
         SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
         RequestConfig config = RequestConfig.custom()
                 .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
                 .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
-                .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
+                .setSocketTimeout(soTimeout == null ? 5000 : soTimeout)
-        httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
+                .setRedirectsEnabled(followRedirects)
+                .build();
+        httpsClient = HttpClients.custom()
+                .setSSLSocketFactory(factory)
+                .setDefaultRequestConfig(config)
+                .build();
         createUriRequest(url, headers);
         String downloadDir = getDirectDownloadTempPath(templateId);
         File tempFile = createTemporaryDirectoryAndFile(downloadDir);
@@ -90,6 +96,7 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
 
     protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
         req = new HttpGet(downloadUrl);
+        setFollowRedirects(this.isFollowRedirects());
         if (MapUtils.isNotEmpty(headers)) {
             for (String headerKey: headers.keySet()) {
                 req.setHeader(headerKey, headers.get(headerKey));
@@ -164,8 +171,9 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
         HttpHead httpHead = new HttpHead(url);
         try {
             CloseableHttpResponse response = httpsClient.execute(httpHead);
-            if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+            int responseCode = response.getStatusLine().getStatusCode();
-                logger.error(String.format("Invalid URL: %s", url));
+            if (responseCode != HttpStatus.SC_OK) {
+                logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
                 return false;
             }
             return true;

core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java

@@ -39,16 +39,15 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm
     private Integer soTimeout;
 
     protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId,
-                                                                    String destPoolPath, String checksum,
+                String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout,
-                                                                    Map<String, String> headers,
+                Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
-                                                                    Integer connectTimeout, Integer soTimeout,
-                                                                    Integer connectionRequestTimeout, String temporaryDownloadPath) {
         if (url.toLowerCase().startsWith("https:")) {
             return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
-                    connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath);
+                    connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath,
+                    this.isFollowRedirects());
         } else if (url.toLowerCase().startsWith("http:")) {
             return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
-                    connectTimeout, soTimeout, temporaryDownloadPath);
+                    connectTimeout, soTimeout, temporaryDownloadPath, this.isFollowRedirects());
         } else if (url.toLowerCase().startsWith("nfs:")) {
             return new NfsDirectTemplateDownloader(url);
         } else {
@@ -57,13 +56,14 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm
         }
     }
 
-    protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
+    protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout, boolean followRedirects) {
-        this(url, null, null, null, null, connectTimeout, socketTimeout, null);
+        this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects);
     }
 
     public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
-                                            Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
+                Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
-        super(url, destPoolPath, templateId, checksum, downloadPath);
+                boolean followRedirects) {
+        super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
         this.headers = headers;
         this.connectTimeout = connectTimeout;
         this.soTimeout = soTimeout;

core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java

@@ -57,8 +57,9 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
         this(url, null, null, null, null);
     }
 
-    public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) {
+    public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum,
-        super(url, destPool, templateId, checksum, downloadPath);
+               String downloadPath) {
+        super(url, destPool, templateId, checksum, downloadPath, false);
         parseUrl();
     }
 

core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java

@@ -49,6 +49,8 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
     private DataStoreTO _store;
     private DataStoreTO cacheStore;
 
+    private boolean followRedirects = false;
+
     protected DownloadCommand() {
     }
 
@@ -65,6 +67,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
         installPath = that.installPath;
         _store = that._store;
         _proxy = that._proxy;
+        followRedirects = that.followRedirects;
     }
 
     public DownloadCommand(TemplateObjectTO template, Long maxDownloadSizeInBytes) {
@@ -80,6 +83,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
             setSecUrl(((NfsTO)_store).getUrl());
         }
         this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
+        this.followRedirects = template.isFollowRedirects();
     }
 
     public DownloadCommand(TemplateObjectTO template, String user, String passwd, Long maxDownloadSizeInBytes) {
@@ -95,6 +99,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
         _store = volume.getDataStore();
         this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
         resourceType = ResourceType.VOLUME;
+        this.followRedirects = volume.isFollowRedirects();
     }
 
     public DownloadCommand(SnapshotObjectTO snapshot, Long maxDownloadSizeInBytes, String url) {
@@ -194,4 +199,12 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
     public DataStoreTO getCacheStore() {
         return cacheStore;
     }
+
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
 }

core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.storage.to;
+
+public class DownloadableObjectTO {
+    protected boolean followRedirects = false;
+
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
+}

core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java

@@ -30,7 +30,7 @@ import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 
-public class SnapshotObjectTO implements DataTO {
+public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO {
     private String path;
     private VolumeObjectTO volume;
     private String parentSnapshotPath;

core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java

@@ -28,7 +28,7 @@ import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.template.VirtualMachineTemplate;
 
-public class TemplateObjectTO implements DataTO {
+public class TemplateObjectTO extends DownloadableObjectTO implements DataTO {
     private String path;
     private String origUrl;
     private String uuid;
@@ -87,6 +87,7 @@ public class TemplateObjectTO implements DataTO {
         this.deployAsIs = template.isDeployAsIs();
         this.deployAsIsConfiguration = template.getDeployAsIsConfiguration();
         this.directDownload = template.isDirectDownload();
+        this.followRedirects = template.isFollowRedirects();
     }
 
     @Override

core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java

@@ -33,7 +33,7 @@ import com.cloud.storage.Volume;
 
 import java.util.Arrays;
 
-public class VolumeObjectTO implements DataTO {
+public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
     private String uuid;
     private Volume.Type volumeType;
     private DataStoreTO dataStore;
@@ -119,6 +119,7 @@ public class VolumeObjectTO implements DataTO {
         this.vSphereStoragePolicyId = volume.getvSphereStoragePolicyId();
         this.passphrase = volume.getPassphrase();
         this.encryptFormat = volume.getEncryptFormat();
+        this.followRedirects = volume.isFollowRedirects();
     }
 
     public String getUuid() {

core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java

@@ -57,7 +57,7 @@ public class BaseDirectTemplateDownloaderTest {
     private HttpEntity httpEntity;
 
     @InjectMocks
-    protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000);
+    protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000, false);
 
     private AutoCloseable closeable;
 

core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java

@@ -25,8 +25,7 @@ import org.mockito.InjectMocks;
 public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
 
     @InjectMocks
-    protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000);
+    protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000, false);
-
     @Test
     public void testCheckUrlMetalink() {
         metalinkDownloader.downloader = httpsDownloader;

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.engine.subsystem.api.storage;
+
+public interface DownloadableDataInfo extends DataObject {
+    default public boolean isFollowRedirects() {
+        return true;
+    }
+}

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java

@@ -21,7 +21,7 @@ package org.apache.cloudstack.engine.subsystem.api.storage;
 import com.cloud.template.VirtualMachineTemplate;
 import com.cloud.user.UserData;
 
-public interface TemplateInfo extends DataObject, VirtualMachineTemplate {
+public interface TemplateInfo extends DownloadableDataInfo, VirtualMachineTemplate {
     @Override
     String getUniqueName();
 

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java

@@ -26,7 +26,7 @@ import com.cloud.storage.Storage;
 import com.cloud.storage.Volume;
 import com.cloud.vm.VirtualMachine;
 
-public interface VolumeInfo extends DataObject, Volume {
+public interface VolumeInfo extends DownloadableDataInfo, Volume {
 
     boolean isAttachedVM();
 

engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java

@@ -276,4 +276,6 @@ public interface ConfigurationManager {
     Pair<String, String> getConfigurationGroupAndSubGroup(String configName);
 
     List<ConfigurationSubGroupVO> getConfigurationSubGroups(Long groupId);
+
+    void validateExtraConfigInServiceOfferingDetail(String detailName);
 }

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -199,6 +199,10 @@ public interface StorageManager extends StorageService {
             true,
             ConfigKey.Scope.Global,
             null);
+    static final ConfigKey<Boolean> DataStoreDownloadFollowRedirects = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED,
+            Boolean.class, "store.download.follow.redirects", "false",
+            "Whether HTTP redirect is followed during store downloads for objects such as template, volume etc.",
+            true, ConfigKey.Scope.Global);
 
     ConfigKey<Long> HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000",
             "The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true);

engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java

@@ -223,4 +223,87 @@ public class DatabaseUpgradeCheckerTest {
         assertEquals("We should have 1 upgrade step", 1, upgrades.length);
         assertTrue(upgrades[0] instanceof NoopDbUpgrade);
     }
+
+    @Test
+    public void testCalculateUpgradePathFrom41800toNextSecurityRelease() {
+
+        final CloudStackVersion dbVersion = CloudStackVersion.parse("4.18.0.0");
+        assertNotNull(dbVersion);
+
+        final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
+        final CloudStackVersion currentVersion = checker.getLatestVersion();
+        assertNotNull(currentVersion);
+
+        final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
+        assertNotNull(upgrades);
+
+        final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "."
+                + currentVersion.getMinorRelease() + "."
+                + currentVersion.getPatchRelease() + "."
+                + (currentVersion.getSecurityRelease() + 1));
+        assertNotNull(nextSecurityRelease);
+
+        final DbUpgrade[] upgradesToNext = checker.calculateUpgradePath(dbVersion, nextSecurityRelease);
+        assertNotNull(upgradesToNext);
+
+        assertEquals(upgrades.length + 1, upgradesToNext.length);
+        assertTrue(upgradesToNext[upgradesToNext.length - 1] instanceof NoopDbUpgrade);
+    }
+
+    @Test
+    public void testCalculateUpgradePathFromSecurityReleaseToLatest() {
+
+        final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.2.0");    // a EOL version
+        assertNotNull(dbVersion);
+
+        final CloudStackVersion oldSecurityRelease = CloudStackVersion.parse(dbVersion.getMajorRelease() + "."
+                + dbVersion.getMinorRelease() + "."
+                + dbVersion.getPatchRelease() + "."
+                + (dbVersion.getSecurityRelease() + 100));
+        assertNotNull(oldSecurityRelease);      // fake security release 4.17.2.100
+
+        final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
+        final CloudStackVersion currentVersion = checker.getLatestVersion();
+        assertNotNull(currentVersion);
+
+        final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
+        assertNotNull(upgrades);
+
+        final DbUpgrade[] upgradesFromSecurityRelease = checker.calculateUpgradePath(oldSecurityRelease, currentVersion);
+        assertNotNull(upgradesFromSecurityRelease);
+
+        assertEquals("The upgrade paths should be the same", upgrades.length, upgradesFromSecurityRelease.length);
+    }
+
+    @Test
+    public void testCalculateUpgradePathFromSecurityReleaseToNextSecurityRelease() {
+
+        final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.2.0");    // a EOL version
+        assertNotNull(dbVersion);
+
+        final CloudStackVersion oldSecurityRelease = CloudStackVersion.parse(dbVersion.getMajorRelease() + "."
+                + dbVersion.getMinorRelease() + "."
+                + dbVersion.getPatchRelease() + "."
+                + (dbVersion.getSecurityRelease() + 100));
+        assertNotNull(oldSecurityRelease);      // fake security release 4.17.2.100
+
+        final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
+        final CloudStackVersion currentVersion = checker.getLatestVersion();
+        assertNotNull(currentVersion);
+
+        final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "."
+                + currentVersion.getMinorRelease() + "."
+                + currentVersion.getPatchRelease() + "."
+                + (currentVersion.getSecurityRelease() + 1));
+        assertNotNull(nextSecurityRelease);     // fake security release
+
+        final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
+        assertNotNull(upgrades);
+
+        final DbUpgrade[] upgradesFromSecurityReleaseToNext = checker.calculateUpgradePath(oldSecurityRelease, nextSecurityRelease);
+        assertNotNull(upgradesFromSecurityReleaseToNext);
+
+        assertEquals(upgrades.length + 1, upgradesFromSecurityReleaseToNext.length);
+        assertTrue(upgradesFromSecurityReleaseToNext[upgradesFromSecurityReleaseToNext.length - 1] instanceof NoopDbUpgrade);
+    }
 }

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -92,6 +92,7 @@ import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.TemplateType;
+import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
@@ -367,6 +368,7 @@ public class TemplateServiceImpl implements TemplateService {
                     toBeDownloaded.addAll(allTemplates);
 
                     final StateMachine2<VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate> stateMachine = VirtualMachineTemplate.State.getStateMachine();
+                    Boolean followRedirect = StorageManager.DataStoreDownloadFollowRedirects.value();
                     for (VMTemplateVO tmplt : allTemplates) {
                         String uniqueName = tmplt.getUniqueName();
                         TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
@@ -447,7 +449,8 @@ public class TemplateServiceImpl implements TemplateService {
                                         try {
                                             _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId),
                                                     com.cloud.configuration.Resource.ResourceType.secondary_storage,
-                                                    tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
+                                                    tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl(),
+                                                            followRedirect));
                                         } catch (ResourceAllocationException e) {
                                             logger.warn(e.getMessage());
                                             _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null, e.getMessage(), e.getMessage());

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java

@@ -23,6 +23,7 @@ import java.util.Map;
 
 import javax.inject.Inject;
 
+import com.cloud.storage.StorageManager;
 import com.cloud.user.UserData;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
@@ -74,8 +75,10 @@ public class TemplateObject implements TemplateInfo {
     VMTemplatePoolDao templatePoolDao;
     @Inject
     TemplateDataStoreDao templateStoreDao;
+    final private boolean followRedirects;
 
     public TemplateObject() {
+        this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
     }
 
     protected void configure(VMTemplateVO template, DataStore dataStore) {
@@ -574,4 +577,9 @@ public class TemplateObject implements TemplateInfo {
         // TODO Auto-generated method stub
         return null;
     }
+
+    @Override
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
 }

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java

@@ -23,6 +23,7 @@ import javax.inject.Inject;
 import com.cloud.configuration.Resource.ResourceType;
 import com.cloud.dc.VsphereStoragePolicyVO;
 import com.cloud.dc.dao.VsphereStoragePolicyDao;
+import com.cloud.storage.StorageManager;
 import com.cloud.utils.db.Transaction;
 import com.cloud.utils.db.TransactionCallbackNoReturn;
 import com.cloud.utils.db.TransactionStatus;
@@ -118,6 +119,7 @@ public class VolumeObject implements VolumeInfo {
     private MigrationOptions migrationOptions;
     private boolean directDownload;
     private String vSphereStoragePolicyId;
+    private boolean followRedirects;
 
     private final List<Volume.State> volumeStatesThatShouldNotTransitWhenDataStoreRoleIsImage = Arrays.asList(Volume.State.Migrating, Volume.State.Uploaded, Volume.State.Copying,
       Volume.State.Expunged);
@@ -128,6 +130,7 @@ public class VolumeObject implements VolumeInfo {
 
     public VolumeObject() {
         _volStateMachine = Volume.State.getStateMachine();
+        this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
     }
 
     protected void configure(DataStore dataStore, VolumeVO volumeVO) {
@@ -931,4 +934,9 @@ public class VolumeObject implements VolumeInfo {
     public void setEncryptFormat(String encryptFormat) {
         volumeVO.setEncryptFormat(encryptFormat);
     }
+
+    @Override
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
 }

framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java

@@ -31,4 +31,5 @@ public interface ConfigDepot {
     <T> void set(ConfigKey<T> key, T value);
 
     <T> void createOrUpdateConfigObject(String componentName, ConfigKey<T> key, String value);
+    boolean isNewConfig(ConfigKey<?> configKey);
 }

framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java

@@ -34,6 +34,7 @@ public class ConfigKey<T> {
 
     public static final String CATEGORY_ADVANCED = "Advanced";
     public static final String CATEGORY_ALERT = "Alert";
+    public static final String CATEGORY_NETWORK = "Network";
 
     public enum Scope {
         Global, Zone, Cluster, StoragePool, Account, ManagementServer, ImageStore, Domain

framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java

@@ -82,6 +82,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
     List<Configurable> _configurables;
     List<ScopedConfigStorage> _scopedStorages;
     Set<Configurable> _configured = Collections.synchronizedSet(new HashSet<Configurable>());
+    Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
 
     private HashMap<String, Pair<String, ConfigKey<?>>> _allKeys = new HashMap<String, Pair<String, ConfigKey<?>>>(1007);
 
@@ -194,6 +195,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
             }
 
             _configDao.persist(vo);
+            newConfigs.add(vo.getName());
         } else {
             boolean configUpdated = false;
             if (vo.isDynamic() != key.isDynamic() || !ObjectUtils.equals(vo.getDescription(), key.description()) || !ObjectUtils.equals(vo.getDefaultValue(), key.defaultValue()) ||
@@ -344,4 +346,9 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
 
         return new Pair<>(groupId, subGroupId);
     }
+
+    @Override
+    public boolean isNewConfig(ConfigKey<?> configKey) {
+        return newConfigs.contains(configKey.key());
+    }
 }

framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java

@@ -18,9 +18,14 @@
 //
 package org.apache.cloudstack.framework.config.impl;
 
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.junit.Assert;
 import org.junit.Test;
+import org.springframework.test.util.ReflectionTestUtils;
 
 public class ConfigDepotImplTest {
 
@@ -40,4 +45,16 @@ public class ConfigDepotImplTest {
         }
     }
 
+    @Test
+    public void testIsNewConfig() {
+        String validNewConfigKey = "CONFIG";
+        ConfigKey<Boolean> validNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG", "true", "", true);
+        ConfigKey<Boolean> invalidNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG1", "true", "", true);
+        Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
+        newConfigs.add(validNewConfigKey);
+        ReflectionTestUtils.setField(configDepotImpl, "newConfigs", newConfigs);
+        Assert.assertTrue(configDepotImpl.isNewConfig(validNewConfig));
+        Assert.assertFalse(configDepotImpl.isNewConfig(invalidNewConfig));
+    }
+
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java

@@ -40,9 +40,9 @@ public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, Chec
         logger.info(String.format("Checking URL: %s, with connect timeout: %d, connect request timeout: %d, socket timeout: %d", url, connectTimeout, connectionRequestTimeout, socketTimeout));
         Long remoteSize = null;
 
-        boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+        boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout, cmd.isFollowRedirects());
         if (checkResult) {
-            remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout);
+            remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout, cmd.isFollowRedirects());
             if (remoteSize == null || remoteSize < 0) {
                 logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
                         "url %s, obtained size = %s", url, remoteSize));

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -2381,7 +2381,7 @@ public class KVMStorageProcessor implements StorageProcessor {
             Long templateSize = null;
             if (StringUtils.isNotBlank(cmd.getUrl())) {
                 String url = cmd.getUrl();
-                templateSize = UriUtils.getRemoteSize(url);
+                templateSize = UriUtils.getRemoteSize(url, cmd.isFollowRedirects());
             }
 
             logger.debug("Checking for free space on the host for downloading the template with physical size: " + templateSize + " and virtual size: " + cmd.getTemplateSize());

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -4853,12 +4853,7 @@ public class ApiResponseHelper implements ResponseGenerator {
     @Override
     public UserDataResponse createUserDataResponse(UserData userData) {
         UserDataResponse response = new UserDataResponse(userData.getUuid(), userData.getName(), userData.getUserData(), userData.getParams());
-        Account account = ApiDBUtils.findAccountById(userData.getAccountId());
+        populateOwner(response, userData);
-        response.setAccountId(account.getUuid());
-        response.setAccountName(account.getAccountName());
-        Domain domain = ApiDBUtils.findDomainById(userData.getDomainId());
-        response.setDomainId(domain.getUuid());
-        response.setDomainName(domain.getName());
         response.setHasAnnotation(annotationDao.hasAnnotations(userData.getUuid(), AnnotationService.EntityType.USER_DATA.name(),
                 _accountMgr.isRootAdmin(CallContext.current().getCallingAccount().getId())));
         return response;

server/src/main/java/com/cloud/api/ApiServer.java

@@ -233,42 +233,42 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
     @Inject
     private MessageBus messageBus;
 
-    private static final ConfigKey<Integer> IntegrationAPIPort = new ConfigKey<Integer>("Advanced"
+    private static final ConfigKey<Integer> IntegrationAPIPort = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
             , Integer.class
             , "integration.api.port"
             , "0"
             , "Integration (unauthenticated) API port. To disable set it to 0 or negative."
             , false
             , ConfigKey.Scope.Global);
-    private static final ConfigKey<Long> ConcurrentSnapshotsThresholdPerHost = new ConfigKey<Long>("Advanced"
+    private static final ConfigKey<Long> ConcurrentSnapshotsThresholdPerHost = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
             , Long.class
             , "concurrent.snapshots.threshold.perhost"
             , null
             , "Limits number of snapshots that can be handled by the host concurrently; default is NULL - unlimited"
             , true // not sure if this is to be dynamic
             , ConfigKey.Scope.Global);
-    private static final ConfigKey<Boolean> EncodeApiResponse = new ConfigKey<Boolean>("Advanced"
+    private static final ConfigKey<Boolean> EncodeApiResponse = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
             , Boolean.class
             , "encode.api.response"
             , "false"
             , "Do URL encoding for the api response, false by default"
             , false
             , ConfigKey.Scope.Global);
-    static final ConfigKey<String> JSONcontentType = new ConfigKey<String>( "Advanced"
+    static final ConfigKey<String> JSONcontentType = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
             , String.class
             , "json.content.type"
             , "application/json; charset=UTF-8"
             , "Http response content type for .js files (default is text/javascript)"
             , false
             , ConfigKey.Scope.Global);
-    static final ConfigKey<Boolean> EnableSecureSessionCookie = new ConfigKey<Boolean>("Advanced"
+    static final ConfigKey<Boolean> EnableSecureSessionCookie = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
             , Boolean.class
             , "enable.secure.session.cookie"
             , "false"
             , "Session cookie is marked as secure if this is enabled. Secure cookies only work when HTTPS is used."
             , false
             , ConfigKey.Scope.Global);
-    private static final ConfigKey<String> JSONDefaultContentType = new ConfigKey<String> ("Advanced"
+    private static final ConfigKey<String> JSONDefaultContentType = new ConfigKey<> (ConfigKey.CATEGORY_ADVANCED
             , String.class
             , "json.content.type"
             , "application/json; charset=UTF-8"
@@ -276,13 +276,34 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
             , false
             , ConfigKey.Scope.Global);
 
-    private static final ConfigKey<Boolean> UseEventAccountInfo = new ConfigKey<Boolean>( "advanced"
+    private static final ConfigKey<Boolean> UseEventAccountInfo = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
             , Boolean.class
             , "event.accountinfo"
             , "false"
             , "use account info in event logging"
             , true
             , ConfigKey.Scope.Global);
+    static final ConfigKey<Boolean> useForwardHeader = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK
+            , Boolean.class
+            , "proxy.header.verify"
+            , "false"
+            , "enables/disables checking of ipaddresses from a proxy set header. See \"proxy.header.names\" for the headers to allow."
+            , true
+            , ConfigKey.Scope.Global);
+    static final ConfigKey<String> listOfForwardHeaders = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK
+            , String.class
+            , "proxy.header.names"
+            , "X-Forwarded-For,HTTP_CLIENT_IP,HTTP_X_FORWARDED_FOR"
+            , "a list of names to check for allowed ipaddresses from a proxy set header. See \"proxy.cidr\" for the proxies allowed to set these headers."
+            , true
+            , ConfigKey.Scope.Global);
+    static final ConfigKey<String> proxyForwardList = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK
+            , String.class
+            , "proxy.cidr"
+            , ""
+            , "a list of cidrs for which \"proxy.header.names\" are honoured if the \"Remote_Addr\" is in this list."
+            , true
+            , ConfigKey.Scope.Global);
 
     @Override
     public boolean configure(final String name, final Map<String, Object> params) throws ConfigurationException {
@@ -1500,7 +1521,10 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
                 ConcurrentSnapshotsThresholdPerHost,
                 EncodeApiResponse,
                 EnableSecureSessionCookie,
-                JSONDefaultContentType
+                JSONDefaultContentType,
+                proxyForwardList,
+                useForwardHeader,
+                listOfForwardHeaders
         };
     }
 }

server/src/main/java/com/cloud/api/ApiServlet.java

@@ -70,7 +70,6 @@ import com.cloud.utils.db.EntityManager;
 import com.cloud.utils.net.NetUtils;
 
 @Component("apiServlet")
-@SuppressWarnings("serial")
 public class ApiServlet extends HttpServlet {
     protected static Logger LOGGER = LogManager.getLogger(ApiServlet.class);
     private final static List<String> s_clientAddressHeaders = Collections
@@ -570,17 +569,39 @@ public class ApiServlet extends HttpServlet {
         }
         return false;
     }
+    boolean doUseForwardHeaders() {
+        return Boolean.TRUE.equals(ApiServer.useForwardHeader.value());
+    }
 
+    String[] proxyNets() {
+        return ApiServer.proxyForwardList.value().split(",");
+    }
     //This method will try to get login IP of user even if servlet is behind reverseProxy or loadBalancer
-    public static InetAddress getClientAddress(final HttpServletRequest request) throws UnknownHostException {
+    public InetAddress getClientAddress(final HttpServletRequest request) throws UnknownHostException {
-        for(final String header : s_clientAddressHeaders) {
+        String ip = null;
-            final String ip = getCorrectIPAddress(request.getHeader(header));
+        InetAddress pretender = InetAddress.getByName(request.getRemoteAddr());
-            if (ip != null) {
+        if(doUseForwardHeaders()) {
-                return InetAddress.getByName(ip);
+            if (NetUtils.isIpInCidrList(pretender, proxyNets())) {
-            }
+                for (String header : getClientAddressHeaders()) {
+                    header = header.trim();
+                    ip = getCorrectIPAddress(request.getHeader(header));
+                    if (StringUtils.isNotBlank(ip)) {
+                        LOGGER.debug(String.format("found ip %s in header %s ", ip, header));
+                        break;
+                    }
+                } // no address found in header so ip is blank and use remote addr
+            } // else not an allowed proxy address, ip is blank and use remote addr
+        }
+        if (StringUtils.isBlank(ip)) {
+            LOGGER.trace(String.format("no ip found in headers, returning remote address %s.", pretender.getHostAddress()));
+            return pretender;
         }
 
-        return InetAddress.getByName(request.getRemoteAddr());
+        return InetAddress.getByName(ip);
+    }
+
+    private String[] getClientAddressHeaders() {
+        return ApiServer.listOfForwardHeaders.value().split(",");
     }
 
     private static String getCorrectIPAddress(String ip) {

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -205,6 +205,7 @@ import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
 import com.cloud.host.dao.HostTagsDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
 import com.cloud.network.IpAddress;
 import com.cloud.network.IpAddressManager;
 import com.cloud.network.Ipv6GuestPrefixSubnetNetworkMapVO;
@@ -3255,6 +3256,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                     }
                 }
                 if (detailEntry.getKey().startsWith(ApiConstants.EXTRA_CONFIG)) {
+                    validateExtraConfigInServiceOfferingDetail(detailEntry.getKey());
                     try {
                         detailEntryValue = URLDecoder.decode(detailEntry.getValue(), "UTF-8");
                     } catch (UnsupportedEncodingException | IllegalArgumentException e) {
@@ -3320,6 +3322,14 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         }
     }
 
+    @Override
+    public void validateExtraConfigInServiceOfferingDetail(String detailName) {
+        if (!detailName.equals(DpdkHelper.DPDK_NUMA) && !detailName.equals(DpdkHelper.DPDK_HUGE_PAGES)
+                && !detailName.startsWith(DpdkHelper.DPDK_INTERFACE_PREFIX)) {
+            throw new InvalidParameterValueException("Only extraconfig for DPDK are supported in service offering details");
+        }
+    }
+
     private DiskOfferingVO createDiskOfferingInternal(final long userId, final boolean isSystem, final VirtualMachine.Type vmType,
                                                       final String name, final Integer cpu, final Integer ramSize, final Integer speed, final String displayText, final ProvisioningType typedProvisioningType, final boolean localStorageRequired,
                                                       final boolean offerHA, final boolean limitResourceUse, final boolean volatileVm, String tags, final List<Long> domainIds, List<Long> zoneIds, final String hostTag,

server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java

@@ -47,6 +47,7 @@ import com.cloud.agent.api.Command;
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.configuration.ConfigurationManager;
 import com.cloud.gpu.GPU;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
@@ -69,6 +70,7 @@ import com.cloud.utils.Pair;
 import com.cloud.utils.component.AdapterBase;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.NicVO;
+import com.cloud.vm.UserVmManager;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineProfile;
@@ -113,6 +115,10 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
     @Inject
     protected
     HostDao hostDao;
+    @Inject
+    private UserVmManager userVmManager;
+    @Inject
+    private ConfigurationManager configurationManager;
 
     public static ConfigKey<Boolean> VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor = new ConfigKey<Boolean>("Advanced", Boolean.class, "vm.min.memory.equals.memory.divided.by.mem.overprovisioning.factor", "true",
             "If we set this to 'true', a minimum memory (memory/ mem.overprovisioning.factor) will be set to the VM, independent of using a scalable service offering or not.", true, ConfigKey.Scope.Cluster);
@@ -224,10 +230,12 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
     /**
      * Add extra configuration from VM details. Extra configuration is stored as details starting with 'extraconfig'
      */
-    private void addExtraConfig(Map<String, String> details, VirtualMachineTO to) {
+    private void addExtraConfig(Map<String, String> details, VirtualMachineTO to, long accountId, Hypervisor.HypervisorType hypervisorType) {
         for (String key : details.keySet()) {
             if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
-                to.addExtraConfig(key, details.get(key));
+                String extraConfig = details.get(key);
+                userVmManager.validateExtraConfig(accountId, hypervisorType, extraConfig);
+                to.addExtraConfig(key, extraConfig);
             }
         }
     }
@@ -243,6 +251,7 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
         if (CollectionUtils.isNotEmpty(details)) {
             for (ServiceOfferingDetailsVO detail : details) {
                 if (detail.getName().startsWith(ApiConstants.EXTRA_CONFIG)) {
+                    configurationManager.validateExtraConfigInServiceOfferingDetail(detail.getName());
                     to.addExtraConfig(detail.getName(), detail.getValue());
                 }
             }
@@ -306,7 +315,7 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
         Map<String, String> detailsInVm = _userVmDetailsDao.listDetailsKeyPairs(vm.getId());
         if (detailsInVm != null) {
             to.setDetails(detailsInVm);
-            addExtraConfig(detailsInVm, to);
+            addExtraConfig(detailsInVm, to, vm.getAccountId(), vm.getHypervisorType());
         }
 
         addServiceOfferingExtraConfiguration(offering, to);

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -97,6 +97,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
+import org.apache.cloudstack.framework.config.ConfigDepot;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
@@ -238,6 +239,7 @@ import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.EntityManager;
+import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericSearchBuilder;
 import com.cloud.utils.db.GlobalLock;
 import com.cloud.utils.db.JoinBuilder;
@@ -380,6 +382,11 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
     @Inject
     protected BucketDao _bucketDao;
+    @Inject
+    ConfigDepot configDepot;
+    @Inject
+    ConfigurationDao configurationDao;
+
     protected List<StoragePoolDiscoverer> _discoverers;
 
     public List<StoragePoolDiscoverer> getDiscoverers() {
@@ -440,6 +447,21 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         }
     }
 
+    protected void enableDefaultDatastoreDownloadRedirectionForExistingInstallations() {
+        if (!configDepot.isNewConfig(DataStoreDownloadFollowRedirects)) {
+            logger.trace("{} is not a new configuration, skipping updating its value",
+                    DataStoreDownloadFollowRedirects.key());
+            return;
+        }
+        List<DataCenterVO> zones =
+                _dcDao.listAll(new Filter(1));
+        if (CollectionUtils.isNotEmpty(zones)) {
+            logger.debug(String.format("Updating value for configuration: %s to true",
+                DataStoreDownloadFollowRedirects.key()));
+            configurationDao.update(DataStoreDownloadFollowRedirects.key(), "true");
+        }
+    }
+
     @Override
     public List<StoragePoolVO> ListByDataCenterHypervisor(long datacenterId, HypervisorType type) {
         List<StoragePoolVO> pools = _storagePoolDao.listByDataCenterId(datacenterId);
@@ -671,7 +693,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         }
 
         _executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS);
-
+        enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
         return true;
     }
 
@@ -3764,7 +3786,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
                 MountDisabledStoragePool,
                 VmwareCreateCloneFull,
                 VmwareAllowParallelExecution,
-                ConvertVmwareInstanceToKvmTimeout
+                ConvertVmwareInstanceToKvmTimeout,
+                DataStoreDownloadFollowRedirects
         };
     }
 

server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java

@@ -554,12 +554,14 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
                 throw new InvalidParameterValueException("File:// type urls are currently unsupported");
             }
             UriUtils.validateUrl(format, url);
+            boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
             if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access
                 logger.debug("Checking url: " + url);
-                DirectDownloadHelper.checkUrlExistence(url);
+                DirectDownloadHelper.checkUrlExistence(url, followRedirects);
             }
             // Check that the resource limit for secondary storage won't be exceeded
-            _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+            _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage,
+                    UriUtils.getRemoteSize(url, followRedirects));
         } else {
             _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage);
         }
@@ -660,8 +662,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
                 _resourceLimitMgr.incrementVolumeResourceCount(volume.getAccountId(), true, null, diskOfferingVO);
                 //url can be null incase of postupload
                 if (url != null) {
-                    _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+                    long remoteSize = UriUtils.getRemoteSize(url, StorageManager.DataStoreDownloadFollowRedirects.value());
-                    volume.setSize(UriUtils.getRemoteSize(url));
+                    _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage,
+                            remoteSize);
+                    volume.setSize(remoteSize);
                 }
 
                 return volume;

server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java

@@ -88,6 +88,7 @@ import com.cloud.server.StatsCollector;
 import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.TemplateType;
+import com.cloud.storage.StorageManager;
 import com.cloud.storage.TemplateProfile;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.storage.VMTemplateVO;
@@ -159,7 +160,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
      * @param url url
      */
     private Long performDirectDownloadUrlValidation(final String format, final Hypervisor.HypervisorType hypervisor,
-                                                    final String url, final List<Long> zoneIds) {
+                                                    final String url, final List<Long> zoneIds, final boolean followRedirects) {
         HostVO host = null;
         if (zoneIds != null && !zoneIds.isEmpty()) {
             for (Long zoneId : zoneIds) {
@@ -178,7 +179,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
         Integer socketTimeout = DirectDownloadManager.DirectDownloadSocketTimeout.value();
         Integer connectRequestTimeout = DirectDownloadManager.DirectDownloadConnectionRequestTimeout.value();
         Integer connectTimeout = DirectDownloadManager.DirectDownloadConnectTimeout.value();
-        CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout);
+        CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout, followRedirects);
         logger.debug("Performing URL " + url + " validation on host " + host.getId());
         Answer answer = _agentMgr.easySend(host.getId(), cmd);
         if (answer == null || !answer.getResult()) {
@@ -202,6 +203,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
         TemplateProfile profile = super.prepare(cmd);
         String url = profile.getUrl();
         UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url);
+        boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
         if (cmd.isDirectDownload()) {
             DigestHelper.validateChecksumString(cmd.getChecksum());
             List<Long> zoneIds = null;
@@ -210,12 +212,14 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
                 zoneIds.add(cmd.getZoneId());
             }
             Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(),
-                    Hypervisor.HypervisorType.KVM, url, zoneIds);
+                    Hypervisor.HypervisorType.KVM, url, zoneIds, followRedirects);
             profile.setSize(templateSize);
         }
         profile.setUrl(url);
         // Check that the resource limit for secondary storage won't be exceeded
-        _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+        _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
+                ResourceType.secondary_storage,
+                UriUtils.getRemoteSize(url, followRedirects));
         return profile;
     }
 
@@ -234,15 +238,18 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
         String url = profile.getUrl();
         UriUtils.validateUrl(cmd.getFormat(), url, cmd.isDirectDownload());
         Hypervisor.HypervisorType hypervisor = Hypervisor.HypervisorType.getType(cmd.getHypervisor());
+        boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
         if (cmd.isDirectDownload()) {
             DigestHelper.validateChecksumString(cmd.getChecksum());
             Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(),
-                    hypervisor, url, cmd.getZoneIds());
+                    hypervisor, url, cmd.getZoneIds(), followRedirects);
             profile.setSize(templateSize);
         }
         profile.setUrl(url);
         // Check that the resource limit for secondary storage won't be exceeded
-        _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+        _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
+                ResourceType.secondary_storage,
+                UriUtils.getRemoteSize(url, followRedirects));
         return profile;
     }
 

server/src/main/java/com/cloud/vm/UserVmManager.java

@@ -30,6 +30,7 @@ import com.cloud.exception.ManagementServerException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.exception.VirtualMachineMigrationException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.service.ServiceOfferingVO;
 import com.cloud.storage.Storage.StoragePoolType;
@@ -96,6 +97,8 @@ public interface UserVmManager extends UserVmService {
 
     String validateUserData(String userData, HTTPMethod httpmethod);
 
+    void validateExtraConfig(long accountId, HypervisorType hypervisorType, String extraConfig);
+
     boolean isVMUsingLocalStorage(VMInstanceVO vm);
 
     boolean expunge(UserVmVO vm);

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -658,7 +658,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             "enable.additional.vm.configuration", "false", "allow additional arbitrary configuration to vm", true, ConfigKey.Scope.Account);
 
     private static final ConfigKey<String> KvmAdditionalConfigAllowList = new ConfigKey<>(String.class,
-    "allow.additional.vm.configuration.list.kvm", "Advanced", "", "Comma separated list of allowed additional configuration options.", true, ConfigKey.Scope.Global, null, null, EnableAdditionalVmConfig.key(), null, null, ConfigKey.Kind.CSV, null);
+    "allow.additional.vm.configuration.list.kvm", "Advanced", "", "Comma separated list of allowed additional configuration options.", true, ConfigKey.Scope.Account, null, null, EnableAdditionalVmConfig.key(), null, null, ConfigKey.Kind.CSV, null);
 
     private static final ConfigKey<String> XenServerAdditionalConfigAllowList = new ConfigKey<>(String.class,
     "allow.additional.vm.configuration.list.xenserver", "Advanced", "", "Comma separated list of allowed additional configuration options", true, ConfigKey.Scope.Global, null, null, EnableAdditionalVmConfig.key(), null, null, ConfigKey.Kind.CSV, null);
@@ -2848,14 +2848,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         if (cleanupDetails){
             if (caller != null && caller.getType() == Account.Type.ADMIN) {
                 for (final UserVmDetailVO detail : existingDetails) {
-                    if (detail != null && detail.isDisplay()) {
+                    if (detail != null && detail.isDisplay() && !isExtraConfig(detail.getName())) {
                         userVmDetailsDao.removeDetail(id, detail.getName());
                     }
                 }
             } else {
                 for (final UserVmDetailVO detail : existingDetails) {
                     if (detail != null && !userDenyListedSettings.contains(detail.getName())
-                            && !userReadOnlySettings.contains(detail.getName()) && detail.isDisplay()) {
+                            && !userReadOnlySettings.contains(detail.getName()) && detail.isDisplay()
+                            && !isExtraConfig(detail.getName())) {
                         userVmDetailsDao.removeDetail(id, detail.getName());
                     }
                 }
@@ -2866,6 +2867,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                     throw new InvalidParameterValueException("'extraconfig' should not be included in details as key");
                 }
 
+                details.entrySet().removeIf(detail -> isExtraConfig(detail.getKey()));
+
                 if (caller != null && caller.getType() != Account.Type.ADMIN) {
                     // Ensure denied or read-only detail is not passed by non-root-admin user
                     for (final String detailName : details.keySet()) {
@@ -2889,7 +2892,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
 
                 // ensure details marked as non-displayable are maintained, regardless of admin or not
                 for (final UserVmDetailVO existingDetail : existingDetails) {
-                    if (!existingDetail.isDisplay()) {
+                    if (!existingDetail.isDisplay() || isExtraConfig(existingDetail.getName())) {
                         details.put(existingDetail.getName(), existingDetail.getValue());
                     }
                 }
@@ -2911,6 +2914,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 cmd.getHttpMethod(), cmd.getCustomId(), hostName, cmd.getInstanceName(), securityGroupIdList, cmd.getDhcpOptionsMap());
     }
 
+    private boolean isExtraConfig(String detailName) {
+        return detailName != null && detailName.startsWith(ApiConstants.EXTRA_CONFIG);
+    }
+
     protected void updateDisplayVmFlag(Boolean isDisplayVm, Long id, UserVmVO vmInstance) {
         vmInstance.setDisplayVm(isDisplayVm);
 
@@ -6301,7 +6308,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
      */
     protected void persistExtraConfigKvm(String decodedUrl, UserVm vm) {
         // validate config against denied cfg commands
-        validateKvmExtraConfig(decodedUrl);
+        validateKvmExtraConfig(decodedUrl, vm.getAccountId());
         String[] extraConfigs = decodedUrl.split("\n\n");
         for (String cfg : extraConfigs) {
             int i = 1;
@@ -6319,6 +6326,18 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             i++;
         }
     }
+    /**
+     * This method is used to validate if extra config is valid
+     */
+    @Override
+    public void validateExtraConfig(long accountId, HypervisorType hypervisorType, String extraConfig) {
+        if (!EnableAdditionalVmConfig.valueIn(accountId)) {
+            throw new CloudRuntimeException("Additional VM configuration is not enabled for this account");
+        }
+        if (HypervisorType.KVM.equals(hypervisorType)) {
+            validateKvmExtraConfig(extraConfig, accountId);
+        }
+    }
 
     /**
      * This method is called by the persistExtraConfigKvm
@@ -6326,8 +6345,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
      * controlled by Root admin
      * @param decodedUrl string containing xml configuration to be validated
      */
-    protected void validateKvmExtraConfig(String decodedUrl) {
+    protected void validateKvmExtraConfig(String decodedUrl, long accountId) {
-        String[] allowedConfigOptionList = KvmAdditionalConfigAllowList.value().split(",");
+        String[] allowedConfigOptionList = KvmAdditionalConfigAllowList.valueIn(accountId).split(",");
         // Skip allowed keys validation for DPDK
         if (!decodedUrl.contains(":")) {
             try {
@@ -6347,7 +6366,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                         }
                     }
                     if (!isValidConfig) {
-                        throw new CloudRuntimeException(String.format("Extra config %s is not on the list of allowed keys for KVM hypervisor hosts", currentConfig));
+                        throw new CloudRuntimeException(String.format("Extra config '%s' is not on the list of allowed keys for KVM hypervisor hosts", currentConfig));
                     }
                 }
             } catch (ParserConfigurationException | IOException | SAXException e) {
@@ -6445,6 +6464,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             if (details.containsKey("extraconfig")) {
                 throw new InvalidParameterValueException("'extraconfig' should not be included in details as key");
             }
+
+            for (String detailName : details.keySet()) {
+                if (isExtraConfig(detailName)) {
+                    throw new InvalidParameterValueException("detail name should not start with extraconfig");
+                }
+            }
         }
     }
 

server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java

@@ -270,7 +270,8 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
         PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO();
 
         DownloadProtocol protocol = getProtocolFromUrl(url);
-        DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
+        DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum,
+                headers);
         cmd.setTemplateSize(template.getSize());
         cmd.setFormat(template.getFormat());
 
@@ -391,19 +392,23 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
     /**
      * Return DirectDownloadCommand according to the protocol
      */
-    private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
+    private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url,
-                                                                       String checksum, Map<String, String> httpHeaders) {
+               Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> httpHeaders) {
         int connectTimeout = DirectDownloadConnectTimeout.value();
         int soTimeout = DirectDownloadSocketTimeout.value();
         int connectionRequestTimeout = DirectDownloadConnectionRequestTimeout.value();
+        boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
         if (protocol.equals(DownloadProtocol.HTTP)) {
-            return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
+            return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+                    soTimeout, followRedirects);
         } else if (protocol.equals(DownloadProtocol.HTTPS)) {
-            return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout, connectionRequestTimeout);
+            return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+                    soTimeout, connectionRequestTimeout, followRedirects);
         } else if (protocol.equals(DownloadProtocol.NFS)) {
             return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
         } else if (protocol.equals(DownloadProtocol.METALINK)) {
-            return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
+            return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+                    soTimeout, followRedirects);
         } else {
             return null;
         }

server/src/test/java/com/cloud/api/ApiServletTest.java

@@ -97,15 +97,17 @@ public class ApiServletTest {
     @Mock
     AccountService accountMgr;
 
+    @Mock ConfigKey<Boolean> useForwardHeader;
     StringWriter responseWriter;
 
     ApiServlet servlet;
-
+    ApiServlet spyServlet;
     @SuppressWarnings("unchecked")
     @Before
     public void setup() throws SecurityException, NoSuchFieldException,
     IllegalArgumentException, IllegalAccessException, IOException, UnknownHostException {
         servlet = new ApiServlet();
+        spyServlet = Mockito.spy(servlet);
         responseWriter = new StringWriter();
         Mockito.when(response.getWriter()).thenReturn(
                 new PrintWriter(responseWriter));
@@ -259,32 +261,43 @@ public class ApiServletTest {
 
     @Test
     public void getClientAddressWithXForwardedFor() throws UnknownHostException {
+        String[] proxynet = {"127.0.0.0/8"};
+        Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
+        Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
         Mockito.when(request.getHeader(Mockito.eq("X-Forwarded-For"))).thenReturn("192.168.1.1");
-        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
+        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request));
     }
 
     @Test
     public void getClientAddressWithHttpXForwardedFor() throws UnknownHostException {
+        String[] proxynet = {"127.0.0.0/8"};
+        Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
+        Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
         Mockito.when(request.getHeader(Mockito.eq("HTTP_X_FORWARDED_FOR"))).thenReturn("192.168.1.1");
-        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
+        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request));
     }
 
     @Test
-    public void getClientAddressWithXRemoteAddr() throws UnknownHostException {
+    public void getClientAddressWithRemoteAddr() throws UnknownHostException {
-        Mockito.when(request.getHeader(Mockito.eq("Remote_Addr"))).thenReturn("192.168.1.1");
+        String[] proxynet = {"127.0.0.0/8"};
-        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
+        Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
+        Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
+        Assert.assertEquals(InetAddress.getByName("127.0.0.1"), spyServlet.getClientAddress(request));
     }
 
     @Test
     public void getClientAddressWithHttpClientIp() throws UnknownHostException {
+        String[] proxynet = {"127.0.0.0/8"};
+        Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
+        Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
         Mockito.when(request.getHeader(Mockito.eq("HTTP_CLIENT_IP"))).thenReturn("192.168.1.1");
-        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
+        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request));
     }
 
     @Test
     public void getClientAddressDefault() throws UnknownHostException {
         Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1");
-        Assert.assertEquals(InetAddress.getByName("127.0.0.1"), ApiServlet.getClientAddress(request));
+        Assert.assertEquals(InetAddress.getByName("127.0.0.1"), spyServlet.getClientAddress(request));
     }
 
     @Test

server/src/test/java/com/cloud/storage/StorageManagerImplTest.java

@@ -23,6 +23,8 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.framework.config.ConfigDepot;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao;
 import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
@@ -42,7 +44,9 @@ import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.StoragePoolInfo;
 import com.cloud.capacity.CapacityManager;
+import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.VsphereStoragePolicyVO;
+import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.dc.dao.VsphereStoragePolicyDao;
 import com.cloud.exception.AgentUnavailableException;
 import com.cloud.exception.ConnectionException;
@@ -77,6 +81,12 @@ public class StorageManagerImplTest {
     HypervisorGuruManager hvGuruMgr;
     @Mock
     AgentManager agentManager;
+    @Mock
+    ConfigDepot configDepot;
+    @Mock
+    ConfigurationDao configurationDao;
+    @Mock
+    DataCenterDao dataCenterDao;
 
     @Spy
     @InjectMocks
@@ -455,4 +465,36 @@ public class StorageManagerImplTest {
                 storageManagerImpl.getCheckDatastorePolicyComplianceAnswer("1", pool);
         Assert.assertTrue(answer.getResult());
     }
+
+    @Test
+    public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNoChange() {
+        Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+                .thenReturn(false);
+        storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+        Mockito.verify(configurationDao, Mockito.never()).update(Mockito.anyString(), Mockito.anyString());
+    }
+
+    @Test
+    public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsOldInstall() {
+        Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+                .thenReturn(true);
+        Mockito.when(dataCenterDao.listAll(Mockito.any()))
+                .thenReturn(List.of(Mockito.mock(DataCenterVO.class)));
+        Mockito.doReturn(true).when(configurationDao).update(Mockito.anyString(), Mockito.anyString());
+        storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+        Mockito.verify(configurationDao, Mockito.times(1))
+                .update(StorageManager.DataStoreDownloadFollowRedirects.key(), "true");
+    }
+
+    @Test
+    public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNewInstall() {
+        Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+                .thenReturn(true);
+        Mockito.when(dataCenterDao.listAll(Mockito.any()))
+                .thenReturn(new ArrayList<>()); //new installation
+        storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+        Mockito.verify(configurationDao, Mockito.never())
+                .update(StorageManager.DataStoreDownloadFollowRedirects.key(),StorageManager.DataStoreDownloadFollowRedirects.defaultValue());
+    }
+
 }

server/src/test/java/com/cloud/vpc/MockConfigurationManagerImpl.java

@@ -677,4 +677,9 @@ public class MockConfigurationManagerImpl extends ManagerBase implements Configu
         // TODO Auto-generated method stub
         return null;
     }
+
+    @Override
+    public void validateExtraConfigInServiceOfferingDetail(String detailName) {
+        // TODO Auto-generated method stub
+    }
 }

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java

@@ -41,17 +41,21 @@ public interface DownloadManager extends Manager {
      * @param hvm  whether the template is a hardware virtual machine
      * @param accountId the accountId of the iso owner (null if public iso)
      * @param descr    description of the template
-     * @param user username used for authentication to the server
+     * @param userName username used for authentication to the server
-     * @param password password used for authentication to the server
+     * @param passwd password used for authentication to the server
      * @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes.
      * @param resourceType signifying the type of resource like template, volume etc.
+     * @param followRedirects whether downloader follows redirections
      * @return job-id that can be used to interrogate the status of the download.
      */
-    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
-        String installPathPrefix, String templatePath, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType);
+             Long accountId, String descr, String cksum, String installPathPrefix, String templatePath,
+             String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType,
+             boolean followRedirects);
 
-    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
-        String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType);
+             Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
+             long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects);
 
     Map<String, Processor> getProcessors();
 

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java

@@ -662,8 +662,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
     }
 
     @Override
-    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
-            String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+            Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
+            long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects) {
         UUID uuid = UUID.randomUUID();
         String jobId = uuid.toString();
 
@@ -683,6 +684,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         } else {
             throw new CloudRuntimeException("Unable to download from URL: " + url);
         }
+        td.setFollowRedirects(followRedirects);
         DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
         dj.setTmpltPath(installPathPrefix);
         jobs.put(jobId, dj);
@@ -718,8 +720,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
     }
 
     @Override
-    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
-            String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+            Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user,
+            String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType,
+            boolean followRedirects) {
         UUID uuid = UUID.randomUUID();
         String jobId = uuid.toString();
         String tmpDir = installPathPrefix;
@@ -766,6 +770,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
                     throw new CloudRuntimeException("Unable to download from URL: " + url);
                 }
             }
+            td.setFollowRedirects(followRedirects);
             // NOTE the difference between installPathPrefix and templatePath
             // here. instalPathPrefix is the absolute path for template
             // including mount directory
@@ -902,12 +907,16 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         String jobId = null;
         if (dstore instanceof S3TO) {
             jobId =
-                    downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
+                    downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(),
-                            cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+                            cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(),
+                            installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
+                            cmd.isFollowRedirects());
         } else {
             jobId =
-                    downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
+                    downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(),
-                            cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+                            cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix,
+                            cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
+                            cmd.isFollowRedirects());
         }
         sleep();
         if (jobId == null) {

ui/public/locales/en.json

@@ -13,6 +13,7 @@
 "error.release.dedicate.host": "Failed to release dedicated host.",
 "error.release.dedicate.pod": "Failed to release dedicated pod.",
 "error.release.dedicate.zone": "Failed to release dedicated zone.",
+"error.unable.to.add.setting.extraconfig": "It is not allowed to add setting for extraconfig. Please update VirtualMachine with extraconfig parameter.",
 "error.unable.to.proceed": "Unable to proceed. Please contact your administrator.",
 "firewall.close": "Firewall",
 "icmp.code.desc": "Please specify -1 if you want to allow all ICMP codes (except NSX zones).",

ui/src/components/view/DetailSettings.vue

@@ -101,7 +101,7 @@
             <tooltip-button
               :tooltip="$t('label.edit')"
               icon="edit-outlined"
-              :disabled="deployasistemplate === true"
+              :disabled="deployasistemplate === true || item.name.startsWith('extraconfig')"
               v-if="!item.edit"
               @onClick="showEditDetail(index)" />
           </div>
@@ -115,7 +115,12 @@
               :cancelText="$t('label.no')"
               placement="left"
             >
-              <tooltip-button :tooltip="$t('label.delete')" :disabled="deployasistemplate === true" type="primary" :danger="true" icon="delete-outlined" />
+              <tooltip-button
+                :tooltip="$t('label.delete')"
+                :disabled="deployasistemplate === true || item.name.startsWith('extraconfig')"
+                type="primary"
+                :danger="true"
+                icon="delete-outlined" />
             </a-popconfirm>
           </div>
         </template>
@@ -307,6 +312,10 @@ export default {
         this.error = this.$t('message.error.provide.setting')
         return
       }
+      if (this.newKey.startsWith('extraconfig')) {
+        this.error = this.$t('error.unable.to.add.setting.extraconfig')
+        return
+      }
       if (!this.allowEditOfDetail(this.newKey)) {
         this.error = this.$t('error.unable.to.proceed')
         return

ui/src/config/section/compute.js

@@ -898,7 +898,12 @@ export default {
         var fields = ['name', 'id']
         if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) {
           fields.push('account')
+          if (store.getters.listAllProjects) {
+            fields.push('project')
+          }
           fields.push('domain')
+        } else if (store.getters.listAllProjects) {
+          fields.push('project')
         }
         return fields
       },

ui/src/views/AutogenView.vue

@@ -793,7 +793,7 @@ export default {
       }
 
       this.projectView = Boolean(store.getters.project && store.getters.project.id)
-      this.hasProjectId = ['vm', 'vmgroup', 'ssh', 'affinitygroup', 'volume', 'snapshot', 'vmsnapshot', 'guestnetwork',
+      this.hasProjectId = ['vm', 'vmgroup', 'ssh', 'affinitygroup', 'userdata', 'volume', 'snapshot', 'vmsnapshot', 'guestnetwork',
         'vpc', 'securitygroups', 'publicip', 'vpncustomergateway', 'template', 'iso', 'event', 'kubernetes',
         'autoscalevmgroup', 'vnfapp'].includes(this.$route.name)
 

ui/src/views/network/GuestIpRanges.vue

@@ -36,14 +36,16 @@
         :rowKey="item => item.id"
         :pagination="false" >
 
-        <template #action="{ record }">
+        <template #bodyCell="{ column, record }">
           <tooltip-button
+            v-if="column.key === 'actions'"
             tooltipPlacement="bottom"
             :tooltip="$t('label.edit')"
             type="primary"
             @click="() => { handleUpdateIpRangeModal(record) }"
             icon="swap-outlined" />
           <a-popconfirm
+            v-if="column.key === 'actions'"
             :title="$t('message.confirm.remove.ip.range')"
             @confirm="removeIpRange(record.id)"
             :okText="$t('label.yes')"
@@ -226,6 +228,10 @@ export default {
     },
     removeIpRange (id) {
       api('deleteVlanIpRange', { id: id }).then(json => {
+        const message = `${this.$t('message.success.delete')} ${this.$t('label.ip.range')}`
+        this.$message.success(message)
+      }).catch((error) => {
+        this.$notifyError(error)
       }).finally(() => {
         this.fetchData()
       })

ui/src/views/network/LoadBalancing.vue

@@ -84,12 +84,18 @@
             <a-select-option value="no">{{ $t('label.no') }}</a-select-option>
           </a-select>
         </div>
-        <div class="form__item" v-if="!newRule.autoscale || newRule.autoscale === 'no' || ('vpcid' in this.resource && !('associatednetworkid' in this.resource))">
+        <div class="form__item" v-if="!newRule.autoscale || newRule.autoscale === 'no'">
           <div class="form__label" style="white-space: nowrap;">{{ $t('label.add.vms') }}</div>
           <a-button :disabled="!('createLoadBalancerRule' in $store.getters.apis)" type="primary" @click="handleOpenAddVMModal">
             {{ $t('label.add') }}
           </a-button>
         </div>
+        <div class="form__item" v-else-if="newRule.autoscale === 'yes' && ('vpcid' in this.resource && !this.associatednetworkid)">
+          <div class="form__label" style="white-space: nowrap;">{{ $t('label.select.tier') }}</div>
+          <a-button :disabled="!('createLoadBalancerRule' in $store.getters.apis)" type="primary" @click="handleOpenAddNetworkModal">
+            {{ $t('label.add') }}
+          </a-button>
+        </div>
         <div class="form__item" v-else-if="newRule.autoscale === 'yes'">
           <div class="form__label" style="white-space: nowrap;">{{ $t('label.add') }}</div>
           <a-button :disabled="!('createLoadBalancerRule' in $store.getters.apis)" type="primary" @click="handleAddNewRule">
@@ -519,7 +525,7 @@
             </template>
 
             <template v-if="column.key === 'actions'" style="text-align: center" :text="text">
-              <a-checkbox v-model:value="record.id" @change="e => fetchNics(e, index)" :disabled="newRule.autoscale"/>
+              <a-checkbox v-model:value="record.id" @change="e => fetchNics(e, index)" />
             </template>
           </template>
         </a-table>
@@ -546,6 +552,71 @@
       </div>
     </a-modal>
 
+    <a-modal
+      :title="$t('label.select.tier')"
+      :maskClosable="false"
+      :closable="true"
+      v-if="addNetworkModalVisible"
+      :visible="addNetworkModalVisible"
+      class="network-modal"
+      width="60vw"
+      :footer="null"
+      @cancel="closeModal"
+    >
+      <div @keyup.ctrl.enter="handleAddNewRule">
+        <a-input-search
+          v-focus="!('vpcid' in resource && !('associatednetworkid' in resource))"
+          class="input-search"
+          :placeholder="$t('label.search')"
+          v-model:value="searchQuery"
+          allowClear
+          @search="onNetworkSearch" />
+        <a-table
+          size="small"
+          class="list-view"
+          :loading="addNetworkModalLoading"
+          :columns="networkColumns"
+          :dataSource="networks"
+          :pagination="false"
+          :rowKey="record => record.id"
+          :scroll="{ y: 300 }">
+          <template #bodyCell="{ column, record }">
+            <template v-if="column.key === 'actions'">
+              <div style="text-align: center">
+                <a-radio-group
+                  class="radio-group"
+                  :key="record.id"
+                  v-model:value="this.selectedTierForAutoScaling"
+                  @change="($event) => this.selectedTierForAutoScaling = $event.target.value">
+                  <a-radio :value="record.id" />
+                </a-radio-group>
+              </div>
+            </template>
+          </template>
+        </a-table>
+        <a-pagination
+          class="pagination"
+          size="small"
+          :current="networkPage"
+          :pageSize="networkPageSize"
+          :total="networkCount"
+          :showTotal="total => `${$t('label.total')} ${total} ${$t('label.items')}`"
+          :pageSizeOptions="['10', '20', '40', '80', '100']"
+          @change="handleChangeNetworkPage"
+          @showSizeChange="handleChangeNetworkPageSize"
+          showSizeChanger>
+          <template #buildOptionText="props">
+            <span>{{ props.value }} / {{ $t('label.page') }}</span>
+          </template>
+        </a-pagination>
+
+        <div :span="24" class="action-button">
+          <a-button @click="closeModal">{{ $t('label.cancel') }}</a-button>
+          <a-button :disabled="this.selectedTierForAutoScaling === null" type="primary" ref="submit" @click="handleAddNewRule">{{ $t('label.ok') }}</a-button>
+        </div>
+      </div>
+    </a-modal>
+
     <a-modal
       v-if="healthMonitorModal"
       :title="$t('label.configure.health.monitor')"
@@ -802,6 +873,41 @@ export default {
       vmPage: 1,
       vmPageSize: 10,
       vmCount: 0,
+      addNetworkModalVisible: false,
+      addNetworkModalLoading: false,
+      networks: [],
+      associatednetworkid: null,
+      selectedTierForAutoScaling: null,
+      networkColumns: [
+        {
+          key: 'name',
+          title: this.$t('label.name'),
+          dataIndex: 'name',
+          width: 220
+        },
+        {
+          key: 'state',
+          title: this.$t('label.state'),
+          dataIndex: 'state'
+        },
+        {
+          title: this.$t('label.gateway'),
+          dataIndex: 'gateway'
+        },
+        {
+          title: this.$t('label.netmask'),
+          dataIndex: 'netmask'
+        },
+        {
+          key: 'actions',
+          title: this.$t('label.select'),
+          dataIndex: 'actions',
+          width: 80
+        }
+      ],
+      networkPage: 1,
+      networkPageSize: 10,
+      networkCount: 0,
       searchQuery: null,
       tungstenHealthMonitors: [],
       healthMonitorModal: false,
@@ -825,6 +931,9 @@ export default {
   beforeCreate () {
     this.createLoadBalancerRuleParams = this.$getApiParams('createLoadBalancerRule')
     this.createLoadBalancerStickinessPolicyParams = this.$getApiParams('createLBStickinessPolicy')
+    if ('associatednetworkid' in this.resource) {
+      this.associatednetworkid = this.resource.associatednetworkid
+    }
   },
   created () {
     this.initForm()
@@ -1489,6 +1598,55 @@ export default {
         this.addVmModalLoading = false
       })
     },
+    handleOpenAddNetworkModal () {
+      if (this.addNetworkModalLoading) return
+      if (!this.checkNewRule()) {
+        return
+      }
+      this.addNetworkModalVisible = true
+      this.fetchNetworks()
+    },
+    fetchNetworks () {
+      this.networkCount = 0
+      this.networks = []
+      this.addNetworkModalLoading = true
+      const vpcid = this.resource.vpcid
+      if (!vpcid) {
+        this.addNetworkModalLoading = false
+        return
+      }
+      api('listNetworks', {
+        listAll: true,
+        keyword: this.searchQuery,
+        page: this.networkPage,
+        pagesize: this.networkPageSize,
+        supportedservices: 'Lb',
+        isrecursive: true,
+        vpcid: vpcid
+      }).then(response => {
+        this.networkCount = response.listnetworksresponse.count || 0
+        this.networks = response.listnetworksresponse.network || []
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
+        this.addNetworkModalLoading = false
+      })
+      this.selectedTierForAutoScaling = null
+    },
+    onNetworkSearch (value) {
+      this.searchQuery = value
+      this.fetchNetworks()
+    },
+    handleChangeNetworkPage (page, pageSize) {
+      this.networkPage = page
+      this.networkPageSize = pageSize
+      this.fetchNetworks()
+    },
+    handleChangeNetworkPageSize (currentPage, pageSize) {
+      this.networkPage = currentPage
+      this.networkPageSize = pageSize
+      this.fetchNetworks()
+    },
     handleAssignToLBRule (data) {
       const vmIDIpMap = {}
 
@@ -1560,7 +1718,8 @@ export default {
         return
       }
 
-      const networkId = ('vpcid' in this.resource && !('associatednetworkid' in this.resource)) ? this.selectedTier : this.resource.associatednetworkid
+      const networkId = this.selectedTierForAutoScaling != null ? this.selectedTierForAutoScaling
+        : ('vpcid' in this.resource && !('associatednetworkid' in this.resource)) ? this.selectedTier : this.resource.associatednetworkid
       api('createLoadBalancerRule', {
         openfirewall: false,
         networkid: networkId,
@@ -1573,7 +1732,9 @@ export default {
         cidrlist: this.newRule.cidrlist
       }).then(response => {
         this.addVmModalVisible = false
+        this.addNetworkModalVisible = false
         this.handleAssignToLBRule(response.createloadbalancerruleresponse.id)
+        this.associatednetworkid = networkId
       }).catch(error => {
         this.$notifyError(error)
         this.loading = false
@@ -1597,6 +1758,9 @@ export default {
       this.nics = []
       this.addVmModalVisible = false
       this.newRule.virtualmachineid = []
+      this.addNetworkModalLoading = false
+      this.addNetworkModalVisible = false
+      this.selectedTierForAutoScaling = null
     },
     handleChangePage (page, pageSize) {
       this.page = page

ui/src/views/tools/ImportUnmanagedInstance.vue

@@ -713,7 +713,7 @@ export default {
         page: 1
       })
       this.fetchKvmHostsForConversion()
-      if (this.resource.disk.length > 1) {
+      if (this.resource?.disk?.length > 1) {
         this.updateSelectedRootDisk()
       }
     },

ui/src/views/tools/ManageInstances.vue

@@ -1074,6 +1074,7 @@ export default {
       this.sourceHypervisor = value
       this.sourceActions = this.AllSourceActions.filter(x => x.sourceDestHypervisors[value])
       this.form.sourceAction = this.sourceActions[0].name || ''
+      this.selectedVmwareVcenter = undefined
       this.onSelectSourceAction(this.form.sourceAction)
     },
     onSelectSourceAction (value) {

utils/src/main/java/com/cloud/utils/StringUtils.java

@@ -31,7 +31,7 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-public class StringUtils {
+public class StringUtils extends org.apache.commons.lang3.StringUtils {
     private static final char[] hexChar = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
 
     private static Charset preferredACSCharset;

utils/src/main/java/com/cloud/utils/UriUtils.java

@@ -214,7 +214,7 @@ public class UriUtils {
     }
 
     // Get the size of a file from URL response header.
-    public static long getRemoteSize(String url) {
+    public static long getRemoteSize(String url, Boolean followRedirect) {
         long remoteSize = 0L;
         final String[] methods = new String[]{"HEAD", "GET"};
         IllegalArgumentException exception = null;
@@ -229,6 +229,7 @@ public class UriUtils {
                 httpConn.setRequestMethod(method);
                 httpConn.setConnectTimeout(2000);
                 httpConn.setReadTimeout(5000);
+                httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect));
                 String contentLength = httpConn.getHeaderField("content-length");
                 if (contentLength != null) {
                     remoteSize = Long.parseLong(contentLength);

utils/src/main/java/com/cloud/utils/script/OutputInterpreter.java

@@ -139,6 +139,11 @@ public abstract class OutputInterpreter {
         public String getLines() {
             return allLines;
         }
+
+        @Override
+        public boolean drain() {
+            return true;
+        }
     }
 
     public static class LineByLineOutputLogger extends OutputInterpreter {

utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java

@@ -22,8 +22,9 @@ package com.cloud.utils.storage;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.MalformedURLException;
+import java.net.HttpURLConnection;
-import java.net.URL;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 
@@ -113,16 +114,23 @@ public final class QCOW2Utils {
         }
     }
 
-    public static long getVirtualSize(String urlStr) {
+    public static long getVirtualSizeFromUrl(String urlStr, boolean followRedirects) {
+        HttpURLConnection httpConn = null;
         try {
-            URL url = new URL(urlStr);
+            URI url = new URI(urlStr);
-            return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr));
+            httpConn = (HttpURLConnection)url.toURL().openConnection();
-        } catch (MalformedURLException e) {
+            httpConn.setInstanceFollowRedirects(followRedirects);
+            return getVirtualSize(httpConn.getInputStream(), UriUtils.isUrlForCompressedFile(urlStr));
+        } catch (URISyntaxException e) {
             LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
             throw new IllegalArgumentException("Invalid URL: " + urlStr);
         }  catch (IOException e) {
             LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
             throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
+        } finally {
+            if (httpConn != null) {
+                httpConn.disconnect();
+            }
         }
     }
 }

utils/src/test/java/com/cloud/utils/ScriptTest.java

@@ -112,6 +112,20 @@ public class ScriptTest {
         Assert.assertNotNull(value);
     }
 
+    @Test
+    public void executeWithOutputInterpreterAllLinesParserLargeOutput() {
+        Assume.assumeTrue(SystemUtils.IS_OS_LINUX);
+        OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
+        Script script = new Script("seq");
+        script.add("-f");
+        script.add("my text to test cloudstack %g");
+        script.add("4096"); // AllLinesParser doesn't work with that amount of data
+        String value = script.execute(parser);
+        // it is a stack trace in this case as string
+        Assert.assertNull(value);
+        Assert.assertEquals(129965, parser.getLines().length());
+    }
+
     @Test
     public void runSimpleBashScriptNotExisting() {
         Assume.assumeTrue(SystemUtils.IS_OS_LINUX);