diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmd.java index 9786a5a1467..56fdf6bc126 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmd.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.EnumUtils; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; @@ -69,9 +70,9 @@ public class CheckAndRepairVolumeCmd extends BaseAsyncCmd { public String getRepair() { if (org.apache.commons.lang3.StringUtils.isNotEmpty(repair)) { - RepairValues repairType = Enum.valueOf(RepairValues.class, repair.toUpperCase()); + RepairValues repairType = EnumUtils.getEnumIgnoreCase(RepairValues.class, repair); if (repairType == null) { - throw new InvalidParameterValueException(String.format("Repair parameter can only take the following values: %s" + Arrays.toString(RepairValues.values()))); + throw new InvalidParameterValueException(String.format("Repair parameter can only take the following values: %s", Arrays.toString(RepairValues.values()))); } return repair.toLowerCase(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserDataResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserDataResponse.java index bbe27f84520..e69094c8f80 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserDataResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserDataResponse.java @@ -24,7 +24,7 @@ import org.apache.cloudstack.api.BaseResponseWithAnnotations; import org.apache.cloudstack.api.EntityReference; @EntityReference(value = UserData.class) -public class UserDataResponse extends BaseResponseWithAnnotations { +public class UserDataResponse extends BaseResponseWithAnnotations implements ControlledEntityResponse { @SerializedName(ApiConstants.ID) @Param(description = "ID of the ssh keypair") @@ -40,6 +40,14 @@ public class UserDataResponse extends BaseResponseWithAnnotations { @SerializedName(ApiConstants.ACCOUNT) @Param(description="the owner of the userdata") private String accountName; + @SerializedName(ApiConstants.PROJECT_ID) + @Param(description = "the project id of the userdata", since = "4.19.1") + private String projectId; + + @SerializedName(ApiConstants.PROJECT) + @Param(description = "the project name of the userdata", since = "4.19.1") + private String projectName; + @SerializedName(ApiConstants.DOMAIN_ID) @Param(description="the domain id of the userdata owner") private String domainId; @@ -118,6 +126,16 @@ public class UserDataResponse extends BaseResponseWithAnnotations { this.accountName = accountName; } + @Override + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + @Override + public void setProjectName(String projectName) { + this.projectName = projectName; + } + public String getDomainName() { return domain; } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmdTest.java new file mode 100644 index 00000000000..226ff882fa7 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmdTest.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.volume; + +import com.cloud.exception.InvalidParameterValueException; +import junit.framework.TestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockitoAnnotations; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class CheckAndRepairVolumeCmdTest extends TestCase { + private CheckAndRepairVolumeCmd checkAndRepairVolumeCmd; + private AutoCloseable closeable; + + @Before + public void setup() { + closeable = MockitoAnnotations.openMocks(this); + checkAndRepairVolumeCmd = new CheckAndRepairVolumeCmd(); + } + + @After + public void tearDown() throws Exception { + closeable.close(); + } + + @Test + public void testGetRepair() { + ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", "all"); + assertEquals("all", checkAndRepairVolumeCmd.getRepair()); + + ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", "LEAKS"); + assertEquals("leaks", checkAndRepairVolumeCmd.getRepair()); + + ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", null); + assertNull(checkAndRepairVolumeCmd.getRepair()); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetRepairInvalid() { + ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", "RANDOM STRING"); + checkAndRepairVolumeCmd.getRepair(); + } +} diff --git a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java index 23312b8517f..d2e4483835e 100644 --- a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java +++ b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java @@ -30,7 +30,6 @@ import org.apache.commons.daemon.Daemon; import org.apache.commons.daemon.DaemonContext; import org.apache.commons.lang3.StringUtils; import org.eclipse.jetty.jmx.MBeanContainer; -import org.eclipse.jetty.server.ForwardedRequestCustomizer; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.NCSARequestLog; @@ -174,7 +173,7 @@ public class ServerDaemon implements Daemon { // HTTP config final HttpConfiguration httpConfig = new HttpConfiguration(); - httpConfig.addCustomizer( new ForwardedRequestCustomizer() ); +// it would be nice to make this dynamic but we take care of this ourselves for now: httpConfig.addCustomizer( new ForwardedRequestCustomizer() ); httpConfig.setSecureScheme("https"); httpConfig.setSecurePort(httpsPort); httpConfig.setOutputBufferSize(32768); diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 92865caeb57..9b126846827 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -28,6 +28,7 @@ import java.io.RandomAccessFile; import java.net.URI; import java.net.URISyntaxException; import java.util.Date; +import java.util.List; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; @@ -78,6 +79,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te private long maxTemplateSizeInBytes; private ResourceType resourceType = ResourceType.TEMPLATE; private final HttpMethodRetryHandler myretryhandler; + private boolean followRedirects = false; public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { @@ -109,7 +111,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te private GetMethod createRequest(String downloadUrl) { GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); - request.setFollowRedirects(true); + request.setFollowRedirects(followRedirects); return request; } @@ -335,6 +337,12 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) { status = Status.UNRECOVERABLE_ERROR; errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) "; + if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) + && !followRedirects) { + errorString = String.format("Failed to download %s due to redirection, response code: %d", + downloadUrl, responseCode); + logger.error(errorString); + } return true; //FIXME: retry? } return false; @@ -536,4 +544,12 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te return this; } } + + @Override + public void setFollowRedirects(boolean followRedirects) { + this.followRedirects = followRedirects; + if (this.request != null) { + this.request.setFollowRedirects(followRedirects); + } + } } diff --git a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java index bf4be5825a1..2e62809dfed 100644 --- a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java @@ -58,7 +58,7 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement protected GetMethod createRequest(String downloadUrl) { GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); - request.setFollowRedirects(true); + request.setFollowRedirects(followRedirects); if (!toFileSet) { String[] parts = downloadUrl.split("/"); String filename = parts[parts.length - 1]; @@ -171,4 +171,12 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement public void setStatus(Status status) { this.status = status; } + + @Override + public void setFollowRedirects(boolean followRedirects) { + super.setFollowRedirects(followRedirects); + if (this.request != null) { + this.request.setFollowRedirects(followRedirects); + } + } } diff --git a/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java index 34ba3c6b1ea..c24a4cc221f 100644 --- a/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java @@ -34,6 +34,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpMethodParams; @@ -43,6 +44,7 @@ import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Date; +import java.util.List; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import static java.util.Arrays.asList; @@ -70,8 +72,8 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp private long downloadTime; private long totalBytes; private long maxTemplateSizeInByte; - private boolean resume = false; + private boolean followRedirects = false; public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback, long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) { @@ -89,7 +91,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5)); // Follow redirects - this.getMethod.setFollowRedirects(true); + this.getMethod.setFollowRedirects(followRedirects); // Set file extension. this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), "."); @@ -122,10 +124,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp return 0; } - if (!HTTPUtils.verifyResponseCode(responseCode)) { + boolean failedDueToRedirection = List.of(HttpStatus.SC_MOVED_PERMANENTLY, + HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) && !followRedirects; + if (!HTTPUtils.verifyResponseCode(responseCode) || failedDueToRedirection) { errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode; logger.warn(errorString); - status = Status.UNRECOVERABLE_ERROR; return 0; } @@ -371,4 +374,12 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp public String getFileExtension() { return fileExtension; } + + @Override + public void setFollowRedirects(boolean followRedirects) { + this.followRedirects = followRedirects; + if (this.getMethod != null) { + this.getMethod.setFollowRedirects(followRedirects); + } + } } diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java index db4dccb1302..8719947cb4f 100644 --- a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java @@ -25,6 +25,7 @@ import java.io.InputStream; import java.io.RandomAccessFile; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.cloudstack.managed.context.ManagedContextRunnable; @@ -71,6 +72,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem private final HttpMethodRetryHandler retryHandler; private HashMap urlFileMap; + private boolean followRedirects = false; public SimpleHttpMultiFileDownloader(StorageLayer storageLayer, String[] downloadUrls, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, @@ -92,7 +94,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem private GetMethod createRequest(String downloadUrl) { GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler); - request.setFollowRedirects(true); + request.setFollowRedirects(followRedirects); return request; } @@ -168,7 +170,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem urlFileMap.put(downloadUrl, currentToFile); file = new File(currentToFile); long localFileSize = checkLocalFileSizeForResume(resume, file); - if (checkServerResponse(localFileSize)) return 0; + if (checkServerResponse(localFileSize, downloadUrl)) return 0; if (!tryAndGetRemoteSize()) return 0; if (!canHandleDownloadSize()) return 0; checkAndSetDownloadSize(); @@ -315,7 +317,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem return true; } - private boolean checkServerResponse(long localFileSize) throws IOException { + private boolean checkServerResponse(long localFileSize, String downloadUrl) throws IOException { int responseCode = 0; if (localFileSize > 0) { @@ -329,6 +331,12 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) { currentStatus = Status.UNRECOVERABLE_ERROR; errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) "; + if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) + && !followRedirects) { + errorString = String.format("Failed to download %s due to redirection, response code: %d", + downloadUrl, responseCode); + logger.error(errorString); + } return true; //FIXME: retry? } return false; @@ -476,4 +484,12 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem public Map getDownloadedFilesMap() { return urlFileMap; } + + @Override + public void setFollowRedirects(boolean followRedirects) { + this.followRedirects = followRedirects; + if (this.request != null) { + this.request.setFollowRedirects(followRedirects); + } + } } diff --git a/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java index 5db3d2425a5..9fb1ca42442 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java @@ -92,4 +92,6 @@ public interface TemplateDownloader extends Runnable { boolean isInited(); long getMaxTemplateSizeInBytes(); + + void setFollowRedirects(boolean followRedirects); } diff --git a/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java b/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java index cf6f4d27ecd..f1cb21a1815 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java @@ -41,6 +41,7 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl protected long _start; protected StorageLayer _storage; protected boolean _inited = false; + protected boolean followRedirects = false; private long maxTemplateSizeInBytes; public TemplateDownloaderBase(StorageLayer storage, String downloadUrl, String toDir, long maxTemplateSizeInBytes, DownloadCompleteCallback callback) { @@ -147,4 +148,9 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl public boolean isInited() { return _inited; } + + @Override + public void setFollowRedirects(boolean followRedirects) { + this.followRedirects = followRedirects; + } } diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java index b1b76da8211..325f61427a9 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java @@ -28,6 +28,7 @@ public class CheckUrlCommand extends Command { private Integer connectTimeout; private Integer connectionRequestTimeout; private Integer socketTimeout; + private boolean followRedirects; public String getFormat() { return format; @@ -43,19 +44,25 @@ public class CheckUrlCommand extends Command { public Integer getSocketTimeout() { return socketTimeout; } - public CheckUrlCommand(final String format,final String url) { + public boolean isFollowRedirects() { + return followRedirects; + } + + public CheckUrlCommand(final String format, final String url, final boolean followRedirects) { super(); this.format = format; this.url = url; + this.followRedirects = followRedirects; } - public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) { + public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, final boolean followRedirects) { super(); this.format = format; this.url = url; this.connectTimeout = connectTimeout; this.socketTimeout = socketTimeout; this.connectionRequestTimeout = connectionRequestTimeout; + this.followRedirects = followRedirects; } @Override diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java index 7e1ff0b34c4..b6748dca434 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java @@ -45,7 +45,11 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand { private Long templateSize; private Storage.ImageFormat format; - protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) { + private boolean followRedirects; + + protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, + final String checksum, final Map headers, final Integer connectTimeout, + final Integer soTimeout, final Integer connectionRequestTimeout, final boolean followRedirects) { this.url = url; this.templateId = templateId; this.destData = destData; @@ -55,6 +59,7 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand { this.connectTimeout = connectTimeout; this.soTimeout = soTimeout; this.connectionRequestTimeout = connectionRequestTimeout; + this.followRedirects = followRedirects; } public String getUrl() { @@ -137,4 +142,12 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand { public int getWaitInMillSeconds() { return getWait() * 1000; } + + public boolean isFollowRedirects() { + return followRedirects; + } + + public void setFollowRedirects(boolean followRedirects) { + this.followRedirects = followRedirects; + } } diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java index f131b3b0a7f..bdc00b3bc47 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java @@ -24,8 +24,10 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; public class HttpDirectDownloadCommand extends DirectDownloadCommand { - public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers, int connectTimeout, int soTimeout) { - super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null); + public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, + Map headers, int connectTimeout, int soTimeout, boolean followRedirects) { + super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, + null, followRedirects); } } diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java index dd88ad2a155..a7e16eac91a 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java @@ -25,7 +25,10 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; public class HttpsDirectDownloadCommand extends DirectDownloadCommand { - public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers, int connectTimeout, int soTimeout, int connectionRequestTimeout) { - super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, connectionRequestTimeout); + public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, + Map headers, int connectTimeout, int soTimeout, int connectionRequestTimeout, + boolean followRedirects) { + super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, + connectionRequestTimeout, followRedirects); } } diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java index a3edcebe7de..7742994c76b 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java @@ -24,8 +24,9 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; public class MetalinkDirectDownloadCommand extends DirectDownloadCommand { - public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map headers, int connectTimeout, int soTimeout) { - super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null); + public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, + Map headers, int connectTimeout, int soTimeout, boolean followRedirects) { + super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null, followRedirects); } } diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java index 0bf9c4d934b..0e51d786230 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java @@ -24,8 +24,9 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; public class NfsDirectDownloadCommand extends DirectDownloadCommand { - public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map headers) { - super(url, templateId, destPool, checksum, headers, null, null, null); + public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, + final String checksum, final Map headers) { + super(url, templateId, destPool, checksum, headers, null, null, null, false); } } diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java index be8841f3c88..0e0e2f0b57b 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java @@ -35,27 +35,30 @@ public class DirectDownloadHelper { * Get direct template downloader from direct download command and destination pool */ public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd, - String destPoolLocalPath, - String temporaryDownloadPath) { + String destPoolLocalPath, String temporaryDownloadPath) { if (cmd instanceof HttpDirectDownloadCommand) { - return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(), - cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath); + return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, + cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(), + temporaryDownloadPath, cmd.isFollowRedirects()); } else if (cmd instanceof HttpsDirectDownloadCommand) { - return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(), - cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath); + return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, + cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(), + cmd.getConnectionRequestTimeout(), temporaryDownloadPath, cmd.isFollowRedirects()); } else if (cmd instanceof NfsDirectDownloadCommand) { - return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath); + return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), + cmd.getChecksum(), temporaryDownloadPath); } else if (cmd instanceof MetalinkDirectDownloadCommand) { - return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(), - cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath); + return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), + cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(), + temporaryDownloadPath, cmd.isFollowRedirects()); } else { throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink"); } } - public static boolean checkUrlExistence(String url) { + public static boolean checkUrlExistence(String url, boolean followRedirects) { try { - DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null); + DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects); return checker.checkUrl(url); } catch (CloudRuntimeException e) { LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e); @@ -63,9 +66,9 @@ public class DirectDownloadHelper { } } - public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) { + public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) { try { - DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout); + DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects); return checker.checkUrl(url); } catch (CloudRuntimeException e) { LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e); @@ -73,27 +76,27 @@ public class DirectDownloadHelper { } } - private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) { + private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) { if (url.toLowerCase().startsWith("https:")) { - return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout); + return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects); } else if (url.toLowerCase().startsWith("http:")) { - return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout); + return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects); } else if (url.toLowerCase().startsWith("nfs:")) { return new NfsDirectTemplateDownloader(url); } else if (url.toLowerCase().endsWith(".metalink")) { - return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout); + return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects); } else { throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url)); } } - public static Long getFileSize(String url, String format) { - DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null); + public static Long getFileSize(String url, String format, boolean followRedirects) { + DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects); return checker.getRemoteFileSize(url, format); } - public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) { - DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout); + public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) { + DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects); return checker.getRemoteFileSize(url, format); } } diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java index b6a872aca7c..d22c803818b 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java @@ -43,16 +43,19 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown private String checksum; private boolean redownload = false; protected String temporaryDownloadPath; + private boolean followRedirects; protected Logger logger = LogManager.getLogger(getClass()); protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, - final String checksum, final String temporaryDownloadPath) { + final String checksum, final String temporaryDownloadPath, + final boolean followRedirects) { this.url = url; this.destPoolPath = destPoolPath; this.templateId = templateId; this.checksum = checksum; this.temporaryDownloadPath = temporaryDownloadPath; + this.followRedirects = followRedirects; } private static String directDownloadDir = "template"; @@ -112,6 +115,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown return redownload; } + public boolean isFollowRedirects() { + return followRedirects; + } + + public void setFollowRedirects(boolean followRedirects) { + this.followRedirects = followRedirects; + } + /** * Create download directory (if it does not exist) */ diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index 70cdb8ed467..8c4147fad60 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -48,13 +48,14 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod request; protected Map reqHeaders = new HashMap<>(); - protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) { - this(url, null, null, null, null, connectTimeout, socketTimeout, null); + protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout, boolean followRedirects) { + this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects); } public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, - Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { - super(url, destPoolPath, templateId, checksum, downloadPath); + Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath, + boolean followRedirects) { + super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects); s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout); s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout); client = new HttpClient(s_httpClientManager); @@ -66,7 +67,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod createRequest(String downloadUrl, Map headers) { GetMethod request = new GetMethod(downloadUrl); - request.setFollowRedirects(true); + request.setFollowRedirects(this.isFollowRedirects()); if (MapUtils.isNotEmpty(headers)) { for (String key : headers.keySet()) { request.setRequestHeader(key, headers.get(key)); @@ -109,9 +110,11 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @Override public boolean checkUrl(String url) { HeadMethod httpHead = new HeadMethod(url); + httpHead.setFollowRedirects(this.isFollowRedirects()); try { - if (client.executeMethod(httpHead) != HttpStatus.SC_OK) { - logger.error(String.format("Invalid URL: %s", url)); + int responseCode = client.executeMethod(httpHead); + if (responseCode != HttpStatus.SC_OK) { + logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode)); return false; } return true; @@ -126,9 +129,9 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @Override public Long getRemoteFileSize(String url, String format) { if ("qcow2".equalsIgnoreCase(format)) { - return QCOW2Utils.getVirtualSize(url); + return QCOW2Utils.getVirtualSizeFromUrl(url, this.isFollowRedirects()); } else { - return UriUtils.getRemoteSize(url); + return UriUtils.getRemoteSize(url, this.isFollowRedirects()); } } diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java index fbf5f728111..b8a25a11b5c 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java @@ -68,20 +68,26 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl protected CloseableHttpClient httpsClient; private HttpUriRequest req; - protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) { - this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null); + protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) { + this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null, followRedirects); } - public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, - Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) { - super(url, destPoolPath, templateId, checksum, temporaryDownloadPath); + public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, + Map headers, Integer connectTimeout, Integer soTimeout, + Integer connectionRequestTimeout, String temporaryDownloadPath, boolean followRedirects) { + super(url, destPoolPath, templateId, checksum, temporaryDownloadPath, followRedirects); SSLContext sslcontext = getSSLContext(); SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); RequestConfig config = RequestConfig.custom() .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout) .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout) - .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build(); - httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build(); + .setSocketTimeout(soTimeout == null ? 5000 : soTimeout) + .setRedirectsEnabled(followRedirects) + .build(); + httpsClient = HttpClients.custom() + .setSSLSocketFactory(factory) + .setDefaultRequestConfig(config) + .build(); createUriRequest(url, headers); String downloadDir = getDirectDownloadTempPath(templateId); File tempFile = createTemporaryDirectoryAndFile(downloadDir); @@ -90,6 +96,7 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl protected void createUriRequest(String downloadUrl, Map headers) { req = new HttpGet(downloadUrl); + setFollowRedirects(this.isFollowRedirects()); if (MapUtils.isNotEmpty(headers)) { for (String headerKey: headers.keySet()) { req.setHeader(headerKey, headers.get(headerKey)); @@ -164,8 +171,9 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl HttpHead httpHead = new HttpHead(url); try { CloseableHttpResponse response = httpsClient.execute(httpHead); - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - logger.error(String.format("Invalid URL: %s", url)); + int responseCode = response.getStatusLine().getStatusCode(); + if (responseCode != HttpStatus.SC_OK) { + logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode)); return false; } return true; diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java index 24d642460c5..5335da99150 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java @@ -39,16 +39,15 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm private Integer soTimeout; protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId, - String destPoolPath, String checksum, - Map headers, - Integer connectTimeout, Integer soTimeout, - Integer connectionRequestTimeout, String temporaryDownloadPath) { + String destPoolPath, String checksum, Map headers, Integer connectTimeout, + Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) { if (url.toLowerCase().startsWith("https:")) { return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers, - connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath); + connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath, + this.isFollowRedirects()); } else if (url.toLowerCase().startsWith("http:")) { return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers, - connectTimeout, soTimeout, temporaryDownloadPath); + connectTimeout, soTimeout, temporaryDownloadPath, this.isFollowRedirects()); } else if (url.toLowerCase().startsWith("nfs:")) { return new NfsDirectTemplateDownloader(url); } else { @@ -57,13 +56,14 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm } } - protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) { - this(url, null, null, null, null, connectTimeout, socketTimeout, null); + protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout, boolean followRedirects) { + this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects); } public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, - Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { - super(url, destPoolPath, templateId, checksum, downloadPath); + Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath, + boolean followRedirects) { + super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects); this.headers = headers; this.connectTimeout = connectTimeout; this.soTimeout = soTimeout; diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java index 0b7866104b1..21184ef07fe 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java @@ -57,8 +57,9 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { this(url, null, null, null, null); } - public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) { - super(url, destPool, templateId, checksum, downloadPath); + public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, + String downloadPath) { + super(url, destPool, templateId, checksum, downloadPath, false); parseUrl(); } diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java index 4032ac0b632..f44220f31d0 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java @@ -49,6 +49,8 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal private DataStoreTO _store; private DataStoreTO cacheStore; + private boolean followRedirects = false; + protected DownloadCommand() { } @@ -65,6 +67,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal installPath = that.installPath; _store = that._store; _proxy = that._proxy; + followRedirects = that.followRedirects; } public DownloadCommand(TemplateObjectTO template, Long maxDownloadSizeInBytes) { @@ -80,6 +83,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal setSecUrl(((NfsTO)_store).getUrl()); } this.maxDownloadSizeInBytes = maxDownloadSizeInBytes; + this.followRedirects = template.isFollowRedirects(); } public DownloadCommand(TemplateObjectTO template, String user, String passwd, Long maxDownloadSizeInBytes) { @@ -95,6 +99,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal _store = volume.getDataStore(); this.maxDownloadSizeInBytes = maxDownloadSizeInBytes; resourceType = ResourceType.VOLUME; + this.followRedirects = volume.isFollowRedirects(); } public DownloadCommand(SnapshotObjectTO snapshot, Long maxDownloadSizeInBytes, String url) { @@ -194,4 +199,12 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal public DataStoreTO getCacheStore() { return cacheStore; } + + public boolean isFollowRedirects() { + return followRedirects; + } + + public void setFollowRedirects(boolean followRedirects) { + this.followRedirects = followRedirects; + } } diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java new file mode 100644 index 00000000000..70db8fafffc --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.to; + +public class DownloadableObjectTO { + protected boolean followRedirects = false; + + public boolean isFollowRedirects() { + return followRedirects; + } + + public void setFollowRedirects(boolean followRedirects) { + this.followRedirects = followRedirects; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java index 70cb6d155b0..76b93909b8c 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java @@ -30,7 +30,7 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.hypervisor.Hypervisor.HypervisorType; -public class SnapshotObjectTO implements DataTO { +public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO { private String path; private VolumeObjectTO volume; private String parentSnapshotPath; diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java index a405785bf64..eafe8f83269 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java @@ -28,7 +28,7 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Storage.ImageFormat; import com.cloud.template.VirtualMachineTemplate; -public class TemplateObjectTO implements DataTO { +public class TemplateObjectTO extends DownloadableObjectTO implements DataTO { private String path; private String origUrl; private String uuid; @@ -87,6 +87,7 @@ public class TemplateObjectTO implements DataTO { this.deployAsIs = template.isDeployAsIs(); this.deployAsIsConfiguration = template.getDeployAsIsConfiguration(); this.directDownload = template.isDirectDownload(); + this.followRedirects = template.isFollowRedirects(); } @Override diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java index 8473ea7a49e..2bb67c80ce4 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java @@ -33,7 +33,7 @@ import com.cloud.storage.Volume; import java.util.Arrays; -public class VolumeObjectTO implements DataTO { +public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { private String uuid; private Volume.Type volumeType; private DataStoreTO dataStore; @@ -119,6 +119,7 @@ public class VolumeObjectTO implements DataTO { this.vSphereStoragePolicyId = volume.getvSphereStoragePolicyId(); this.passphrase = volume.getPassphrase(); this.encryptFormat = volume.getEncryptFormat(); + this.followRedirects = volume.isFollowRedirects(); } public String getUuid() { diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java index 24ca268913c..4b0f4cd9744 100644 --- a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java +++ b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java @@ -57,7 +57,7 @@ public class BaseDirectTemplateDownloaderTest { private HttpEntity httpEntity; @InjectMocks - protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000); + protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000, false); private AutoCloseable closeable; diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java index 68982fb915f..2fc0f7b00a0 100644 --- a/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java +++ b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java @@ -25,8 +25,7 @@ import org.mockito.InjectMocks; public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest { @InjectMocks - protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000); - + protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000, false); @Test public void testCheckUrlMetalink() { metalinkDownloader.downloader = httpsDownloader; diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java new file mode 100644 index 00000000000..63b0867d1ab --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.engine.subsystem.api.storage; + +public interface DownloadableDataInfo extends DataObject { + default public boolean isFollowRedirects() { + return true; + } +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java index 1d49e19d2da..3bd3100e84e 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java @@ -21,7 +21,7 @@ package org.apache.cloudstack.engine.subsystem.api.storage; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.UserData; -public interface TemplateInfo extends DataObject, VirtualMachineTemplate { +public interface TemplateInfo extends DownloadableDataInfo, VirtualMachineTemplate { @Override String getUniqueName(); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java index be16c20d173..19b852bd912 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java @@ -26,7 +26,7 @@ import com.cloud.storage.Storage; import com.cloud.storage.Volume; import com.cloud.vm.VirtualMachine; -public interface VolumeInfo extends DataObject, Volume { +public interface VolumeInfo extends DownloadableDataInfo, Volume { boolean isAttachedVM(); diff --git a/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java b/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java index bbddd8fd471..ebbae0b31c2 100644 --- a/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java +++ b/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java @@ -276,4 +276,6 @@ public interface ConfigurationManager { Pair getConfigurationGroupAndSubGroup(String configName); List getConfigurationSubGroups(Long groupId); + + void validateExtraConfigInServiceOfferingDetail(String detailName); } diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 427a58669e6..daeb4b19a18 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -199,6 +199,10 @@ public interface StorageManager extends StorageService { true, ConfigKey.Scope.Global, null); + static final ConfigKey DataStoreDownloadFollowRedirects = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Boolean.class, "store.download.follow.redirects", "false", + "Whether HTTP redirect is followed during store downloads for objects such as template, volume etc.", + true, ConfigKey.Scope.Global); ConfigKey HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000", "The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true); diff --git a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java index d7ef18a4f32..1a9372f73ea 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java @@ -223,4 +223,87 @@ public class DatabaseUpgradeCheckerTest { assertEquals("We should have 1 upgrade step", 1, upgrades.length); assertTrue(upgrades[0] instanceof NoopDbUpgrade); } + + @Test + public void testCalculateUpgradePathFrom41800toNextSecurityRelease() { + + final CloudStackVersion dbVersion = CloudStackVersion.parse("4.18.0.0"); + assertNotNull(dbVersion); + + final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); + final CloudStackVersion currentVersion = checker.getLatestVersion(); + assertNotNull(currentVersion); + + final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); + assertNotNull(upgrades); + + final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "." + + currentVersion.getMinorRelease() + "." + + currentVersion.getPatchRelease() + "." + + (currentVersion.getSecurityRelease() + 1)); + assertNotNull(nextSecurityRelease); + + final DbUpgrade[] upgradesToNext = checker.calculateUpgradePath(dbVersion, nextSecurityRelease); + assertNotNull(upgradesToNext); + + assertEquals(upgrades.length + 1, upgradesToNext.length); + assertTrue(upgradesToNext[upgradesToNext.length - 1] instanceof NoopDbUpgrade); + } + + @Test + public void testCalculateUpgradePathFromSecurityReleaseToLatest() { + + final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.2.0"); // a EOL version + assertNotNull(dbVersion); + + final CloudStackVersion oldSecurityRelease = CloudStackVersion.parse(dbVersion.getMajorRelease() + "." + + dbVersion.getMinorRelease() + "." + + dbVersion.getPatchRelease() + "." + + (dbVersion.getSecurityRelease() + 100)); + assertNotNull(oldSecurityRelease); // fake security release 4.17.2.100 + + final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); + final CloudStackVersion currentVersion = checker.getLatestVersion(); + assertNotNull(currentVersion); + + final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); + assertNotNull(upgrades); + + final DbUpgrade[] upgradesFromSecurityRelease = checker.calculateUpgradePath(oldSecurityRelease, currentVersion); + assertNotNull(upgradesFromSecurityRelease); + + assertEquals("The upgrade paths should be the same", upgrades.length, upgradesFromSecurityRelease.length); + } + + @Test + public void testCalculateUpgradePathFromSecurityReleaseToNextSecurityRelease() { + + final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.2.0"); // a EOL version + assertNotNull(dbVersion); + + final CloudStackVersion oldSecurityRelease = CloudStackVersion.parse(dbVersion.getMajorRelease() + "." + + dbVersion.getMinorRelease() + "." + + dbVersion.getPatchRelease() + "." + + (dbVersion.getSecurityRelease() + 100)); + assertNotNull(oldSecurityRelease); // fake security release 4.17.2.100 + + final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); + final CloudStackVersion currentVersion = checker.getLatestVersion(); + assertNotNull(currentVersion); + + final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "." + + currentVersion.getMinorRelease() + "." + + currentVersion.getPatchRelease() + "." + + (currentVersion.getSecurityRelease() + 1)); + assertNotNull(nextSecurityRelease); // fake security release + + final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); + assertNotNull(upgrades); + + final DbUpgrade[] upgradesFromSecurityReleaseToNext = checker.calculateUpgradePath(oldSecurityRelease, nextSecurityRelease); + assertNotNull(upgradesFromSecurityReleaseToNext); + + assertEquals(upgrades.length + 1, upgradesFromSecurityReleaseToNext.length); + assertTrue(upgradesFromSecurityReleaseToNext[upgradesFromSecurityReleaseToNext.length - 1] instanceof NoopDbUpgrade); + } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 7b14b0c26c7..39d4618f81c 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -92,6 +92,7 @@ import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; +import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; @@ -367,6 +368,7 @@ public class TemplateServiceImpl implements TemplateService { toBeDownloaded.addAll(allTemplates); final StateMachine2 stateMachine = VirtualMachineTemplate.State.getStateMachine(); + Boolean followRedirect = StorageManager.DataStoreDownloadFollowRedirects.value(); for (VMTemplateVO tmplt : allTemplates) { String uniqueName = tmplt.getUniqueName(); TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId()); @@ -447,7 +449,8 @@ public class TemplateServiceImpl implements TemplateService { try { _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId), com.cloud.configuration.Resource.ResourceType.secondary_storage, - tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl())); + tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl(), + followRedirect)); } catch (ResourceAllocationException e) { logger.warn(e.getMessage()); _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null, e.getMessage(), e.getMessage()); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java index c04ba2207df..fdb4fe6753a 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -23,6 +23,7 @@ import java.util.Map; import javax.inject.Inject; +import com.cloud.storage.StorageManager; import com.cloud.user.UserData; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -74,8 +75,10 @@ public class TemplateObject implements TemplateInfo { VMTemplatePoolDao templatePoolDao; @Inject TemplateDataStoreDao templateStoreDao; + final private boolean followRedirects; public TemplateObject() { + this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value(); } protected void configure(VMTemplateVO template, DataStore dataStore) { @@ -574,4 +577,9 @@ public class TemplateObject implements TemplateInfo { // TODO Auto-generated method stub return null; } + + @Override + public boolean isFollowRedirects() { + return followRedirects; + } } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 8bb9d40fcfe..1b3bec0e907 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -23,6 +23,7 @@ import javax.inject.Inject; import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.dc.dao.VsphereStoragePolicyDao; +import com.cloud.storage.StorageManager; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; @@ -118,6 +119,7 @@ public class VolumeObject implements VolumeInfo { private MigrationOptions migrationOptions; private boolean directDownload; private String vSphereStoragePolicyId; + private boolean followRedirects; private final List volumeStatesThatShouldNotTransitWhenDataStoreRoleIsImage = Arrays.asList(Volume.State.Migrating, Volume.State.Uploaded, Volume.State.Copying, Volume.State.Expunged); @@ -128,6 +130,7 @@ public class VolumeObject implements VolumeInfo { public VolumeObject() { _volStateMachine = Volume.State.getStateMachine(); + this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value(); } protected void configure(DataStore dataStore, VolumeVO volumeVO) { @@ -931,4 +934,9 @@ public class VolumeObject implements VolumeInfo { public void setEncryptFormat(String encryptFormat) { volumeVO.setEncryptFormat(encryptFormat); } + + @Override + public boolean isFollowRedirects() { + return followRedirects; + } } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java index 1ed37ab9969..b38b30e88b8 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java @@ -31,4 +31,5 @@ public interface ConfigDepot { void set(ConfigKey key, T value); void createOrUpdateConfigObject(String componentName, ConfigKey key, String value); + boolean isNewConfig(ConfigKey configKey); } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 192c556bd2a..df93f78fa83 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -34,6 +34,7 @@ public class ConfigKey { public static final String CATEGORY_ADVANCED = "Advanced"; public static final String CATEGORY_ALERT = "Alert"; + public static final String CATEGORY_NETWORK = "Network"; public enum Scope { Global, Zone, Cluster, StoragePool, Account, ManagementServer, ImageStore, Domain diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java index fa48670ce0c..6884043cae2 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java @@ -82,6 +82,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { List _configurables; List _scopedStorages; Set _configured = Collections.synchronizedSet(new HashSet()); + Set newConfigs = Collections.synchronizedSet(new HashSet<>()); private HashMap>> _allKeys = new HashMap>>(1007); @@ -194,6 +195,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { } _configDao.persist(vo); + newConfigs.add(vo.getName()); } else { boolean configUpdated = false; if (vo.isDynamic() != key.isDynamic() || !ObjectUtils.equals(vo.getDescription(), key.description()) || !ObjectUtils.equals(vo.getDefaultValue(), key.defaultValue()) || @@ -344,4 +346,9 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { return new Pair<>(groupId, subGroupId); } + + @Override + public boolean isNewConfig(ConfigKey configKey) { + return newConfigs.contains(configKey.key()); + } } diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java index fed784cfe2d..8dd6f71af3c 100644 --- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java +++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java @@ -18,9 +18,14 @@ // package org.apache.cloudstack.framework.config.impl; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + import org.apache.cloudstack.framework.config.ConfigKey; import org.junit.Assert; import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; public class ConfigDepotImplTest { @@ -40,4 +45,16 @@ public class ConfigDepotImplTest { } } + @Test + public void testIsNewConfig() { + String validNewConfigKey = "CONFIG"; + ConfigKey validNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG", "true", "", true); + ConfigKey invalidNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG1", "true", "", true); + Set newConfigs = Collections.synchronizedSet(new HashSet<>()); + newConfigs.add(validNewConfigKey); + ReflectionTestUtils.setField(configDepotImpl, "newConfigs", newConfigs); + Assert.assertTrue(configDepotImpl.isNewConfig(validNewConfig)); + Assert.assertFalse(configDepotImpl.isNewConfig(invalidNewConfig)); + } + } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java index 90befe6ed64..c7dbf8a59bf 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java @@ -40,9 +40,9 @@ public class LibvirtCheckUrlCommand extends CommandWrapper IntegrationAPIPort = new ConfigKey("Advanced" + private static final ConfigKey IntegrationAPIPort = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED , Integer.class , "integration.api.port" , "0" , "Integration (unauthenticated) API port. To disable set it to 0 or negative." , false , ConfigKey.Scope.Global); - private static final ConfigKey ConcurrentSnapshotsThresholdPerHost = new ConfigKey("Advanced" + private static final ConfigKey ConcurrentSnapshotsThresholdPerHost = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED , Long.class , "concurrent.snapshots.threshold.perhost" , null , "Limits number of snapshots that can be handled by the host concurrently; default is NULL - unlimited" , true // not sure if this is to be dynamic , ConfigKey.Scope.Global); - private static final ConfigKey EncodeApiResponse = new ConfigKey("Advanced" + private static final ConfigKey EncodeApiResponse = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED , Boolean.class , "encode.api.response" , "false" , "Do URL encoding for the api response, false by default" , false , ConfigKey.Scope.Global); - static final ConfigKey JSONcontentType = new ConfigKey( "Advanced" + static final ConfigKey JSONcontentType = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED , String.class , "json.content.type" , "application/json; charset=UTF-8" , "Http response content type for .js files (default is text/javascript)" , false , ConfigKey.Scope.Global); - static final ConfigKey EnableSecureSessionCookie = new ConfigKey("Advanced" + static final ConfigKey EnableSecureSessionCookie = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED , Boolean.class , "enable.secure.session.cookie" , "false" , "Session cookie is marked as secure if this is enabled. Secure cookies only work when HTTPS is used." , false , ConfigKey.Scope.Global); - private static final ConfigKey JSONDefaultContentType = new ConfigKey ("Advanced" + private static final ConfigKey JSONDefaultContentType = new ConfigKey<> (ConfigKey.CATEGORY_ADVANCED , String.class , "json.content.type" , "application/json; charset=UTF-8" @@ -276,13 +276,34 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer , false , ConfigKey.Scope.Global); - private static final ConfigKey UseEventAccountInfo = new ConfigKey( "advanced" + private static final ConfigKey UseEventAccountInfo = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED , Boolean.class , "event.accountinfo" , "false" , "use account info in event logging" , true , ConfigKey.Scope.Global); + static final ConfigKey useForwardHeader = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK + , Boolean.class + , "proxy.header.verify" + , "false" + , "enables/disables checking of ipaddresses from a proxy set header. See \"proxy.header.names\" for the headers to allow." + , true + , ConfigKey.Scope.Global); + static final ConfigKey listOfForwardHeaders = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK + , String.class + , "proxy.header.names" + , "X-Forwarded-For,HTTP_CLIENT_IP,HTTP_X_FORWARDED_FOR" + , "a list of names to check for allowed ipaddresses from a proxy set header. See \"proxy.cidr\" for the proxies allowed to set these headers." + , true + , ConfigKey.Scope.Global); + static final ConfigKey proxyForwardList = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK + , String.class + , "proxy.cidr" + , "" + , "a list of cidrs for which \"proxy.header.names\" are honoured if the \"Remote_Addr\" is in this list." + , true + , ConfigKey.Scope.Global); @Override public boolean configure(final String name, final Map params) throws ConfigurationException { @@ -1500,7 +1521,10 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer ConcurrentSnapshotsThresholdPerHost, EncodeApiResponse, EnableSecureSessionCookie, - JSONDefaultContentType + JSONDefaultContentType, + proxyForwardList, + useForwardHeader, + listOfForwardHeaders }; } } diff --git a/server/src/main/java/com/cloud/api/ApiServlet.java b/server/src/main/java/com/cloud/api/ApiServlet.java index 7bc2bc54a62..f2b5d3c4797 100644 --- a/server/src/main/java/com/cloud/api/ApiServlet.java +++ b/server/src/main/java/com/cloud/api/ApiServlet.java @@ -70,7 +70,6 @@ import com.cloud.utils.db.EntityManager; import com.cloud.utils.net.NetUtils; @Component("apiServlet") -@SuppressWarnings("serial") public class ApiServlet extends HttpServlet { protected static Logger LOGGER = LogManager.getLogger(ApiServlet.class); private final static List s_clientAddressHeaders = Collections @@ -570,17 +569,39 @@ public class ApiServlet extends HttpServlet { } return false; } + boolean doUseForwardHeaders() { + return Boolean.TRUE.equals(ApiServer.useForwardHeader.value()); + } + String[] proxyNets() { + return ApiServer.proxyForwardList.value().split(","); + } //This method will try to get login IP of user even if servlet is behind reverseProxy or loadBalancer - public static InetAddress getClientAddress(final HttpServletRequest request) throws UnknownHostException { - for(final String header : s_clientAddressHeaders) { - final String ip = getCorrectIPAddress(request.getHeader(header)); - if (ip != null) { - return InetAddress.getByName(ip); - } + public InetAddress getClientAddress(final HttpServletRequest request) throws UnknownHostException { + String ip = null; + InetAddress pretender = InetAddress.getByName(request.getRemoteAddr()); + if(doUseForwardHeaders()) { + if (NetUtils.isIpInCidrList(pretender, proxyNets())) { + for (String header : getClientAddressHeaders()) { + header = header.trim(); + ip = getCorrectIPAddress(request.getHeader(header)); + if (StringUtils.isNotBlank(ip)) { + LOGGER.debug(String.format("found ip %s in header %s ", ip, header)); + break; + } + } // no address found in header so ip is blank and use remote addr + } // else not an allowed proxy address, ip is blank and use remote addr + } + if (StringUtils.isBlank(ip)) { + LOGGER.trace(String.format("no ip found in headers, returning remote address %s.", pretender.getHostAddress())); + return pretender; } - return InetAddress.getByName(request.getRemoteAddr()); + return InetAddress.getByName(ip); + } + + private String[] getClientAddressHeaders() { + return ApiServer.listOfForwardHeaders.value().split(","); } private static String getCorrectIPAddress(String ip) { diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 67dffa5b0a0..719017cc922 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -205,6 +205,7 @@ import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostTagsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.hypervisor.kvm.dpdk.DpdkHelper; import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManager; import com.cloud.network.Ipv6GuestPrefixSubnetNetworkMapVO; @@ -3255,6 +3256,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } if (detailEntry.getKey().startsWith(ApiConstants.EXTRA_CONFIG)) { + validateExtraConfigInServiceOfferingDetail(detailEntry.getKey()); try { detailEntryValue = URLDecoder.decode(detailEntry.getValue(), "UTF-8"); } catch (UnsupportedEncodingException | IllegalArgumentException e) { @@ -3320,6 +3322,14 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } + @Override + public void validateExtraConfigInServiceOfferingDetail(String detailName) { + if (!detailName.equals(DpdkHelper.DPDK_NUMA) && !detailName.equals(DpdkHelper.DPDK_HUGE_PAGES) + && !detailName.startsWith(DpdkHelper.DPDK_INTERFACE_PREFIX)) { + throw new InvalidParameterValueException("Only extraconfig for DPDK are supported in service offering details"); + } + } + private DiskOfferingVO createDiskOfferingInternal(final long userId, final boolean isSystem, final VirtualMachine.Type vmType, final String name, final Integer cpu, final Integer ramSize, final Integer speed, final String displayText, final ProvisioningType typedProvisioningType, final boolean localStorageRequired, final boolean offerHA, final boolean limitResourceUse, final boolean volatileVm, String tags, final List domainIds, List zoneIds, final String hostTag, diff --git a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java index 236a2869421..6242289caea 100644 --- a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java +++ b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java @@ -47,6 +47,7 @@ import com.cloud.agent.api.Command; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.configuration.ConfigurationManager; import com.cloud.gpu.GPU; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; @@ -69,6 +70,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.vm.NicProfile; import com.cloud.vm.NicVO; +import com.cloud.vm.UserVmManager; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; @@ -113,6 +115,10 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis @Inject protected HostDao hostDao; + @Inject + private UserVmManager userVmManager; + @Inject + private ConfigurationManager configurationManager; public static ConfigKey VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor = new ConfigKey("Advanced", Boolean.class, "vm.min.memory.equals.memory.divided.by.mem.overprovisioning.factor", "true", "If we set this to 'true', a minimum memory (memory/ mem.overprovisioning.factor) will be set to the VM, independent of using a scalable service offering or not.", true, ConfigKey.Scope.Cluster); @@ -224,10 +230,12 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis /** * Add extra configuration from VM details. Extra configuration is stored as details starting with 'extraconfig' */ - private void addExtraConfig(Map details, VirtualMachineTO to) { + private void addExtraConfig(Map details, VirtualMachineTO to, long accountId, Hypervisor.HypervisorType hypervisorType) { for (String key : details.keySet()) { if (key.startsWith(ApiConstants.EXTRA_CONFIG)) { - to.addExtraConfig(key, details.get(key)); + String extraConfig = details.get(key); + userVmManager.validateExtraConfig(accountId, hypervisorType, extraConfig); + to.addExtraConfig(key, extraConfig); } } } @@ -243,6 +251,7 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis if (CollectionUtils.isNotEmpty(details)) { for (ServiceOfferingDetailsVO detail : details) { if (detail.getName().startsWith(ApiConstants.EXTRA_CONFIG)) { + configurationManager.validateExtraConfigInServiceOfferingDetail(detail.getName()); to.addExtraConfig(detail.getName(), detail.getValue()); } } @@ -306,7 +315,7 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis Map detailsInVm = _userVmDetailsDao.listDetailsKeyPairs(vm.getId()); if (detailsInVm != null) { to.setDetails(detailsInVm); - addExtraConfig(detailsInVm, to); + addExtraConfig(detailsInVm, to, vm.getAccountId(), vm.getHypervisorType()); } addServiceOfferingExtraConfiguration(offering, to); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 75d7e8d3a0a..9a6f8563223 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -97,6 +97,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -238,6 +239,7 @@ import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.JoinBuilder; @@ -380,6 +382,11 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C @Inject protected BucketDao _bucketDao; + @Inject + ConfigDepot configDepot; + @Inject + ConfigurationDao configurationDao; + protected List _discoverers; public List getDiscoverers() { @@ -440,6 +447,21 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } } + protected void enableDefaultDatastoreDownloadRedirectionForExistingInstallations() { + if (!configDepot.isNewConfig(DataStoreDownloadFollowRedirects)) { + logger.trace("{} is not a new configuration, skipping updating its value", + DataStoreDownloadFollowRedirects.key()); + return; + } + List zones = + _dcDao.listAll(new Filter(1)); + if (CollectionUtils.isNotEmpty(zones)) { + logger.debug(String.format("Updating value for configuration: %s to true", + DataStoreDownloadFollowRedirects.key())); + configurationDao.update(DataStoreDownloadFollowRedirects.key(), "true"); + } + } + @Override public List ListByDataCenterHypervisor(long datacenterId, HypervisorType type) { List pools = _storagePoolDao.listByDataCenterId(datacenterId); @@ -671,7 +693,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } _executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS); - + enableDefaultDatastoreDownloadRedirectionForExistingInstallations(); return true; } @@ -3764,7 +3786,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C MountDisabledStoragePool, VmwareCreateCloneFull, VmwareAllowParallelExecution, - ConvertVmwareInstanceToKvmTimeout + ConvertVmwareInstanceToKvmTimeout, + DataStoreDownloadFollowRedirects }; } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 9ab446e55da..0000638f911 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -554,12 +554,14 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new InvalidParameterValueException("File:// type urls are currently unsupported"); } UriUtils.validateUrl(format, url); + boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value(); if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access logger.debug("Checking url: " + url); - DirectDownloadHelper.checkUrlExistence(url); + DirectDownloadHelper.checkUrlExistence(url, followRedirects); } // Check that the resource limit for secondary storage won't be exceeded - _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, + UriUtils.getRemoteSize(url, followRedirects)); } else { _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage); } @@ -660,8 +662,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic _resourceLimitMgr.incrementVolumeResourceCount(volume.getAccountId(), true, null, diskOfferingVO); //url can be null incase of postupload if (url != null) { - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); - volume.setSize(UriUtils.getRemoteSize(url)); + long remoteSize = UriUtils.getRemoteSize(url, StorageManager.DataStoreDownloadFollowRedirects.value()); + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, + remoteSize); + volume.setSize(remoteSize); } return volume; diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index 2974cc90e6e..56d7382e84c 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -88,6 +88,7 @@ import com.cloud.server.StatsCollector; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; +import com.cloud.storage.StorageManager; import com.cloud.storage.TemplateProfile; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; @@ -159,7 +160,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { * @param url url */ private Long performDirectDownloadUrlValidation(final String format, final Hypervisor.HypervisorType hypervisor, - final String url, final List zoneIds) { + final String url, final List zoneIds, final boolean followRedirects) { HostVO host = null; if (zoneIds != null && !zoneIds.isEmpty()) { for (Long zoneId : zoneIds) { @@ -178,7 +179,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { Integer socketTimeout = DirectDownloadManager.DirectDownloadSocketTimeout.value(); Integer connectRequestTimeout = DirectDownloadManager.DirectDownloadConnectionRequestTimeout.value(); Integer connectTimeout = DirectDownloadManager.DirectDownloadConnectTimeout.value(); - CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout); + CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout, followRedirects); logger.debug("Performing URL " + url + " validation on host " + host.getId()); Answer answer = _agentMgr.easySend(host.getId(), cmd); if (answer == null || !answer.getResult()) { @@ -202,6 +203,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { TemplateProfile profile = super.prepare(cmd); String url = profile.getUrl(); UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url); + boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value(); if (cmd.isDirectDownload()) { DigestHelper.validateChecksumString(cmd.getChecksum()); List zoneIds = null; @@ -210,12 +212,14 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { zoneIds.add(cmd.getZoneId()); } Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(), - Hypervisor.HypervisorType.KVM, url, zoneIds); + Hypervisor.HypervisorType.KVM, url, zoneIds, followRedirects); profile.setSize(templateSize); } profile.setUrl(url); // Check that the resource limit for secondary storage won't be exceeded - _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), + ResourceType.secondary_storage, + UriUtils.getRemoteSize(url, followRedirects)); return profile; } @@ -234,15 +238,18 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { String url = profile.getUrl(); UriUtils.validateUrl(cmd.getFormat(), url, cmd.isDirectDownload()); Hypervisor.HypervisorType hypervisor = Hypervisor.HypervisorType.getType(cmd.getHypervisor()); + boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value(); if (cmd.isDirectDownload()) { DigestHelper.validateChecksumString(cmd.getChecksum()); Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(), - hypervisor, url, cmd.getZoneIds()); + hypervisor, url, cmd.getZoneIds(), followRedirects); profile.setSize(templateSize); } profile.setUrl(url); // Check that the resource limit for secondary storage won't be exceeded - _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), + ResourceType.secondary_storage, + UriUtils.getRemoteSize(url, followRedirects)); return profile; } diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index 4f1396913cc..b107a520205 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -30,6 +30,7 @@ import com.cloud.exception.ManagementServerException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.VirtualMachineMigrationException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.offering.ServiceOffering; import com.cloud.service.ServiceOfferingVO; import com.cloud.storage.Storage.StoragePoolType; @@ -96,6 +97,8 @@ public interface UserVmManager extends UserVmService { String validateUserData(String userData, HTTPMethod httpmethod); + void validateExtraConfig(long accountId, HypervisorType hypervisorType, String extraConfig); + boolean isVMUsingLocalStorage(VMInstanceVO vm); boolean expunge(UserVmVO vm); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 47c26757949..97bfef9a95d 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -658,7 +658,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir "enable.additional.vm.configuration", "false", "allow additional arbitrary configuration to vm", true, ConfigKey.Scope.Account); private static final ConfigKey KvmAdditionalConfigAllowList = new ConfigKey<>(String.class, - "allow.additional.vm.configuration.list.kvm", "Advanced", "", "Comma separated list of allowed additional configuration options.", true, ConfigKey.Scope.Global, null, null, EnableAdditionalVmConfig.key(), null, null, ConfigKey.Kind.CSV, null); + "allow.additional.vm.configuration.list.kvm", "Advanced", "", "Comma separated list of allowed additional configuration options.", true, ConfigKey.Scope.Account, null, null, EnableAdditionalVmConfig.key(), null, null, ConfigKey.Kind.CSV, null); private static final ConfigKey XenServerAdditionalConfigAllowList = new ConfigKey<>(String.class, "allow.additional.vm.configuration.list.xenserver", "Advanced", "", "Comma separated list of allowed additional configuration options", true, ConfigKey.Scope.Global, null, null, EnableAdditionalVmConfig.key(), null, null, ConfigKey.Kind.CSV, null); @@ -2848,14 +2848,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (cleanupDetails){ if (caller != null && caller.getType() == Account.Type.ADMIN) { for (final UserVmDetailVO detail : existingDetails) { - if (detail != null && detail.isDisplay()) { + if (detail != null && detail.isDisplay() && !isExtraConfig(detail.getName())) { userVmDetailsDao.removeDetail(id, detail.getName()); } } } else { for (final UserVmDetailVO detail : existingDetails) { if (detail != null && !userDenyListedSettings.contains(detail.getName()) - && !userReadOnlySettings.contains(detail.getName()) && detail.isDisplay()) { + && !userReadOnlySettings.contains(detail.getName()) && detail.isDisplay() + && !isExtraConfig(detail.getName())) { userVmDetailsDao.removeDetail(id, detail.getName()); } } @@ -2866,6 +2867,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir throw new InvalidParameterValueException("'extraconfig' should not be included in details as key"); } + details.entrySet().removeIf(detail -> isExtraConfig(detail.getKey())); + if (caller != null && caller.getType() != Account.Type.ADMIN) { // Ensure denied or read-only detail is not passed by non-root-admin user for (final String detailName : details.keySet()) { @@ -2889,7 +2892,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir // ensure details marked as non-displayable are maintained, regardless of admin or not for (final UserVmDetailVO existingDetail : existingDetails) { - if (!existingDetail.isDisplay()) { + if (!existingDetail.isDisplay() || isExtraConfig(existingDetail.getName())) { details.put(existingDetail.getName(), existingDetail.getValue()); } } @@ -2911,6 +2914,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir cmd.getHttpMethod(), cmd.getCustomId(), hostName, cmd.getInstanceName(), securityGroupIdList, cmd.getDhcpOptionsMap()); } + private boolean isExtraConfig(String detailName) { + return detailName != null && detailName.startsWith(ApiConstants.EXTRA_CONFIG); + } + protected void updateDisplayVmFlag(Boolean isDisplayVm, Long id, UserVmVO vmInstance) { vmInstance.setDisplayVm(isDisplayVm); @@ -6301,7 +6308,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir */ protected void persistExtraConfigKvm(String decodedUrl, UserVm vm) { // validate config against denied cfg commands - validateKvmExtraConfig(decodedUrl); + validateKvmExtraConfig(decodedUrl, vm.getAccountId()); String[] extraConfigs = decodedUrl.split("\n\n"); for (String cfg : extraConfigs) { int i = 1; @@ -6319,6 +6326,18 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir i++; } } + /** + * This method is used to validate if extra config is valid + */ + @Override + public void validateExtraConfig(long accountId, HypervisorType hypervisorType, String extraConfig) { + if (!EnableAdditionalVmConfig.valueIn(accountId)) { + throw new CloudRuntimeException("Additional VM configuration is not enabled for this account"); + } + if (HypervisorType.KVM.equals(hypervisorType)) { + validateKvmExtraConfig(extraConfig, accountId); + } + } /** * This method is called by the persistExtraConfigKvm @@ -6326,8 +6345,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir * controlled by Root admin * @param decodedUrl string containing xml configuration to be validated */ - protected void validateKvmExtraConfig(String decodedUrl) { - String[] allowedConfigOptionList = KvmAdditionalConfigAllowList.value().split(","); + protected void validateKvmExtraConfig(String decodedUrl, long accountId) { + String[] allowedConfigOptionList = KvmAdditionalConfigAllowList.valueIn(accountId).split(","); // Skip allowed keys validation for DPDK if (!decodedUrl.contains(":")) { try { @@ -6347,7 +6366,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } if (!isValidConfig) { - throw new CloudRuntimeException(String.format("Extra config %s is not on the list of allowed keys for KVM hypervisor hosts", currentConfig)); + throw new CloudRuntimeException(String.format("Extra config '%s' is not on the list of allowed keys for KVM hypervisor hosts", currentConfig)); } } } catch (ParserConfigurationException | IOException | SAXException e) { @@ -6445,6 +6464,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (details.containsKey("extraconfig")) { throw new InvalidParameterValueException("'extraconfig' should not be included in details as key"); } + + for (String detailName : details.keySet()) { + if (isExtraConfig(detailName)) { + throw new InvalidParameterValueException("detail name should not start with extraconfig"); + } + } } } diff --git a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java index 0428a9e6907..3194af03ac4 100644 --- a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java @@ -270,7 +270,8 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO(); DownloadProtocol protocol = getProtocolFromUrl(url); - DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers); + DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, + headers); cmd.setTemplateSize(template.getSize()); cmd.setFormat(template.getFormat()); @@ -391,19 +392,23 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown /** * Return DirectDownloadCommand according to the protocol */ - private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool, - String checksum, Map httpHeaders) { + private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, + Long templateId, PrimaryDataStoreTO destPool, String checksum, Map httpHeaders) { int connectTimeout = DirectDownloadConnectTimeout.value(); int soTimeout = DirectDownloadSocketTimeout.value(); int connectionRequestTimeout = DirectDownloadConnectionRequestTimeout.value(); + boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value(); if (protocol.equals(DownloadProtocol.HTTP)) { - return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout); + return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, + soTimeout, followRedirects); } else if (protocol.equals(DownloadProtocol.HTTPS)) { - return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout, connectionRequestTimeout); + return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, + soTimeout, connectionRequestTimeout, followRedirects); } else if (protocol.equals(DownloadProtocol.NFS)) { return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders); } else if (protocol.equals(DownloadProtocol.METALINK)) { - return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout); + return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, + soTimeout, followRedirects); } else { return null; } diff --git a/server/src/test/java/com/cloud/api/ApiServletTest.java b/server/src/test/java/com/cloud/api/ApiServletTest.java index cefbbe2d630..4d4f0a12098 100644 --- a/server/src/test/java/com/cloud/api/ApiServletTest.java +++ b/server/src/test/java/com/cloud/api/ApiServletTest.java @@ -97,15 +97,17 @@ public class ApiServletTest { @Mock AccountService accountMgr; + @Mock ConfigKey useForwardHeader; StringWriter responseWriter; ApiServlet servlet; - + ApiServlet spyServlet; @SuppressWarnings("unchecked") @Before public void setup() throws SecurityException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException, IOException, UnknownHostException { servlet = new ApiServlet(); + spyServlet = Mockito.spy(servlet); responseWriter = new StringWriter(); Mockito.when(response.getWriter()).thenReturn( new PrintWriter(responseWriter)); @@ -259,32 +261,43 @@ public class ApiServletTest { @Test public void getClientAddressWithXForwardedFor() throws UnknownHostException { + String[] proxynet = {"127.0.0.0/8"}; + Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet); + Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true); Mockito.when(request.getHeader(Mockito.eq("X-Forwarded-For"))).thenReturn("192.168.1.1"); - Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request)); + Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request)); } @Test public void getClientAddressWithHttpXForwardedFor() throws UnknownHostException { + String[] proxynet = {"127.0.0.0/8"}; + Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet); + Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true); Mockito.when(request.getHeader(Mockito.eq("HTTP_X_FORWARDED_FOR"))).thenReturn("192.168.1.1"); - Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request)); + Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request)); } @Test - public void getClientAddressWithXRemoteAddr() throws UnknownHostException { - Mockito.when(request.getHeader(Mockito.eq("Remote_Addr"))).thenReturn("192.168.1.1"); - Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request)); + public void getClientAddressWithRemoteAddr() throws UnknownHostException { + String[] proxynet = {"127.0.0.0/8"}; + Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet); + Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true); + Assert.assertEquals(InetAddress.getByName("127.0.0.1"), spyServlet.getClientAddress(request)); } @Test public void getClientAddressWithHttpClientIp() throws UnknownHostException { + String[] proxynet = {"127.0.0.0/8"}; + Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet); + Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true); Mockito.when(request.getHeader(Mockito.eq("HTTP_CLIENT_IP"))).thenReturn("192.168.1.1"); - Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request)); + Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request)); } @Test public void getClientAddressDefault() throws UnknownHostException { Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1"); - Assert.assertEquals(InetAddress.getByName("127.0.0.1"), ApiServlet.getClientAddress(request)); + Assert.assertEquals(InetAddress.getByName("127.0.0.1"), spyServlet.getClientAddress(request)); } @Test diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 18643d0711a..2910476c18e 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.framework.config.ConfigDepot; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -42,7 +44,9 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Command; import com.cloud.agent.api.StoragePoolInfo; import com.cloud.capacity.CapacityManager; +import com.cloud.dc.DataCenterVO; import com.cloud.dc.VsphereStoragePolicyVO; +import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.VsphereStoragePolicyDao; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.ConnectionException; @@ -77,6 +81,12 @@ public class StorageManagerImplTest { HypervisorGuruManager hvGuruMgr; @Mock AgentManager agentManager; + @Mock + ConfigDepot configDepot; + @Mock + ConfigurationDao configurationDao; + @Mock + DataCenterDao dataCenterDao; @Spy @InjectMocks @@ -455,4 +465,36 @@ public class StorageManagerImplTest { storageManagerImpl.getCheckDatastorePolicyComplianceAnswer("1", pool); Assert.assertTrue(answer.getResult()); } + + @Test + public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNoChange() { + Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects)) + .thenReturn(false); + storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations(); + Mockito.verify(configurationDao, Mockito.never()).update(Mockito.anyString(), Mockito.anyString()); + } + + @Test + public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsOldInstall() { + Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects)) + .thenReturn(true); + Mockito.when(dataCenterDao.listAll(Mockito.any())) + .thenReturn(List.of(Mockito.mock(DataCenterVO.class))); + Mockito.doReturn(true).when(configurationDao).update(Mockito.anyString(), Mockito.anyString()); + storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations(); + Mockito.verify(configurationDao, Mockito.times(1)) + .update(StorageManager.DataStoreDownloadFollowRedirects.key(), "true"); + } + + @Test + public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNewInstall() { + Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects)) + .thenReturn(true); + Mockito.when(dataCenterDao.listAll(Mockito.any())) + .thenReturn(new ArrayList<>()); //new installation + storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations(); + Mockito.verify(configurationDao, Mockito.never()) + .update(StorageManager.DataStoreDownloadFollowRedirects.key(),StorageManager.DataStoreDownloadFollowRedirects.defaultValue()); + } + } diff --git a/server/src/test/java/com/cloud/vpc/MockConfigurationManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockConfigurationManagerImpl.java index f9f66550657..14e65634947 100644 --- a/server/src/test/java/com/cloud/vpc/MockConfigurationManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockConfigurationManagerImpl.java @@ -677,4 +677,9 @@ public class MockConfigurationManagerImpl extends ManagerBase implements Configu // TODO Auto-generated method stub return null; } + + @Override + public void validateExtraConfigInServiceOfferingDetail(String detailName) { + // TODO Auto-generated method stub + } } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java index a0417704028..5526835b8f2 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java @@ -41,17 +41,21 @@ public interface DownloadManager extends Manager { * @param hvm whether the template is a hardware virtual machine * @param accountId the accountId of the iso owner (null if public iso) * @param descr description of the template - * @param user username used for authentication to the server - * @param password password used for authentication to the server + * @param userName username used for authentication to the server + * @param passwd password used for authentication to the server * @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes. * @param resourceType signifying the type of resource like template, volume etc. + * @param followRedirects whether downloader follows redirections * @return job-id that can be used to interrogate the status of the download. */ - public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, - String installPathPrefix, String templatePath, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType); + public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, + Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, + String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType, + boolean followRedirects); - public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, - String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType); + public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, + Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, + long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects); Map getProcessors(); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index adb1a4b2ddd..fd5c9e43ac6 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -662,8 +662,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } @Override - public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, - String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { + public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, + Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, + long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects) { UUID uuid = UUID.randomUUID(); String jobId = uuid.toString(); @@ -683,6 +684,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } else { throw new CloudRuntimeException("Unable to download from URL: " + url); } + td.setFollowRedirects(followRedirects); DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); dj.setTmpltPath(installPathPrefix); jobs.put(jobId, dj); @@ -718,8 +720,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } @Override - public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, - String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { + public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, + Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user, + String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, + boolean followRedirects) { UUID uuid = UUID.randomUUID(); String jobId = uuid.toString(); String tmpDir = installPathPrefix; @@ -766,6 +770,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager throw new CloudRuntimeException("Unable to download from URL: " + url); } } + td.setFollowRedirects(followRedirects); // NOTE the difference between installPathPrefix and templatePath // here. instalPathPrefix is the absolute path for template // including mount directory @@ -902,12 +907,16 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager String jobId = null; if (dstore instanceof S3TO) { jobId = - downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), - cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), + cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), + installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType, + cmd.isFollowRedirects()); } else { jobId = - downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), - cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), + cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, + cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType, + cmd.isFollowRedirects()); } sleep(); if (jobId == null) { diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 08d7d82b1a1..81a304d48e2 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -13,6 +13,7 @@ "error.release.dedicate.host": "Failed to release dedicated host.", "error.release.dedicate.pod": "Failed to release dedicated pod.", "error.release.dedicate.zone": "Failed to release dedicated zone.", +"error.unable.to.add.setting.extraconfig": "It is not allowed to add setting for extraconfig. Please update VirtualMachine with extraconfig parameter.", "error.unable.to.proceed": "Unable to proceed. Please contact your administrator.", "firewall.close": "Firewall", "icmp.code.desc": "Please specify -1 if you want to allow all ICMP codes (except NSX zones).", diff --git a/ui/src/components/view/DetailSettings.vue b/ui/src/components/view/DetailSettings.vue index ba96ad6ee8b..ba9f61860e4 100644 --- a/ui/src/components/view/DetailSettings.vue +++ b/ui/src/components/view/DetailSettings.vue @@ -101,7 +101,7 @@ @@ -115,7 +115,12 @@ :cancelText="$t('label.no')" placement="left" > - + @@ -307,6 +312,10 @@ export default { this.error = this.$t('message.error.provide.setting') return } + if (this.newKey.startsWith('extraconfig')) { + this.error = this.$t('error.unable.to.add.setting.extraconfig') + return + } if (!this.allowEditOfDetail(this.newKey)) { this.error = this.$t('error.unable.to.proceed') return diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index f1b3e1b2c99..bd18535f380 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -898,7 +898,12 @@ export default { var fields = ['name', 'id'] if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { fields.push('account') + if (store.getters.listAllProjects) { + fields.push('project') + } fields.push('domain') + } else if (store.getters.listAllProjects) { + fields.push('project') } return fields }, diff --git a/ui/src/views/AutogenView.vue b/ui/src/views/AutogenView.vue index 2404bc0b6d4..408a48727d7 100644 --- a/ui/src/views/AutogenView.vue +++ b/ui/src/views/AutogenView.vue @@ -793,7 +793,7 @@ export default { } this.projectView = Boolean(store.getters.project && store.getters.project.id) - this.hasProjectId = ['vm', 'vmgroup', 'ssh', 'affinitygroup', 'volume', 'snapshot', 'vmsnapshot', 'guestnetwork', + this.hasProjectId = ['vm', 'vmgroup', 'ssh', 'affinitygroup', 'userdata', 'volume', 'snapshot', 'vmsnapshot', 'guestnetwork', 'vpc', 'securitygroups', 'publicip', 'vpncustomergateway', 'template', 'iso', 'event', 'kubernetes', 'autoscalevmgroup', 'vnfapp'].includes(this.$route.name) diff --git a/ui/src/views/network/GuestIpRanges.vue b/ui/src/views/network/GuestIpRanges.vue index 0ecc033119f..815b3d9132f 100644 --- a/ui/src/views/network/GuestIpRanges.vue +++ b/ui/src/views/network/GuestIpRanges.vue @@ -36,14 +36,16 @@ :rowKey="item => item.id" :pagination="false" > - @@ -546,6 +552,71 @@ + +
+ + + + + + + + +
+ {{ $t('label.cancel') }} + {{ $t('label.ok') }} +
+
+ + { + this.networkCount = response.listnetworksresponse.count || 0 + this.networks = response.listnetworksresponse.network || [] + }).catch(error => { + this.$notifyError(error) + }).finally(() => { + this.addNetworkModalLoading = false + }) + this.selectedTierForAutoScaling = null + }, + onNetworkSearch (value) { + this.searchQuery = value + this.fetchNetworks() + }, + handleChangeNetworkPage (page, pageSize) { + this.networkPage = page + this.networkPageSize = pageSize + this.fetchNetworks() + }, + handleChangeNetworkPageSize (currentPage, pageSize) { + this.networkPage = currentPage + this.networkPageSize = pageSize + this.fetchNetworks() + }, handleAssignToLBRule (data) { const vmIDIpMap = {} @@ -1560,7 +1718,8 @@ export default { return } - const networkId = ('vpcid' in this.resource && !('associatednetworkid' in this.resource)) ? this.selectedTier : this.resource.associatednetworkid + const networkId = this.selectedTierForAutoScaling != null ? this.selectedTierForAutoScaling + : ('vpcid' in this.resource && !('associatednetworkid' in this.resource)) ? this.selectedTier : this.resource.associatednetworkid api('createLoadBalancerRule', { openfirewall: false, networkid: networkId, @@ -1573,7 +1732,9 @@ export default { cidrlist: this.newRule.cidrlist }).then(response => { this.addVmModalVisible = false + this.addNetworkModalVisible = false this.handleAssignToLBRule(response.createloadbalancerruleresponse.id) + this.associatednetworkid = networkId }).catch(error => { this.$notifyError(error) this.loading = false @@ -1597,6 +1758,9 @@ export default { this.nics = [] this.addVmModalVisible = false this.newRule.virtualmachineid = [] + this.addNetworkModalLoading = false + this.addNetworkModalVisible = false + this.selectedTierForAutoScaling = null }, handleChangePage (page, pageSize) { this.page = page diff --git a/ui/src/views/tools/ImportUnmanagedInstance.vue b/ui/src/views/tools/ImportUnmanagedInstance.vue index a03ef3866ec..9eb74877365 100644 --- a/ui/src/views/tools/ImportUnmanagedInstance.vue +++ b/ui/src/views/tools/ImportUnmanagedInstance.vue @@ -713,7 +713,7 @@ export default { page: 1 }) this.fetchKvmHostsForConversion() - if (this.resource.disk.length > 1) { + if (this.resource?.disk?.length > 1) { this.updateSelectedRootDisk() } }, diff --git a/ui/src/views/tools/ManageInstances.vue b/ui/src/views/tools/ManageInstances.vue index a869768bf39..b61c8a9efa5 100644 --- a/ui/src/views/tools/ManageInstances.vue +++ b/ui/src/views/tools/ManageInstances.vue @@ -1074,6 +1074,7 @@ export default { this.sourceHypervisor = value this.sourceActions = this.AllSourceActions.filter(x => x.sourceDestHypervisors[value]) this.form.sourceAction = this.sourceActions[0].name || '' + this.selectedVmwareVcenter = undefined this.onSelectSourceAction(this.form.sourceAction) }, onSelectSourceAction (value) { diff --git a/utils/src/main/java/com/cloud/utils/StringUtils.java b/utils/src/main/java/com/cloud/utils/StringUtils.java index fd4f7aba698..01b6f833271 100644 --- a/utils/src/main/java/com/cloud/utils/StringUtils.java +++ b/utils/src/main/java/com/cloud/utils/StringUtils.java @@ -31,7 +31,7 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -public class StringUtils { +public class StringUtils extends org.apache.commons.lang3.StringUtils { private static final char[] hexChar = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; private static Charset preferredACSCharset; diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index 8264d442d46..e1766e691c2 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -214,7 +214,7 @@ public class UriUtils { } // Get the size of a file from URL response header. - public static long getRemoteSize(String url) { + public static long getRemoteSize(String url, Boolean followRedirect) { long remoteSize = 0L; final String[] methods = new String[]{"HEAD", "GET"}; IllegalArgumentException exception = null; @@ -229,6 +229,7 @@ public class UriUtils { httpConn.setRequestMethod(method); httpConn.setConnectTimeout(2000); httpConn.setReadTimeout(5000); + httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect)); String contentLength = httpConn.getHeaderField("content-length"); if (contentLength != null) { remoteSize = Long.parseLong(contentLength); diff --git a/utils/src/main/java/com/cloud/utils/script/OutputInterpreter.java b/utils/src/main/java/com/cloud/utils/script/OutputInterpreter.java index dc3c51402d8..130e9551419 100644 --- a/utils/src/main/java/com/cloud/utils/script/OutputInterpreter.java +++ b/utils/src/main/java/com/cloud/utils/script/OutputInterpreter.java @@ -139,6 +139,11 @@ public abstract class OutputInterpreter { public String getLines() { return allLines; } + + @Override + public boolean drain() { + return true; + } } public static class LineByLineOutputLogger extends OutputInterpreter { diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java index 67907f40a22..34d748b0708 100644 --- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java +++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java @@ -22,8 +22,9 @@ package com.cloud.utils.storage; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -113,16 +114,23 @@ public final class QCOW2Utils { } } - public static long getVirtualSize(String urlStr) { + public static long getVirtualSizeFromUrl(String urlStr, boolean followRedirects) { + HttpURLConnection httpConn = null; try { - URL url = new URL(urlStr); - return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr)); - } catch (MalformedURLException e) { + URI url = new URI(urlStr); + httpConn = (HttpURLConnection)url.toURL().openConnection(); + httpConn.setInstanceFollowRedirects(followRedirects); + return getVirtualSize(httpConn.getInputStream(), UriUtils.isUrlForCompressedFile(urlStr)); + } catch (URISyntaxException e) { LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage()); throw new IllegalArgumentException("Invalid URL: " + urlStr); } catch (IOException e) { LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage()); throw new IllegalArgumentException("Failed to connect URL: " + urlStr); + } finally { + if (httpConn != null) { + httpConn.disconnect(); + } } } } diff --git a/utils/src/test/java/com/cloud/utils/ScriptTest.java b/utils/src/test/java/com/cloud/utils/ScriptTest.java index 3952674566c..2d2b3d54b2e 100644 --- a/utils/src/test/java/com/cloud/utils/ScriptTest.java +++ b/utils/src/test/java/com/cloud/utils/ScriptTest.java @@ -112,6 +112,20 @@ public class ScriptTest { Assert.assertNotNull(value); } + @Test + public void executeWithOutputInterpreterAllLinesParserLargeOutput() { + Assume.assumeTrue(SystemUtils.IS_OS_LINUX); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + Script script = new Script("seq"); + script.add("-f"); + script.add("my text to test cloudstack %g"); + script.add("4096"); // AllLinesParser doesn't work with that amount of data + String value = script.execute(parser); + // it is a stack trace in this case as string + Assert.assertNull(value); + Assert.assertEquals(129965, parser.getLines().length()); + } + @Test public void runSimpleBashScriptNotExisting() { Assume.assumeTrue(SystemUtils.IS_OS_LINUX);