apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'apache/4.19'

Browse Source
This commit is contained in:
Abhishek Kumar 2024-04-04 17:36:05 +05:30
commit 02305fbc5f
73 changed files with 986 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/src/main/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmd.java
View File

@ -29,6 +29,7 @@ import org.apache.cloudstack.api.ResponseObject.ResponseView;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.VolumeResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.commons.lang3.EnumUtils;
import com.cloud.event.EventTypes;
import com.cloud.exception.InvalidParameterValueException;
@ -69,9 +70,9 @@ public class CheckAndRepairVolumeCmd extends BaseAsyncCmd {
public String getRepair() {
if (org.apache.commons.lang3.StringUtils.isNotEmpty(repair)) {
RepairValues repairType = Enum.valueOf(RepairValues.class, repair.toUpperCase());
RepairValues repairType = EnumUtils.getEnumIgnoreCase(RepairValues.class, repair);
if (repairType == null) {
throw new InvalidParameterValueException(String.format("Repair parameter can only take the following values: %s" + Arrays.toString(RepairValues.values())));
throw new InvalidParameterValueException(String.format("Repair parameter can only take the following values: %s", Arrays.toString(RepairValues.values())));
}
return repair.toLowerCase();
}

20
api/src/main/java/org/apache/cloudstack/api/response/UserDataResponse.java
View File

@ -24,7 +24,7 @@ import org.apache.cloudstack.api.BaseResponseWithAnnotations;
import org.apache.cloudstack.api.EntityReference;
@EntityReference(value = UserData.class)
public class UserDataResponse extends BaseResponseWithAnnotations {
public class UserDataResponse extends BaseResponseWithAnnotations implements ControlledEntityResponse {
@SerializedName(ApiConstants.ID)
@Param(description = "ID of the ssh keypair")
@ -40,6 +40,14 @@ public class UserDataResponse extends BaseResponseWithAnnotations {
@SerializedName(ApiConstants.ACCOUNT) @Param(description="the owner of the userdata")
private String accountName;
@SerializedName(ApiConstants.PROJECT_ID)
@Param(description = "the project id of the userdata", since = "4.19.1")
private String projectId;
@SerializedName(ApiConstants.PROJECT)
@Param(description = "the project name of the userdata", since = "4.19.1")
private String projectName;
@SerializedName(ApiConstants.DOMAIN_ID) @Param(description="the domain id of the userdata owner")
private String domainId;
@ -118,6 +126,16 @@ public class UserDataResponse extends BaseResponseWithAnnotations {
this.accountName = accountName;
}
@Override
public void setProjectId(String projectId) {
this.projectId = projectId;
}
@Override
public void setProjectName(String projectName) {
this.projectName = projectName;
}
public String getDomainName() {
return domain;
}

63
api/src/test/java/org/apache/cloudstack/api/command/user/volume/CheckAndRepairVolumeCmdTest.java Normal file
View File

@ -0,0 +1,63 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.user.volume;
import com.cloud.exception.InvalidParameterValueException;
import junit.framework.TestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.MockitoAnnotations;
import org.mockito.junit.MockitoJUnitRunner;
import org.springframework.test.util.ReflectionTestUtils;
@RunWith(MockitoJUnitRunner.class)
public class CheckAndRepairVolumeCmdTest extends TestCase {
private CheckAndRepairVolumeCmd checkAndRepairVolumeCmd;
private AutoCloseable closeable;
@Before
public void setup() {
closeable = MockitoAnnotations.openMocks(this);
checkAndRepairVolumeCmd = new CheckAndRepairVolumeCmd();
}
@After
public void tearDown() throws Exception {
closeable.close();
}
@Test
public void testGetRepair() {
ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", "all");
assertEquals("all", checkAndRepairVolumeCmd.getRepair());
ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", "LEAKS");
assertEquals("leaks", checkAndRepairVolumeCmd.getRepair());
ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", null);
assertNull(checkAndRepairVolumeCmd.getRepair());
}
@Test(expected = InvalidParameterValueException.class)
public void testGetRepairInvalid() {
ReflectionTestUtils.setField(checkAndRepairVolumeCmd, "repair", "RANDOM STRING");
checkAndRepairVolumeCmd.getRepair();
}
}

3
client/src/main/java/org/apache/cloudstack/ServerDaemon.java
View File

@ -30,7 +30,6 @@ import org.apache.commons.daemon.Daemon;
import org.apache.commons.daemon.DaemonContext;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jetty.jmx.MBeanContainer;
import org.eclipse.jetty.server.ForwardedRequestCustomizer;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.NCSARequestLog;
@ -174,7 +173,7 @@ public class ServerDaemon implements Daemon {
// HTTP config
final HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.addCustomizer( new ForwardedRequestCustomizer() );
// it would be nice to make this dynamic but we take care of this ourselves for now: httpConfig.addCustomizer( new ForwardedRequestCustomizer() );
httpConfig.setSecureScheme("https");
httpConfig.setSecurePort(httpsPort);
httpConfig.setOutputBufferSize(32768);

18
core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java
View File

@ -28,6 +28,7 @@ import java.io.RandomAccessFile;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Date;
import java.util.List;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
@ -78,6 +79,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
private long maxTemplateSizeInBytes;
private ResourceType resourceType = ResourceType.TEMPLATE;
private final HttpMethodRetryHandler myretryhandler;
private boolean followRedirects = false;
public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
String user, String password, Proxy proxy, ResourceType resourceType) {
@ -109,7 +111,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
private GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
request.setFollowRedirects(true);
request.setFollowRedirects(followRedirects);
return request;
}
@ -335,6 +337,12 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
} else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
status = Status.UNRECOVERABLE_ERROR;
errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
&& !followRedirects) {
errorString = String.format("Failed to download %s due to redirection, response code: %d",
downloadUrl, responseCode);
logger.error(errorString);
}
return true; //FIXME: retry?
}
return false;
@ -536,4 +544,12 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
return this;
}
}
@Override
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
if (this.request != null) {
this.request.setFollowRedirects(followRedirects);
}
}
}

10
core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java
View File

@ -58,7 +58,7 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement
protected GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
request.setFollowRedirects(true);
request.setFollowRedirects(followRedirects);
if (!toFileSet) {
String[] parts = downloadUrl.split("/");
String filename = parts[parts.length - 1];
@ -171,4 +171,12 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement
public void setStatus(Status status) {
this.status = status;
}
@Override
public void setFollowRedirects(boolean followRedirects) {
super.setFollowRedirects(followRedirects);
if (this.request != null) {
this.request.setFollowRedirects(followRedirects);
}
}
}

19
core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java
View File

@ -34,6 +34,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
@ -43,6 +44,7 @@ import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import java.util.List;
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
import static java.util.Arrays.asList;
@ -70,8 +72,8 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
private long downloadTime;
private long totalBytes;
private long maxTemplateSizeInByte;
private boolean resume = false;
private boolean followRedirects = false;
public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
@ -89,7 +91,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
// Follow redirects
this.getMethod.setFollowRedirects(true);
this.getMethod.setFollowRedirects(followRedirects);
// Set file extension.
this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
@ -122,10 +124,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
return 0;
}
if (!HTTPUtils.verifyResponseCode(responseCode)) {
boolean failedDueToRedirection = List.of(HttpStatus.SC_MOVED_PERMANENTLY,
HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) && !followRedirects;
if (!HTTPUtils.verifyResponseCode(responseCode) || failedDueToRedirection) {
errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
logger.warn(errorString);
status = Status.UNRECOVERABLE_ERROR;
return 0;
}
@ -371,4 +374,12 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
public String getFileExtension() {
return fileExtension;
}
@Override
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
if (this.getMethod != null) {
this.getMethod.setFollowRedirects(followRedirects);
}
}
}

22
core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java
View File

@ -25,6 +25,7 @@ import java.io.InputStream;
import java.io.RandomAccessFile;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
@ -71,6 +72,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
private final HttpMethodRetryHandler retryHandler;
private HashMap<String, String> urlFileMap;
private boolean followRedirects = false;
public SimpleHttpMultiFileDownloader(StorageLayer storageLayer, String[] downloadUrls, String toDir,
DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
@ -92,7 +94,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
private GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler);
request.setFollowRedirects(true);
request.setFollowRedirects(followRedirects);
return request;
}
@ -168,7 +170,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
urlFileMap.put(downloadUrl, currentToFile);
file = new File(currentToFile);
long localFileSize = checkLocalFileSizeForResume(resume, file);
if (checkServerResponse(localFileSize)) return 0;
if (checkServerResponse(localFileSize, downloadUrl)) return 0;
if (!tryAndGetRemoteSize()) return 0;
if (!canHandleDownloadSize()) return 0;
checkAndSetDownloadSize();
@ -315,7 +317,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
return true;
}
private boolean checkServerResponse(long localFileSize) throws IOException {
private boolean checkServerResponse(long localFileSize, String downloadUrl) throws IOException {
int responseCode = 0;
if (localFileSize > 0) {
@ -329,6 +331,12 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
} else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
currentStatus = Status.UNRECOVERABLE_ERROR;
errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
&& !followRedirects) {
errorString = String.format("Failed to download %s due to redirection, response code: %d",
downloadUrl, responseCode);
logger.error(errorString);
}
return true; //FIXME: retry?
}
return false;
@ -476,4 +484,12 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem
public Map<String, String> getDownloadedFilesMap() {
return urlFileMap;
}
@Override
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
if (this.request != null) {
this.request.setFollowRedirects(followRedirects);
}
}
}

2
core/src/main/java/com/cloud/storage/template/TemplateDownloader.java
View File

@ -92,4 +92,6 @@ public interface TemplateDownloader extends Runnable {
boolean isInited();
long getMaxTemplateSizeInBytes();
void setFollowRedirects(boolean followRedirects);
}

6
core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java
View File

@ -41,6 +41,7 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl
protected long _start;
protected StorageLayer _storage;
protected boolean _inited = false;
protected boolean followRedirects = false;
private long maxTemplateSizeInBytes;
public TemplateDownloaderBase(StorageLayer storage, String downloadUrl, String toDir, long maxTemplateSizeInBytes, DownloadCompleteCallback callback) {
@ -147,4 +148,9 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl
public boolean isInited() {
return _inited;
}
@Override
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
}
}

11
core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
View File

@ -28,6 +28,7 @@ public class CheckUrlCommand extends Command {
private Integer connectTimeout;
private Integer connectionRequestTimeout;
private Integer socketTimeout;
private boolean followRedirects;
public String getFormat() {
return format;
@ -43,19 +44,25 @@ public class CheckUrlCommand extends Command {
public Integer getSocketTimeout() { return socketTimeout; }
public CheckUrlCommand(final String format,final String url) {
public boolean isFollowRedirects() {
return followRedirects;
}
public CheckUrlCommand(final String format, final String url, final boolean followRedirects) {
super();
this.format = format;
this.url = url;
this.followRedirects = followRedirects;
}
public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, final boolean followRedirects) {
super();
this.format = format;
this.url = url;
this.connectTimeout = connectTimeout;
this.socketTimeout = socketTimeout;
this.connectionRequestTimeout = connectionRequestTimeout;
this.followRedirects = followRedirects;
}
@Override

15
core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
View File

@ -45,7 +45,11 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
private Long templateSize;
private Storage.ImageFormat format;
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) {
private boolean followRedirects;
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool,
final String checksum, final Map<String, String> headers, final Integer connectTimeout,
final Integer soTimeout, final Integer connectionRequestTimeout, final boolean followRedirects) {
this.url = url;
this.templateId = templateId;
this.destData = destData;
@ -55,6 +59,7 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;
this.connectionRequestTimeout = connectionRequestTimeout;
this.followRedirects = followRedirects;
}
public String getUrl() {
@ -137,4 +142,12 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
public int getWaitInMillSeconds() {
return getWait() * 1000;
}
public boolean isFollowRedirects() {
return followRedirects;
}
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
}
}

6
core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
View File

@ -24,8 +24,10 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class HttpDirectDownloadCommand extends DirectDownloadCommand {
public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
null, followRedirects);
}
}

7
core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
View File

@ -25,7 +25,10 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, connectionRequestTimeout);
public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout,
boolean followRedirects) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
connectionRequestTimeout, followRedirects);
}
}

5
core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
View File

@ -24,8 +24,9 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null, followRedirects);
}
}

5
core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
View File

@ -24,8 +24,9 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class NfsDirectDownloadCommand extends DirectDownloadCommand {
public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
super(url, templateId, destPool, checksum, headers, null, null, null);
public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool,
final String checksum, final Map<String, String> headers) {
super(url, templateId, destPool, checksum, headers, null, null, null, false);
}
}

45
core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
View File

@ -35,27 +35,30 @@ public class DirectDownloadHelper {
* Get direct template downloader from direct download command and destination pool
*/
public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
String destPoolLocalPath,
String temporaryDownloadPath) {
String destPoolLocalPath, String temporaryDownloadPath) {
if (cmd instanceof HttpDirectDownloadCommand) {
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
temporaryDownloadPath, cmd.isFollowRedirects());
} else if (cmd instanceof HttpsDirectDownloadCommand) {
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
cmd.getConnectionRequestTimeout(), temporaryDownloadPath, cmd.isFollowRedirects());
} else if (cmd instanceof NfsDirectDownloadCommand) {
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
cmd.getChecksum(), temporaryDownloadPath);
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
temporaryDownloadPath, cmd.isFollowRedirects());
} else {
throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}
}
public static boolean checkUrlExistence(String url) {
public static boolean checkUrlExistence(String url, boolean followRedirects) {
try {
DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects);
return checker.checkUrl(url);
} catch (CloudRuntimeException e) {
LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@ -63,9 +66,9 @@ public class DirectDownloadHelper {
}
}
public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
try {
DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
return checker.checkUrl(url);
} catch (CloudRuntimeException e) {
LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@ -73,27 +76,27 @@ public class DirectDownloadHelper {
}
}
private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
if (url.toLowerCase().startsWith("https:")) {
return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
} else if (url.toLowerCase().startsWith("http:")) {
return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout);
return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects);
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else if (url.toLowerCase().endsWith(".metalink")) {
return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout);
return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects);
} else {
throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url));
}
}
public static Long getFileSize(String url, String format) {
DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
public static Long getFileSize(String url, String format, boolean followRedirects) {
DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects);
return checker.getRemoteFileSize(url, format);
}
public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
return checker.getRemoteFileSize(url, format);
}
}

13
core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java
View File

@ -43,16 +43,19 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
private String checksum;
private boolean redownload = false;
protected String temporaryDownloadPath;
private boolean followRedirects;
protected Logger logger = LogManager.getLogger(getClass());
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId,
final String checksum, final String temporaryDownloadPath) {
final String checksum, final String temporaryDownloadPath,
final boolean followRedirects) {
this.url = url;
this.destPoolPath = destPoolPath;
this.templateId = templateId;
this.checksum = checksum;
this.temporaryDownloadPath = temporaryDownloadPath;
this.followRedirects = followRedirects;
}
private static String directDownloadDir = "template";
@ -112,6 +115,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
return redownload;
}
public boolean isFollowRedirects() {
return followRedirects;
}
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
}
/**
* Create download directory (if it does not exist)
*/

21
core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
View File

@ -48,13 +48,14 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected GetMethod request;
protected Map<String, String> reqHeaders = new HashMap<>();
protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
this(url, null, null, null, null, connectTimeout, socketTimeout, null);
protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout, boolean followRedirects) {
this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects);
}
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, destPoolPath, templateId, checksum, downloadPath);
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
boolean followRedirects) {
super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout);
s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout);
client = new HttpClient(s_httpClientManager);
@ -66,7 +67,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
GetMethod request = new GetMethod(downloadUrl);
request.setFollowRedirects(true);
request.setFollowRedirects(this.isFollowRedirects());
if (MapUtils.isNotEmpty(headers)) {
for (String key : headers.keySet()) {
request.setRequestHeader(key, headers.get(key));
@ -109,9 +110,11 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
@Override
public boolean checkUrl(String url) {
HeadMethod httpHead = new HeadMethod(url);
httpHead.setFollowRedirects(this.isFollowRedirects());
try {
if (client.executeMethod(httpHead) != HttpStatus.SC_OK) {
logger.error(String.format("Invalid URL: %s", url));
int responseCode = client.executeMethod(httpHead);
if (responseCode != HttpStatus.SC_OK) {
logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
return false;
}
return true;
@ -126,9 +129,9 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
@Override
public Long getRemoteFileSize(String url, String format) {
if ("qcow2".equalsIgnoreCase(format)) {
return QCOW2Utils.getVirtualSize(url);
return QCOW2Utils.getVirtualSizeFromUrl(url, this.isFollowRedirects());
} else {
return UriUtils.getRemoteSize(url);
return UriUtils.getRemoteSize(url, this.isFollowRedirects());
}
}

26
core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
View File

@ -68,20 +68,26 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
protected CloseableHttpClient httpsClient;
private HttpUriRequest req;
protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null);
protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null, followRedirects);
}
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
super(url, destPoolPath, templateId, checksum, temporaryDownloadPath);
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout,
Integer connectionRequestTimeout, String temporaryDownloadPath, boolean followRedirects) {
super(url, destPoolPath, templateId, checksum, temporaryDownloadPath, followRedirects);
SSLContext sslcontext = getSSLContext();
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
.setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
.setSocketTimeout(soTimeout == null ? 5000 : soTimeout)
.setRedirectsEnabled(followRedirects)
.build();
httpsClient = HttpClients.custom()
.setSSLSocketFactory(factory)
.setDefaultRequestConfig(config)
.build();
createUriRequest(url, headers);
String downloadDir = getDirectDownloadTempPath(templateId);
File tempFile = createTemporaryDirectoryAndFile(downloadDir);
@ -90,6 +96,7 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
req = new HttpGet(downloadUrl);
setFollowRedirects(this.isFollowRedirects());
if (MapUtils.isNotEmpty(headers)) {
for (String headerKey: headers.keySet()) {
req.setHeader(headerKey, headers.get(headerKey));
@ -164,8 +171,9 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
HttpHead httpHead = new HttpHead(url);
try {
CloseableHttpResponse response = httpsClient.execute(httpHead);
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
logger.error(String.format("Invalid URL: %s", url));
int responseCode = response.getStatusLine().getStatusCode();
if (responseCode != HttpStatus.SC_OK) {
logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
return false;
}
return true;

20
core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
View File

@ -39,16 +39,15 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm
private Integer soTimeout;
protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId,
String destPoolPath, String checksum,
Map<String, String> headers,
Integer connectTimeout, Integer soTimeout,
Integer connectionRequestTimeout, String temporaryDownloadPath) {
String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout,
Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
if (url.toLowerCase().startsWith("https:")) {
return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath);
connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath,
this.isFollowRedirects());
} else if (url.toLowerCase().startsWith("http:")) {
return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
connectTimeout, soTimeout, temporaryDownloadPath);
connectTimeout, soTimeout, temporaryDownloadPath, this.isFollowRedirects());
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else {
@ -57,13 +56,14 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm
}
}
protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
this(url, null, null, null, null, connectTimeout, socketTimeout, null);
protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout, boolean followRedirects) {
this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects);
}
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, destPoolPath, templateId, checksum, downloadPath);
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
boolean followRedirects) {
super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
this.headers = headers;
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;

5
core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java
View File

@ -57,8 +57,9 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
this(url, null, null, null, null);
}
public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) {
super(url, destPool, templateId, checksum, downloadPath);
public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum,
String downloadPath) {
super(url, destPool, templateId, checksum, downloadPath, false);
parseUrl();
}

13
core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java
View File

@ -49,6 +49,8 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
private DataStoreTO _store;
private DataStoreTO cacheStore;
private boolean followRedirects = false;
protected DownloadCommand() {
}
@ -65,6 +67,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
installPath = that.installPath;
_store = that._store;
_proxy = that._proxy;
followRedirects = that.followRedirects;
}
public DownloadCommand(TemplateObjectTO template, Long maxDownloadSizeInBytes) {
@ -80,6 +83,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
setSecUrl(((NfsTO)_store).getUrl());
}
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
this.followRedirects = template.isFollowRedirects();
}
public DownloadCommand(TemplateObjectTO template, String user, String passwd, Long maxDownloadSizeInBytes) {
@ -95,6 +99,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
_store = volume.getDataStore();
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
resourceType = ResourceType.VOLUME;
this.followRedirects = volume.isFollowRedirects();
}
public DownloadCommand(SnapshotObjectTO snapshot, Long maxDownloadSizeInBytes, String url) {
@ -194,4 +199,12 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
public DataStoreTO getCacheStore() {
return cacheStore;
}
public boolean isFollowRedirects() {
return followRedirects;
}
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
}
}

30
core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java Normal file
View File

@ -0,0 +1,30 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.to;
public class DownloadableObjectTO {
protected boolean followRedirects = false;
public boolean isFollowRedirects() {
return followRedirects;
}
public void setFollowRedirects(boolean followRedirects) {
this.followRedirects = followRedirects;
}
}

2
core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java
View File

@ -30,7 +30,7 @@ import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
public class SnapshotObjectTO implements DataTO {
public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO {
private String path;
private VolumeObjectTO volume;
private String parentSnapshotPath;

3
core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java
View File

@ -28,7 +28,7 @@ import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.template.VirtualMachineTemplate;
public class TemplateObjectTO implements DataTO {
public class TemplateObjectTO extends DownloadableObjectTO implements DataTO {
private String path;
private String origUrl;
private String uuid;
@ -87,6 +87,7 @@ public class TemplateObjectTO implements DataTO {
this.deployAsIs = template.isDeployAsIs();
this.deployAsIsConfiguration = template.getDeployAsIsConfiguration();
this.directDownload = template.isDirectDownload();
this.followRedirects = template.isFollowRedirects();
}
@Override

3
core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
View File

@ -33,7 +33,7 @@ import com.cloud.storage.Volume;
import java.util.Arrays;
public class VolumeObjectTO implements DataTO {
public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
private String uuid;
private Volume.Type volumeType;
private DataStoreTO dataStore;
@ -119,6 +119,7 @@ public class VolumeObjectTO implements DataTO {
this.vSphereStoragePolicyId = volume.getvSphereStoragePolicyId();
this.passphrase = volume.getPassphrase();
this.encryptFormat = volume.getEncryptFormat();
this.followRedirects = volume.isFollowRedirects();
}
public String getUuid() {

2
core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
View File

@ -57,7 +57,7 @@ public class BaseDirectTemplateDownloaderTest {
private HttpEntity httpEntity;
@InjectMocks
protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000);
protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000, false);
private AutoCloseable closeable;

3
core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
View File

@ -25,8 +25,7 @@ import org.mockito.InjectMocks;
public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
@InjectMocks
protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000);
protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000, false);
@Test
public void testCheckUrlMetalink() {
metalinkDownloader.downloader = httpsDownloader;

24
engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java Normal file
View File

@ -0,0 +1,24 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.engine.subsystem.api.storage;
public interface DownloadableDataInfo extends DataObject {
default public boolean isFollowRedirects() {
return true;
}
}

2
engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
View File

@ -21,7 +21,7 @@ package org.apache.cloudstack.engine.subsystem.api.storage;
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.user.UserData;
public interface TemplateInfo extends DataObject, VirtualMachineTemplate {
public interface TemplateInfo extends DownloadableDataInfo, VirtualMachineTemplate {
@Override
String getUniqueName();

2
engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java
View File

@ -26,7 +26,7 @@ import com.cloud.storage.Storage;
import com.cloud.storage.Volume;
import com.cloud.vm.VirtualMachine;
public interface VolumeInfo extends DataObject, Volume {
public interface VolumeInfo extends DownloadableDataInfo, Volume {
boolean isAttachedVM();

2
engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java
View File

@ -276,4 +276,6 @@ public interface ConfigurationManager {
Pair<String, String> getConfigurationGroupAndSubGroup(String configName);
List<ConfigurationSubGroupVO> getConfigurationSubGroups(Long groupId);
void validateExtraConfigInServiceOfferingDetail(String detailName);
}

4
engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
View File

@ -199,6 +199,10 @@ public interface StorageManager extends StorageService {
true,
ConfigKey.Scope.Global,
null);
static final ConfigKey<Boolean> DataStoreDownloadFollowRedirects = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED,
Boolean.class, "store.download.follow.redirects", "false",
"Whether HTTP redirect is followed during store downloads for objects such as template, volume etc.",
true, ConfigKey.Scope.Global);
ConfigKey<Long> HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000",
"The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true);

83
engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java
View File

@ -223,4 +223,87 @@ public class DatabaseUpgradeCheckerTest {
assertEquals("We should have 1 upgrade step", 1, upgrades.length);
assertTrue(upgrades[0] instanceof NoopDbUpgrade);
}
@Test
public void testCalculateUpgradePathFrom41800toNextSecurityRelease() {
final CloudStackVersion dbVersion = CloudStackVersion.parse("4.18.0.0");
assertNotNull(dbVersion);
final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
final CloudStackVersion currentVersion = checker.getLatestVersion();
assertNotNull(currentVersion);
final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
assertNotNull(upgrades);
final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "."
+ currentVersion.getMinorRelease() + "."
+ currentVersion.getPatchRelease() + "."
+ (currentVersion.getSecurityRelease() + 1));
assertNotNull(nextSecurityRelease);
final DbUpgrade[] upgradesToNext = checker.calculateUpgradePath(dbVersion, nextSecurityRelease);
assertNotNull(upgradesToNext);
assertEquals(upgrades.length + 1, upgradesToNext.length);
assertTrue(upgradesToNext[upgradesToNext.length - 1] instanceof NoopDbUpgrade);
}
@Test
public void testCalculateUpgradePathFromSecurityReleaseToLatest() {
final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.2.0"); // a EOL version
assertNotNull(dbVersion);
final CloudStackVersion oldSecurityRelease = CloudStackVersion.parse(dbVersion.getMajorRelease() + "."
+ dbVersion.getMinorRelease() + "."
+ dbVersion.getPatchRelease() + "."
+ (dbVersion.getSecurityRelease() + 100));
assertNotNull(oldSecurityRelease); // fake security release 4.17.2.100
final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
final CloudStackVersion currentVersion = checker.getLatestVersion();
assertNotNull(currentVersion);
final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
assertNotNull(upgrades);
final DbUpgrade[] upgradesFromSecurityRelease = checker.calculateUpgradePath(oldSecurityRelease, currentVersion);
assertNotNull(upgradesFromSecurityRelease);
assertEquals("The upgrade paths should be the same", upgrades.length, upgradesFromSecurityRelease.length);
}
@Test
public void testCalculateUpgradePathFromSecurityReleaseToNextSecurityRelease() {
final CloudStackVersion dbVersion = CloudStackVersion.parse("4.17.2.0"); // a EOL version
assertNotNull(dbVersion);
final CloudStackVersion oldSecurityRelease = CloudStackVersion.parse(dbVersion.getMajorRelease() + "."
+ dbVersion.getMinorRelease() + "."
+ dbVersion.getPatchRelease() + "."
+ (dbVersion.getSecurityRelease() + 100));
assertNotNull(oldSecurityRelease); // fake security release 4.17.2.100
final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker();
final CloudStackVersion currentVersion = checker.getLatestVersion();
assertNotNull(currentVersion);
final CloudStackVersion nextSecurityRelease = CloudStackVersion.parse(currentVersion.getMajorRelease() + "."
+ currentVersion.getMinorRelease() + "."
+ currentVersion.getPatchRelease() + "."
+ (currentVersion.getSecurityRelease() + 1));
assertNotNull(nextSecurityRelease); // fake security release
final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion);
assertNotNull(upgrades);
final DbUpgrade[] upgradesFromSecurityReleaseToNext = checker.calculateUpgradePath(oldSecurityRelease, nextSecurityRelease);
assertNotNull(upgradesFromSecurityReleaseToNext);
assertEquals(upgrades.length + 1, upgradesFromSecurityReleaseToNext.length);
assertTrue(upgradesFromSecurityReleaseToNext[upgradesFromSecurityReleaseToNext.length - 1] instanceof NoopDbUpgrade);
}
}

5
engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
View File

@ -92,6 +92,7 @@ import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
@ -367,6 +368,7 @@ public class TemplateServiceImpl implements TemplateService {
toBeDownloaded.addAll(allTemplates);
final StateMachine2<VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate> stateMachine = VirtualMachineTemplate.State.getStateMachine();
Boolean followRedirect = StorageManager.DataStoreDownloadFollowRedirects.value();
for (VMTemplateVO tmplt : allTemplates) {
String uniqueName = tmplt.getUniqueName();
TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
@ -447,7 +449,8 @@ public class TemplateServiceImpl implements TemplateService {
try {
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId),
com.cloud.configuration.Resource.ResourceType.secondary_storage,
tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl(),
followRedirect));
} catch (ResourceAllocationException e) {
logger.warn(e.getMessage());
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null, e.getMessage(), e.getMessage());

8
engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java
View File

@ -23,6 +23,7 @@ import java.util.Map;
import javax.inject.Inject;
import com.cloud.storage.StorageManager;
import com.cloud.user.UserData;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@ -74,8 +75,10 @@ public class TemplateObject implements TemplateInfo {
VMTemplatePoolDao templatePoolDao;
@Inject
TemplateDataStoreDao templateStoreDao;
final private boolean followRedirects;
public TemplateObject() {
this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
}
protected void configure(VMTemplateVO template, DataStore dataStore) {
@ -574,4 +577,9 @@ public class TemplateObject implements TemplateInfo {
// TODO Auto-generated method stub
return null;
}
@Override
public boolean isFollowRedirects() {
return followRedirects;
}
}

8
engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
View File

@ -23,6 +23,7 @@ import javax.inject.Inject;
import com.cloud.configuration.Resource.ResourceType;
import com.cloud.dc.VsphereStoragePolicyVO;
import com.cloud.dc.dao.VsphereStoragePolicyDao;
import com.cloud.storage.StorageManager;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;
@ -118,6 +119,7 @@ public class VolumeObject implements VolumeInfo {
private MigrationOptions migrationOptions;
private boolean directDownload;
private String vSphereStoragePolicyId;
private boolean followRedirects;
private final List<Volume.State> volumeStatesThatShouldNotTransitWhenDataStoreRoleIsImage = Arrays.asList(Volume.State.Migrating, Volume.State.Uploaded, Volume.State.Copying,
Volume.State.Expunged);
@ -128,6 +130,7 @@ public class VolumeObject implements VolumeInfo {
public VolumeObject() {
_volStateMachine = Volume.State.getStateMachine();
this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
}
protected void configure(DataStore dataStore, VolumeVO volumeVO) {
@ -931,4 +934,9 @@ public class VolumeObject implements VolumeInfo {
public void setEncryptFormat(String encryptFormat) {
volumeVO.setEncryptFormat(encryptFormat);
}
@Override
public boolean isFollowRedirects() {
return followRedirects;
}
}

1
framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java
View File

@ -31,4 +31,5 @@ public interface ConfigDepot {
<T> void set(ConfigKey<T> key, T value);
<T> void createOrUpdateConfigObject(String componentName, ConfigKey<T> key, String value);
boolean isNewConfig(ConfigKey<?> configKey);
}

1
framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java
View File

@ -34,6 +34,7 @@ public class ConfigKey<T> {
public static final String CATEGORY_ADVANCED = "Advanced";
public static final String CATEGORY_ALERT = "Alert";
public static final String CATEGORY_NETWORK = "Network";
public enum Scope {
Global, Zone, Cluster, StoragePool, Account, ManagementServer, ImageStore, Domain

7
framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java
View File

@ -82,6 +82,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
List<Configurable> _configurables;
List<ScopedConfigStorage> _scopedStorages;
Set<Configurable> _configured = Collections.synchronizedSet(new HashSet<Configurable>());
Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
private HashMap<String, Pair<String, ConfigKey<?>>> _allKeys = new HashMap<String, Pair<String, ConfigKey<?>>>(1007);
@ -194,6 +195,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
}
_configDao.persist(vo);
newConfigs.add(vo.getName());
} else {
boolean configUpdated = false;
if (vo.isDynamic() != key.isDynamic() || !ObjectUtils.equals(vo.getDescription(), key.description()) || !ObjectUtils.equals(vo.getDefaultValue(), key.defaultValue()) ||
@ -344,4 +346,9 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
return new Pair<>(groupId, subGroupId);
}
@Override
public boolean isNewConfig(ConfigKey<?> configKey) {
return newConfigs.contains(configKey.key());
}
}

17
framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java
View File

@ -18,9 +18,14 @@
//
package org.apache.cloudstack.framework.config.impl;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.test.util.ReflectionTestUtils;
public class ConfigDepotImplTest {
@ -40,4 +45,16 @@ public class ConfigDepotImplTest {
}
}
@Test
public void testIsNewConfig() {
String validNewConfigKey = "CONFIG";
ConfigKey<Boolean> validNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG", "true", "", true);
ConfigKey<Boolean> invalidNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG1", "true", "", true);
Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
newConfigs.add(validNewConfigKey);
ReflectionTestUtils.setField(configDepotImpl, "newConfigs", newConfigs);
Assert.assertTrue(configDepotImpl.isNewConfig(validNewConfig));
Assert.assertFalse(configDepotImpl.isNewConfig(invalidNewConfig));
}
}

4
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
View File

@ -40,9 +40,9 @@ public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, Chec
logger.info(String.format("Checking URL: %s, with connect timeout: %d, connect request timeout: %d, socket timeout: %d", url, connectTimeout, connectionRequestTimeout, socketTimeout));
Long remoteSize = null;
boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout);
boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout, cmd.isFollowRedirects());
if (checkResult) {
remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout);
remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout, cmd.isFollowRedirects());
if (remoteSize == null || remoteSize < 0) {
logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
"url %s, obtained size = %s", url, remoteSize));

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -2381,7 +2381,7 @@ public class KVMStorageProcessor implements StorageProcessor {
Long templateSize = null;
if (StringUtils.isNotBlank(cmd.getUrl())) {
String url = cmd.getUrl();
templateSize = UriUtils.getRemoteSize(url);
templateSize = UriUtils.getRemoteSize(url, cmd.isFollowRedirects());
}
logger.debug("Checking for free space on the host for downloading the template with physical size: " + templateSize + " and virtual size: " + cmd.getTemplateSize());

7
server/src/main/java/com/cloud/api/ApiResponseHelper.java
View File

@ -4853,12 +4853,7 @@ public class ApiResponseHelper implements ResponseGenerator {
@Override
public UserDataResponse createUserDataResponse(UserData userData) {
UserDataResponse response = new UserDataResponse(userData.getUuid(), userData.getName(), userData.getUserData(), userData.getParams());
Account account = ApiDBUtils.findAccountById(userData.getAccountId());
response.setAccountId(account.getUuid());
response.setAccountName(account.getAccountName());
Domain domain = ApiDBUtils.findDomainById(userData.getDomainId());
response.setDomainId(domain.getUuid());
response.setDomainName(domain.getName());
populateOwner(response, userData);
response.setHasAnnotation(annotationDao.hasAnnotations(userData.getUuid(), AnnotationService.EntityType.USER_DATA.name(),
_accountMgr.isRootAdmin(CallContext.current().getCallingAccount().getId())));
return response;

40
server/src/main/java/com/cloud/api/ApiServer.java
View File

@ -233,42 +233,42 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
@Inject
private MessageBus messageBus;
private static final ConfigKey<Integer> IntegrationAPIPort = new ConfigKey<Integer>("Advanced"
private static final ConfigKey<Integer> IntegrationAPIPort = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
, Integer.class
, "integration.api.port"
, "0"
, "Integration (unauthenticated) API port. To disable set it to 0 or negative."
, false
, ConfigKey.Scope.Global);
private static final ConfigKey<Long> ConcurrentSnapshotsThresholdPerHost = new ConfigKey<Long>("Advanced"
private static final ConfigKey<Long> ConcurrentSnapshotsThresholdPerHost = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
, Long.class
, "concurrent.snapshots.threshold.perhost"
, null
, "Limits number of snapshots that can be handled by the host concurrently; default is NULL - unlimited"
, true // not sure if this is to be dynamic
, ConfigKey.Scope.Global);
private static final ConfigKey<Boolean> EncodeApiResponse = new ConfigKey<Boolean>("Advanced"
private static final ConfigKey<Boolean> EncodeApiResponse = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
, Boolean.class
, "encode.api.response"
, "false"
, "Do URL encoding for the api response, false by default"
, false
, ConfigKey.Scope.Global);
static final ConfigKey<String> JSONcontentType = new ConfigKey<String>( "Advanced"
static final ConfigKey<String> JSONcontentType = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
, String.class
, "json.content.type"
, "application/json; charset=UTF-8"
, "Http response content type for .js files (default is text/javascript)"
, false
, ConfigKey.Scope.Global);
static final ConfigKey<Boolean> EnableSecureSessionCookie = new ConfigKey<Boolean>("Advanced"
static final ConfigKey<Boolean> EnableSecureSessionCookie = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
, Boolean.class
, "enable.secure.session.cookie"
, "false"
, "Session cookie is marked as secure if this is enabled. Secure cookies only work when HTTPS is used."
, false
, ConfigKey.Scope.Global);
private static final ConfigKey<String> JSONDefaultContentType = new ConfigKey<String> ("Advanced"
private static final ConfigKey<String> JSONDefaultContentType = new ConfigKey<> (ConfigKey.CATEGORY_ADVANCED
, String.class
, "json.content.type"
, "application/json; charset=UTF-8"
@ -276,13 +276,34 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
, false
, ConfigKey.Scope.Global);
private static final ConfigKey<Boolean> UseEventAccountInfo = new ConfigKey<Boolean>( "advanced"
private static final ConfigKey<Boolean> UseEventAccountInfo = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED
, Boolean.class
, "event.accountinfo"
, "false"
, "use account info in event logging"
, true
, ConfigKey.Scope.Global);
static final ConfigKey<Boolean> useForwardHeader = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK
, Boolean.class
, "proxy.header.verify"
, "false"
, "enables/disables checking of ipaddresses from a proxy set header. See \"proxy.header.names\" for the headers to allow."
, true
, ConfigKey.Scope.Global);
static final ConfigKey<String> listOfForwardHeaders = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK
, String.class
, "proxy.header.names"
, "X-Forwarded-For,HTTP_CLIENT_IP,HTTP_X_FORWARDED_FOR"
, "a list of names to check for allowed ipaddresses from a proxy set header. See \"proxy.cidr\" for the proxies allowed to set these headers."
, true
, ConfigKey.Scope.Global);
static final ConfigKey<String> proxyForwardList = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK
, String.class
, "proxy.cidr"
, ""
, "a list of cidrs for which \"proxy.header.names\" are honoured if the \"Remote_Addr\" is in this list."
, true
, ConfigKey.Scope.Global);
@Override
public boolean configure(final String name, final Map<String, Object> params) throws ConfigurationException {
@ -1500,7 +1521,10 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
ConcurrentSnapshotsThresholdPerHost,
EncodeApiResponse,
EnableSecureSessionCookie,
JSONDefaultContentType
JSONDefaultContentType,
proxyForwardList,
useForwardHeader,
listOfForwardHeaders
};
}
}

35
server/src/main/java/com/cloud/api/ApiServlet.java
View File

@ -70,7 +70,6 @@ import com.cloud.utils.db.EntityManager;
import com.cloud.utils.net.NetUtils;
@Component("apiServlet")
@SuppressWarnings("serial")
public class ApiServlet extends HttpServlet {
protected static Logger LOGGER = LogManager.getLogger(ApiServlet.class);
private final static List<String> s_clientAddressHeaders = Collections
@ -570,17 +569,39 @@ public class ApiServlet extends HttpServlet {
}
return false;
}
boolean doUseForwardHeaders() {
return Boolean.TRUE.equals(ApiServer.useForwardHeader.value());
}
String[] proxyNets() {
return ApiServer.proxyForwardList.value().split(",");
}
//This method will try to get login IP of user even if servlet is behind reverseProxy or loadBalancer
public static InetAddress getClientAddress(final HttpServletRequest request) throws UnknownHostException {
for(final String header : s_clientAddressHeaders) {
final String ip = getCorrectIPAddress(request.getHeader(header));
if (ip != null) {
public InetAddress getClientAddress(final HttpServletRequest request) throws UnknownHostException {
String ip = null;
InetAddress pretender = InetAddress.getByName(request.getRemoteAddr());
if(doUseForwardHeaders()) {
if (NetUtils.isIpInCidrList(pretender, proxyNets())) {
for (String header : getClientAddressHeaders()) {
header = header.trim();
ip = getCorrectIPAddress(request.getHeader(header));
if (StringUtils.isNotBlank(ip)) {
LOGGER.debug(String.format("found ip %s in header %s ", ip, header));
break;
}
} // no address found in header so ip is blank and use remote addr
} // else not an allowed proxy address, ip is blank and use remote addr
}
if (StringUtils.isBlank(ip)) {
LOGGER.trace(String.format("no ip found in headers, returning remote address %s.", pretender.getHostAddress()));
return pretender;
}
return InetAddress.getByName(ip);
}
}
return InetAddress.getByName(request.getRemoteAddr());
private String[] getClientAddressHeaders() {
return ApiServer.listOfForwardHeaders.value().split(",");
}
private static String getCorrectIPAddress(String ip) {

10
server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
View File

@ -205,6 +205,7 @@ import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.host.dao.HostTagsDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
import com.cloud.network.IpAddress;
import com.cloud.network.IpAddressManager;
import com.cloud.network.Ipv6GuestPrefixSubnetNetworkMapVO;
@ -3255,6 +3256,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
}
}
if (detailEntry.getKey().startsWith(ApiConstants.EXTRA_CONFIG)) {
validateExtraConfigInServiceOfferingDetail(detailEntry.getKey());
try {
detailEntryValue = URLDecoder.decode(detailEntry.getValue(), "UTF-8");
} catch (UnsupportedEncodingException | IllegalArgumentException e) {
@ -3320,6 +3322,14 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
}
}
@Override
public void validateExtraConfigInServiceOfferingDetail(String detailName) {
if (!detailName.equals(DpdkHelper.DPDK_NUMA) && !detailName.equals(DpdkHelper.DPDK_HUGE_PAGES)
&& !detailName.startsWith(DpdkHelper.DPDK_INTERFACE_PREFIX)) {
throw new InvalidParameterValueException("Only extraconfig for DPDK are supported in service offering details");
}
}
private DiskOfferingVO createDiskOfferingInternal(final long userId, final boolean isSystem, final VirtualMachine.Type vmType,
final String name, final Integer cpu, final Integer ramSize, final Integer speed, final String displayText, final ProvisioningType typedProvisioningType, final boolean localStorageRequired,
final boolean offerHA, final boolean limitResourceUse, final boolean volatileVm, String tags, final List<Long> domainIds, List<Long> zoneIds, final String hostTag,

15
server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java
View File

@ -47,6 +47,7 @@ import com.cloud.agent.api.Command;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.NicTO;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.gpu.GPU;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
@ -69,6 +70,7 @@ import com.cloud.utils.Pair;
import com.cloud.utils.component.AdapterBase;
import com.cloud.vm.NicProfile;
import com.cloud.vm.NicVO;
import com.cloud.vm.UserVmManager;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineProfile;
@ -113,6 +115,10 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
@Inject
protected
HostDao hostDao;
@Inject
private UserVmManager userVmManager;
@Inject
private ConfigurationManager configurationManager;
public static ConfigKey<Boolean> VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor = new ConfigKey<Boolean>("Advanced", Boolean.class, "vm.min.memory.equals.memory.divided.by.mem.overprovisioning.factor", "true",
"If we set this to 'true', a minimum memory (memory/ mem.overprovisioning.factor) will be set to the VM, independent of using a scalable service offering or not.", true, ConfigKey.Scope.Cluster);
@ -224,10 +230,12 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
/**
* Add extra configuration from VM details. Extra configuration is stored as details starting with 'extraconfig'
*/
private void addExtraConfig(Map<String, String> details, VirtualMachineTO to) {
private void addExtraConfig(Map<String, String> details, VirtualMachineTO to, long accountId, Hypervisor.HypervisorType hypervisorType) {
for (String key : details.keySet()) {
if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
to.addExtraConfig(key, details.get(key));
String extraConfig = details.get(key);
userVmManager.validateExtraConfig(accountId, hypervisorType, extraConfig);
to.addExtraConfig(key, extraConfig);
}
}
}
@ -243,6 +251,7 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
if (CollectionUtils.isNotEmpty(details)) {
for (ServiceOfferingDetailsVO detail : details) {
if (detail.getName().startsWith(ApiConstants.EXTRA_CONFIG)) {
configurationManager.validateExtraConfigInServiceOfferingDetail(detail.getName());
to.addExtraConfig(detail.getName(), detail.getValue());
}
}
@ -306,7 +315,7 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
Map<String, String> detailsInVm = _userVmDetailsDao.listDetailsKeyPairs(vm.getId());
if (detailsInVm != null) {
to.setDetails(detailsInVm);
addExtraConfig(detailsInVm, to);
addExtraConfig(detailsInVm, to, vm.getAccountId(), vm.getHypervisorType());
}
addServiceOfferingExtraConfiguration(offering, to);

27
server/src/main/java/com/cloud/storage/StorageManagerImpl.java
View File

@ -97,6 +97,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
@ -238,6 +239,7 @@ import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.EntityManager;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.db.JoinBuilder;
@ -380,6 +382,11 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
@Inject
protected BucketDao _bucketDao;
@Inject
ConfigDepot configDepot;
@Inject
ConfigurationDao configurationDao;
protected List<StoragePoolDiscoverer> _discoverers;
public List<StoragePoolDiscoverer> getDiscoverers() {
@ -440,6 +447,21 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
}
protected void enableDefaultDatastoreDownloadRedirectionForExistingInstallations() {
if (!configDepot.isNewConfig(DataStoreDownloadFollowRedirects)) {
logger.trace("{} is not a new configuration, skipping updating its value",
DataStoreDownloadFollowRedirects.key());
return;
}
List<DataCenterVO> zones =
_dcDao.listAll(new Filter(1));
if (CollectionUtils.isNotEmpty(zones)) {
logger.debug(String.format("Updating value for configuration: %s to true",
DataStoreDownloadFollowRedirects.key()));
configurationDao.update(DataStoreDownloadFollowRedirects.key(), "true");
}
}
@Override
public List<StoragePoolVO> ListByDataCenterHypervisor(long datacenterId, HypervisorType type) {
List<StoragePoolVO> pools = _storagePoolDao.listByDataCenterId(datacenterId);
@ -671,7 +693,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
_executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS);
enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
return true;
}
@ -3764,7 +3786,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
MountDisabledStoragePool,
VmwareCreateCloneFull,
VmwareAllowParallelExecution,
ConvertVmwareInstanceToKvmTimeout
ConvertVmwareInstanceToKvmTimeout,
DataStoreDownloadFollowRedirects
};
}

12
server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
View File

@ -554,12 +554,14 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
throw new InvalidParameterValueException("File:// type urls are currently unsupported");
}
UriUtils.validateUrl(format, url);
boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access
logger.debug("Checking url: " + url);
DirectDownloadHelper.checkUrlExistence(url);
DirectDownloadHelper.checkUrlExistence(url, followRedirects);
}
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage,
UriUtils.getRemoteSize(url, followRedirects));
} else {
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage);
}
@ -660,8 +662,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
_resourceLimitMgr.incrementVolumeResourceCount(volume.getAccountId(), true, null, diskOfferingVO);
//url can be null incase of postupload
if (url != null) {
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
volume.setSize(UriUtils.getRemoteSize(url));
long remoteSize = UriUtils.getRemoteSize(url, StorageManager.DataStoreDownloadFollowRedirects.value());
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage,
remoteSize);
volume.setSize(remoteSize);
}
return volume;

19
server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
View File

@ -88,6 +88,7 @@ import com.cloud.server.StatsCollector;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.TemplateProfile;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateVO;
@ -159,7 +160,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
* @param url url
*/
private Long performDirectDownloadUrlValidation(final String format, final Hypervisor.HypervisorType hypervisor,
final String url, final List<Long> zoneIds) {
final String url, final List<Long> zoneIds, final boolean followRedirects) {
HostVO host = null;
if (zoneIds != null && !zoneIds.isEmpty()) {
for (Long zoneId : zoneIds) {
@ -178,7 +179,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
Integer socketTimeout = DirectDownloadManager.DirectDownloadSocketTimeout.value();
Integer connectRequestTimeout = DirectDownloadManager.DirectDownloadConnectionRequestTimeout.value();
Integer connectTimeout = DirectDownloadManager.DirectDownloadConnectTimeout.value();
CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout);
CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout, followRedirects);
logger.debug("Performing URL " + url + " validation on host " + host.getId());
Answer answer = _agentMgr.easySend(host.getId(), cmd);
if (answer == null || !answer.getResult()) {
@ -202,6 +203,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url);
boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (cmd.isDirectDownload()) {
DigestHelper.validateChecksumString(cmd.getChecksum());
List<Long> zoneIds = null;
@ -210,12 +212,14 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
zoneIds.add(cmd.getZoneId());
}
Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(),
Hypervisor.HypervisorType.KVM, url, zoneIds);
Hypervisor.HypervisorType.KVM, url, zoneIds, followRedirects);
profile.setSize(templateSize);
}
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
ResourceType.secondary_storage,
UriUtils.getRemoteSize(url, followRedirects));
return profile;
}
@ -234,15 +238,18 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
String url = profile.getUrl();
UriUtils.validateUrl(cmd.getFormat(), url, cmd.isDirectDownload());
Hypervisor.HypervisorType hypervisor = Hypervisor.HypervisorType.getType(cmd.getHypervisor());
boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (cmd.isDirectDownload()) {
DigestHelper.validateChecksumString(cmd.getChecksum());
Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(),
hypervisor, url, cmd.getZoneIds());
hypervisor, url, cmd.getZoneIds(), followRedirects);
profile.setSize(templateSize);
}
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
ResourceType.secondary_storage,
UriUtils.getRemoteSize(url, followRedirects));
return profile;
}

3
server/src/main/java/com/cloud/vm/UserVmManager.java
View File

@ -30,6 +30,7 @@ import com.cloud.exception.ManagementServerException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.exception.VirtualMachineMigrationException;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.storage.Storage.StoragePoolType;
@ -96,6 +97,8 @@ public interface UserVmManager extends UserVmService {
String validateUserData(String userData, HTTPMethod httpmethod);
void validateExtraConfig(long accountId, HypervisorType hypervisorType, String extraConfig);
boolean isVMUsingLocalStorage(VMInstanceVO vm);
boolean expunge(UserVmVO vm);

41
server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
View File

@ -658,7 +658,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
"enable.additional.vm.configuration", "false", "allow additional arbitrary configuration to vm", true, ConfigKey.Scope.Account);
private static final ConfigKey<String> KvmAdditionalConfigAllowList = new ConfigKey<>(String.class,
"allow.additional.vm.configuration.list.kvm", "Advanced", "", "Comma separated list of allowed additional configuration options.", true, ConfigKey.Scope.Global, null, null, EnableAdditionalVmConfig.key(), null, null, ConfigKey.Kind.CSV, null);
"allow.additional.vm.configuration.list.kvm", "Advanced", "", "Comma separated list of allowed additional configuration options.", true, ConfigKey.Scope.Account, null, null, EnableAdditionalVmConfig.key(), null, null, ConfigKey.Kind.CSV, null);
private static final ConfigKey<String> XenServerAdditionalConfigAllowList = new ConfigKey<>(String.class,
"allow.additional.vm.configuration.list.xenserver", "Advanced", "", "Comma separated list of allowed additional configuration options", true, ConfigKey.Scope.Global, null, null, EnableAdditionalVmConfig.key(), null, null, ConfigKey.Kind.CSV, null);
@ -2848,14 +2848,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
if (cleanupDetails){
if (caller != null && caller.getType() == Account.Type.ADMIN) {
for (final UserVmDetailVO detail : existingDetails) {
if (detail != null && detail.isDisplay()) {
if (detail != null && detail.isDisplay() && !isExtraConfig(detail.getName())) {
userVmDetailsDao.removeDetail(id, detail.getName());
}
}
} else {
for (final UserVmDetailVO detail : existingDetails) {
if (detail != null && !userDenyListedSettings.contains(detail.getName())
&& !userReadOnlySettings.contains(detail.getName()) && detail.isDisplay()) {
&& !userReadOnlySettings.contains(detail.getName()) && detail.isDisplay()
&& !isExtraConfig(detail.getName())) {
userVmDetailsDao.removeDetail(id, detail.getName());
}
}
@ -2866,6 +2867,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
throw new InvalidParameterValueException("'extraconfig' should not be included in details as key");
}
details.entrySet().removeIf(detail -> isExtraConfig(detail.getKey()));
if (caller != null && caller.getType() != Account.Type.ADMIN) {
// Ensure denied or read-only detail is not passed by non-root-admin user
for (final String detailName : details.keySet()) {
@ -2889,7 +2892,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
// ensure details marked as non-displayable are maintained, regardless of admin or not
for (final UserVmDetailVO existingDetail : existingDetails) {
if (!existingDetail.isDisplay()) {
if (!existingDetail.isDisplay() || isExtraConfig(existingDetail.getName())) {
details.put(existingDetail.getName(), existingDetail.getValue());
}
}
@ -2911,6 +2914,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
cmd.getHttpMethod(), cmd.getCustomId(), hostName, cmd.getInstanceName(), securityGroupIdList, cmd.getDhcpOptionsMap());
}
private boolean isExtraConfig(String detailName) {
return detailName != null && detailName.startsWith(ApiConstants.EXTRA_CONFIG);
}
protected void updateDisplayVmFlag(Boolean isDisplayVm, Long id, UserVmVO vmInstance) {
vmInstance.setDisplayVm(isDisplayVm);
@ -6301,7 +6308,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
*/
protected void persistExtraConfigKvm(String decodedUrl, UserVm vm) {
// validate config against denied cfg commands
validateKvmExtraConfig(decodedUrl);
validateKvmExtraConfig(decodedUrl, vm.getAccountId());
String[] extraConfigs = decodedUrl.split("\n\n");
for (String cfg : extraConfigs) {
int i = 1;
@ -6319,6 +6326,18 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
i++;
}
}
/**
* This method is used to validate if extra config is valid
*/
@Override
public void validateExtraConfig(long accountId, HypervisorType hypervisorType, String extraConfig) {
if (!EnableAdditionalVmConfig.valueIn(accountId)) {
throw new CloudRuntimeException("Additional VM configuration is not enabled for this account");
}
if (HypervisorType.KVM.equals(hypervisorType)) {
validateKvmExtraConfig(extraConfig, accountId);
}
}
/**
* This method is called by the persistExtraConfigKvm
@ -6326,8 +6345,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
* controlled by Root admin
* @param decodedUrl string containing xml configuration to be validated
*/
protected void validateKvmExtraConfig(String decodedUrl) {
String[] allowedConfigOptionList = KvmAdditionalConfigAllowList.value().split(",");
protected void validateKvmExtraConfig(String decodedUrl, long accountId) {
String[] allowedConfigOptionList = KvmAdditionalConfigAllowList.valueIn(accountId).split(",");
// Skip allowed keys validation for DPDK
if (!decodedUrl.contains(":")) {
try {
@ -6347,7 +6366,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
}
}
if (!isValidConfig) {
throw new CloudRuntimeException(String.format("Extra config %s is not on the list of allowed keys for KVM hypervisor hosts", currentConfig));
throw new CloudRuntimeException(String.format("Extra config '%s' is not on the list of allowed keys for KVM hypervisor hosts", currentConfig));
}
}
} catch (ParserConfigurationException | IOException | SAXException e) {
@ -6445,6 +6464,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
if (details.containsKey("extraconfig")) {
throw new InvalidParameterValueException("'extraconfig' should not be included in details as key");
}
for (String detailName : details.keySet()) {
if (isExtraConfig(detailName)) {
throw new InvalidParameterValueException("detail name should not start with extraconfig");
}
}
}
}

17
server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
View File

@ -270,7 +270,8 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO();
DownloadProtocol protocol = getProtocolFromUrl(url);
DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum,
headers);
cmd.setTemplateSize(template.getSize());
cmd.setFormat(template.getFormat());
@ -391,19 +392,23 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
/**
* Return DirectDownloadCommand according to the protocol
*/
private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
String checksum, Map<String, String> httpHeaders) {
private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url,
Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> httpHeaders) {
int connectTimeout = DirectDownloadConnectTimeout.value();
int soTimeout = DirectDownloadSocketTimeout.value();
int connectionRequestTimeout = DirectDownloadConnectionRequestTimeout.value();
boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (protocol.equals(DownloadProtocol.HTTP)) {
return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
soTimeout, followRedirects);
} else if (protocol.equals(DownloadProtocol.HTTPS)) {
return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout, connectionRequestTimeout);
return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
soTimeout, connectionRequestTimeout, followRedirects);
} else if (protocol.equals(DownloadProtocol.NFS)) {
return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
} else if (protocol.equals(DownloadProtocol.METALINK)) {
return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
soTimeout, followRedirects);
} else {
return null;
}

29
server/src/test/java/com/cloud/api/ApiServletTest.java
View File

@ -97,15 +97,17 @@ public class ApiServletTest {
@Mock
AccountService accountMgr;
@Mock ConfigKey<Boolean> useForwardHeader;
StringWriter responseWriter;
ApiServlet servlet;
ApiServlet spyServlet;
@SuppressWarnings("unchecked")
@Before
public void setup() throws SecurityException, NoSuchFieldException,
IllegalArgumentException, IllegalAccessException, IOException, UnknownHostException {
servlet = new ApiServlet();
spyServlet = Mockito.spy(servlet);
responseWriter = new StringWriter();
Mockito.when(response.getWriter()).thenReturn(
new PrintWriter(responseWriter));
@ -259,32 +261,43 @@ public class ApiServletTest {
@Test
public void getClientAddressWithXForwardedFor() throws UnknownHostException {
String[] proxynet = {"127.0.0.0/8"};
Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
Mockito.when(request.getHeader(Mockito.eq("X-Forwarded-For"))).thenReturn("192.168.1.1");
Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request));
}
@Test
public void getClientAddressWithHttpXForwardedFor() throws UnknownHostException {
String[] proxynet = {"127.0.0.0/8"};
Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
Mockito.when(request.getHeader(Mockito.eq("HTTP_X_FORWARDED_FOR"))).thenReturn("192.168.1.1");
Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request));
}
@Test
public void getClientAddressWithXRemoteAddr() throws UnknownHostException {
Mockito.when(request.getHeader(Mockito.eq("Remote_Addr"))).thenReturn("192.168.1.1");
Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
public void getClientAddressWithRemoteAddr() throws UnknownHostException {
String[] proxynet = {"127.0.0.0/8"};
Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
Assert.assertEquals(InetAddress.getByName("127.0.0.1"), spyServlet.getClientAddress(request));
}
@Test
public void getClientAddressWithHttpClientIp() throws UnknownHostException {
String[] proxynet = {"127.0.0.0/8"};
Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
Mockito.when(request.getHeader(Mockito.eq("HTTP_CLIENT_IP"))).thenReturn("192.168.1.1");
Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request));
}
@Test
public void getClientAddressDefault() throws UnknownHostException {
Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1");
Assert.assertEquals(InetAddress.getByName("127.0.0.1"), ApiServlet.getClientAddress(request));
Assert.assertEquals(InetAddress.getByName("127.0.0.1"), spyServlet.getClientAddress(request));
}
@Test

42
server/src/test/java/com/cloud/storage/StorageManagerImplTest.java
View File

@ -23,6 +23,8 @@ import java.util.List;
import java.util.Map;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao;
import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
@ -42,7 +44,9 @@ import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.capacity.CapacityManager;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.VsphereStoragePolicyVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.dc.dao.VsphereStoragePolicyDao;
import com.cloud.exception.AgentUnavailableException;
import com.cloud.exception.ConnectionException;
@ -77,6 +81,12 @@ public class StorageManagerImplTest {
HypervisorGuruManager hvGuruMgr;
@Mock
AgentManager agentManager;
@Mock
ConfigDepot configDepot;
@Mock
ConfigurationDao configurationDao;
@Mock
DataCenterDao dataCenterDao;
@Spy
@InjectMocks
@ -455,4 +465,36 @@ public class StorageManagerImplTest {
storageManagerImpl.getCheckDatastorePolicyComplianceAnswer("1", pool);
Assert.assertTrue(answer.getResult());
}
@Test
public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNoChange() {
Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
.thenReturn(false);
storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
Mockito.verify(configurationDao, Mockito.never()).update(Mockito.anyString(), Mockito.anyString());
}
@Test
public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsOldInstall() {
Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
.thenReturn(true);
Mockito.when(dataCenterDao.listAll(Mockito.any()))
.thenReturn(List.of(Mockito.mock(DataCenterVO.class)));
Mockito.doReturn(true).when(configurationDao).update(Mockito.anyString(), Mockito.anyString());
storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
Mockito.verify(configurationDao, Mockito.times(1))
.update(StorageManager.DataStoreDownloadFollowRedirects.key(), "true");
}
@Test
public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNewInstall() {
Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
.thenReturn(true);
Mockito.when(dataCenterDao.listAll(Mockito.any()))
.thenReturn(new ArrayList<>()); //new installation
storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
Mockito.verify(configurationDao, Mockito.never())
.update(StorageManager.DataStoreDownloadFollowRedirects.key(),StorageManager.DataStoreDownloadFollowRedirects.defaultValue());
}
}

5
server/src/test/java/com/cloud/vpc/MockConfigurationManagerImpl.java
View File

@ -677,4 +677,9 @@ public class MockConfigurationManagerImpl extends ManagerBase implements Configu
// TODO Auto-generated method stub
return null;
}
@Override
public void validateExtraConfigInServiceOfferingDetail(String detailName) {
// TODO Auto-generated method stub
}
}

16
services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java
View File

@ -41,17 +41,21 @@ public interface DownloadManager extends Manager {
* @param hvm whether the template is a hardware virtual machine
* @param accountId the accountId of the iso owner (null if public iso)
* @param descr description of the template
* @param user username used for authentication to the server
* @param password password used for authentication to the server
* @param userName username used for authentication to the server
* @param passwd password used for authentication to the server
* @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes.
* @param resourceType signifying the type of resource like template, volume etc.
* @param followRedirects whether downloader follows redirections
* @return job-id that can be used to interrogate the status of the download.
*/
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
String installPathPrefix, String templatePath, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType);
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
Long accountId, String descr, String cksum, String installPathPrefix, String templatePath,
String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType,
boolean followRedirects);
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType);
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects);
Map<String, Processor> getProcessors();

25
services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
View File

@ -662,8 +662,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
@Override
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects) {
UUID uuid = UUID.randomUUID();
String jobId = uuid.toString();
@ -683,6 +684,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
} else {
throw new CloudRuntimeException("Unable to download from URL: " + url);
}
td.setFollowRedirects(followRedirects);
DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
dj.setTmpltPath(installPathPrefix);
jobs.put(jobId, dj);
@ -718,8 +720,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
@Override
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user,
String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType,
boolean followRedirects) {
UUID uuid = UUID.randomUUID();
String jobId = uuid.toString();
String tmpDir = installPathPrefix;
@ -766,6 +770,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
throw new CloudRuntimeException("Unable to download from URL: " + url);
}
}
td.setFollowRedirects(followRedirects);
// NOTE the difference between installPathPrefix and templatePath
// here. instalPathPrefix is the absolute path for template
// including mount directory
@ -902,12 +907,16 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
String jobId = null;
if (dstore instanceof S3TO) {
jobId =
downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(),
cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(),
installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
cmd.isFollowRedirects());
} else {
jobId =
downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(),
cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix,
cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
cmd.isFollowRedirects());
}
sleep();
if (jobId == null) {

1
ui/public/locales/en.json
View File

@ -13,6 +13,7 @@
"error.release.dedicate.host": "Failed to release dedicated host.",
"error.release.dedicate.pod": "Failed to release dedicated pod.",
"error.release.dedicate.zone": "Failed to release dedicated zone.",
"error.unable.to.add.setting.extraconfig": "It is not allowed to add setting for extraconfig. Please update VirtualMachine with extraconfig parameter.",
"error.unable.to.proceed": "Unable to proceed. Please contact your administrator.",
"firewall.close": "Firewall",
"icmp.code.desc": "Please specify -1 if you want to allow all ICMP codes (except NSX zones).",

13
ui/src/components/view/DetailSettings.vue
View File

@ -101,7 +101,7 @@
<tooltip-button
:tooltip="$t('label.edit')"
icon="edit-outlined"
:disabled="deployasistemplate === true"
:disabled="deployasistemplate === true || item.name.startsWith('extraconfig')"
v-if="!item.edit"
@onClick="showEditDetail(index)" />
</div>
@ -115,7 +115,12 @@
:cancelText="$t('label.no')"
placement="left"
>
<tooltip-button :tooltip="$t('label.delete')" :disabled="deployasistemplate === true" type="primary" :danger="true" icon="delete-outlined" />
<tooltip-button
:tooltip="$t('label.delete')"
:disabled="deployasistemplate === true || item.name.startsWith('extraconfig')"
type="primary"
:danger="true"
icon="delete-outlined" />
</a-popconfirm>
</div>
</template>
@ -307,6 +312,10 @@ export default {
this.error = this.$t('message.error.provide.setting')
return
}
if (this.newKey.startsWith('extraconfig')) {
this.error = this.$t('error.unable.to.add.setting.extraconfig')
return
}
if (!this.allowEditOfDetail(this.newKey)) {
this.error = this.$t('error.unable.to.proceed')
return

5
ui/src/config/section/compute.js
View File

@ -898,7 +898,12 @@ export default {
var fields = ['name', 'id']
if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) {
fields.push('account')
if (store.getters.listAllProjects) {
fields.push('project')
}
fields.push('domain')
} else if (store.getters.listAllProjects) {
fields.push('project')
}
return fields
},

2
ui/src/views/AutogenView.vue
View File

@ -793,7 +793,7 @@ export default {
}
this.projectView = Boolean(store.getters.project && store.getters.project.id)
this.hasProjectId = ['vm', 'vmgroup', 'ssh', 'affinitygroup', 'volume', 'snapshot', 'vmsnapshot', 'guestnetwork',
this.hasProjectId = ['vm', 'vmgroup', 'ssh', 'affinitygroup', 'userdata', 'volume', 'snapshot', 'vmsnapshot', 'guestnetwork',
'vpc', 'securitygroups', 'publicip', 'vpncustomergateway', 'template', 'iso', 'event', 'kubernetes',
'autoscalevmgroup', 'vnfapp'].includes(this.$route.name)

8
ui/src/views/network/GuestIpRanges.vue
View File

@ -36,14 +36,16 @@
:rowKey="item => item.id"
:pagination="false" >
<template #action="{ record }">
<template #bodyCell="{ column, record }">
<tooltip-button
v-if="column.key === 'actions'"
tooltipPlacement="bottom"
:tooltip="$t('label.edit')"
type="primary"
@click="() => { handleUpdateIpRangeModal(record) }"
icon="swap-outlined" />
<a-popconfirm
v-if="column.key === 'actions'"
:title="$t('message.confirm.remove.ip.range')"
@confirm="removeIpRange(record.id)"
:okText="$t('label.yes')"
@ -226,6 +228,10 @@ export default {
},
removeIpRange (id) {
api('deleteVlanIpRange', { id: id }).then(json => {
const message = `${this.$t('message.success.delete')} ${this.$t('label.ip.range')}`
this.$message.success(message)
}).catch((error) => {
this.$notifyError(error)
}).finally(() => {
this.fetchData()
})

170
ui/src/views/network/LoadBalancing.vue
View File

@ -84,12 +84,18 @@
<a-select-option value="no">{{ $t('label.no') }}</a-select-option>
</a-select>
</div>
<div class="form__item" v-if="!newRule.autoscale || newRule.autoscale === 'no' || ('vpcid' in this.resource && !('associatednetworkid' in this.resource))">
<div class="form__item" v-if="!newRule.autoscale || newRule.autoscale === 'no'">
<div class="form__label" style="white-space: nowrap;">{{ $t('label.add.vms') }}</div>
<a-button :disabled="!('createLoadBalancerRule' in $store.getters.apis)" type="primary" @click="handleOpenAddVMModal">
{{ $t('label.add') }}
</a-button>
</div>
<div class="form__item" v-else-if="newRule.autoscale === 'yes' && ('vpcid' in this.resource && !this.associatednetworkid)">
<div class="form__label" style="white-space: nowrap;">{{ $t('label.select.tier') }}</div>
<a-button :disabled="!('createLoadBalancerRule' in $store.getters.apis)" type="primary" @click="handleOpenAddNetworkModal">
{{ $t('label.add') }}
</a-button>
</div>
<div class="form__item" v-else-if="newRule.autoscale === 'yes'">
<div class="form__label" style="white-space: nowrap;">{{ $t('label.add') }}</div>
<a-button :disabled="!('createLoadBalancerRule' in $store.getters.apis)" type="primary" @click="handleAddNewRule">
@ -519,7 +525,7 @@
</template>
<template v-if="column.key === 'actions'" style="text-align: center" :text="text">
<a-checkbox v-model:value="record.id" @change="e => fetchNics(e, index)" :disabled="newRule.autoscale"/>
<a-checkbox v-model:value="record.id" @change="e => fetchNics(e, index)" />
</template>
</template>
</a-table>
@ -546,6 +552,71 @@
</div>
</a-modal>
<a-modal
:title="$t('label.select.tier')"
:maskClosable="false"
:closable="true"
v-if="addNetworkModalVisible"
:visible="addNetworkModalVisible"
class="network-modal"
width="60vw"
:footer="null"
@cancel="closeModal"
>
<div @keyup.ctrl.enter="handleAddNewRule">
<a-input-search
v-focus="!('vpcid' in resource && !('associatednetworkid' in resource))"
class="input-search"
:placeholder="$t('label.search')"
v-model:value="searchQuery"
allowClear
@search="onNetworkSearch" />
<a-table
size="small"
class="list-view"
:loading="addNetworkModalLoading"
:columns="networkColumns"
:dataSource="networks"
:pagination="false"
:rowKey="record => record.id"
:scroll="{ y: 300 }">
<template #bodyCell="{ column, record }">
<template v-if="column.key === 'actions'">
<div style="text-align: center">
<a-radio-group
class="radio-group"
:key="record.id"
v-model:value="this.selectedTierForAutoScaling"
@change="($event) => this.selectedTierForAutoScaling = $event.target.value">
<a-radio :value="record.id" />
</a-radio-group>
</div>
</template>
</template>
</a-table>
<a-pagination
class="pagination"
size="small"
:current="networkPage"
:pageSize="networkPageSize"
:total="networkCount"
:showTotal="total => `${$t('label.total')} ${total} ${$t('label.items')}`"
:pageSizeOptions="['10', '20', '40', '80', '100']"
@change="handleChangeNetworkPage"
@showSizeChange="handleChangeNetworkPageSize"
showSizeChanger>
<template #buildOptionText="props">
<span>{{ props.value }} / {{ $t('label.page') }}</span>
</template>
</a-pagination>
<div :span="24" class="action-button">
<a-button @click="closeModal">{{ $t('label.cancel') }}</a-button>
<a-button :disabled="this.selectedTierForAutoScaling === null" type="primary" ref="submit" @click="handleAddNewRule">{{ $t('label.ok') }}</a-button>
</div>
</div>
</a-modal>
<a-modal
v-if="healthMonitorModal"
:title="$t('label.configure.health.monitor')"
@ -802,6 +873,41 @@ export default {
vmPage: 1,
vmPageSize: 10,
vmCount: 0,
addNetworkModalVisible: false,
addNetworkModalLoading: false,
networks: [],
associatednetworkid: null,
selectedTierForAutoScaling: null,
networkColumns: [
{
key: 'name',
title: this.$t('label.name'),
dataIndex: 'name',
width: 220
},
{
key: 'state',
title: this.$t('label.state'),
dataIndex: 'state'
},
{
title: this.$t('label.gateway'),
dataIndex: 'gateway'
},
{
title: this.$t('label.netmask'),
dataIndex: 'netmask'
},
{
key: 'actions',
title: this.$t('label.select'),
dataIndex: 'actions',
width: 80
}
],
networkPage: 1,
networkPageSize: 10,
networkCount: 0,
searchQuery: null,
tungstenHealthMonitors: [],
healthMonitorModal: false,
@ -825,6 +931,9 @@ export default {
beforeCreate () {
this.createLoadBalancerRuleParams = this.$getApiParams('createLoadBalancerRule')
this.createLoadBalancerStickinessPolicyParams = this.$getApiParams('createLBStickinessPolicy')
if ('associatednetworkid' in this.resource) {
this.associatednetworkid = this.resource.associatednetworkid
}
},
created () {
this.initForm()
@ -1489,6 +1598,55 @@ export default {
this.addVmModalLoading = false
})
},
handleOpenAddNetworkModal () {
if (this.addNetworkModalLoading) return
if (!this.checkNewRule()) {
return
}
this.addNetworkModalVisible = true
this.fetchNetworks()
},
fetchNetworks () {
this.networkCount = 0
this.networks = []
this.addNetworkModalLoading = true
const vpcid = this.resource.vpcid
if (!vpcid) {
this.addNetworkModalLoading = false
return
}
api('listNetworks', {
listAll: true,
keyword: this.searchQuery,
page: this.networkPage,
pagesize: this.networkPageSize,
supportedservices: 'Lb',
isrecursive: true,
vpcid: vpcid
}).then(response => {
this.networkCount = response.listnetworksresponse.count || 0
this.networks = response.listnetworksresponse.network || []
}).catch(error => {
this.$notifyError(error)
}).finally(() => {
this.addNetworkModalLoading = false
})
this.selectedTierForAutoScaling = null
},
onNetworkSearch (value) {
this.searchQuery = value
this.fetchNetworks()
},
handleChangeNetworkPage (page, pageSize) {
this.networkPage = page
this.networkPageSize = pageSize
this.fetchNetworks()
},
handleChangeNetworkPageSize (currentPage, pageSize) {
this.networkPage = currentPage
this.networkPageSize = pageSize
this.fetchNetworks()
},
handleAssignToLBRule (data) {
const vmIDIpMap = {}
@ -1560,7 +1718,8 @@ export default {
return
}
const networkId = ('vpcid' in this.resource && !('associatednetworkid' in this.resource)) ? this.selectedTier : this.resource.associatednetworkid
const networkId = this.selectedTierForAutoScaling != null ? this.selectedTierForAutoScaling
: ('vpcid' in this.resource && !('associatednetworkid' in this.resource)) ? this.selectedTier : this.resource.associatednetworkid
api('createLoadBalancerRule', {
openfirewall: false,
networkid: networkId,
@ -1573,7 +1732,9 @@ export default {
cidrlist: this.newRule.cidrlist
}).then(response => {
this.addVmModalVisible = false
this.addNetworkModalVisible = false
this.handleAssignToLBRule(response.createloadbalancerruleresponse.id)
this.associatednetworkid = networkId
}).catch(error => {
this.$notifyError(error)
this.loading = false
@ -1597,6 +1758,9 @@ export default {
this.nics = []
this.addVmModalVisible = false
this.newRule.virtualmachineid = []
this.addNetworkModalLoading = false
this.addNetworkModalVisible = false
this.selectedTierForAutoScaling = null
},
handleChangePage (page, pageSize) {
this.page = page

2
ui/src/views/tools/ImportUnmanagedInstance.vue
View File

@ -713,7 +713,7 @@ export default {
page: 1
})
this.fetchKvmHostsForConversion()
if (this.resource.disk.length > 1) {
if (this.resource?.disk?.length > 1) {
this.updateSelectedRootDisk()
}
},

1
ui/src/views/tools/ManageInstances.vue
View File

@ -1074,6 +1074,7 @@ export default {
this.sourceHypervisor = value
this.sourceActions = this.AllSourceActions.filter(x => x.sourceDestHypervisors[value])
this.form.sourceAction = this.sourceActions[0].name || ''
this.selectedVmwareVcenter = undefined
this.onSelectSourceAction(this.form.sourceAction)
},
onSelectSourceAction (value) {

2
utils/src/main/java/com/cloud/utils/StringUtils.java
View File

@ -31,7 +31,7 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class StringUtils {
public class StringUtils extends org.apache.commons.lang3.StringUtils {
private static final char[] hexChar = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
private static Charset preferredACSCharset;

3
utils/src/main/java/com/cloud/utils/UriUtils.java
View File

@ -214,7 +214,7 @@ public class UriUtils {
}
// Get the size of a file from URL response header.
public static long getRemoteSize(String url) {
public static long getRemoteSize(String url, Boolean followRedirect) {
long remoteSize = 0L;
final String[] methods = new String[]{"HEAD", "GET"};
IllegalArgumentException exception = null;
@ -229,6 +229,7 @@ public class UriUtils {
httpConn.setRequestMethod(method);
httpConn.setConnectTimeout(2000);
httpConn.setReadTimeout(5000);
httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect));
String contentLength = httpConn.getHeaderField("content-length");
if (contentLength != null) {
remoteSize = Long.parseLong(contentLength);

5
utils/src/main/java/com/cloud/utils/script/OutputInterpreter.java
View File

@ -139,6 +139,11 @@ public abstract class OutputInterpreter {
public String getLines() {
return allLines;
}
@Override
public boolean drain() {
return true;
}
}
public static class LineByLineOutputLogger extends OutputInterpreter {

20
utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
View File

@ -22,8 +22,9 @@ package com.cloud.utils.storage;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@ -113,16 +114,23 @@ public final class QCOW2Utils {
}
}
public static long getVirtualSize(String urlStr) {
public static long getVirtualSizeFromUrl(String urlStr, boolean followRedirects) {
HttpURLConnection httpConn = null;
try {
URL url = new URL(urlStr);
return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr));
} catch (MalformedURLException e) {
URI url = new URI(urlStr);
httpConn = (HttpURLConnection)url.toURL().openConnection();
httpConn.setInstanceFollowRedirects(followRedirects);
return getVirtualSize(httpConn.getInputStream(), UriUtils.isUrlForCompressedFile(urlStr));
} catch (URISyntaxException e) {
LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
throw new IllegalArgumentException("Invalid URL: " + urlStr);
} catch (IOException e) {
LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
} finally {
if (httpConn != null) {
httpConn.disconnect();
}
}
}
}

14
utils/src/test/java/com/cloud/utils/ScriptTest.java
View File

@ -112,6 +112,20 @@ public class ScriptTest {
Assert.assertNotNull(value);
}
@Test
public void executeWithOutputInterpreterAllLinesParserLargeOutput() {
Assume.assumeTrue(SystemUtils.IS_OS_LINUX);
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
Script script = new Script("seq");
script.add("-f");
script.add("my text to test cloudstack %g");
script.add("4096"); // AllLinesParser doesn't work with that amount of data
String value = script.execute(parser);
// it is a stack trace in this case as string
Assert.assertNull(value);
Assert.assertEquals(129965, parser.getLines().length());
}
@Test
public void runSimpleBashScriptNotExisting() {
Assume.assumeTrue(SystemUtils.IS_OS_LINUX);