apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '4.18'

Browse Source
This commit is contained in:
Daan Hoogland 2023-07-06 11:24:08 +02:00
commit 2132f46fcb
27 changed files with 1253 additions and 612 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java
View File

@ -1,131 +0,0 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.commons.collections.MapUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.ssl.SSLContexts;
import javax.net.ssl.SSLContext;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Map;
public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader {
private CloseableHttpClient httpsClient;
private HttpUriRequest req;
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, temporaryDownloadPath);
SSLContext sslcontext = null;
try {
sslcontext = getSSLContext();
} catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) {
throw new CloudRuntimeException("Failure getting SSL context for HTTPS downloader: " + e.getMessage());
}
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
.setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
createUriRequest(url, headers);
}
protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
req = new HttpGet(downloadUrl);
if (MapUtils.isNotEmpty(headers)) {
for (String headerKey: headers.keySet()) {
req.setHeader(headerKey, headers.get(headerKey));
}
}
}
private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException {
KeyStore trustStore = KeyStore.getInstance("jks");
FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"));
try {
String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null";
String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties");
String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
trustStore.load(instream, privatePassword.toCharArray());
} finally {
instream.close();
}
return SSLContexts.custom()
.loadTrustMaterial(trustStore, new TrustSelfSignedStrategy())
.build();
}
@Override
public Pair<Boolean, String> downloadTemplate() {
CloseableHttpResponse response;
try {
response = httpsClient.execute(req);
} catch (IOException e) {
throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage());
}
return consumeResponse(response);
}
/**
* Consume response and persist it on getDownloadedFilePath() file
*/
protected Pair<Boolean, String> consumeResponse(CloseableHttpResponse response) {
s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
if (response.getStatusLine().getStatusCode() != 200) {
throw new CloudRuntimeException("Error on HTTPS response");
}
try {
HttpEntity entity = response.getEntity();
InputStream in = entity.getContent();
OutputStream out = new FileOutputStream(getDownloadedFilePath());
IOUtils.copy(in, out);
} catch (Exception e) {
s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage());
return new Pair<>(false, null);
}
return new Pair<>(true, getDownloadedFilePath());
}
}

101
agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java
View File

@ -1,101 +0,0 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader {
private String metalinkUrl;
private List<String> metalinkUrls;
private List<String> metalinkChecksums;
private Random random = new Random();
private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, downloadPath);
metalinkUrl = url;
metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl);
metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl);
if (CollectionUtils.isEmpty(metalinkUrls)) {
throw new CloudRuntimeException("No urls found on metalink file: " + metalinkUrl + ". Not possible to download template " + templateId);
}
setUrl(metalinkUrls.get(0));
s_logger.info("Metalink downloader created, metalink url: " + metalinkUrl + " parsed - " +
metalinkUrls.size() + " urls and " +
(CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found");
}
@Override
public Pair<Boolean, String> downloadTemplate() {
if (StringUtils.isBlank(getUrl())) {
throw new CloudRuntimeException("Download url has not been set, aborting");
}
boolean downloaded = false;
int i = 0;
String downloadDir = getDirectDownloadTempPath(getTemplateId());
do {
if (!isRedownload()) {
setUrl(metalinkUrls.get(i));
}
s_logger.info("Trying to download template from url: " + getUrl());
try {
setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl());
File f = new File(getDownloadedFilePath());
if (f.exists()) {
f.delete();
f.createNewFile();
}
request = createRequest(getUrl(), reqHeaders);
Pair<Boolean, String> downloadResult = super.downloadTemplate();
downloaded = downloadResult.first();
if (downloaded) {
s_logger.info("Successfully downloaded template from url: " + getUrl());
}
} catch (Exception e) {
s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage());
}
i++;
}
while (!downloaded && !isRedownload() && i < metalinkUrls.size());
return new Pair<>(downloaded, getDownloadedFilePath());
}
@Override
public boolean validateChecksum() {
if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) {
String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size()));
setChecksum(chk);
s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk);
}
return super.validateChecksum();
}
}

9
core/src/main/java/com/cloud/storage/JavaStorageLayer.java
View File

@ -22,8 +22,11 @@ package com.cloud.storage;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -186,8 +189,12 @@ public class JavaStorageLayer implements StorageLayer {
@Override
public File createUniqDir() throws IOException {
String dirName = System.getProperty("java.io.tmpdir");
String subDirNamePrefix = "";
FileAttribute<Set<PosixFilePermission>> perms = PosixFilePermissions
.asFileAttribute(PosixFilePermissions.fromString("rwxrwx---"));
if (dirName != null) {
File dir = new File(dirName);
Path p = Files.createTempDirectory(Path.of(dirName), subDirNamePrefix, perms);
File dir = p.toFile();
if (dir.exists()) {
if (isWorldReadable(dir)) {
if (STD_TMP_DIR_PATH.equals(dir.getAbsolutePath())) {

83
core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java Normal file
View File

@ -0,0 +1,83 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
import org.apache.log4j.Logger;
public class DirectDownloadHelper {
public static final Logger LOGGER = Logger.getLogger(DirectDownloadHelper.class.getName());
/**
* Get direct template downloader from direct download command and destination pool
*/
public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
String destPoolLocalPath,
String temporaryDownloadPath) {
if (cmd instanceof HttpDirectDownloadCommand) {
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
} else if (cmd instanceof HttpsDirectDownloadCommand) {
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
} else if (cmd instanceof NfsDirectDownloadCommand) {
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
} else {
throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}
}
public static boolean checkUrlExistence(String url) {
try {
DirectTemplateDownloader checker = getCheckerDownloader(url);
return checker.checkUrl(url);
} catch (CloudRuntimeException e) {
LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
return false;
}
}
private static DirectTemplateDownloader getCheckerDownloader(String url) {
if (url.toLowerCase().startsWith("https:")) {
return new HttpsDirectTemplateDownloader(url);
} else if (url.toLowerCase().startsWith("http:")) {
return new HttpDirectTemplateDownloader(url);
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else if (url.toLowerCase().endsWith(".metalink")) {
return new MetalinkDirectTemplateDownloader(url);
} else {
throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url));
}
}
public static Long getFileSize(String url, String format) {
DirectTemplateDownloader checker = getCheckerDownloader(url);
return checker.getRemoteFileSize(url, format);
}
}

61
core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,61 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import com.cloud.utils.Pair;
import java.util.List;
public interface DirectTemplateDownloader {
/**
* Perform template download to pool specified on downloader creation
* @return (true if successful, false if not, download file path)
*/
Pair<Boolean, String> downloadTemplate();
/**
* Perform checksum validation of previously downloaded template
* @return true if successful, false if not
*/
boolean validateChecksum();
/**
* Validate if the URL is reachable and returns HTTP.OK status code
* @return true if the URL is reachable, false if not
*/
boolean checkUrl(String url);
/**
* Obtain the remote file size (and virtual size in case format is qcow2)
*/
Long getRemoteFileSize(String url, String format);
/**
* Get list of urls within metalink content ordered by ascending priority
* (for those which priority tag is not defined, highest priority value is assumed)
*/
List<String> getMetalinkUrls(String metalinkUrl);
/**
* Get the list of checksums within a metalink content
*/
List<String> getMetalinkChecksums(String metalinkUrl);
}

34
agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java → core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java
View File

@ -16,8 +16,9 @@
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
package org.apache.cloudstack.direct.download;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.utils.security.DigestHelper;
import org.apache.commons.lang3.StringUtils;
@ -26,7 +27,11 @@ import org.apache.log4j.Logger;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
@ -106,6 +111,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
return redownload;
}
/**
* Create download directory (if it does not exist)
*/
protected File createTemporaryDirectoryAndFile(String downloadDir) {
createFolder(downloadDir);
return new File(downloadDir + File.separator + getFileNameFromUrl());
}
/**
* Return filename from url
*/
@ -160,4 +173,23 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
}
}
protected void addMetalinkUrlsToListFromInputStream(InputStream inputStream, List<String> urls) {
Map<String, List<String>> metalinkUrlsMap = UriUtils.getMultipleValuesFromXML(inputStream, new String[] {"url"});
if (metalinkUrlsMap.containsKey("url")) {
List<String> metalinkUrls = metalinkUrlsMap.get("url");
urls.addAll(metalinkUrls);
}
}
protected List<String> generateChecksumListFromInputStream(InputStream is) {
Map<String, List<String>> checksums = UriUtils.getMultipleValuesFromXML(is, new String[] {"hash"});
if (checksums.containsKey("hash")) {
List<String> listChksum = new ArrayList<>();
for (String chk : checksums.get("hash")) {
listChksum.add(chk.replaceAll("\n", "").replaceAll(" ", "").trim());
}
return listChksum;
}
return null;
}
}

89
agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java → core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
View File

@ -17,23 +17,28 @@
// under the License.
//
package com.cloud.agent.direct.download;
package org.apache.cloudstack.direct.download;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.storage.QCOW2Utils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;
@ -45,6 +50,10 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected GetMethod request;
protected Map<String, String> reqHeaders = new HashMap<>();
protected HttpDirectTemplateDownloader(String url) {
this(url, null, null, null, null, null, null, null);
}
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, destPoolPath, templateId, checksum, downloadPath);
@ -57,15 +66,6 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
setDownloadedFilePath(tempFile.getAbsolutePath());
}
/**
* Create download directory (if it does not exist) and set the download file
* @return
*/
protected File createTemporaryDirectoryAndFile(String downloadDir) {
createFolder(downloadDir);
return new File(downloadDir + File.separator + getFileNameFromUrl());
}
protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
GetMethod request = new GetMethod(downloadUrl);
request.setFollowRedirects(true);
@ -98,7 +98,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
try (
InputStream in = request.getResponseBodyAsStream();
OutputStream out = new FileOutputStream(getDownloadedFilePath());
OutputStream out = new FileOutputStream(getDownloadedFilePath())
) {
IOUtils.copy(in, out);
} catch (IOException e) {
@ -107,4 +107,71 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
}
return new Pair<>(true, getDownloadedFilePath());
}
@Override
public boolean checkUrl(String url) {
HeadMethod httpHead = new HeadMethod(url);
try {
if (client.executeMethod(httpHead) != HttpStatus.SC_OK) {
s_logger.error(String.format("Invalid URL: %s", url));
return false;
}
return true;
} catch (IOException e) {
s_logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e);
return false;
} finally {
httpHead.releaseConnection();
}
}
@Override
public Long getRemoteFileSize(String url, String format) {
if ("qcow2".equalsIgnoreCase(format)) {
return QCOW2Utils.getVirtualSize(url);
} else {
return UriUtils.getRemoteSize(url);
}
}
@Override
public List<String> getMetalinkUrls(String metalinkUrl) {
GetMethod getMethod = new GetMethod(metalinkUrl);
List<String> urls = new ArrayList<>();
int status;
try {
status = client.executeMethod(getMethod);
} catch (IOException e) {
s_logger.error("Error retrieving urls form metalink: " + metalinkUrl);
getMethod.releaseConnection();
return null;
}
try {
InputStream is = getMethod.getResponseBodyAsStream();
if (status == HttpStatus.SC_OK) {
addMetalinkUrlsToListFromInputStream(is, urls);
}
} catch (IOException e) {
s_logger.warn(e.getMessage());
} finally {
getMethod.releaseConnection();
}
return urls;
}
@Override
public List<String> getMetalinkChecksums(String metalinkUrl) {
GetMethod getMethod = new GetMethod(metalinkUrl);
try {
if (client.executeMethod(getMethod) == HttpStatus.SC_OK) {
InputStream is = getMethod.getResponseBodyAsStream();
return generateChecksumListFromInputStream(is);
}
} catch (IOException e) {
s_logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e);
} finally {
getMethod.releaseConnection();
}
return null;
}
}

252
core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,252 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.QCOW2Utils;
import org.apache.cloudstack.utils.security.SSLUtils;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.commons.collections.MapUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected CloseableHttpClient httpsClient;
private HttpUriRequest req;
protected HttpsDirectTemplateDownloader(String url) {
this(url, null, null, null, null, null, null, null, null);
}
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
super(url, destPoolPath, templateId, checksum, temporaryDownloadPath);
SSLContext sslcontext = getSSLContext();
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
.setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
createUriRequest(url, headers);
String downloadDir = getDirectDownloadTempPath(templateId);
File tempFile = createTemporaryDirectoryAndFile(downloadDir);
setDownloadedFilePath(tempFile.getAbsolutePath());
}
protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
req = new HttpGet(downloadUrl);
if (MapUtils.isNotEmpty(headers)) {
for (String headerKey: headers.keySet()) {
req.setHeader(headerKey, headers.get(headerKey));
}
}
}
private SSLContext getSSLContext() {
try {
KeyStore customKeystore = KeyStore.getInstance("jks");
try (FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"))) {
String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null";
String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties");
String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
customKeystore.load(instream, privatePassword.toCharArray());
}
KeyStore defaultKeystore = KeyStore.getInstance(KeyStore.getDefaultType());
String relativeCacertsPath = "/lib/security/cacerts".replace("/", File.separator);
String filename = System.getProperty("java.home") + relativeCacertsPath;
try (FileInputStream is = new FileInputStream(filename)) {
String password = "changeit";
defaultKeystore.load(is, password.toCharArray());
}
TrustManager[] tm = HttpsMultiTrustManager.getTrustManagersFromKeyStores(customKeystore, defaultKeystore);
SSLContext sslContext = SSLUtils.getSSLContext();
sslContext.init(null, tm, null);
return sslContext;
} catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) {
s_logger.error(String.format("Failure getting SSL context for HTTPS downloader, using default SSL context: %s", e.getMessage()), e);
try {
return SSLContext.getDefault();
} catch (NoSuchAlgorithmException ex) {
throw new CloudRuntimeException(String.format("Cannot return the default SSL context due to: %s", ex.getMessage()), e);
}
}
}
@Override
public Pair<Boolean, String> downloadTemplate() {
CloseableHttpResponse response;
try {
response = httpsClient.execute(req);
} catch (IOException e) {
throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage());
}
return consumeResponse(response);
}
/**
* Consume response and persist it on getDownloadedFilePath() file
*/
protected Pair<Boolean, String> consumeResponse(CloseableHttpResponse response) {
s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
if (response.getStatusLine().getStatusCode() != 200) {
throw new CloudRuntimeException("Error on HTTPS response");
}
try {
HttpEntity entity = response.getEntity();
InputStream in = entity.getContent();
OutputStream out = new FileOutputStream(getDownloadedFilePath());
IOUtils.copy(in, out);
} catch (Exception e) {
s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage());
return new Pair<>(false, null);
}
return new Pair<>(true, getDownloadedFilePath());
}
@Override
public boolean checkUrl(String url) {
HttpHead httpHead = new HttpHead(url);
try {
CloseableHttpResponse response = httpsClient.execute(httpHead);
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
s_logger.error(String.format("Invalid URL: %s", url));
return false;
}
return true;
} catch (IOException e) {
s_logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e);
return false;
} finally {
httpHead.releaseConnection();
}
}
@Override
public Long getRemoteFileSize(String url, String format) {
if ("qcow2".equalsIgnoreCase(format)) {
try {
URL urlObj = new URL(url);
HttpsURLConnection urlConnection = (HttpsURLConnection)urlObj.openConnection();
SSLContext context = getSSLContext();
urlConnection.setSSLSocketFactory(context.getSocketFactory());
urlConnection.connect();
return QCOW2Utils.getVirtualSize(urlConnection.getInputStream());
} catch (IOException e) {
throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
}
} else {
HttpHead httpHead = new HttpHead(url);
CloseableHttpResponse response = null;
try {
response = httpsClient.execute(httpHead);
Header[] headers = response.getHeaders("Content-Length");
for (Header header : headers) {
return Long.parseLong(header.getValue());
}
} catch (IOException e) {
throw new RuntimeException(e);
}
return null;
}
}
@Override
public List<String> getMetalinkUrls(String metalinkUrl) {
HttpGet getMethod = new HttpGet(metalinkUrl);
List<String> urls = new ArrayList<>();
CloseableHttpResponse response;
try {
response = httpsClient.execute(getMethod);
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
String msg = String.format("Cannot access metalink content on URL %s", metalinkUrl);
s_logger.error(msg);
throw new IOException(msg);
}
} catch (IOException e) {
s_logger.error(String.format("Error retrieving urls form metalink URL %s: %s", metalinkUrl, e.getMessage()), e);
getMethod.releaseConnection();
return null;
}
try {
String responseStr = EntityUtils.toString(response.getEntity());
ByteArrayInputStream inputStream = new ByteArrayInputStream(responseStr.getBytes(StandardCharsets.UTF_8));
addMetalinkUrlsToListFromInputStream(inputStream, urls);
} catch (IOException e) {
s_logger.warn(e.getMessage(), e);
} finally {
getMethod.releaseConnection();
}
return urls;
}
@Override
public List<String> getMetalinkChecksums(String metalinkUrl) {
HttpGet getMethod = new HttpGet(metalinkUrl);
try {
CloseableHttpResponse response = httpsClient.execute(getMethod);
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
InputStream is = response.getEntity().getContent();
return generateChecksumListFromInputStream(is);
}
} catch (IOException e) {
s_logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e);
} finally {
getMethod.releaseConnection();
}
return null;
}
}

102
core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java Normal file
View File

@ -0,0 +1,102 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.direct.download;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
public class HttpsMultiTrustManager implements X509TrustManager {
private final List<X509TrustManager> trustManagers;
public HttpsMultiTrustManager(KeyStore... keystores) {
List<X509TrustManager> trustManagers = new ArrayList<>();
trustManagers.add(getTrustManager(null));
for (KeyStore keystore : keystores) {
trustManagers.add(getTrustManager(keystore));
}
this.trustManagers = ImmutableList.copyOf(trustManagers);
}
public static TrustManager[] getTrustManagersFromKeyStores(KeyStore... keyStore) {
return new TrustManager[] { new HttpsMultiTrustManager(keyStore) };
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
for (X509TrustManager trustManager : trustManagers) {
try {
trustManager.checkClientTrusted(chain, authType);
return;
} catch (CertificateException ignored) {}
}
throw new CertificateException("None of the TrustManagers trust this certificate chain");
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
for (X509TrustManager trustManager : trustManagers) {
try {
trustManager.checkServerTrusted(chain, authType);
return;
} catch (CertificateException ignored) {}
}
throw new CertificateException("None of the TrustManagers trust this certificate chain");
}
@Override
public X509Certificate[] getAcceptedIssuers() {
ImmutableList.Builder<X509Certificate> certificates = ImmutableList.builder();
for (X509TrustManager trustManager : trustManagers) {
for (X509Certificate cert : trustManager.getAcceptedIssuers()) {
certificates.add(cert);
}
}
return Iterables.toArray(certificates.build(), X509Certificate.class);
}
public X509TrustManager getTrustManager(KeyStore keystore) {
return getTrustManager(TrustManagerFactory.getDefaultAlgorithm(), keystore);
}
public X509TrustManager getTrustManager(String algorithm, KeyStore keystore) {
TrustManagerFactory factory;
try {
factory = TrustManagerFactory.getInstance(algorithm);
factory.init(keystore);
return Iterables.getFirst(Iterables.filter(
Arrays.asList(factory.getTrustManagers()), X509TrustManager.class), null);
} catch (NoSuchAlgorithmException | KeyStoreException e) {
e.printStackTrace();
}
return null;
}
}

177
core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,177 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
private List<String> metalinkUrls;
private List<String> metalinkChecksums;
private Random random = new Random();
protected DirectTemplateDownloader downloader;
private Map<String, String> headers;
private Integer connectTimeout;
private Integer soTimeout;
private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId,
String destPoolPath, String checksum,
Map<String, String> headers,
Integer connectTimeout, Integer soTimeout,
Integer connectionRequestTimeout, String temporaryDownloadPath) {
if (url.toLowerCase().startsWith("https:")) {
return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath);
} else if (url.toLowerCase().startsWith("http:")) {
return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
connectTimeout, soTimeout, temporaryDownloadPath);
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else {
s_logger.error(String.format("Cannot find a suitable downloader to handle the metalink URL %s", url));
return null;
}
}
protected MetalinkDirectTemplateDownloader(String url) {
this(url, null, null, null, null, null, null, null);
}
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, destPoolPath, templateId, checksum, downloadPath);
this.headers = headers;
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;
downloader = createDownloaderForMetalinks(url, templateId, destPoolPath, checksum, headers,
connectTimeout, soTimeout, null, downloadPath);
metalinkUrls = downloader.getMetalinkUrls(url);
metalinkChecksums = downloader.getMetalinkChecksums(url);
if (CollectionUtils.isEmpty(metalinkUrls)) {
s_logger.error(String.format("No urls found on metalink file: %s. Not possible to download template %s ", url, templateId));
} else {
setUrl(metalinkUrls.get(0));
s_logger.info("Metalink downloader created, metalink url: " + url + " parsed - " +
metalinkUrls.size() + " urls and " +
(CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found");
}
}
@Override
public Pair<Boolean, String> downloadTemplate() {
if (StringUtils.isBlank(getUrl())) {
throw new CloudRuntimeException("Download url has not been set, aborting");
}
boolean downloaded = false;
int i = 0;
String downloadDir = getDirectDownloadTempPath(getTemplateId());
do {
if (!isRedownload()) {
setUrl(metalinkUrls.get(i));
}
s_logger.info("Trying to download template from url: " + getUrl());
DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(getUrl(), getTemplateId(), getDestPoolPath(),
getChecksum(), headers, connectTimeout, soTimeout, null, temporaryDownloadPath);
try {
setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl());
File f = new File(getDownloadedFilePath());
if (f.exists()) {
f.delete();
f.createNewFile();
}
Pair<Boolean, String> downloadResult = urlDownloader.downloadTemplate();
downloaded = downloadResult.first();
if (downloaded) {
s_logger.info("Successfully downloaded template from url: " + getUrl());
}
} catch (Exception e) {
s_logger.error(String.format("Error downloading template: %s from URL: %s due to: %s", getTemplateId(), getUrl(), e.getMessage()), e);
}
i++;
}
while (!downloaded && !isRedownload() && i < metalinkUrls.size());
return new Pair<>(downloaded, getDownloadedFilePath());
}
@Override
public boolean validateChecksum() {
if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) {
String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size()));
setChecksum(chk);
s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk);
}
return super.validateChecksum();
}
@Override
public boolean checkUrl(String metalinkUrl) {
if (!downloader.checkUrl(metalinkUrl)) {
s_logger.error(String.format("Metalink URL check failed for: %s", metalinkUrl));
return false;
}
List<String> metalinkUrls = downloader.getMetalinkUrls(metalinkUrl);
for (String url : metalinkUrls) {
if (url.endsWith(".torrent")) {
continue;
}
DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(url, null, null, null, headers, connectTimeout, soTimeout, null, null);
if (!urlDownloader.checkUrl(url)) {
return false;
}
}
return true;
}
@Override
public Long getRemoteFileSize(String metalinkUrl, String format) {
List<String> urls = downloader.getMetalinkUrls(metalinkUrl);
for (String url : urls) {
if (url.endsWith("torrent")) {
continue;
}
if (downloader.checkUrl(url)) {
return downloader.getRemoteFileSize(url, format);
}
}
return null;
}
@Override
public List<String> getMetalinkUrls(String metalinkUrl) {
return downloader.getMetalinkUrls(metalinkUrl);
}
@Override
public List<String> getMetalinkChecksums(String metalinkUrl) {
return downloader.getMetalinkChecksums(metalinkUrl);
}
}

33
agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java → core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java
View File

@ -16,7 +16,7 @@
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
package org.apache.cloudstack.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
@ -26,6 +26,7 @@ import com.cloud.utils.script.Script;
import java.io.File;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.UUID;
public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
@ -52,6 +53,10 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
}
}
protected NfsDirectTemplateDownloader(String url) {
this(url, null, null, null, null);
}
public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) {
super(url, destPool, templateId, checksum, downloadPath);
parseUrl();
@ -68,4 +73,30 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid);
return new Pair<>(true, getDownloadedFilePath());
}
@Override
public boolean checkUrl(String url) {
try {
parseUrl();
return true;
} catch (CloudRuntimeException e) {
s_logger.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
return false;
}
}
@Override
public Long getRemoteFileSize(String url, String format) {
return null;
}
@Override
public List<String> getMetalinkUrls(String metalinkUrl) {
return null;
}
@Override
public List<String> getMetalinkChecksums(String metalinkUrl) {
return null;
}
}

46
core/src/test/java/com/cloud/storage/JavaStorageLayerTest.java Normal file
View File

@ -0,0 +1,46 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.storage;
import java.io.File;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
public class JavaStorageLayerTest {
JavaStorageLayer jsl = new JavaStorageLayer();
@Test
public void createUniqDir() {
try {
File one = jsl.createUniqDir();
Assert.assertTrue(one.isDirectory());
Assert.assertTrue(one.canRead());
Assert.assertTrue(one.canWrite());
Assert.assertTrue(one.canExecute());
} catch (IOException e) {
Assert.fail("creation of a unique dir should succeed.");
}
}
}

72
core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java Normal file
View File

@ -0,0 +1,72 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.HttpVersion;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicStatusLine;
import org.junit.Before;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
public class BaseDirectTemplateDownloaderTest {
protected static final String httpUrl = "http://server/path";
protected static final String httpsUrl = "https://server:443/path";
protected static final String httpMetalinkUrl = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2";
protected static final String httpMetalinkContent = "<metalink xmlns=\"urn:ietf:params:xml:ns:metalink\">\n" +
"<file name=\"macchinina-kvm.qcow2.bz2\">\n" +
"<url location=\"pr\">" + httpMetalinkUrl + "</url>\n" +
"</file>\n" +
"</metalink>";
@Mock
private CloseableHttpClient httpsClient;
@Mock
private CloseableHttpResponse response;
@Mock
private HttpEntity httpEntity;
@InjectMocks
protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl);
@Before
public void init() throws IOException {
MockitoAnnotations.initMocks(this);
Mockito.when(httpsClient.execute(Mockito.any(HttpGet.class))).thenReturn(response);
Mockito.when(httpsClient.execute(Mockito.any(HttpHead.class))).thenReturn(response);
StatusLine statusLine = new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
Mockito.when(response.getStatusLine()).thenReturn(statusLine);
Mockito.when(response.getEntity()).thenReturn(httpEntity);
ByteArrayInputStream inputStream = new ByteArrayInputStream(httpMetalinkContent.getBytes(StandardCharsets.UTF_8));
Mockito.when(httpEntity.getContent()).thenReturn(inputStream);
}
}

36
core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java Normal file
View File

@ -0,0 +1,36 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import org.apache.commons.collections.CollectionUtils;
import org.junit.Assert;
import org.junit.Test;
import java.util.List;
public class HttpsDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
@Test
public void testGetMetalinkUrls() {
List<String> metalinkUrls = httpsDownloader.getMetalinkUrls(httpUrl);
Assert.assertTrue(CollectionUtils.isNotEmpty(metalinkUrls));
Assert.assertEquals(1, metalinkUrls.size());
Assert.assertEquals(httpMetalinkUrl, metalinkUrls.get(0));
}
}

30
agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java → core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
View File

@ -16,22 +16,20 @@
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
package com.cloud.agent.direct.download;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.InjectMocks;
import com.cloud.utils.Pair;
public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
public interface DirectTemplateDownloader {
/**
* Perform template download to pool specified on downloader creation
* @return (true if successful, false if not, download file path)
*/
Pair<Boolean, String> downloadTemplate();
/**
* Perform checksum validation of previously downloadeed template
* @return true if successful, false if not
*/
boolean validateChecksum();
}
@InjectMocks
protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl);
@Test
public void testCheckUrlMetalink() {
metalinkDownloader.downloader = httpsDownloader;
boolean result = metalinkDownloader.checkUrl(httpsUrl);
Assert.assertTrue(result);
}
}

113
engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
View File

@ -458,72 +458,79 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
throws InsufficientCapacityException {
s_logger.info(String.format("allocating virtual machine from template:%s with hostname:%s and %d networks", template.getUuid(), vmInstanceName, auxiliaryNetworks.size()));
VMInstanceVO persistedVm = null;
try {
final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName);
final Account owner = _entityMgr.findById(Account.class, vm.getAccountId());
final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName);
final Account owner = _entityMgr.findById(Account.class, vm.getAccountId());
if (s_logger.isDebugEnabled()) {
s_logger.debug("Allocating entries for VM: " + vm);
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Allocating entries for VM: " + vm);
}
vm.setDataCenterId(plan.getDataCenterId());
if (plan.getPodId() != null) {
vm.setPodIdToDeployIn(plan.getPodId());
}
assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet";
persistedVm = _vmDao.persist(vm);
vm.setDataCenterId(plan.getDataCenterId());
if (plan.getPodId() != null) {
vm.setPodIdToDeployIn(plan.getPodId());
}
assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet";
final VMInstanceVO vmFinal = _vmDao.persist(vm);
final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(persistedVm, template, serviceOffering, null, null);
final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmFinal, template, serviceOffering, null, null);
Long rootDiskSize = rootDiskOfferingInfo.getSize();
if (vm.getType().isUsedBySystem() && SystemVmRootDiskSize.value() != null && SystemVmRootDiskSize.value() > 0L) {
rootDiskSize = SystemVmRootDiskSize.value();
}
final Long rootDiskSizeFinal = rootDiskSize;
Long rootDiskSize = rootDiskOfferingInfo.getSize();
if (vm.getType().isUsedBySystem() && SystemVmRootDiskSize.value() != null && SystemVmRootDiskSize.value() > 0L) {
rootDiskSize = SystemVmRootDiskSize.value();
}
final Long rootDiskSizeFinal = rootDiskSize;
if (s_logger.isDebugEnabled()) {
s_logger.debug("Allocating nics for " + persistedVm);
}
Transaction.execute(new TransactionCallbackWithExceptionNoReturn<InsufficientCapacityException>() {
@Override
public void doInTransactionWithoutResult(final TransactionStatus status) throws InsufficientCapacityException {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Allocating nics for " + vmFinal);
try {
if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) {
_networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions);
}
} catch (final ConcurrentOperationException e) {
throw new CloudRuntimeException("Concurrent operation while trying to allocate resources for the VM", e);
}
try {
if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) {
_networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions);
}
} catch (final ConcurrentOperationException e) {
throw new CloudRuntimeException("Concurrent operation while trying to allocate resources for the VM", e);
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Allocating disks for " + persistedVm);
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Allocating disks for " + vmFinal);
}
allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal);
allocateRootVolume(vmFinal, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal);
if (dataDiskOfferings != null) {
for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) {
volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vmFinal.getId(), dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(),
dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), vmFinal, template, owner, null);
}
}
if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) {
int diskNumber = 1;
for (Entry<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) {
DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue();
long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024);
VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey());
volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vmFinal.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null,
vmFinal, dataDiskTemplate, owner, Long.valueOf(diskNumber));
diskNumber++;
}
if (dataDiskOfferings != null) {
for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) {
volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId(), dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(),
dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, null);
}
}
if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) {
int diskNumber = 1;
for (Entry<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) {
DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue();
long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024);
VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey());
volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null,
persistedVm, dataDiskTemplate, owner, Long.valueOf(diskNumber));
diskNumber++;
}
}
});
if (s_logger.isDebugEnabled()) {
s_logger.debug("Allocation completed for VM: " + vmFinal);
if (s_logger.isDebugEnabled()) {
s_logger.debug("Allocation completed for VM: " + persistedVm);
}
} catch (InsufficientCapacityException | CloudRuntimeException e) {
// Failed VM will be in Stopped. Transition it to Error, so it can be expunged by ExpungeTask or similar
try {
if (persistedVm != null) {
stateTransitTo(persistedVm, VirtualMachine.Event.OperationFailedToError, null);
}
} catch (NoTransitionException nte) {
s_logger.error(String.format("Failed to transition %s in %s state to Error state", persistedVm, persistedVm.getState().toString()));
}
throw e;
}
}

11
engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java
View File

@ -26,6 +26,17 @@ public class DatabaseAccessObject {
private static Logger s_logger = Logger.getLogger(DatabaseAccessObject.class);
public void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) {
String addForeignKeyStmt = String.format("ALTER TABLE `cloud`.`%s` ADD CONSTRAINT `fk_%s__%s` FOREIGN KEY `fk_%s__%s`(`%s`) REFERENCES `%s`(`%s`)", tableName, tableName, tableColumn, tableName, tableColumn, tableColumn, foreignTableName, foreignColumnName);
try(PreparedStatement pstmt = conn.prepareStatement(addForeignKeyStmt);)
{
pstmt.executeUpdate();
s_logger.debug(String.format("Foreign key is added successfully from the table %s", tableName));
} catch (SQLException e) {
s_logger.error("Ignored SQL Exception when trying to add foreign key on table " + tableName + " exception: " + e.getMessage());
}
}
public void dropKey(Connection conn, String tableName, String key, boolean isForeignKey)
{
String alter_sql_str;

5
engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java
View File

@ -23,7 +23,10 @@ public class DbUpgradeUtils {
private static DatabaseAccessObject dao = new DatabaseAccessObject();
public static void dropKeysIfExist(Connection conn, String tableName, List<String> keys, boolean isForeignKey) {
public static void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) {
dao.addForeignKey(conn, tableName, tableColumn, foreignTableName, foreignColumnName);
}
public static void dropKeysIfExist(Connection conn, String tableName, List<String> keys, boolean isForeignKey) {
for (String key : keys) {
dao.dropKey(conn, tableName, key, isForeignKey);
}

29
engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java
View File

@ -22,6 +22,8 @@ import org.apache.log4j.Logger;
import java.io.InputStream;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.List;
public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate {
final static Logger LOG = Logger.getLogger(Upgrade41800to41810.class);
@ -55,6 +57,7 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate
@Override
public void performDataMigration(Connection conn) {
fixForeignKeyNames(conn);
}
@Override
@ -82,4 +85,30 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate
throw new CloudRuntimeException("Failed to find / register SystemVM template(s)");
}
}
private void fixForeignKeyNames(Connection conn) {
//Alter foreign key name for user_vm table from fk_user_data_id to fk_user_vm__user_data_id (if exists)
List<String> keys = new ArrayList<String>();
keys.add("fk_user_data_id");
keys.add("fk_user_vm__user_data_id");
DbUpgradeUtils.dropKeysIfExist(conn, "cloud.user_vm", keys, true);
DbUpgradeUtils.dropKeysIfExist(conn, "cloud.user_vm", keys, false);
DbUpgradeUtils.addForeignKey(conn, "user_vm", "user_data_id", "user_data", "id");
//Alter foreign key name for vm_template table from fk_user_data_id to fk_vm_template__user_data_id (if exists)
keys = new ArrayList<>();
keys.add("fk_user_data_id");
keys.add("fk_vm_template__user_data_id");
DbUpgradeUtils.dropKeysIfExist(conn, "cloud.vm_template", keys, true);
DbUpgradeUtils.dropKeysIfExist(conn, "cloud.vm_template", keys, false);
DbUpgradeUtils.addForeignKey(conn, "vm_template", "user_data_id", "user_data", "id");
//Alter foreign key name for volumes table from fk_passphrase_id to fk_volumes__passphrase_id (if exists)
keys = new ArrayList<>();
keys.add("fk_passphrase_id");
keys.add("fk_volumes__passphrase_id");
DbUpgradeUtils.dropKeysIfExist(conn, "cloud.volumes", keys, true);
DbUpgradeUtils.dropKeysIfExist(conn, "cloud.volumes", keys, false);
DbUpgradeUtils.addForeignKey(conn, "volumes", "passphrase_id","passphrase", "id");
}
}

2
engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
View File

@ -277,7 +277,7 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
}
public long getSize() {
return size;
return size == null ? 0l : size.longValue();
}
public void setPhysicalSize(long physicalSize) {

14
engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql
View File

@ -224,17 +224,6 @@ CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` (
BEGIN
DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-- Add foreign key procedure to link volumes to passphrase table
DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`;
CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY` (
IN in_table_name VARCHAR(200),
IN in_foreign_table_name VARCHAR(200),
IN in_foreign_column_name VARCHAR(200)
)
BEGIN
DECLARE CONTINUE HANDLER FOR 1005,1826 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', ' ADD CONSTRAINT '); SET @ddl = CONCAT(@ddl, 'fk_', in_foreign_table_name, '_', in_foreign_column_name); SET @ddl = CONCAT(@ddl, ' FOREIGN KEY (', in_foreign_table_name, '_', in_foreign_column_name, ')'); SET @ddl = CONCAT(@ddl, ' REFERENCES ', in_foreign_table_name, '(', in_foreign_column_name, ')'); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-- Add passphrase table
CREATE TABLE IF NOT EXISTS `cloud`.`passphrase` (
`id` bigint unsigned NOT NULL auto_increment,
@ -244,7 +233,6 @@ CREATE TABLE IF NOT EXISTS `cloud`.`passphrase` (
-- Add passphrase column to volumes table
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'passphrase_id', 'bigint unsigned DEFAULT NULL COMMENT "encryption passphrase id" ');
CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.volumes', 'passphrase', 'id');
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'encrypt_format', 'varchar(64) DEFAULT NULL COMMENT "encryption format" ');
-- Add encrypt column to disk_offering
@ -653,11 +641,9 @@ CREATE TABLE IF NOT EXISTS `cloud`.`user_data` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_vm', 'user_data_id', 'bigint unsigned DEFAULT NULL COMMENT "id of the user data" AFTER `user_data`');
CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.user_vm', 'user_data', 'id');
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_vm', 'user_data_details', 'mediumtext DEFAULT NULL COMMENT "value of the comma-separated list of parameters" AFTER `user_data_id`');
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_template', 'user_data_id', 'bigint unsigned DEFAULT NULL COMMENT "id of the user data"');
CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.vm_template', 'user_data', 'id');
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_template', 'user_data_link_policy', 'varchar(255) DEFAULT NULL COMMENT "user data link policy with template"');
-- Added userdata details to template

19
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
View File

@ -18,6 +18,7 @@
//
package com.cloud.hypervisor.kvm.resource.wrapper;
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer;
import org.apache.cloudstack.agent.directdownload.CheckUrlCommand;
import org.apache.log4j.Logger;
@ -25,8 +26,6 @@ import org.apache.log4j.Logger;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import com.cloud.utils.UriUtils;
import com.cloud.utils.storage.QCOW2Utils;
@ResourceWrapper(handles = CheckUrlCommand.class)
public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, CheckUrlAnswer, LibvirtComputingResource> {
@ -37,20 +36,10 @@ public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, Chec
public CheckUrlAnswer execute(CheckUrlCommand cmd, LibvirtComputingResource serverResource) {
final String url = cmd.getUrl();
s_logger.info("Checking URL: " + url);
boolean checkResult = true;
Long remoteSize = null;
try {
UriUtils.checkUrlExistence(url);
if ("qcow2".equalsIgnoreCase(cmd.getFormat())) {
remoteSize = QCOW2Utils.getVirtualSize(url);
} else {
remoteSize = UriUtils.getRemoteSize(url);
}
}
catch (IllegalArgumentException e) {
s_logger.warn(e.getMessage());
checkResult = false;
boolean checkResult = DirectDownloadHelper.checkUrlExistence(url);
if (checkResult) {
remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat());
}
return new CheckUrlAnswer(checkResult, remoteSize);
}

35
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -36,14 +36,12 @@ import java.util.UUID;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
import org.apache.cloudstack.direct.download.DirectTemplateDownloader;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Volume;
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
@ -97,11 +95,6 @@ import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.direct.download.DirectTemplateDownloader;
import com.cloud.agent.direct.download.HttpDirectTemplateDownloader;
import com.cloud.agent.direct.download.HttpsDirectTemplateDownloader;
import com.cloud.agent.direct.download.MetalinkDirectTemplateDownloader;
import com.cloud.agent.direct.download.NfsDirectTemplateDownloader;
import com.cloud.agent.properties.AgentProperties;
import com.cloud.agent.properties.AgentPropertiesFileHandler;
import com.cloud.exception.InternalErrorException;
@ -2313,28 +2306,6 @@ public class KVMStorageProcessor implements StorageProcessor {
return new Answer(cmd, false, "not implememented yet");
}
/**
* Get direct template downloader from direct download command and destination pool
*/
private DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
KVMStoragePool destPool,
String temporaryDownloadPath) {
if (cmd instanceof HttpDirectDownloadCommand) {
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
} else if (cmd instanceof HttpsDirectDownloadCommand) {
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
} else if (cmd instanceof NfsDirectDownloadCommand) {
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
} else {
throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}
}
@Override
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
final PrimaryDataStoreTO pool = cmd.getDestPool();
@ -2366,7 +2337,7 @@ public class KVMStorageProcessor implements StorageProcessor {
}
destPool = storagePoolMgr.getStoragePool(pool.getPoolType(), pool.getUuid());
downloader = getDirectTemplateDownloaderFromCommand(cmd, destPool, temporaryDownloadPath);
downloader = DirectDownloadHelper.getDirectTemplateDownloaderFromCommand(cmd, destPool.getLocalPath(), temporaryDownloadPath);
s_logger.debug("Trying to download template");
Pair<Boolean, String> result = downloader.downloadTemplate();
if (!result.first()) {

3
server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
View File

@ -52,6 +52,7 @@ import org.apache.cloudstack.api.response.GetUploadParamsResponse;
import org.apache.cloudstack.backup.Backup;
import org.apache.cloudstack.backup.BackupManager;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@ -533,7 +534,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
UriUtils.validateUrl(format, url);
if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access
s_logger.debug("Checking url: " + url);
UriUtils.checkUrlExistence(url);
DirectDownloadHelper.checkUrlExistence(url);
}
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));

288
server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
View File

@ -349,7 +349,6 @@ import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionCallbackWithException;
import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.db.UUIDManager;
@ -4372,161 +4371,156 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map<String, String> customParameters,
final Map<String, Map<Integer, String>> extraDhcpOptionMap, final Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap,
final Map<String, String> userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState, final boolean dynamicScalingEnabled, String vmType, final Long rootDiskOfferingId, String sshkeypairs) throws InsufficientCapacityException {
return Transaction.execute(new TransactionCallbackWithException<UserVmVO, InsufficientCapacityException>() {
@Override
public UserVmVO doInTransaction(TransactionStatus status) throws InsufficientCapacityException {
UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, template.getGuestOSId(), offering.isOfferHA(),
offering.getLimitCpuUse(), owner.getDomainId(), owner.getId(), userId, offering.getId(), userData, userDataId, userDataDetails, hostName);
vm.setUuid(uuidName);
vm.setDynamicallyScalable(dynamicScalingEnabled);
UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, template.getGuestOSId(), offering.isOfferHA(),
offering.getLimitCpuUse(), owner.getDomainId(), owner.getId(), userId, offering.getId(), userData, userDataId, userDataDetails, hostName);
vm.setUuid(uuidName);
vm.setDynamicallyScalable(dynamicScalingEnabled);
Map<String, String> details = template.getDetails();
if (details != null && !details.isEmpty()) {
vm.details.putAll(details);
}
Map<String, String> details = template.getDetails();
if (details != null && !details.isEmpty()) {
vm.details.putAll(details);
}
if (StringUtils.isNotBlank(sshPublicKeys)) {
vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKeys);
}
if (StringUtils.isNotBlank(sshPublicKeys)) {
vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKeys);
}
if (StringUtils.isNotBlank(sshkeypairs)) {
vm.setDetail(VmDetailConstants.SSH_KEY_PAIR_NAMES, sshkeypairs);
}
if (StringUtils.isNotBlank(sshkeypairs)) {
vm.setDetail(VmDetailConstants.SSH_KEY_PAIR_NAMES, sshkeypairs);
}
if (keyboard != null && !keyboard.isEmpty()) {
vm.setDetail(VmDetailConstants.KEYBOARD, keyboard);
}
if (keyboard != null && !keyboard.isEmpty()) {
vm.setDetail(VmDetailConstants.KEYBOARD, keyboard);
}
if (!isImport && isIso) {
vm.setIsoId(template.getId());
}
if (!isImport && isIso) {
vm.setIsoId(template.getId());
}
long guestOSId = template.getGuestOSId();
GuestOSVO guestOS = _guestOSDao.findById(guestOSId);
long guestOSCategoryId = guestOS.getCategoryId();
GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId);
if (hypervisorType.equals(HypervisorType.VMware)) {
updateVMDiskController(vm, customParameters, guestOS);
}
long guestOSId = template.getGuestOSId();
GuestOSVO guestOS = _guestOSDao.findById(guestOSId);
long guestOSCategoryId = guestOS.getCategoryId();
GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId);
if (hypervisorType.equals(HypervisorType.VMware)) {
updateVMDiskController(vm, customParameters, guestOS);
}
Long rootDiskSize = null;
// custom root disk size, resizes base template to larger size
if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) {
// already verified for positive number
rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE));
Long rootDiskSize = null;
// custom root disk size, resizes base template to larger size
if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) {
// already verified for positive number
rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE));
VMTemplateVO templateVO = _templateDao.findById(template.getId());
if (templateVO == null) {
throw new InvalidParameterValueException("Unable to look up template by id " + template.getId());
}
validateRootDiskResize(hypervisorType, rootDiskSize, templateVO, vm, customParameters);
}
if (isDisplayVm != null) {
vm.setDisplayVm(isDisplayVm);
} else {
vm.setDisplayVm(true);
}
if (isImport) {
vm.setDataCenterId(zone.getId());
vm.setHostId(host.getId());
if (lastHost != null) {
vm.setLastHostId(lastHost.getId());
}
vm.setPowerState(powerState);
if (powerState == VirtualMachine.PowerState.PowerOn) {
vm.setState(State.Running);
}
}
vm.setUserVmType(vmType);
_vmDao.persist(vm);
for (String key : customParameters.keySet()) {
// BIOS was explicitly passed as the boot type, so honour it
if (key.equalsIgnoreCase(ApiConstants.BootType.BIOS.toString())) {
vm.details.remove(ApiConstants.BootType.UEFI.toString());
continue;
}
// Deploy as is, Don't care about the boot type or template settings
if (key.equalsIgnoreCase(ApiConstants.BootType.UEFI.toString()) && template.isDeployAsIs()) {
vm.details.remove(ApiConstants.BootType.UEFI.toString());
continue;
}
if (!hypervisorType.equals(HypervisorType.KVM)) {
if (key.equalsIgnoreCase(VmDetailConstants.IOTHREADS)) {
vm.details.remove(VmDetailConstants.IOTHREADS);
continue;
}
if (key.equalsIgnoreCase(VmDetailConstants.IO_POLICY)) {
vm.details.remove(VmDetailConstants.IO_POLICY);
continue;
}
}
if (key.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) ||
key.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) ||
key.equalsIgnoreCase(VmDetailConstants.MEMORY)) {
// handle double byte strings.
vm.setDetail(key, Integer.toString(Integer.parseInt(customParameters.get(key))));
} else {
vm.setDetail(key, customParameters.get(key));
}
}
vm.setDetail(VmDetailConstants.DEPLOY_VM, "true");
persistVMDeployAsIsProperties(vm, userVmOVFPropertiesMap);
List<String> hiddenDetails = new ArrayList<>();
if (customParameters.containsKey(VmDetailConstants.NAME_ON_HYPERVISOR)) {
hiddenDetails.add(VmDetailConstants.NAME_ON_HYPERVISOR);
}
_vmDao.saveDetails(vm, hiddenDetails);
if (!isImport) {
s_logger.debug("Allocating in the DB for vm");
DataCenterDeployment plan = new DataCenterDeployment(zone.getId());
List<String> computeTags = new ArrayList<String>();
computeTags.add(offering.getHostTag());
List<String> rootDiskTags = new ArrayList<String>();
DiskOfferingVO rootDiskOfferingVO = _diskOfferingDao.findById(rootDiskOfferingId);
rootDiskTags.add(rootDiskOfferingVO.getTags());
if (isIso) {
_orchSrvc.createVirtualMachineFromScratch(vm.getUuid(), Long.toString(owner.getAccountId()), vm.getIsoId().toString(), hostName, displayName,
hypervisorType.name(), guestOSCategory.getName(), offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags,
networkNicMap, plan, extraDhcpOptionMap, rootDiskOfferingId);
} else {
_orchSrvc.createVirtualMachine(vm.getUuid(), Long.toString(owner.getAccountId()), Long.toString(template.getId()), hostName, displayName, hypervisorType.name(),
offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, networkNicMap, plan, rootDiskSize, extraDhcpOptionMap,
dataDiskTemplateToDiskOfferingMap, diskOfferingId, rootDiskOfferingId);
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Successfully allocated DB entry for " + vm);
}
}
CallContext.current().setEventDetails("Vm Id: " + vm.getUuid());
if (!isImport) {
if (!offering.isDynamic()) {
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(),
hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm());
} else {
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(),
hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), customParameters, vm.isDisplayVm());
}
//Update Resource Count for the given account
resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize()));
}
return vm;
VMTemplateVO templateVO = _templateDao.findById(template.getId());
if (templateVO == null) {
throw new InvalidParameterValueException("Unable to look up template by id " + template.getId());
}
});
validateRootDiskResize(hypervisorType, rootDiskSize, templateVO, vm, customParameters);
}
if (isDisplayVm != null) {
vm.setDisplayVm(isDisplayVm);
} else {
vm.setDisplayVm(true);
}
if (isImport) {
vm.setDataCenterId(zone.getId());
vm.setHostId(host.getId());
if (lastHost != null) {
vm.setLastHostId(lastHost.getId());
}
vm.setPowerState(powerState);
if (powerState == VirtualMachine.PowerState.PowerOn) {
vm.setState(State.Running);
}
}
vm.setUserVmType(vmType);
_vmDao.persist(vm);
for (String key : customParameters.keySet()) {
// BIOS was explicitly passed as the boot type, so honour it
if (key.equalsIgnoreCase(ApiConstants.BootType.BIOS.toString())) {
vm.details.remove(ApiConstants.BootType.UEFI.toString());
continue;
}
// Deploy as is, Don't care about the boot type or template settings
if (key.equalsIgnoreCase(ApiConstants.BootType.UEFI.toString()) && template.isDeployAsIs()) {
vm.details.remove(ApiConstants.BootType.UEFI.toString());
continue;
}
if (!hypervisorType.equals(HypervisorType.KVM)) {
if (key.equalsIgnoreCase(VmDetailConstants.IOTHREADS)) {
vm.details.remove(VmDetailConstants.IOTHREADS);
continue;
}
if (key.equalsIgnoreCase(VmDetailConstants.IO_POLICY)) {
vm.details.remove(VmDetailConstants.IO_POLICY);
continue;
}
}
if (key.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) ||
key.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) ||
key.equalsIgnoreCase(VmDetailConstants.MEMORY)) {
// handle double byte strings.
vm.setDetail(key, Integer.toString(Integer.parseInt(customParameters.get(key))));
} else {
vm.setDetail(key, customParameters.get(key));
}
}
vm.setDetail(VmDetailConstants.DEPLOY_VM, "true");
persistVMDeployAsIsProperties(vm, userVmOVFPropertiesMap);
List<String> hiddenDetails = new ArrayList<>();
if (customParameters.containsKey(VmDetailConstants.NAME_ON_HYPERVISOR)) {
hiddenDetails.add(VmDetailConstants.NAME_ON_HYPERVISOR);
}
_vmDao.saveDetails(vm, hiddenDetails);
if (!isImport) {
s_logger.debug("Allocating in the DB for vm");
DataCenterDeployment plan = new DataCenterDeployment(zone.getId());
List<String> computeTags = new ArrayList<String>();
computeTags.add(offering.getHostTag());
List<String> rootDiskTags = new ArrayList<String>();
DiskOfferingVO rootDiskOfferingVO = _diskOfferingDao.findById(rootDiskOfferingId);
rootDiskTags.add(rootDiskOfferingVO.getTags());
if (isIso) {
_orchSrvc.createVirtualMachineFromScratch(vm.getUuid(), Long.toString(owner.getAccountId()), vm.getIsoId().toString(), hostName, displayName,
hypervisorType.name(), guestOSCategory.getName(), offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags,
networkNicMap, plan, extraDhcpOptionMap, rootDiskOfferingId);
} else {
_orchSrvc.createVirtualMachine(vm.getUuid(), Long.toString(owner.getAccountId()), Long.toString(template.getId()), hostName, displayName, hypervisorType.name(),
offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, networkNicMap, plan, rootDiskSize, extraDhcpOptionMap,
dataDiskTemplateToDiskOfferingMap, diskOfferingId, rootDiskOfferingId);
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Successfully allocated DB entry for " + vm);
}
}
CallContext.current().setEventDetails("Vm Id: " + vm.getUuid());
if (!isImport) {
if (!offering.isDynamic()) {
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(),
hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm());
} else {
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(),
hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), customParameters, vm.isDisplayVm());
}
//Update Resource Count for the given account
resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize()));
}
return vm;
}
private void updateVMDiskController(UserVmVO vm, Map<String, String> customParameters, GuestOSVO guestOS) {

3
ui/src/views/network/CreateIsolatedNetworkForm.vue
View File

@ -646,6 +646,9 @@ export default {
if (this.isValidTextValueForKey(values, 'privatemtu')) {
params.privatemtu = values.privatemtu
}
if ('vpcid' in values) {
params.vpcid = this.selectedVpc.id
}
if ('domainid' in values && values.domainid > 0) {
params.domainid = this.selectedDomain.id
if (this.isValidTextValueForKey(values, 'account') && this.selectedAccount.id !== '-1') {

87
utils/src/main/java/com/cloud/utils/UriUtils.java
View File

@ -46,13 +46,11 @@ import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.cloudstack.utils.security.ParserUtils;
import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.commons.httpclient.util.URIUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair;
@ -348,32 +346,10 @@ public class UriUtils {
return new HttpClient(s_httpClientManager);
}
public static List<String> getMetalinkChecksums(String url) {
HttpClient httpClient = getHttpClient();
GetMethod getMethod = new GetMethod(url);
try {
if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) {
InputStream is = getMethod.getResponseBodyAsStream();
Map<String, List<String>> checksums = getMultipleValuesFromXML(is, new String[] {"hash"});
if (checksums.containsKey("hash")) {
List<String> listChksum = new ArrayList<>();
for (String chk : checksums.get("hash")) {
listChksum.add(chk.replaceAll("\n", "").replaceAll(" ", "").trim());
}
return listChksum;
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
getMethod.releaseConnection();
}
return null;
}
/**
* Retrieve values from XML documents ordered by ascending priority for each tag name
*/
protected static Map<String, List<String>> getMultipleValuesFromXML(InputStream is, String[] tagNames) {
public static Map<String, List<String>> getMultipleValuesFromXML(InputStream is, String[] tagNames) {
Map<String, List<String>> returnValues = new HashMap<String, List<String>>();
try {
DocumentBuilderFactory factory = ParserUtils.getSaferDocumentBuilderFactory();
@ -400,45 +376,6 @@ public class UriUtils {
return returnValues;
}
/**
* Check if there is at least one existent URL defined on metalink
* @param url metalink url
* @return true if at least one existent URL defined on metalink, false if not
*/
protected static boolean checkUrlExistenceMetalink(String url) {
HttpClient httpClient = getHttpClient();
GetMethod getMethod = new GetMethod(url);
try {
if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) {
InputStream is = getMethod.getResponseBodyAsStream();
Map<String, List<String>> metalinkUrls = getMultipleValuesFromXML(is, new String[] {"url"});
if (metalinkUrls.containsKey("url")) {
List<String> urls = metalinkUrls.get("url");
boolean validUrl = false;
for (String u : urls) {
if (u.endsWith("torrent")) {
continue;
}
try {
UriUtils.checkUrlExistence(u);
validUrl = true;
break;
}
catch (IllegalArgumentException e) {
s_logger.warn(e.getMessage());
}
}
return validUrl;
}
}
} catch (IOException e) {
s_logger.warn(e.getMessage());
} finally {
getMethod.releaseConnection();
}
return false;
}
/**
* Get list of urls on metalink ordered by ascending priority (for those which priority tag is not defined, highest priority value is assumed)
*/
@ -471,28 +408,6 @@ public class UriUtils {
return urls;
}
// use http HEAD method to validate url
public static void checkUrlExistence(String url) {
if (url.toLowerCase().startsWith("http") || url.toLowerCase().startsWith("https")) {
HttpClient httpClient = getHttpClient();
HeadMethod httphead = new HeadMethod(url);
try {
if (httpClient.executeMethod(httphead) != HttpStatus.SC_OK) {
throw new IllegalArgumentException("Invalid URL: " + url);
}
if (url.endsWith("metalink") && !checkUrlExistenceMetalink(url)) {
throw new IllegalArgumentException("Invalid URLs defined on metalink: " + url);
}
} catch (HttpException hte) {
throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + hte.getMessage());
} catch (IOException ioe) {
throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + ioe.getMessage());
} finally {
httphead.releaseConnection();
}
}
}
public static final Set<String> COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) {