apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix direct download URL checks (#7693)

Browse Source 
This PR fixes the URL check for direct downloads, in the case of HTTPS URLs the certificates were not loaded into the SSL context
This commit is contained in:
Nicolas Vazquez 2023-07-06 05:17:13 -03:00 committed by GitHub
parent e6ef8a5225
commit c733a23c90
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 950 additions and 395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java
View File

@ -1,131 +0,0 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.commons.collections.MapUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.ssl.SSLContexts;
import javax.net.ssl.SSLContext;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Map;
public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader {
private CloseableHttpClient httpsClient;
private HttpUriRequest req;
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, temporaryDownloadPath);
SSLContext sslcontext = null;
try {
sslcontext = getSSLContext();
} catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) {
throw new CloudRuntimeException("Failure getting SSL context for HTTPS downloader: " + e.getMessage());
}
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
.setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
createUriRequest(url, headers);
}
protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
req = new HttpGet(downloadUrl);
if (MapUtils.isNotEmpty(headers)) {
for (String headerKey: headers.keySet()) {
req.setHeader(headerKey, headers.get(headerKey));
}
}
}
private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException {
KeyStore trustStore = KeyStore.getInstance("jks");
FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"));
try {
String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null";
String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties");
String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
trustStore.load(instream, privatePassword.toCharArray());
} finally {
instream.close();
}
return SSLContexts.custom()
.loadTrustMaterial(trustStore, new TrustSelfSignedStrategy())
.build();
}
@Override
public Pair<Boolean, String> downloadTemplate() {
CloseableHttpResponse response;
try {
response = httpsClient.execute(req);
} catch (IOException e) {
throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage());
}
return consumeResponse(response);
}
/**
* Consume response and persist it on getDownloadedFilePath() file
*/
protected Pair<Boolean, String> consumeResponse(CloseableHttpResponse response) {
s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
if (response.getStatusLine().getStatusCode() != 200) {
throw new CloudRuntimeException("Error on HTTPS response");
}
try {
HttpEntity entity = response.getEntity();
InputStream in = entity.getContent();
OutputStream out = new FileOutputStream(getDownloadedFilePath());
IOUtils.copy(in, out);
} catch (Exception e) {
s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage());
return new Pair<>(false, null);
}
return new Pair<>(true, getDownloadedFilePath());
}
}

101
agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java
View File

@ -1,101 +0,0 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader {
private String metalinkUrl;
private List<String> metalinkUrls;
private List<String> metalinkChecksums;
private Random random = new Random();
private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, downloadPath);
metalinkUrl = url;
metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl);
metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl);
if (CollectionUtils.isEmpty(metalinkUrls)) {
throw new CloudRuntimeException("No urls found on metalink file: " + metalinkUrl + ". Not possible to download template " + templateId);
}
setUrl(metalinkUrls.get(0));
s_logger.info("Metalink downloader created, metalink url: " + metalinkUrl + " parsed - " +
metalinkUrls.size() + " urls and " +
(CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found");
}
@Override
public Pair<Boolean, String> downloadTemplate() {
if (StringUtils.isBlank(getUrl())) {
throw new CloudRuntimeException("Download url has not been set, aborting");
}
boolean downloaded = false;
int i = 0;
String downloadDir = getDirectDownloadTempPath(getTemplateId());
do {
if (!isRedownload()) {
setUrl(metalinkUrls.get(i));
}
s_logger.info("Trying to download template from url: " + getUrl());
try {
setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl());
File f = new File(getDownloadedFilePath());
if (f.exists()) {
f.delete();
f.createNewFile();
}
request = createRequest(getUrl(), reqHeaders);
Pair<Boolean, String> downloadResult = super.downloadTemplate();
downloaded = downloadResult.first();
if (downloaded) {
s_logger.info("Successfully downloaded template from url: " + getUrl());
}
} catch (Exception e) {
s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage());
}
i++;
}
while (!downloaded && !isRedownload() && i < metalinkUrls.size());
return new Pair<>(downloaded, getDownloadedFilePath());
}
@Override
public boolean validateChecksum() {
if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) {
String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size()));
setChecksum(chk);
s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk);
}
return super.validateChecksum();
}
}

83
core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java Normal file
View File

@ -0,0 +1,83 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
import org.apache.log4j.Logger;
public class DirectDownloadHelper {
public static final Logger LOGGER = Logger.getLogger(DirectDownloadHelper.class.getName());
/**
* Get direct template downloader from direct download command and destination pool
*/
public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
String destPoolLocalPath,
String temporaryDownloadPath) {
if (cmd instanceof HttpDirectDownloadCommand) {
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
} else if (cmd instanceof HttpsDirectDownloadCommand) {
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
} else if (cmd instanceof NfsDirectDownloadCommand) {
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
} else {
throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}
}
public static boolean checkUrlExistence(String url) {
try {
DirectTemplateDownloader checker = getCheckerDownloader(url);
return checker.checkUrl(url);
} catch (CloudRuntimeException e) {
LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
return false;
}
}
private static DirectTemplateDownloader getCheckerDownloader(String url) {
if (url.toLowerCase().startsWith("https:")) {
return new HttpsDirectTemplateDownloader(url);
} else if (url.toLowerCase().startsWith("http:")) {
return new HttpDirectTemplateDownloader(url);
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else if (url.toLowerCase().endsWith(".metalink")) {
return new MetalinkDirectTemplateDownloader(url);
} else {
throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url));
}
}
public static Long getFileSize(String url, String format) {
DirectTemplateDownloader checker = getCheckerDownloader(url);
return checker.getRemoteFileSize(url, format);
}
}

61
core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,61 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import com.cloud.utils.Pair;
import java.util.List;
public interface DirectTemplateDownloader {
/**
* Perform template download to pool specified on downloader creation
* @return (true if successful, false if not, download file path)
*/
Pair<Boolean, String> downloadTemplate();
/**
* Perform checksum validation of previously downloaded template
* @return true if successful, false if not
*/
boolean validateChecksum();
/**
* Validate if the URL is reachable and returns HTTP.OK status code
* @return true if the URL is reachable, false if not
*/
boolean checkUrl(String url);
/**
* Obtain the remote file size (and virtual size in case format is qcow2)
*/
Long getRemoteFileSize(String url, String format);
/**
* Get list of urls within metalink content ordered by ascending priority
* (for those which priority tag is not defined, highest priority value is assumed)
*/
List<String> getMetalinkUrls(String metalinkUrl);
/**
* Get the list of checksums within a metalink content
*/
List<String> getMetalinkChecksums(String metalinkUrl);
}

34
agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java → core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java
View File

@ -16,8 +16,9 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
// //
package com.cloud.agent.direct.download; package org.apache.cloudstack.direct.download;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.cloudstack.utils.security.DigestHelper;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -26,7 +27,11 @@ import org.apache.log4j.Logger;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader { public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
@ -106,6 +111,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
return redownload; return redownload;
} }
/**
* Create download directory (if it does not exist)
*/
protected File createTemporaryDirectoryAndFile(String downloadDir) {
createFolder(downloadDir);
return new File(downloadDir + File.separator + getFileNameFromUrl());
}
/** /**
* Return filename from url * Return filename from url
*/ */
@ -160,4 +173,23 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
} }
} }
protected void addMetalinkUrlsToListFromInputStream(InputStream inputStream, List<String> urls) {
Map<String, List<String>> metalinkUrlsMap = UriUtils.getMultipleValuesFromXML(inputStream, new String[] {"url"});
if (metalinkUrlsMap.containsKey("url")) {
List<String> metalinkUrls = metalinkUrlsMap.get("url");
urls.addAll(metalinkUrls);
}
}
protected List<String> generateChecksumListFromInputStream(InputStream is) {
Map<String, List<String>> checksums = UriUtils.getMultipleValuesFromXML(is, new String[] {"hash"});
if (checksums.containsKey("hash")) {
List<String> listChksum = new ArrayList<>();
for (String chk : checksums.get("hash")) {
listChksum.add(chk.replaceAll("\n", "").replaceAll(" ", "").trim());
}
return listChksum;
}
return null;
}
} }

89
agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java → core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
View File

@ -17,23 +17,28 @@
// under the License. // under the License.
// //
package com.cloud.agent.direct.download; package org.apache.cloudstack.direct.download;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import com.cloud.utils.Pair; import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.storage.QCOW2Utils;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
@ -45,6 +50,10 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected GetMethod request; protected GetMethod request;
protected Map<String, String> reqHeaders = new HashMap<>(); protected Map<String, String> reqHeaders = new HashMap<>();
protected HttpDirectTemplateDownloader(String url) {
this(url, null, null, null, null, null, null, null);
}
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, destPoolPath, templateId, checksum, downloadPath); super(url, destPoolPath, templateId, checksum, downloadPath);
@ -57,15 +66,6 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
setDownloadedFilePath(tempFile.getAbsolutePath()); setDownloadedFilePath(tempFile.getAbsolutePath());
} }
/**
* Create download directory (if it does not exist) and set the download file
* @return
*/
protected File createTemporaryDirectoryAndFile(String downloadDir) {
createFolder(downloadDir);
return new File(downloadDir + File.separator + getFileNameFromUrl());
}
protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) { protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
GetMethod request = new GetMethod(downloadUrl); GetMethod request = new GetMethod(downloadUrl);
request.setFollowRedirects(true); request.setFollowRedirects(true);
@ -98,7 +98,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
try ( try (
InputStream in = request.getResponseBodyAsStream(); InputStream in = request.getResponseBodyAsStream();
OutputStream out = new FileOutputStream(getDownloadedFilePath()); OutputStream out = new FileOutputStream(getDownloadedFilePath())
) { ) {
IOUtils.copy(in, out); IOUtils.copy(in, out);
} catch (IOException e) { } catch (IOException e) {
@ -107,4 +107,71 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
} }
return new Pair<>(true, getDownloadedFilePath()); return new Pair<>(true, getDownloadedFilePath());
} }
@Override
public boolean checkUrl(String url) {
HeadMethod httpHead = new HeadMethod(url);
try {
if (client.executeMethod(httpHead) != HttpStatus.SC_OK) {
s_logger.error(String.format("Invalid URL: %s", url));
return false;
}
return true;
} catch (IOException e) {
s_logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e);
return false;
} finally {
httpHead.releaseConnection();
}
}
@Override
public Long getRemoteFileSize(String url, String format) {
if ("qcow2".equalsIgnoreCase(format)) {
return QCOW2Utils.getVirtualSize(url);
} else {
return UriUtils.getRemoteSize(url);
}
}
@Override
public List<String> getMetalinkUrls(String metalinkUrl) {
GetMethod getMethod = new GetMethod(metalinkUrl);
List<String> urls = new ArrayList<>();
int status;
try {
status = client.executeMethod(getMethod);
} catch (IOException e) {
s_logger.error("Error retrieving urls form metalink: " + metalinkUrl);
getMethod.releaseConnection();
return null;
}
try {
InputStream is = getMethod.getResponseBodyAsStream();
if (status == HttpStatus.SC_OK) {
addMetalinkUrlsToListFromInputStream(is, urls);
}
} catch (IOException e) {
s_logger.warn(e.getMessage());
} finally {
getMethod.releaseConnection();
}
return urls;
}
@Override
public List<String> getMetalinkChecksums(String metalinkUrl) {
GetMethod getMethod = new GetMethod(metalinkUrl);
try {
if (client.executeMethod(getMethod) == HttpStatus.SC_OK) {
InputStream is = getMethod.getResponseBodyAsStream();
return generateChecksumListFromInputStream(is);
}
} catch (IOException e) {
s_logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e);
} finally {
getMethod.releaseConnection();
}
return null;
}
} }

252
core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,252 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.QCOW2Utils;
import org.apache.cloudstack.utils.security.SSLUtils;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.commons.collections.MapUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected CloseableHttpClient httpsClient;
private HttpUriRequest req;
protected HttpsDirectTemplateDownloader(String url) {
this(url, null, null, null, null, null, null, null, null);
}
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
super(url, destPoolPath, templateId, checksum, temporaryDownloadPath);
SSLContext sslcontext = getSSLContext();
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
.setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
createUriRequest(url, headers);
String downloadDir = getDirectDownloadTempPath(templateId);
File tempFile = createTemporaryDirectoryAndFile(downloadDir);
setDownloadedFilePath(tempFile.getAbsolutePath());
}
protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
req = new HttpGet(downloadUrl);
if (MapUtils.isNotEmpty(headers)) {
for (String headerKey: headers.keySet()) {
req.setHeader(headerKey, headers.get(headerKey));
}
}
}
private SSLContext getSSLContext() {
try {
KeyStore customKeystore = KeyStore.getInstance("jks");
try (FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"))) {
String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null";
String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties");
String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
customKeystore.load(instream, privatePassword.toCharArray());
}
KeyStore defaultKeystore = KeyStore.getInstance(KeyStore.getDefaultType());
String relativeCacertsPath = "/lib/security/cacerts".replace("/", File.separator);
String filename = System.getProperty("java.home") + relativeCacertsPath;
try (FileInputStream is = new FileInputStream(filename)) {
String password = "changeit";
defaultKeystore.load(is, password.toCharArray());
}
TrustManager[] tm = HttpsMultiTrustManager.getTrustManagersFromKeyStores(customKeystore, defaultKeystore);
SSLContext sslContext = SSLUtils.getSSLContext();
sslContext.init(null, tm, null);
return sslContext;
} catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) {
s_logger.error(String.format("Failure getting SSL context for HTTPS downloader, using default SSL context: %s", e.getMessage()), e);
try {
return SSLContext.getDefault();
} catch (NoSuchAlgorithmException ex) {
throw new CloudRuntimeException(String.format("Cannot return the default SSL context due to: %s", ex.getMessage()), e);
}
}
}
@Override
public Pair<Boolean, String> downloadTemplate() {
CloseableHttpResponse response;
try {
response = httpsClient.execute(req);
} catch (IOException e) {
throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage());
}
return consumeResponse(response);
}
/**
* Consume response and persist it on getDownloadedFilePath() file
*/
protected Pair<Boolean, String> consumeResponse(CloseableHttpResponse response) {
s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
if (response.getStatusLine().getStatusCode() != 200) {
throw new CloudRuntimeException("Error on HTTPS response");
}
try {
HttpEntity entity = response.getEntity();
InputStream in = entity.getContent();
OutputStream out = new FileOutputStream(getDownloadedFilePath());
IOUtils.copy(in, out);
} catch (Exception e) {
s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage());
return new Pair<>(false, null);
}
return new Pair<>(true, getDownloadedFilePath());
}
@Override
public boolean checkUrl(String url) {
HttpHead httpHead = new HttpHead(url);
try {
CloseableHttpResponse response = httpsClient.execute(httpHead);
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
s_logger.error(String.format("Invalid URL: %s", url));
return false;
}
return true;
} catch (IOException e) {
s_logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e);
return false;
} finally {
httpHead.releaseConnection();
}
}
@Override
public Long getRemoteFileSize(String url, String format) {
if ("qcow2".equalsIgnoreCase(format)) {
try {
URL urlObj = new URL(url);
HttpsURLConnection urlConnection = (HttpsURLConnection)urlObj.openConnection();
SSLContext context = getSSLContext();
urlConnection.setSSLSocketFactory(context.getSocketFactory());
urlConnection.connect();
return QCOW2Utils.getVirtualSize(urlConnection.getInputStream());
} catch (IOException e) {
throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
}
} else {
HttpHead httpHead = new HttpHead(url);
CloseableHttpResponse response = null;
try {
response = httpsClient.execute(httpHead);
Header[] headers = response.getHeaders("Content-Length");
for (Header header : headers) {
return Long.parseLong(header.getValue());
}
} catch (IOException e) {
throw new RuntimeException(e);
}
return null;
}
}
@Override
public List<String> getMetalinkUrls(String metalinkUrl) {
HttpGet getMethod = new HttpGet(metalinkUrl);
List<String> urls = new ArrayList<>();
CloseableHttpResponse response;
try {
response = httpsClient.execute(getMethod);
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
String msg = String.format("Cannot access metalink content on URL %s", metalinkUrl);
s_logger.error(msg);
throw new IOException(msg);
}
} catch (IOException e) {
s_logger.error(String.format("Error retrieving urls form metalink URL %s: %s", metalinkUrl, e.getMessage()), e);
getMethod.releaseConnection();
return null;
}
try {
String responseStr = EntityUtils.toString(response.getEntity());
ByteArrayInputStream inputStream = new ByteArrayInputStream(responseStr.getBytes(StandardCharsets.UTF_8));
addMetalinkUrlsToListFromInputStream(inputStream, urls);
} catch (IOException e) {
s_logger.warn(e.getMessage(), e);
} finally {
getMethod.releaseConnection();
}
return urls;
}
@Override
public List<String> getMetalinkChecksums(String metalinkUrl) {
HttpGet getMethod = new HttpGet(metalinkUrl);
try {
CloseableHttpResponse response = httpsClient.execute(getMethod);
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
InputStream is = response.getEntity().getContent();
return generateChecksumListFromInputStream(is);
}
} catch (IOException e) {
s_logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e);
} finally {
getMethod.releaseConnection();
}
return null;
}
}

102
core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java Normal file
View File

@ -0,0 +1,102 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.direct.download;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
public class HttpsMultiTrustManager implements X509TrustManager {
private final List<X509TrustManager> trustManagers;
public HttpsMultiTrustManager(KeyStore... keystores) {
List<X509TrustManager> trustManagers = new ArrayList<>();
trustManagers.add(getTrustManager(null));
for (KeyStore keystore : keystores) {
trustManagers.add(getTrustManager(keystore));
}
this.trustManagers = ImmutableList.copyOf(trustManagers);
}
public static TrustManager[] getTrustManagersFromKeyStores(KeyStore... keyStore) {
return new TrustManager[] { new HttpsMultiTrustManager(keyStore) };
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
for (X509TrustManager trustManager : trustManagers) {
try {
trustManager.checkClientTrusted(chain, authType);
return;
} catch (CertificateException ignored) {}
}
throw new CertificateException("None of the TrustManagers trust this certificate chain");
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
for (X509TrustManager trustManager : trustManagers) {
try {
trustManager.checkServerTrusted(chain, authType);
return;
} catch (CertificateException ignored) {}
}
throw new CertificateException("None of the TrustManagers trust this certificate chain");
}
@Override
public X509Certificate[] getAcceptedIssuers() {
ImmutableList.Builder<X509Certificate> certificates = ImmutableList.builder();
for (X509TrustManager trustManager : trustManagers) {
for (X509Certificate cert : trustManager.getAcceptedIssuers()) {
certificates.add(cert);
}
}
return Iterables.toArray(certificates.build(), X509Certificate.class);
}
public X509TrustManager getTrustManager(KeyStore keystore) {
return getTrustManager(TrustManagerFactory.getDefaultAlgorithm(), keystore);
}
public X509TrustManager getTrustManager(String algorithm, KeyStore keystore) {
TrustManagerFactory factory;
try {
factory = TrustManagerFactory.getInstance(algorithm);
factory.init(keystore);
return Iterables.getFirst(Iterables.filter(
Arrays.asList(factory.getTrustManagers()), X509TrustManager.class), null);
} catch (NoSuchAlgorithmException | KeyStoreException e) {
e.printStackTrace();
}
return null;
}
}

177
core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,177 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
private List<String> metalinkUrls;
private List<String> metalinkChecksums;
private Random random = new Random();
protected DirectTemplateDownloader downloader;
private Map<String, String> headers;
private Integer connectTimeout;
private Integer soTimeout;
private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId,
String destPoolPath, String checksum,
Map<String, String> headers,
Integer connectTimeout, Integer soTimeout,
Integer connectionRequestTimeout, String temporaryDownloadPath) {
if (url.toLowerCase().startsWith("https:")) {
return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath);
} else if (url.toLowerCase().startsWith("http:")) {
return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
connectTimeout, soTimeout, temporaryDownloadPath);
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else {
s_logger.error(String.format("Cannot find a suitable downloader to handle the metalink URL %s", url));
return null;
}
}
protected MetalinkDirectTemplateDownloader(String url) {
this(url, null, null, null, null, null, null, null);
}
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, destPoolPath, templateId, checksum, downloadPath);
this.headers = headers;
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;
downloader = createDownloaderForMetalinks(url, templateId, destPoolPath, checksum, headers,
connectTimeout, soTimeout, null, downloadPath);
metalinkUrls = downloader.getMetalinkUrls(url);
metalinkChecksums = downloader.getMetalinkChecksums(url);
if (CollectionUtils.isEmpty(metalinkUrls)) {
s_logger.error(String.format("No urls found on metalink file: %s. Not possible to download template %s ", url, templateId));
} else {
setUrl(metalinkUrls.get(0));
s_logger.info("Metalink downloader created, metalink url: " + url + " parsed - " +
metalinkUrls.size() + " urls and " +
(CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found");
}
}
@Override
public Pair<Boolean, String> downloadTemplate() {
if (StringUtils.isBlank(getUrl())) {
throw new CloudRuntimeException("Download url has not been set, aborting");
}
boolean downloaded = false;
int i = 0;
String downloadDir = getDirectDownloadTempPath(getTemplateId());
do {
if (!isRedownload()) {
setUrl(metalinkUrls.get(i));
}
s_logger.info("Trying to download template from url: " + getUrl());
DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(getUrl(), getTemplateId(), getDestPoolPath(),
getChecksum(), headers, connectTimeout, soTimeout, null, temporaryDownloadPath);
try {
setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl());
File f = new File(getDownloadedFilePath());
if (f.exists()) {
f.delete();
f.createNewFile();
}
Pair<Boolean, String> downloadResult = urlDownloader.downloadTemplate();
downloaded = downloadResult.first();
if (downloaded) {
s_logger.info("Successfully downloaded template from url: " + getUrl());
}
} catch (Exception e) {
s_logger.error(String.format("Error downloading template: %s from URL: %s due to: %s", getTemplateId(), getUrl(), e.getMessage()), e);
}
i++;
}
while (!downloaded && !isRedownload() && i < metalinkUrls.size());
return new Pair<>(downloaded, getDownloadedFilePath());
}
@Override
public boolean validateChecksum() {
if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) {
String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size()));
setChecksum(chk);
s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk);
}
return super.validateChecksum();
}
@Override
public boolean checkUrl(String metalinkUrl) {
if (!downloader.checkUrl(metalinkUrl)) {
s_logger.error(String.format("Metalink URL check failed for: %s", metalinkUrl));
return false;
}
List<String> metalinkUrls = downloader.getMetalinkUrls(metalinkUrl);
for (String url : metalinkUrls) {
if (url.endsWith(".torrent")) {
continue;
}
DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(url, null, null, null, headers, connectTimeout, soTimeout, null, null);
if (!urlDownloader.checkUrl(url)) {
return false;
}
}
return true;
}
@Override
public Long getRemoteFileSize(String metalinkUrl, String format) {
List<String> urls = downloader.getMetalinkUrls(metalinkUrl);
for (String url : urls) {
if (url.endsWith("torrent")) {
continue;
}
if (downloader.checkUrl(url)) {
return downloader.getRemoteFileSize(url, format);
}
}
return null;
}
@Override
public List<String> getMetalinkUrls(String metalinkUrl) {
return downloader.getMetalinkUrls(metalinkUrl);
}
@Override
public List<String> getMetalinkChecksums(String metalinkUrl) {
return downloader.getMetalinkChecksums(metalinkUrl);
}
}

33
agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java → core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java
View File

@ -16,7 +16,7 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
// //
package com.cloud.agent.direct.download; package org.apache.cloudstack.direct.download;
import com.cloud.utils.Pair; import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils; import com.cloud.utils.UriUtils;
@ -26,6 +26,7 @@ import com.cloud.utils.script.Script;
import java.io.File; import java.io.File;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.util.List;
import java.util.UUID; import java.util.UUID;
public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
@ -52,6 +53,10 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
} }
} }
protected NfsDirectTemplateDownloader(String url) {
this(url, null, null, null, null);
}
public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) { public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) {
super(url, destPool, templateId, checksum, downloadPath); super(url, destPool, templateId, checksum, downloadPath);
parseUrl(); parseUrl();
@ -68,4 +73,30 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid); Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid);
return new Pair<>(true, getDownloadedFilePath()); return new Pair<>(true, getDownloadedFilePath());
} }
@Override
public boolean checkUrl(String url) {
try {
parseUrl();
return true;
} catch (CloudRuntimeException e) {
s_logger.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
return false;
}
}
@Override
public Long getRemoteFileSize(String url, String format) {
return null;
}
@Override
public List<String> getMetalinkUrls(String metalinkUrl) {
return null;
}
@Override
public List<String> getMetalinkChecksums(String metalinkUrl) {
return null;
}
} }

72
core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java Normal file
View File

@ -0,0 +1,72 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.HttpVersion;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicStatusLine;
import org.junit.Before;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
public class BaseDirectTemplateDownloaderTest {
protected static final String httpUrl = "http://server/path";
protected static final String httpsUrl = "https://server:443/path";
protected static final String httpMetalinkUrl = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2";
protected static final String httpMetalinkContent = "<metalink xmlns=\"urn:ietf:params:xml:ns:metalink\">\n" +
"<file name=\"macchinina-kvm.qcow2.bz2\">\n" +
"<url location=\"pr\">" + httpMetalinkUrl + "</url>\n" +
"</file>\n" +
"</metalink>";
@Mock
private CloseableHttpClient httpsClient;
@Mock
private CloseableHttpResponse response;
@Mock
private HttpEntity httpEntity;
@InjectMocks
protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl);
@Before
public void init() throws IOException {
MockitoAnnotations.initMocks(this);
Mockito.when(httpsClient.execute(Mockito.any(HttpGet.class))).thenReturn(response);
Mockito.when(httpsClient.execute(Mockito.any(HttpHead.class))).thenReturn(response);
StatusLine statusLine = new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
Mockito.when(response.getStatusLine()).thenReturn(statusLine);
Mockito.when(response.getEntity()).thenReturn(httpEntity);
ByteArrayInputStream inputStream = new ByteArrayInputStream(httpMetalinkContent.getBytes(StandardCharsets.UTF_8));
Mockito.when(httpEntity.getContent()).thenReturn(inputStream);
}
}

36
core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java Normal file
View File

@ -0,0 +1,36 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import org.apache.commons.collections.CollectionUtils;
import org.junit.Assert;
import org.junit.Test;
import java.util.List;
public class HttpsDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
@Test
public void testGetMetalinkUrls() {
List<String> metalinkUrls = httpsDownloader.getMetalinkUrls(httpUrl);
Assert.assertTrue(CollectionUtils.isNotEmpty(metalinkUrls));
Assert.assertEquals(1, metalinkUrls.size());
Assert.assertEquals(httpMetalinkUrl, metalinkUrls.get(0));
}
}

30
agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java → core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
View File

@ -16,22 +16,20 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
// //
package org.apache.cloudstack.direct.download;
package com.cloud.agent.direct.download; import org.junit.Assert;
import org.junit.Test;
import org.mockito.InjectMocks;
import com.cloud.utils.Pair; public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
public interface DirectTemplateDownloader { @InjectMocks
protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl);
/** @Test
* Perform template download to pool specified on downloader creation public void testCheckUrlMetalink() {
* @return (true if successful, false if not, download file path) metalinkDownloader.downloader = httpsDownloader;
*/ boolean result = metalinkDownloader.checkUrl(httpsUrl);
Pair<Boolean, String> downloadTemplate(); Assert.assertTrue(result);
}
/** }
* Perform checksum validation of previously downloadeed template
* @return true if successful, false if not
*/
boolean validateChecksum();
}

19
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
View File

@ -18,6 +18,7 @@
// //
package com.cloud.hypervisor.kvm.resource.wrapper; package com.cloud.hypervisor.kvm.resource.wrapper;
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer; import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer;
import org.apache.cloudstack.agent.directdownload.CheckUrlCommand; import org.apache.cloudstack.agent.directdownload.CheckUrlCommand;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
@ -25,8 +26,6 @@ import org.apache.log4j.Logger;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.resource.CommandWrapper; import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper; import com.cloud.resource.ResourceWrapper;
import com.cloud.utils.UriUtils;
import com.cloud.utils.storage.QCOW2Utils;
@ResourceWrapper(handles = CheckUrlCommand.class) @ResourceWrapper(handles = CheckUrlCommand.class)
public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, CheckUrlAnswer, LibvirtComputingResource> { public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, CheckUrlAnswer, LibvirtComputingResource> {
@ -37,20 +36,10 @@ public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, Chec
public CheckUrlAnswer execute(CheckUrlCommand cmd, LibvirtComputingResource serverResource) { public CheckUrlAnswer execute(CheckUrlCommand cmd, LibvirtComputingResource serverResource) {
final String url = cmd.getUrl(); final String url = cmd.getUrl();
s_logger.info("Checking URL: " + url); s_logger.info("Checking URL: " + url);
boolean checkResult = true;
Long remoteSize = null; Long remoteSize = null;
try { boolean checkResult = DirectDownloadHelper.checkUrlExistence(url);
UriUtils.checkUrlExistence(url); if (checkResult) {
remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat());
if ("qcow2".equalsIgnoreCase(cmd.getFormat())) {
remoteSize = QCOW2Utils.getVirtualSize(url);
} else {
remoteSize = UriUtils.getRemoteSize(url);
}
}
catch (IllegalArgumentException e) {
s_logger.warn(e.getMessage());
checkResult = false;
} }
return new CheckUrlAnswer(checkResult, remoteSize); return new CheckUrlAnswer(checkResult, remoteSize);
} }

35
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -36,14 +36,12 @@ import java.util.UUID;
import javax.naming.ConfigurationException; import javax.naming.ConfigurationException;
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
import org.apache.cloudstack.direct.download.DirectTemplateDownloader;
import com.cloud.storage.ScopeType; import com.cloud.storage.ScopeType;
import com.cloud.storage.Volume; import com.cloud.storage.Volume;
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.AttachCommand;
@ -97,11 +95,6 @@ import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.direct.download.DirectTemplateDownloader;
import com.cloud.agent.direct.download.HttpDirectTemplateDownloader;
import com.cloud.agent.direct.download.HttpsDirectTemplateDownloader;
import com.cloud.agent.direct.download.MetalinkDirectTemplateDownloader;
import com.cloud.agent.direct.download.NfsDirectTemplateDownloader;
import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentProperties;
import com.cloud.agent.properties.AgentPropertiesFileHandler; import com.cloud.agent.properties.AgentPropertiesFileHandler;
import com.cloud.exception.InternalErrorException; import com.cloud.exception.InternalErrorException;
@ -2313,28 +2306,6 @@ public class KVMStorageProcessor implements StorageProcessor {
return new Answer(cmd, false, "not implememented yet"); return new Answer(cmd, false, "not implememented yet");
} }
/**
* Get direct template downloader from direct download command and destination pool
*/
private DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
KVMStoragePool destPool,
String temporaryDownloadPath) {
if (cmd instanceof HttpDirectDownloadCommand) {
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
} else if (cmd instanceof HttpsDirectDownloadCommand) {
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
} else if (cmd instanceof NfsDirectDownloadCommand) {
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
} else {
throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}
}
@Override @Override
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) { public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
final PrimaryDataStoreTO pool = cmd.getDestPool(); final PrimaryDataStoreTO pool = cmd.getDestPool();
@ -2366,7 +2337,7 @@ public class KVMStorageProcessor implements StorageProcessor {
} }
destPool = storagePoolMgr.getStoragePool(pool.getPoolType(), pool.getUuid()); destPool = storagePoolMgr.getStoragePool(pool.getPoolType(), pool.getUuid());
downloader = getDirectTemplateDownloaderFromCommand(cmd, destPool, temporaryDownloadPath); downloader = DirectDownloadHelper.getDirectTemplateDownloaderFromCommand(cmd, destPool.getLocalPath(), temporaryDownloadPath);
s_logger.debug("Trying to download template"); s_logger.debug("Trying to download template");
Pair<Boolean, String> result = downloader.downloadTemplate(); Pair<Boolean, String> result = downloader.downloadTemplate();
if (!result.first()) { if (!result.first()) {

3
server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
View File

@ -52,6 +52,7 @@ import org.apache.cloudstack.api.response.GetUploadParamsResponse;
import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.Backup;
import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.backup.BackupManager;
import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@ -533,7 +534,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
UriUtils.validateUrl(format, url); UriUtils.validateUrl(format, url);
if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access
s_logger.debug("Checking url: " + url); s_logger.debug("Checking url: " + url);
UriUtils.checkUrlExistence(url); DirectDownloadHelper.checkUrlExistence(url);
} }
// Check that the resource limit for secondary storage won't be exceeded // Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));

87
utils/src/main/java/com/cloud/utils/UriUtils.java
View File

@ -46,13 +46,11 @@ import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.cloudstack.utils.security.ParserUtils;
import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.commons.httpclient.util.URIUtil; import org.apache.commons.httpclient.util.URIUtil;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair; import org.apache.http.NameValuePair;
@ -348,32 +346,10 @@ public class UriUtils {
return new HttpClient(s_httpClientManager); return new HttpClient(s_httpClientManager);
} }
public static List<String> getMetalinkChecksums(String url) {
HttpClient httpClient = getHttpClient();
GetMethod getMethod = new GetMethod(url);
try {
if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) {
InputStream is = getMethod.getResponseBodyAsStream();
Map<String, List<String>> checksums = getMultipleValuesFromXML(is, new String[] {"hash"});
if (checksums.containsKey("hash")) {
List<String> listChksum = new ArrayList<>();
for (String chk : checksums.get("hash")) {
listChksum.add(chk.replaceAll("\n", "").replaceAll(" ", "").trim());
}
return listChksum;
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
getMethod.releaseConnection();
}
return null;
}
/** /**
* Retrieve values from XML documents ordered by ascending priority for each tag name * Retrieve values from XML documents ordered by ascending priority for each tag name
*/ */
protected static Map<String, List<String>> getMultipleValuesFromXML(InputStream is, String[] tagNames) { public static Map<String, List<String>> getMultipleValuesFromXML(InputStream is, String[] tagNames) {
Map<String, List<String>> returnValues = new HashMap<String, List<String>>(); Map<String, List<String>> returnValues = new HashMap<String, List<String>>();
try { try {
DocumentBuilderFactory factory = ParserUtils.getSaferDocumentBuilderFactory(); DocumentBuilderFactory factory = ParserUtils.getSaferDocumentBuilderFactory();
@ -400,45 +376,6 @@ public class UriUtils {
return returnValues; return returnValues;
} }
/**
* Check if there is at least one existent URL defined on metalink
* @param url metalink url
* @return true if at least one existent URL defined on metalink, false if not
*/
protected static boolean checkUrlExistenceMetalink(String url) {
HttpClient httpClient = getHttpClient();
GetMethod getMethod = new GetMethod(url);
try {
if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) {
InputStream is = getMethod.getResponseBodyAsStream();
Map<String, List<String>> metalinkUrls = getMultipleValuesFromXML(is, new String[] {"url"});
if (metalinkUrls.containsKey("url")) {
List<String> urls = metalinkUrls.get("url");
boolean validUrl = false;
for (String u : urls) {
if (u.endsWith("torrent")) {
continue;
}
try {
UriUtils.checkUrlExistence(u);
validUrl = true;
break;
}
catch (IllegalArgumentException e) {
s_logger.warn(e.getMessage());
}
}
return validUrl;
}
}
} catch (IOException e) {
s_logger.warn(e.getMessage());
} finally {
getMethod.releaseConnection();
}
return false;
}
/** /**
* Get list of urls on metalink ordered by ascending priority (for those which priority tag is not defined, highest priority value is assumed) * Get list of urls on metalink ordered by ascending priority (for those which priority tag is not defined, highest priority value is assumed)
*/ */
@ -471,28 +408,6 @@ public class UriUtils {
return urls; return urls;
} }
// use http HEAD method to validate url
public static void checkUrlExistence(String url) {
if (url.toLowerCase().startsWith("http") || url.toLowerCase().startsWith("https")) {
HttpClient httpClient = getHttpClient();
HeadMethod httphead = new HeadMethod(url);
try {
if (httpClient.executeMethod(httphead) != HttpStatus.SC_OK) {
throw new IllegalArgumentException("Invalid URL: " + url);
}
if (url.endsWith("metalink") && !checkUrlExistenceMetalink(url)) {
throw new IllegalArgumentException("Invalid URLs defined on metalink: " + url);
}
} catch (HttpException hte) {
throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + hte.getMessage());
} catch (IOException ioe) {
throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + ioe.getMessage());
} finally {
httphead.releaseConnection();
}
}
}
public static final Set<String> COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz"); public static final Set<String> COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) { public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) {