From c733a23c9056735c9922a701ddee7e7ef0efefa0 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 6 Jul 2023 05:17:13 -0300 Subject: [PATCH] Fix direct download URL checks (#7693) This PR fixes the URL check for direct downloads, in the case of HTTPS URLs the certificates were not loaded into the SSL context --- .../HttpsDirectTemplateDownloader.java | 131 --------- .../MetalinkDirectTemplateDownloader.java | 101 ------- .../direct/download/DirectDownloadHelper.java | 83 ++++++ .../download/DirectTemplateDownloader.java | 61 +++++ .../DirectTemplateDownloaderImpl.java | 34 ++- .../HttpDirectTemplateDownloader.java | 89 ++++++- .../HttpsDirectTemplateDownloader.java | 252 ++++++++++++++++++ .../download/HttpsMultiTrustManager.java | 102 +++++++ .../MetalinkDirectTemplateDownloader.java | 177 ++++++++++++ .../download/NfsDirectTemplateDownloader.java | 33 ++- .../BaseDirectTemplateDownloaderTest.java | 72 +++++ .../HttpsDirectTemplateDownloaderTest.java | 36 +++ .../MetalinkDirectTemplateDownloaderTest.java | 30 +-- .../wrapper/LibvirtCheckUrlCommand.java | 19 +- .../kvm/storage/KVMStorageProcessor.java | 35 +-- .../cloud/storage/VolumeApiServiceImpl.java | 3 +- .../main/java/com/cloud/utils/UriUtils.java | 87 +----- 17 files changed, 950 insertions(+), 395 deletions(-) delete mode 100644 agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java delete mode 100644 agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java create mode 100644 core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java create mode 100644 core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java rename {agent/src/main/java/com/cloud/agent => core/src/main/java/org/apache/cloudstack}/direct/download/DirectTemplateDownloaderImpl.java (80%) rename {agent/src/main/java/com/cloud/agent => core/src/main/java/org/apache/cloudstack}/direct/download/HttpDirectTemplateDownloader.java (61%) create mode 100644 core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java create mode 100644 core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java create mode 100644 core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java rename {agent/src/main/java/com/cloud/agent => core/src/main/java/org/apache/cloudstack}/direct/download/NfsDirectTemplateDownloader.java (77%) create mode 100644 core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java create mode 100644 core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java rename agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java => core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java (58%) diff --git a/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java deleted file mode 100644 index d788310f68e..00000000000 --- a/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java +++ /dev/null @@ -1,131 +0,0 @@ -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// - -package com.cloud.agent.direct.download; - -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.commons.collections.MapUtils; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContexts; - -import javax.net.ssl.SSLContext; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.util.Map; - -public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader { - - private CloseableHttpClient httpsClient; - private HttpUriRequest req; - - public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, - Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) { - super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, temporaryDownloadPath); - SSLContext sslcontext = null; - try { - sslcontext = getSSLContext(); - } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) { - throw new CloudRuntimeException("Failure getting SSL context for HTTPS downloader: " + e.getMessage()); - } - SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); - RequestConfig config = RequestConfig.custom() - .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout) - .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout) - .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build(); - httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build(); - createUriRequest(url, headers); - } - - protected void createUriRequest(String downloadUrl, Map headers) { - req = new HttpGet(downloadUrl); - if (MapUtils.isNotEmpty(headers)) { - for (String headerKey: headers.keySet()) { - req.setHeader(headerKey, headers.get(headerKey)); - } - } - } - - private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException { - KeyStore trustStore = KeyStore.getInstance("jks"); - FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks")); - try { - String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null"; - String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties"); - String privatePassword = Script.runSimpleBashScript(privatePasswordCmd); - trustStore.load(instream, privatePassword.toCharArray()); - } finally { - instream.close(); - } - return SSLContexts.custom() - .loadTrustMaterial(trustStore, new TrustSelfSignedStrategy()) - .build(); - } - - @Override - public Pair downloadTemplate() { - CloseableHttpResponse response; - try { - response = httpsClient.execute(req); - } catch (IOException e) { - throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage()); - } - return consumeResponse(response); - } - - /** - * Consume response and persist it on getDownloadedFilePath() file - */ - protected Pair consumeResponse(CloseableHttpResponse response) { - s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); - if (response.getStatusLine().getStatusCode() != 200) { - throw new CloudRuntimeException("Error on HTTPS response"); - } - try { - HttpEntity entity = response.getEntity(); - InputStream in = entity.getContent(); - OutputStream out = new FileOutputStream(getDownloadedFilePath()); - IOUtils.copy(in, out); - } catch (Exception e) { - s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage()); - return new Pair<>(false, null); - } - return new Pair<>(true, getDownloadedFilePath()); - } - -} diff --git a/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java deleted file mode 100644 index 40e77c37110..00000000000 --- a/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java +++ /dev/null @@ -1,101 +0,0 @@ -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -package com.cloud.agent.direct.download; - -import com.cloud.utils.Pair; -import com.cloud.utils.UriUtils; -import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; - -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Random; - -public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader { - - private String metalinkUrl; - private List metalinkUrls; - private List metalinkChecksums; - private Random random = new Random(); - private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName()); - - public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, - Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { - super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, downloadPath); - metalinkUrl = url; - metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl); - metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl); - if (CollectionUtils.isEmpty(metalinkUrls)) { - throw new CloudRuntimeException("No urls found on metalink file: " + metalinkUrl + ". Not possible to download template " + templateId); - } - setUrl(metalinkUrls.get(0)); - s_logger.info("Metalink downloader created, metalink url: " + metalinkUrl + " parsed - " + - metalinkUrls.size() + " urls and " + - (CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found"); - } - - @Override - public Pair downloadTemplate() { - if (StringUtils.isBlank(getUrl())) { - throw new CloudRuntimeException("Download url has not been set, aborting"); - } - boolean downloaded = false; - int i = 0; - String downloadDir = getDirectDownloadTempPath(getTemplateId()); - do { - if (!isRedownload()) { - setUrl(metalinkUrls.get(i)); - } - s_logger.info("Trying to download template from url: " + getUrl()); - try { - setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); - File f = new File(getDownloadedFilePath()); - if (f.exists()) { - f.delete(); - f.createNewFile(); - } - request = createRequest(getUrl(), reqHeaders); - Pair downloadResult = super.downloadTemplate(); - downloaded = downloadResult.first(); - if (downloaded) { - s_logger.info("Successfully downloaded template from url: " + getUrl()); - } - - } catch (Exception e) { - s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage()); - } - i++; - } - while (!downloaded && !isRedownload() && i < metalinkUrls.size()); - return new Pair<>(downloaded, getDownloadedFilePath()); - } - - @Override - public boolean validateChecksum() { - if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) { - String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size())); - setChecksum(chk); - s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk); - } - return super.validateChecksum(); - } -} diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java new file mode 100644 index 00000000000..e06483ac44d --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java @@ -0,0 +1,83 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand; +import org.apache.log4j.Logger; + +public class DirectDownloadHelper { + + public static final Logger LOGGER = Logger.getLogger(DirectDownloadHelper.class.getName()); + + /** + * Get direct template downloader from direct download command and destination pool + */ + public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd, + String destPoolLocalPath, + String temporaryDownloadPath) { + if (cmd instanceof HttpDirectDownloadCommand) { + return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath); + } else if (cmd instanceof HttpsDirectDownloadCommand) { + return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath); + } else if (cmd instanceof NfsDirectDownloadCommand) { + return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath); + } else if (cmd instanceof MetalinkDirectDownloadCommand) { + return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath); + } else { + throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink"); + } + } + + public static boolean checkUrlExistence(String url) { + try { + DirectTemplateDownloader checker = getCheckerDownloader(url); + return checker.checkUrl(url); + } catch (CloudRuntimeException e) { + LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e); + return false; + } + } + + private static DirectTemplateDownloader getCheckerDownloader(String url) { + if (url.toLowerCase().startsWith("https:")) { + return new HttpsDirectTemplateDownloader(url); + } else if (url.toLowerCase().startsWith("http:")) { + return new HttpDirectTemplateDownloader(url); + } else if (url.toLowerCase().startsWith("nfs:")) { + return new NfsDirectTemplateDownloader(url); + } else if (url.toLowerCase().endsWith(".metalink")) { + return new MetalinkDirectTemplateDownloader(url); + } else { + throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url)); + } + } + + public static Long getFileSize(String url, String format) { + DirectTemplateDownloader checker = getCheckerDownloader(url); + return checker.getRemoteFileSize(url, format); + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java new file mode 100644 index 00000000000..c9dd32f72e8 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java @@ -0,0 +1,61 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.Pair; + +import java.util.List; + +public interface DirectTemplateDownloader { + + /** + * Perform template download to pool specified on downloader creation + * @return (true if successful, false if not, download file path) + */ + Pair downloadTemplate(); + + /** + * Perform checksum validation of previously downloaded template + * @return true if successful, false if not + */ + boolean validateChecksum(); + + /** + * Validate if the URL is reachable and returns HTTP.OK status code + * @return true if the URL is reachable, false if not + */ + boolean checkUrl(String url); + + /** + * Obtain the remote file size (and virtual size in case format is qcow2) + */ + Long getRemoteFileSize(String url, String format); + + /** + * Get list of urls within metalink content ordered by ascending priority + * (for those which priority tag is not defined, highest priority value is assumed) + */ + List getMetalinkUrls(String metalinkUrl); + + /** + * Get the list of checksums within a metalink content + */ + List getMetalinkChecksums(String metalinkUrl); +} \ No newline at end of file diff --git a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java similarity index 80% rename from agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java rename to core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java index eb816192288..9476dbaa5ce 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java @@ -16,8 +16,9 @@ // specific language governing permissions and limitations // under the License. // -package com.cloud.agent.direct.download; +package org.apache.cloudstack.direct.download; +import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.lang3.StringUtils; @@ -26,7 +27,11 @@ import org.apache.log4j.Logger; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader { @@ -106,6 +111,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown return redownload; } + /** + * Create download directory (if it does not exist) + */ + protected File createTemporaryDirectoryAndFile(String downloadDir) { + createFolder(downloadDir); + return new File(downloadDir + File.separator + getFileNameFromUrl()); + } + /** * Return filename from url */ @@ -160,4 +173,23 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown } } + protected void addMetalinkUrlsToListFromInputStream(InputStream inputStream, List urls) { + Map> metalinkUrlsMap = UriUtils.getMultipleValuesFromXML(inputStream, new String[] {"url"}); + if (metalinkUrlsMap.containsKey("url")) { + List metalinkUrls = metalinkUrlsMap.get("url"); + urls.addAll(metalinkUrls); + } + } + + protected List generateChecksumListFromInputStream(InputStream is) { + Map> checksums = UriUtils.getMultipleValuesFromXML(is, new String[] {"hash"}); + if (checksums.containsKey("hash")) { + List listChksum = new ArrayList<>(); + for (String chk : checksums.get("hash")) { + listChksum.add(chk.replaceAll("\n", "").replaceAll(" ", "").trim()); + } + return listChksum; + } + return null; + } } diff --git a/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java similarity index 61% rename from agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java rename to core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index fc236034404..11ba6a5aab0 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -17,23 +17,28 @@ // under the License. // -package com.cloud.agent.direct.download; +package org.apache.cloudstack.direct.download; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import com.cloud.utils.Pair; +import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.storage.QCOW2Utils; import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.HeadMethod; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; @@ -45,6 +50,10 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod request; protected Map reqHeaders = new HashMap<>(); + protected HttpDirectTemplateDownloader(String url) { + this(url, null, null, null, null, null, null, null); + } + public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { super(url, destPoolPath, templateId, checksum, downloadPath); @@ -57,15 +66,6 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { setDownloadedFilePath(tempFile.getAbsolutePath()); } - /** - * Create download directory (if it does not exist) and set the download file - * @return - */ - protected File createTemporaryDirectoryAndFile(String downloadDir) { - createFolder(downloadDir); - return new File(downloadDir + File.separator + getFileNameFromUrl()); - } - protected GetMethod createRequest(String downloadUrl, Map headers) { GetMethod request = new GetMethod(downloadUrl); request.setFollowRedirects(true); @@ -98,7 +98,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); try ( InputStream in = request.getResponseBodyAsStream(); - OutputStream out = new FileOutputStream(getDownloadedFilePath()); + OutputStream out = new FileOutputStream(getDownloadedFilePath()) ) { IOUtils.copy(in, out); } catch (IOException e) { @@ -107,4 +107,71 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { } return new Pair<>(true, getDownloadedFilePath()); } + + @Override + public boolean checkUrl(String url) { + HeadMethod httpHead = new HeadMethod(url); + try { + if (client.executeMethod(httpHead) != HttpStatus.SC_OK) { + s_logger.error(String.format("Invalid URL: %s", url)); + return false; + } + return true; + } catch (IOException e) { + s_logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e); + return false; + } finally { + httpHead.releaseConnection(); + } + } + + @Override + public Long getRemoteFileSize(String url, String format) { + if ("qcow2".equalsIgnoreCase(format)) { + return QCOW2Utils.getVirtualSize(url); + } else { + return UriUtils.getRemoteSize(url); + } + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + GetMethod getMethod = new GetMethod(metalinkUrl); + List urls = new ArrayList<>(); + int status; + try { + status = client.executeMethod(getMethod); + } catch (IOException e) { + s_logger.error("Error retrieving urls form metalink: " + metalinkUrl); + getMethod.releaseConnection(); + return null; + } + try { + InputStream is = getMethod.getResponseBodyAsStream(); + if (status == HttpStatus.SC_OK) { + addMetalinkUrlsToListFromInputStream(is, urls); + } + } catch (IOException e) { + s_logger.warn(e.getMessage()); + } finally { + getMethod.releaseConnection(); + } + return urls; + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + GetMethod getMethod = new GetMethod(metalinkUrl); + try { + if (client.executeMethod(getMethod) == HttpStatus.SC_OK) { + InputStream is = getMethod.getResponseBodyAsStream(); + return generateChecksumListFromInputStream(is); + } + } catch (IOException e) { + s_logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e); + } finally { + getMethod.releaseConnection(); + } + return null; + } } \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java new file mode 100644 index 00000000000..a274eb562e4 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java @@ -0,0 +1,252 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import com.cloud.utils.storage.QCOW2Utils; +import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.commons.collections.MapUtils; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { + + protected CloseableHttpClient httpsClient; + private HttpUriRequest req; + + protected HttpsDirectTemplateDownloader(String url) { + this(url, null, null, null, null, null, null, null, null); + } + + public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, + Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) { + super(url, destPoolPath, templateId, checksum, temporaryDownloadPath); + SSLContext sslcontext = getSSLContext(); + SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout) + .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout) + .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build(); + httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build(); + createUriRequest(url, headers); + String downloadDir = getDirectDownloadTempPath(templateId); + File tempFile = createTemporaryDirectoryAndFile(downloadDir); + setDownloadedFilePath(tempFile.getAbsolutePath()); + } + + protected void createUriRequest(String downloadUrl, Map headers) { + req = new HttpGet(downloadUrl); + if (MapUtils.isNotEmpty(headers)) { + for (String headerKey: headers.keySet()) { + req.setHeader(headerKey, headers.get(headerKey)); + } + } + } + + private SSLContext getSSLContext() { + try { + KeyStore customKeystore = KeyStore.getInstance("jks"); + try (FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"))) { + String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null"; + String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties"); + String privatePassword = Script.runSimpleBashScript(privatePasswordCmd); + customKeystore.load(instream, privatePassword.toCharArray()); + } + KeyStore defaultKeystore = KeyStore.getInstance(KeyStore.getDefaultType()); + String relativeCacertsPath = "/lib/security/cacerts".replace("/", File.separator); + String filename = System.getProperty("java.home") + relativeCacertsPath; + try (FileInputStream is = new FileInputStream(filename)) { + String password = "changeit"; + defaultKeystore.load(is, password.toCharArray()); + } + TrustManager[] tm = HttpsMultiTrustManager.getTrustManagersFromKeyStores(customKeystore, defaultKeystore); + SSLContext sslContext = SSLUtils.getSSLContext(); + sslContext.init(null, tm, null); + return sslContext; + } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) { + s_logger.error(String.format("Failure getting SSL context for HTTPS downloader, using default SSL context: %s", e.getMessage()), e); + try { + return SSLContext.getDefault(); + } catch (NoSuchAlgorithmException ex) { + throw new CloudRuntimeException(String.format("Cannot return the default SSL context due to: %s", ex.getMessage()), e); + } + } + + } + + @Override + public Pair downloadTemplate() { + CloseableHttpResponse response; + try { + response = httpsClient.execute(req); + } catch (IOException e) { + throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage()); + } + return consumeResponse(response); + } + + /** + * Consume response and persist it on getDownloadedFilePath() file + */ + protected Pair consumeResponse(CloseableHttpResponse response) { + s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); + if (response.getStatusLine().getStatusCode() != 200) { + throw new CloudRuntimeException("Error on HTTPS response"); + } + try { + HttpEntity entity = response.getEntity(); + InputStream in = entity.getContent(); + OutputStream out = new FileOutputStream(getDownloadedFilePath()); + IOUtils.copy(in, out); + } catch (Exception e) { + s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage()); + return new Pair<>(false, null); + } + return new Pair<>(true, getDownloadedFilePath()); + } + + @Override + public boolean checkUrl(String url) { + HttpHead httpHead = new HttpHead(url); + try { + CloseableHttpResponse response = httpsClient.execute(httpHead); + if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { + s_logger.error(String.format("Invalid URL: %s", url)); + return false; + } + return true; + } catch (IOException e) { + s_logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e); + return false; + } finally { + httpHead.releaseConnection(); + } + } + + @Override + public Long getRemoteFileSize(String url, String format) { + if ("qcow2".equalsIgnoreCase(format)) { + try { + URL urlObj = new URL(url); + HttpsURLConnection urlConnection = (HttpsURLConnection)urlObj.openConnection(); + SSLContext context = getSSLContext(); + urlConnection.setSSLSocketFactory(context.getSocketFactory()); + urlConnection.connect(); + return QCOW2Utils.getVirtualSize(urlConnection.getInputStream()); + } catch (IOException e) { + throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e); + } + } else { + HttpHead httpHead = new HttpHead(url); + CloseableHttpResponse response = null; + try { + response = httpsClient.execute(httpHead); + Header[] headers = response.getHeaders("Content-Length"); + for (Header header : headers) { + return Long.parseLong(header.getValue()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return null; + } + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + HttpGet getMethod = new HttpGet(metalinkUrl); + List urls = new ArrayList<>(); + CloseableHttpResponse response; + try { + response = httpsClient.execute(getMethod); + if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { + String msg = String.format("Cannot access metalink content on URL %s", metalinkUrl); + s_logger.error(msg); + throw new IOException(msg); + } + } catch (IOException e) { + s_logger.error(String.format("Error retrieving urls form metalink URL %s: %s", metalinkUrl, e.getMessage()), e); + getMethod.releaseConnection(); + return null; + } + + try { + String responseStr = EntityUtils.toString(response.getEntity()); + ByteArrayInputStream inputStream = new ByteArrayInputStream(responseStr.getBytes(StandardCharsets.UTF_8)); + addMetalinkUrlsToListFromInputStream(inputStream, urls); + } catch (IOException e) { + s_logger.warn(e.getMessage(), e); + } finally { + getMethod.releaseConnection(); + } + return urls; + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + HttpGet getMethod = new HttpGet(metalinkUrl); + try { + CloseableHttpResponse response = httpsClient.execute(getMethod); + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + InputStream is = response.getEntity().getContent(); + return generateChecksumListFromInputStream(is); + } + } catch (IOException e) { + s_logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e); + } finally { + getMethod.releaseConnection(); + } + return null; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java new file mode 100644 index 00000000000..f991bfdfb3c --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.direct.download; + +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +public class HttpsMultiTrustManager implements X509TrustManager { + + private final List trustManagers; + + public HttpsMultiTrustManager(KeyStore... keystores) { + List trustManagers = new ArrayList<>(); + trustManagers.add(getTrustManager(null)); + for (KeyStore keystore : keystores) { + trustManagers.add(getTrustManager(keystore)); + } + this.trustManagers = ImmutableList.copyOf(trustManagers); + } + + public static TrustManager[] getTrustManagersFromKeyStores(KeyStore... keyStore) { + return new TrustManager[] { new HttpsMultiTrustManager(keyStore) }; + + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + for (X509TrustManager trustManager : trustManagers) { + try { + trustManager.checkClientTrusted(chain, authType); + return; + } catch (CertificateException ignored) {} + } + throw new CertificateException("None of the TrustManagers trust this certificate chain"); + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + for (X509TrustManager trustManager : trustManagers) { + try { + trustManager.checkServerTrusted(chain, authType); + return; + } catch (CertificateException ignored) {} + } + throw new CertificateException("None of the TrustManagers trust this certificate chain"); + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + ImmutableList.Builder certificates = ImmutableList.builder(); + for (X509TrustManager trustManager : trustManagers) { + for (X509Certificate cert : trustManager.getAcceptedIssuers()) { + certificates.add(cert); + } + } + return Iterables.toArray(certificates.build(), X509Certificate.class); + } + + public X509TrustManager getTrustManager(KeyStore keystore) { + return getTrustManager(TrustManagerFactory.getDefaultAlgorithm(), keystore); + } + + public X509TrustManager getTrustManager(String algorithm, KeyStore keystore) { + TrustManagerFactory factory; + try { + factory = TrustManagerFactory.getInstance(algorithm); + factory.init(keystore); + return Iterables.getFirst(Iterables.filter( + Arrays.asList(factory.getTrustManagers()), X509TrustManager.class), null); + } catch (NoSuchAlgorithmException | KeyStoreException e) { + e.printStackTrace(); + } + return null; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java new file mode 100644 index 00000000000..8051313f968 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java @@ -0,0 +1,177 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.collections.CollectionUtils; + +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Random; + +public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl { + private List metalinkUrls; + private List metalinkChecksums; + private Random random = new Random(); + protected DirectTemplateDownloader downloader; + private Map headers; + private Integer connectTimeout; + private Integer soTimeout; + + private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName()); + + protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId, + String destPoolPath, String checksum, + Map headers, + Integer connectTimeout, Integer soTimeout, + Integer connectionRequestTimeout, String temporaryDownloadPath) { + if (url.toLowerCase().startsWith("https:")) { + return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers, + connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath); + } else if (url.toLowerCase().startsWith("http:")) { + return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers, + connectTimeout, soTimeout, temporaryDownloadPath); + } else if (url.toLowerCase().startsWith("nfs:")) { + return new NfsDirectTemplateDownloader(url); + } else { + s_logger.error(String.format("Cannot find a suitable downloader to handle the metalink URL %s", url)); + return null; + } + } + + protected MetalinkDirectTemplateDownloader(String url) { + this(url, null, null, null, null, null, null, null); + } + + public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, + Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { + super(url, destPoolPath, templateId, checksum, downloadPath); + this.headers = headers; + this.connectTimeout = connectTimeout; + this.soTimeout = soTimeout; + downloader = createDownloaderForMetalinks(url, templateId, destPoolPath, checksum, headers, + connectTimeout, soTimeout, null, downloadPath); + metalinkUrls = downloader.getMetalinkUrls(url); + metalinkChecksums = downloader.getMetalinkChecksums(url); + if (CollectionUtils.isEmpty(metalinkUrls)) { + s_logger.error(String.format("No urls found on metalink file: %s. Not possible to download template %s ", url, templateId)); + } else { + setUrl(metalinkUrls.get(0)); + s_logger.info("Metalink downloader created, metalink url: " + url + " parsed - " + + metalinkUrls.size() + " urls and " + + (CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found"); + } + } + + @Override + public Pair downloadTemplate() { + if (StringUtils.isBlank(getUrl())) { + throw new CloudRuntimeException("Download url has not been set, aborting"); + } + boolean downloaded = false; + int i = 0; + String downloadDir = getDirectDownloadTempPath(getTemplateId()); + do { + if (!isRedownload()) { + setUrl(metalinkUrls.get(i)); + } + s_logger.info("Trying to download template from url: " + getUrl()); + DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(getUrl(), getTemplateId(), getDestPoolPath(), + getChecksum(), headers, connectTimeout, soTimeout, null, temporaryDownloadPath); + try { + setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); + File f = new File(getDownloadedFilePath()); + if (f.exists()) { + f.delete(); + f.createNewFile(); + } + Pair downloadResult = urlDownloader.downloadTemplate(); + downloaded = downloadResult.first(); + if (downloaded) { + s_logger.info("Successfully downloaded template from url: " + getUrl()); + } + } catch (Exception e) { + s_logger.error(String.format("Error downloading template: %s from URL: %s due to: %s", getTemplateId(), getUrl(), e.getMessage()), e); + } + i++; + } + while (!downloaded && !isRedownload() && i < metalinkUrls.size()); + return new Pair<>(downloaded, getDownloadedFilePath()); + } + + @Override + public boolean validateChecksum() { + if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) { + String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size())); + setChecksum(chk); + s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk); + } + return super.validateChecksum(); + } + + @Override + public boolean checkUrl(String metalinkUrl) { + if (!downloader.checkUrl(metalinkUrl)) { + s_logger.error(String.format("Metalink URL check failed for: %s", metalinkUrl)); + return false; + } + + List metalinkUrls = downloader.getMetalinkUrls(metalinkUrl); + for (String url : metalinkUrls) { + if (url.endsWith(".torrent")) { + continue; + } + DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(url, null, null, null, headers, connectTimeout, soTimeout, null, null); + if (!urlDownloader.checkUrl(url)) { + return false; + } + } + return true; + } + + @Override + public Long getRemoteFileSize(String metalinkUrl, String format) { + List urls = downloader.getMetalinkUrls(metalinkUrl); + for (String url : urls) { + if (url.endsWith("torrent")) { + continue; + } + if (downloader.checkUrl(url)) { + return downloader.getRemoteFileSize(url, format); + } + } + return null; + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + return downloader.getMetalinkUrls(metalinkUrl); + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + return downloader.getMetalinkChecksums(metalinkUrl); + } + +} \ No newline at end of file diff --git a/agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java similarity index 77% rename from agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java rename to core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java index 932477031d6..d606136e297 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java @@ -16,7 +16,7 @@ // specific language governing permissions and limitations // under the License. // -package com.cloud.agent.direct.download; +package org.apache.cloudstack.direct.download; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; @@ -26,6 +26,7 @@ import com.cloud.utils.script.Script; import java.io.File; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; import java.util.UUID; public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @@ -52,6 +53,10 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { } } + protected NfsDirectTemplateDownloader(String url) { + this(url, null, null, null, null); + } + public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) { super(url, destPool, templateId, checksum, downloadPath); parseUrl(); @@ -68,4 +73,30 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid); return new Pair<>(true, getDownloadedFilePath()); } + + @Override + public boolean checkUrl(String url) { + try { + parseUrl(); + return true; + } catch (CloudRuntimeException e) { + s_logger.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e); + return false; + } + } + + @Override + public Long getRemoteFileSize(String url, String format) { + return null; + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + return null; + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + return null; + } } diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java new file mode 100644 index 00000000000..8b70d29b965 --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java @@ -0,0 +1,72 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpStatus; +import org.apache.http.HttpVersion; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicStatusLine; +import org.junit.Before; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class BaseDirectTemplateDownloaderTest { + protected static final String httpUrl = "http://server/path"; + protected static final String httpsUrl = "https://server:443/path"; + protected static final String httpMetalinkUrl = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2"; + protected static final String httpMetalinkContent = "\n" + + "\n" + + "" + httpMetalinkUrl + "\n" + + "\n" + + ""; + + @Mock + private CloseableHttpClient httpsClient; + + @Mock + private CloseableHttpResponse response; + @Mock + private HttpEntity httpEntity; + + @InjectMocks + protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl); + + @Before + public void init() throws IOException { + MockitoAnnotations.initMocks(this); + Mockito.when(httpsClient.execute(Mockito.any(HttpGet.class))).thenReturn(response); + Mockito.when(httpsClient.execute(Mockito.any(HttpHead.class))).thenReturn(response); + StatusLine statusLine = new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK"); + Mockito.when(response.getStatusLine()).thenReturn(statusLine); + Mockito.when(response.getEntity()).thenReturn(httpEntity); + ByteArrayInputStream inputStream = new ByteArrayInputStream(httpMetalinkContent.getBytes(StandardCharsets.UTF_8)); + Mockito.when(httpEntity.getContent()).thenReturn(inputStream); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java new file mode 100644 index 00000000000..589ec77902f --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java @@ -0,0 +1,36 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class HttpsDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest { + + @Test + public void testGetMetalinkUrls() { + List metalinkUrls = httpsDownloader.getMetalinkUrls(httpUrl); + Assert.assertTrue(CollectionUtils.isNotEmpty(metalinkUrls)); + Assert.assertEquals(1, metalinkUrls.size()); + Assert.assertEquals(httpMetalinkUrl, metalinkUrls.get(0)); + } +} \ No newline at end of file diff --git a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java similarity index 58% rename from agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java rename to core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java index 32b84a34436..fec23c5533c 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java +++ b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java @@ -16,22 +16,20 @@ // specific language governing permissions and limitations // under the License. // +package org.apache.cloudstack.direct.download; -package com.cloud.agent.direct.download; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.InjectMocks; -import com.cloud.utils.Pair; +public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest { -public interface DirectTemplateDownloader { - - /** - * Perform template download to pool specified on downloader creation - * @return (true if successful, false if not, download file path) - */ - Pair downloadTemplate(); - - /** - * Perform checksum validation of previously downloadeed template - * @return true if successful, false if not - */ - boolean validateChecksum(); -} + @InjectMocks + protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl); + @Test + public void testCheckUrlMetalink() { + metalinkDownloader.downloader = httpsDownloader; + boolean result = metalinkDownloader.checkUrl(httpsUrl); + Assert.assertTrue(result); + } +} \ No newline at end of file diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java index 2618f20fae1..939d43086f4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java @@ -18,6 +18,7 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import org.apache.cloudstack.direct.download.DirectDownloadHelper; import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer; import org.apache.cloudstack.agent.directdownload.CheckUrlCommand; import org.apache.log4j.Logger; @@ -25,8 +26,6 @@ import org.apache.log4j.Logger; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; -import com.cloud.utils.UriUtils; -import com.cloud.utils.storage.QCOW2Utils; @ResourceWrapper(handles = CheckUrlCommand.class) public class LibvirtCheckUrlCommand extends CommandWrapper { @@ -37,20 +36,10 @@ public class LibvirtCheckUrlCommand extends CommandWrapper result = downloader.downloadTemplate(); if (!result.first()) { diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 6b1d170be9c..d6325b43134 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -52,6 +52,7 @@ import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.direct.download.DirectDownloadHelper; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -533,7 +534,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic UriUtils.validateUrl(format, url); if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access s_logger.debug("Checking url: " + url); - UriUtils.checkUrlExistence(url); + DirectDownloadHelper.checkUrlExistence(url); } // Check that the resource limit for secondary storage won't be exceeded _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index 11e62255fae..e68b5307f0e 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -46,13 +46,11 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.httpclient.methods.HeadMethod; import org.apache.commons.httpclient.util.URIUtil; import org.apache.commons.lang3.StringUtils; import org.apache.http.NameValuePair; @@ -348,32 +346,10 @@ public class UriUtils { return new HttpClient(s_httpClientManager); } - public static List getMetalinkChecksums(String url) { - HttpClient httpClient = getHttpClient(); - GetMethod getMethod = new GetMethod(url); - try { - if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) { - InputStream is = getMethod.getResponseBodyAsStream(); - Map> checksums = getMultipleValuesFromXML(is, new String[] {"hash"}); - if (checksums.containsKey("hash")) { - List listChksum = new ArrayList<>(); - for (String chk : checksums.get("hash")) { - listChksum.add(chk.replaceAll("\n", "").replaceAll(" ", "").trim()); - } - return listChksum; - } - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - getMethod.releaseConnection(); - } - return null; - } /** * Retrieve values from XML documents ordered by ascending priority for each tag name */ - protected static Map> getMultipleValuesFromXML(InputStream is, String[] tagNames) { + public static Map> getMultipleValuesFromXML(InputStream is, String[] tagNames) { Map> returnValues = new HashMap>(); try { DocumentBuilderFactory factory = ParserUtils.getSaferDocumentBuilderFactory(); @@ -400,45 +376,6 @@ public class UriUtils { return returnValues; } - /** - * Check if there is at least one existent URL defined on metalink - * @param url metalink url - * @return true if at least one existent URL defined on metalink, false if not - */ - protected static boolean checkUrlExistenceMetalink(String url) { - HttpClient httpClient = getHttpClient(); - GetMethod getMethod = new GetMethod(url); - try { - if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) { - InputStream is = getMethod.getResponseBodyAsStream(); - Map> metalinkUrls = getMultipleValuesFromXML(is, new String[] {"url"}); - if (metalinkUrls.containsKey("url")) { - List urls = metalinkUrls.get("url"); - boolean validUrl = false; - for (String u : urls) { - if (u.endsWith("torrent")) { - continue; - } - try { - UriUtils.checkUrlExistence(u); - validUrl = true; - break; - } - catch (IllegalArgumentException e) { - s_logger.warn(e.getMessage()); - } - } - return validUrl; - } - } - } catch (IOException e) { - s_logger.warn(e.getMessage()); - } finally { - getMethod.releaseConnection(); - } - return false; - } - /** * Get list of urls on metalink ordered by ascending priority (for those which priority tag is not defined, highest priority value is assumed) */ @@ -471,28 +408,6 @@ public class UriUtils { return urls; } - // use http HEAD method to validate url - public static void checkUrlExistence(String url) { - if (url.toLowerCase().startsWith("http") || url.toLowerCase().startsWith("https")) { - HttpClient httpClient = getHttpClient(); - HeadMethod httphead = new HeadMethod(url); - try { - if (httpClient.executeMethod(httphead) != HttpStatus.SC_OK) { - throw new IllegalArgumentException("Invalid URL: " + url); - } - if (url.endsWith("metalink") && !checkUrlExistenceMetalink(url)) { - throw new IllegalArgumentException("Invalid URLs defined on metalink: " + url); - } - } catch (HttpException hte) { - throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + hte.getMessage()); - } catch (IOException ioe) { - throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + ioe.getMessage()); - } finally { - httphead.releaseConnection(); - } - } - } - public static final Set COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz"); public static final Set buildExtensionSet(boolean metalink, String... baseExtensions) {