diff --git a/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java deleted file mode 100644 index d788310f68e..00000000000 --- a/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java +++ /dev/null @@ -1,131 +0,0 @@ -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// - -package com.cloud.agent.direct.download; - -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.commons.collections.MapUtils; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContexts; - -import javax.net.ssl.SSLContext; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.util.Map; - -public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader { - - private CloseableHttpClient httpsClient; - private HttpUriRequest req; - - public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, - Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) { - super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, temporaryDownloadPath); - SSLContext sslcontext = null; - try { - sslcontext = getSSLContext(); - } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) { - throw new CloudRuntimeException("Failure getting SSL context for HTTPS downloader: " + e.getMessage()); - } - SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); - RequestConfig config = RequestConfig.custom() - .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout) - .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout) - .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build(); - httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build(); - createUriRequest(url, headers); - } - - protected void createUriRequest(String downloadUrl, Map headers) { - req = new HttpGet(downloadUrl); - if (MapUtils.isNotEmpty(headers)) { - for (String headerKey: headers.keySet()) { - req.setHeader(headerKey, headers.get(headerKey)); - } - } - } - - private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException { - KeyStore trustStore = KeyStore.getInstance("jks"); - FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks")); - try { - String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null"; - String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties"); - String privatePassword = Script.runSimpleBashScript(privatePasswordCmd); - trustStore.load(instream, privatePassword.toCharArray()); - } finally { - instream.close(); - } - return SSLContexts.custom() - .loadTrustMaterial(trustStore, new TrustSelfSignedStrategy()) - .build(); - } - - @Override - public Pair downloadTemplate() { - CloseableHttpResponse response; - try { - response = httpsClient.execute(req); - } catch (IOException e) { - throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage()); - } - return consumeResponse(response); - } - - /** - * Consume response and persist it on getDownloadedFilePath() file - */ - protected Pair consumeResponse(CloseableHttpResponse response) { - s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); - if (response.getStatusLine().getStatusCode() != 200) { - throw new CloudRuntimeException("Error on HTTPS response"); - } - try { - HttpEntity entity = response.getEntity(); - InputStream in = entity.getContent(); - OutputStream out = new FileOutputStream(getDownloadedFilePath()); - IOUtils.copy(in, out); - } catch (Exception e) { - s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage()); - return new Pair<>(false, null); - } - return new Pair<>(true, getDownloadedFilePath()); - } - -} diff --git a/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java deleted file mode 100644 index 40e77c37110..00000000000 --- a/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java +++ /dev/null @@ -1,101 +0,0 @@ -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -package com.cloud.agent.direct.download; - -import com.cloud.utils.Pair; -import com.cloud.utils.UriUtils; -import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; - -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Random; - -public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader { - - private String metalinkUrl; - private List metalinkUrls; - private List metalinkChecksums; - private Random random = new Random(); - private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName()); - - public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, - Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { - super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, downloadPath); - metalinkUrl = url; - metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl); - metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl); - if (CollectionUtils.isEmpty(metalinkUrls)) { - throw new CloudRuntimeException("No urls found on metalink file: " + metalinkUrl + ". Not possible to download template " + templateId); - } - setUrl(metalinkUrls.get(0)); - s_logger.info("Metalink downloader created, metalink url: " + metalinkUrl + " parsed - " + - metalinkUrls.size() + " urls and " + - (CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found"); - } - - @Override - public Pair downloadTemplate() { - if (StringUtils.isBlank(getUrl())) { - throw new CloudRuntimeException("Download url has not been set, aborting"); - } - boolean downloaded = false; - int i = 0; - String downloadDir = getDirectDownloadTempPath(getTemplateId()); - do { - if (!isRedownload()) { - setUrl(metalinkUrls.get(i)); - } - s_logger.info("Trying to download template from url: " + getUrl()); - try { - setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); - File f = new File(getDownloadedFilePath()); - if (f.exists()) { - f.delete(); - f.createNewFile(); - } - request = createRequest(getUrl(), reqHeaders); - Pair downloadResult = super.downloadTemplate(); - downloaded = downloadResult.first(); - if (downloaded) { - s_logger.info("Successfully downloaded template from url: " + getUrl()); - } - - } catch (Exception e) { - s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage()); - } - i++; - } - while (!downloaded && !isRedownload() && i < metalinkUrls.size()); - return new Pair<>(downloaded, getDownloadedFilePath()); - } - - @Override - public boolean validateChecksum() { - if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) { - String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size())); - setChecksum(chk); - s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk); - } - return super.validateChecksum(); - } -} diff --git a/core/src/main/java/com/cloud/storage/JavaStorageLayer.java b/core/src/main/java/com/cloud/storage/JavaStorageLayer.java index b65a76b6c25..d4c2639d478 100644 --- a/core/src/main/java/com/cloud/storage/JavaStorageLayer.java +++ b/core/src/main/java/com/cloud/storage/JavaStorageLayer.java @@ -22,8 +22,11 @@ package com.cloud.storage; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -186,8 +189,12 @@ public class JavaStorageLayer implements StorageLayer { @Override public File createUniqDir() throws IOException { String dirName = System.getProperty("java.io.tmpdir"); + String subDirNamePrefix = ""; + FileAttribute> perms = PosixFilePermissions + .asFileAttribute(PosixFilePermissions.fromString("rwxrwx---")); if (dirName != null) { - File dir = new File(dirName); + Path p = Files.createTempDirectory(Path.of(dirName), subDirNamePrefix, perms); + File dir = p.toFile(); if (dir.exists()) { if (isWorldReadable(dir)) { if (STD_TMP_DIR_PATH.equals(dir.getAbsolutePath())) { diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java new file mode 100644 index 00000000000..e06483ac44d --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java @@ -0,0 +1,83 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand; +import org.apache.log4j.Logger; + +public class DirectDownloadHelper { + + public static final Logger LOGGER = Logger.getLogger(DirectDownloadHelper.class.getName()); + + /** + * Get direct template downloader from direct download command and destination pool + */ + public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd, + String destPoolLocalPath, + String temporaryDownloadPath) { + if (cmd instanceof HttpDirectDownloadCommand) { + return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath); + } else if (cmd instanceof HttpsDirectDownloadCommand) { + return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath); + } else if (cmd instanceof NfsDirectDownloadCommand) { + return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath); + } else if (cmd instanceof MetalinkDirectDownloadCommand) { + return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath); + } else { + throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink"); + } + } + + public static boolean checkUrlExistence(String url) { + try { + DirectTemplateDownloader checker = getCheckerDownloader(url); + return checker.checkUrl(url); + } catch (CloudRuntimeException e) { + LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e); + return false; + } + } + + private static DirectTemplateDownloader getCheckerDownloader(String url) { + if (url.toLowerCase().startsWith("https:")) { + return new HttpsDirectTemplateDownloader(url); + } else if (url.toLowerCase().startsWith("http:")) { + return new HttpDirectTemplateDownloader(url); + } else if (url.toLowerCase().startsWith("nfs:")) { + return new NfsDirectTemplateDownloader(url); + } else if (url.toLowerCase().endsWith(".metalink")) { + return new MetalinkDirectTemplateDownloader(url); + } else { + throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url)); + } + } + + public static Long getFileSize(String url, String format) { + DirectTemplateDownloader checker = getCheckerDownloader(url); + return checker.getRemoteFileSize(url, format); + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java new file mode 100644 index 00000000000..c9dd32f72e8 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java @@ -0,0 +1,61 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.Pair; + +import java.util.List; + +public interface DirectTemplateDownloader { + + /** + * Perform template download to pool specified on downloader creation + * @return (true if successful, false if not, download file path) + */ + Pair downloadTemplate(); + + /** + * Perform checksum validation of previously downloaded template + * @return true if successful, false if not + */ + boolean validateChecksum(); + + /** + * Validate if the URL is reachable and returns HTTP.OK status code + * @return true if the URL is reachable, false if not + */ + boolean checkUrl(String url); + + /** + * Obtain the remote file size (and virtual size in case format is qcow2) + */ + Long getRemoteFileSize(String url, String format); + + /** + * Get list of urls within metalink content ordered by ascending priority + * (for those which priority tag is not defined, highest priority value is assumed) + */ + List getMetalinkUrls(String metalinkUrl); + + /** + * Get the list of checksums within a metalink content + */ + List getMetalinkChecksums(String metalinkUrl); +} \ No newline at end of file diff --git a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java similarity index 80% rename from agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java rename to core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java index eb816192288..9476dbaa5ce 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java @@ -16,8 +16,9 @@ // specific language governing permissions and limitations // under the License. // -package com.cloud.agent.direct.download; +package org.apache.cloudstack.direct.download; +import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.lang3.StringUtils; @@ -26,7 +27,11 @@ import org.apache.log4j.Logger; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader { @@ -106,6 +111,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown return redownload; } + /** + * Create download directory (if it does not exist) + */ + protected File createTemporaryDirectoryAndFile(String downloadDir) { + createFolder(downloadDir); + return new File(downloadDir + File.separator + getFileNameFromUrl()); + } + /** * Return filename from url */ @@ -160,4 +173,23 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown } } + protected void addMetalinkUrlsToListFromInputStream(InputStream inputStream, List urls) { + Map> metalinkUrlsMap = UriUtils.getMultipleValuesFromXML(inputStream, new String[] {"url"}); + if (metalinkUrlsMap.containsKey("url")) { + List metalinkUrls = metalinkUrlsMap.get("url"); + urls.addAll(metalinkUrls); + } + } + + protected List generateChecksumListFromInputStream(InputStream is) { + Map> checksums = UriUtils.getMultipleValuesFromXML(is, new String[] {"hash"}); + if (checksums.containsKey("hash")) { + List listChksum = new ArrayList<>(); + for (String chk : checksums.get("hash")) { + listChksum.add(chk.replaceAll("\n", "").replaceAll(" ", "").trim()); + } + return listChksum; + } + return null; + } } diff --git a/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java similarity index 61% rename from agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java rename to core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index db387b28538..093f0604a44 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -17,23 +17,28 @@ // under the License. // -package com.cloud.agent.direct.download; +package org.apache.cloudstack.direct.download; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import com.cloud.utils.Pair; +import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.storage.QCOW2Utils; import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.HeadMethod; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; @@ -45,6 +50,10 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod request; protected Map reqHeaders = new HashMap<>(); + protected HttpDirectTemplateDownloader(String url) { + this(url, null, null, null, null, null, null, null); + } + public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { super(url, destPoolPath, templateId, checksum, downloadPath); @@ -57,15 +66,6 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { setDownloadedFilePath(tempFile.getAbsolutePath()); } - /** - * Create download directory (if it does not exist) and set the download file - * @return - */ - protected File createTemporaryDirectoryAndFile(String downloadDir) { - createFolder(downloadDir); - return new File(downloadDir + File.separator + getFileNameFromUrl()); - } - protected GetMethod createRequest(String downloadUrl, Map headers) { GetMethod request = new GetMethod(downloadUrl); request.setFollowRedirects(true); @@ -98,7 +98,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); try ( InputStream in = request.getResponseBodyAsStream(); - OutputStream out = new FileOutputStream(getDownloadedFilePath()); + OutputStream out = new FileOutputStream(getDownloadedFilePath()) ) { IOUtils.copy(in, out); } catch (IOException e) { @@ -107,4 +107,71 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { } return new Pair<>(true, getDownloadedFilePath()); } + + @Override + public boolean checkUrl(String url) { + HeadMethod httpHead = new HeadMethod(url); + try { + if (client.executeMethod(httpHead) != HttpStatus.SC_OK) { + s_logger.error(String.format("Invalid URL: %s", url)); + return false; + } + return true; + } catch (IOException e) { + s_logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e); + return false; + } finally { + httpHead.releaseConnection(); + } + } + + @Override + public Long getRemoteFileSize(String url, String format) { + if ("qcow2".equalsIgnoreCase(format)) { + return QCOW2Utils.getVirtualSize(url); + } else { + return UriUtils.getRemoteSize(url); + } + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + GetMethod getMethod = new GetMethod(metalinkUrl); + List urls = new ArrayList<>(); + int status; + try { + status = client.executeMethod(getMethod); + } catch (IOException e) { + s_logger.error("Error retrieving urls form metalink: " + metalinkUrl); + getMethod.releaseConnection(); + return null; + } + try { + InputStream is = getMethod.getResponseBodyAsStream(); + if (status == HttpStatus.SC_OK) { + addMetalinkUrlsToListFromInputStream(is, urls); + } + } catch (IOException e) { + s_logger.warn(e.getMessage()); + } finally { + getMethod.releaseConnection(); + } + return urls; + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + GetMethod getMethod = new GetMethod(metalinkUrl); + try { + if (client.executeMethod(getMethod) == HttpStatus.SC_OK) { + InputStream is = getMethod.getResponseBodyAsStream(); + return generateChecksumListFromInputStream(is); + } + } catch (IOException e) { + s_logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e); + } finally { + getMethod.releaseConnection(); + } + return null; + } } diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java new file mode 100644 index 00000000000..a274eb562e4 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java @@ -0,0 +1,252 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import com.cloud.utils.storage.QCOW2Utils; +import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.commons.collections.MapUtils; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { + + protected CloseableHttpClient httpsClient; + private HttpUriRequest req; + + protected HttpsDirectTemplateDownloader(String url) { + this(url, null, null, null, null, null, null, null, null); + } + + public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, + Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) { + super(url, destPoolPath, templateId, checksum, temporaryDownloadPath); + SSLContext sslcontext = getSSLContext(); + SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout) + .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout) + .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build(); + httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build(); + createUriRequest(url, headers); + String downloadDir = getDirectDownloadTempPath(templateId); + File tempFile = createTemporaryDirectoryAndFile(downloadDir); + setDownloadedFilePath(tempFile.getAbsolutePath()); + } + + protected void createUriRequest(String downloadUrl, Map headers) { + req = new HttpGet(downloadUrl); + if (MapUtils.isNotEmpty(headers)) { + for (String headerKey: headers.keySet()) { + req.setHeader(headerKey, headers.get(headerKey)); + } + } + } + + private SSLContext getSSLContext() { + try { + KeyStore customKeystore = KeyStore.getInstance("jks"); + try (FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"))) { + String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null"; + String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties"); + String privatePassword = Script.runSimpleBashScript(privatePasswordCmd); + customKeystore.load(instream, privatePassword.toCharArray()); + } + KeyStore defaultKeystore = KeyStore.getInstance(KeyStore.getDefaultType()); + String relativeCacertsPath = "/lib/security/cacerts".replace("/", File.separator); + String filename = System.getProperty("java.home") + relativeCacertsPath; + try (FileInputStream is = new FileInputStream(filename)) { + String password = "changeit"; + defaultKeystore.load(is, password.toCharArray()); + } + TrustManager[] tm = HttpsMultiTrustManager.getTrustManagersFromKeyStores(customKeystore, defaultKeystore); + SSLContext sslContext = SSLUtils.getSSLContext(); + sslContext.init(null, tm, null); + return sslContext; + } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) { + s_logger.error(String.format("Failure getting SSL context for HTTPS downloader, using default SSL context: %s", e.getMessage()), e); + try { + return SSLContext.getDefault(); + } catch (NoSuchAlgorithmException ex) { + throw new CloudRuntimeException(String.format("Cannot return the default SSL context due to: %s", ex.getMessage()), e); + } + } + + } + + @Override + public Pair downloadTemplate() { + CloseableHttpResponse response; + try { + response = httpsClient.execute(req); + } catch (IOException e) { + throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage()); + } + return consumeResponse(response); + } + + /** + * Consume response and persist it on getDownloadedFilePath() file + */ + protected Pair consumeResponse(CloseableHttpResponse response) { + s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); + if (response.getStatusLine().getStatusCode() != 200) { + throw new CloudRuntimeException("Error on HTTPS response"); + } + try { + HttpEntity entity = response.getEntity(); + InputStream in = entity.getContent(); + OutputStream out = new FileOutputStream(getDownloadedFilePath()); + IOUtils.copy(in, out); + } catch (Exception e) { + s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage()); + return new Pair<>(false, null); + } + return new Pair<>(true, getDownloadedFilePath()); + } + + @Override + public boolean checkUrl(String url) { + HttpHead httpHead = new HttpHead(url); + try { + CloseableHttpResponse response = httpsClient.execute(httpHead); + if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { + s_logger.error(String.format("Invalid URL: %s", url)); + return false; + } + return true; + } catch (IOException e) { + s_logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e); + return false; + } finally { + httpHead.releaseConnection(); + } + } + + @Override + public Long getRemoteFileSize(String url, String format) { + if ("qcow2".equalsIgnoreCase(format)) { + try { + URL urlObj = new URL(url); + HttpsURLConnection urlConnection = (HttpsURLConnection)urlObj.openConnection(); + SSLContext context = getSSLContext(); + urlConnection.setSSLSocketFactory(context.getSocketFactory()); + urlConnection.connect(); + return QCOW2Utils.getVirtualSize(urlConnection.getInputStream()); + } catch (IOException e) { + throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e); + } + } else { + HttpHead httpHead = new HttpHead(url); + CloseableHttpResponse response = null; + try { + response = httpsClient.execute(httpHead); + Header[] headers = response.getHeaders("Content-Length"); + for (Header header : headers) { + return Long.parseLong(header.getValue()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return null; + } + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + HttpGet getMethod = new HttpGet(metalinkUrl); + List urls = new ArrayList<>(); + CloseableHttpResponse response; + try { + response = httpsClient.execute(getMethod); + if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { + String msg = String.format("Cannot access metalink content on URL %s", metalinkUrl); + s_logger.error(msg); + throw new IOException(msg); + } + } catch (IOException e) { + s_logger.error(String.format("Error retrieving urls form metalink URL %s: %s", metalinkUrl, e.getMessage()), e); + getMethod.releaseConnection(); + return null; + } + + try { + String responseStr = EntityUtils.toString(response.getEntity()); + ByteArrayInputStream inputStream = new ByteArrayInputStream(responseStr.getBytes(StandardCharsets.UTF_8)); + addMetalinkUrlsToListFromInputStream(inputStream, urls); + } catch (IOException e) { + s_logger.warn(e.getMessage(), e); + } finally { + getMethod.releaseConnection(); + } + return urls; + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + HttpGet getMethod = new HttpGet(metalinkUrl); + try { + CloseableHttpResponse response = httpsClient.execute(getMethod); + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + InputStream is = response.getEntity().getContent(); + return generateChecksumListFromInputStream(is); + } + } catch (IOException e) { + s_logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e); + } finally { + getMethod.releaseConnection(); + } + return null; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java new file mode 100644 index 00000000000..f991bfdfb3c --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.direct.download; + +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +public class HttpsMultiTrustManager implements X509TrustManager { + + private final List trustManagers; + + public HttpsMultiTrustManager(KeyStore... keystores) { + List trustManagers = new ArrayList<>(); + trustManagers.add(getTrustManager(null)); + for (KeyStore keystore : keystores) { + trustManagers.add(getTrustManager(keystore)); + } + this.trustManagers = ImmutableList.copyOf(trustManagers); + } + + public static TrustManager[] getTrustManagersFromKeyStores(KeyStore... keyStore) { + return new TrustManager[] { new HttpsMultiTrustManager(keyStore) }; + + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + for (X509TrustManager trustManager : trustManagers) { + try { + trustManager.checkClientTrusted(chain, authType); + return; + } catch (CertificateException ignored) {} + } + throw new CertificateException("None of the TrustManagers trust this certificate chain"); + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + for (X509TrustManager trustManager : trustManagers) { + try { + trustManager.checkServerTrusted(chain, authType); + return; + } catch (CertificateException ignored) {} + } + throw new CertificateException("None of the TrustManagers trust this certificate chain"); + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + ImmutableList.Builder certificates = ImmutableList.builder(); + for (X509TrustManager trustManager : trustManagers) { + for (X509Certificate cert : trustManager.getAcceptedIssuers()) { + certificates.add(cert); + } + } + return Iterables.toArray(certificates.build(), X509Certificate.class); + } + + public X509TrustManager getTrustManager(KeyStore keystore) { + return getTrustManager(TrustManagerFactory.getDefaultAlgorithm(), keystore); + } + + public X509TrustManager getTrustManager(String algorithm, KeyStore keystore) { + TrustManagerFactory factory; + try { + factory = TrustManagerFactory.getInstance(algorithm); + factory.init(keystore); + return Iterables.getFirst(Iterables.filter( + Arrays.asList(factory.getTrustManagers()), X509TrustManager.class), null); + } catch (NoSuchAlgorithmException | KeyStoreException e) { + e.printStackTrace(); + } + return null; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java new file mode 100644 index 00000000000..8051313f968 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java @@ -0,0 +1,177 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.collections.CollectionUtils; + +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Random; + +public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl { + private List metalinkUrls; + private List metalinkChecksums; + private Random random = new Random(); + protected DirectTemplateDownloader downloader; + private Map headers; + private Integer connectTimeout; + private Integer soTimeout; + + private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName()); + + protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId, + String destPoolPath, String checksum, + Map headers, + Integer connectTimeout, Integer soTimeout, + Integer connectionRequestTimeout, String temporaryDownloadPath) { + if (url.toLowerCase().startsWith("https:")) { + return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers, + connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath); + } else if (url.toLowerCase().startsWith("http:")) { + return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers, + connectTimeout, soTimeout, temporaryDownloadPath); + } else if (url.toLowerCase().startsWith("nfs:")) { + return new NfsDirectTemplateDownloader(url); + } else { + s_logger.error(String.format("Cannot find a suitable downloader to handle the metalink URL %s", url)); + return null; + } + } + + protected MetalinkDirectTemplateDownloader(String url) { + this(url, null, null, null, null, null, null, null); + } + + public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, + Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { + super(url, destPoolPath, templateId, checksum, downloadPath); + this.headers = headers; + this.connectTimeout = connectTimeout; + this.soTimeout = soTimeout; + downloader = createDownloaderForMetalinks(url, templateId, destPoolPath, checksum, headers, + connectTimeout, soTimeout, null, downloadPath); + metalinkUrls = downloader.getMetalinkUrls(url); + metalinkChecksums = downloader.getMetalinkChecksums(url); + if (CollectionUtils.isEmpty(metalinkUrls)) { + s_logger.error(String.format("No urls found on metalink file: %s. Not possible to download template %s ", url, templateId)); + } else { + setUrl(metalinkUrls.get(0)); + s_logger.info("Metalink downloader created, metalink url: " + url + " parsed - " + + metalinkUrls.size() + " urls and " + + (CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found"); + } + } + + @Override + public Pair downloadTemplate() { + if (StringUtils.isBlank(getUrl())) { + throw new CloudRuntimeException("Download url has not been set, aborting"); + } + boolean downloaded = false; + int i = 0; + String downloadDir = getDirectDownloadTempPath(getTemplateId()); + do { + if (!isRedownload()) { + setUrl(metalinkUrls.get(i)); + } + s_logger.info("Trying to download template from url: " + getUrl()); + DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(getUrl(), getTemplateId(), getDestPoolPath(), + getChecksum(), headers, connectTimeout, soTimeout, null, temporaryDownloadPath); + try { + setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); + File f = new File(getDownloadedFilePath()); + if (f.exists()) { + f.delete(); + f.createNewFile(); + } + Pair downloadResult = urlDownloader.downloadTemplate(); + downloaded = downloadResult.first(); + if (downloaded) { + s_logger.info("Successfully downloaded template from url: " + getUrl()); + } + } catch (Exception e) { + s_logger.error(String.format("Error downloading template: %s from URL: %s due to: %s", getTemplateId(), getUrl(), e.getMessage()), e); + } + i++; + } + while (!downloaded && !isRedownload() && i < metalinkUrls.size()); + return new Pair<>(downloaded, getDownloadedFilePath()); + } + + @Override + public boolean validateChecksum() { + if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) { + String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size())); + setChecksum(chk); + s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk); + } + return super.validateChecksum(); + } + + @Override + public boolean checkUrl(String metalinkUrl) { + if (!downloader.checkUrl(metalinkUrl)) { + s_logger.error(String.format("Metalink URL check failed for: %s", metalinkUrl)); + return false; + } + + List metalinkUrls = downloader.getMetalinkUrls(metalinkUrl); + for (String url : metalinkUrls) { + if (url.endsWith(".torrent")) { + continue; + } + DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(url, null, null, null, headers, connectTimeout, soTimeout, null, null); + if (!urlDownloader.checkUrl(url)) { + return false; + } + } + return true; + } + + @Override + public Long getRemoteFileSize(String metalinkUrl, String format) { + List urls = downloader.getMetalinkUrls(metalinkUrl); + for (String url : urls) { + if (url.endsWith("torrent")) { + continue; + } + if (downloader.checkUrl(url)) { + return downloader.getRemoteFileSize(url, format); + } + } + return null; + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + return downloader.getMetalinkUrls(metalinkUrl); + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + return downloader.getMetalinkChecksums(metalinkUrl); + } + +} \ No newline at end of file diff --git a/agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java similarity index 77% rename from agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java rename to core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java index 932477031d6..d606136e297 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java @@ -16,7 +16,7 @@ // specific language governing permissions and limitations // under the License. // -package com.cloud.agent.direct.download; +package org.apache.cloudstack.direct.download; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; @@ -26,6 +26,7 @@ import com.cloud.utils.script.Script; import java.io.File; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; import java.util.UUID; public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @@ -52,6 +53,10 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { } } + protected NfsDirectTemplateDownloader(String url) { + this(url, null, null, null, null); + } + public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) { super(url, destPool, templateId, checksum, downloadPath); parseUrl(); @@ -68,4 +73,30 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid); return new Pair<>(true, getDownloadedFilePath()); } + + @Override + public boolean checkUrl(String url) { + try { + parseUrl(); + return true; + } catch (CloudRuntimeException e) { + s_logger.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e); + return false; + } + } + + @Override + public Long getRemoteFileSize(String url, String format) { + return null; + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + return null; + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + return null; + } } diff --git a/core/src/test/java/com/cloud/storage/JavaStorageLayerTest.java b/core/src/test/java/com/cloud/storage/JavaStorageLayerTest.java new file mode 100644 index 00000000000..241afcd5a92 --- /dev/null +++ b/core/src/test/java/com/cloud/storage/JavaStorageLayerTest.java @@ -0,0 +1,46 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.storage; + +import java.io.File; +import java.io.IOException; + +import org.junit.Assert; +import org.junit.Test; + +public class JavaStorageLayerTest { + + JavaStorageLayer jsl = new JavaStorageLayer(); + + @Test + public void createUniqDir() { + + try { + File one = jsl.createUniqDir(); + Assert.assertTrue(one.isDirectory()); + Assert.assertTrue(one.canRead()); + Assert.assertTrue(one.canWrite()); + Assert.assertTrue(one.canExecute()); + } catch (IOException e) { + Assert.fail("creation of a unique dir should succeed."); + } + } +} + diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java new file mode 100644 index 00000000000..8b70d29b965 --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java @@ -0,0 +1,72 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpStatus; +import org.apache.http.HttpVersion; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicStatusLine; +import org.junit.Before; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class BaseDirectTemplateDownloaderTest { + protected static final String httpUrl = "http://server/path"; + protected static final String httpsUrl = "https://server:443/path"; + protected static final String httpMetalinkUrl = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2"; + protected static final String httpMetalinkContent = "\n" + + "\n" + + "" + httpMetalinkUrl + "\n" + + "\n" + + ""; + + @Mock + private CloseableHttpClient httpsClient; + + @Mock + private CloseableHttpResponse response; + @Mock + private HttpEntity httpEntity; + + @InjectMocks + protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl); + + @Before + public void init() throws IOException { + MockitoAnnotations.initMocks(this); + Mockito.when(httpsClient.execute(Mockito.any(HttpGet.class))).thenReturn(response); + Mockito.when(httpsClient.execute(Mockito.any(HttpHead.class))).thenReturn(response); + StatusLine statusLine = new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK"); + Mockito.when(response.getStatusLine()).thenReturn(statusLine); + Mockito.when(response.getEntity()).thenReturn(httpEntity); + ByteArrayInputStream inputStream = new ByteArrayInputStream(httpMetalinkContent.getBytes(StandardCharsets.UTF_8)); + Mockito.when(httpEntity.getContent()).thenReturn(inputStream); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java new file mode 100644 index 00000000000..589ec77902f --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java @@ -0,0 +1,36 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class HttpsDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest { + + @Test + public void testGetMetalinkUrls() { + List metalinkUrls = httpsDownloader.getMetalinkUrls(httpUrl); + Assert.assertTrue(CollectionUtils.isNotEmpty(metalinkUrls)); + Assert.assertEquals(1, metalinkUrls.size()); + Assert.assertEquals(httpMetalinkUrl, metalinkUrls.get(0)); + } +} \ No newline at end of file diff --git a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java similarity index 58% rename from agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java rename to core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java index 32b84a34436..fec23c5533c 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java +++ b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java @@ -16,22 +16,20 @@ // specific language governing permissions and limitations // under the License. // +package org.apache.cloudstack.direct.download; -package com.cloud.agent.direct.download; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.InjectMocks; -import com.cloud.utils.Pair; +public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest { -public interface DirectTemplateDownloader { - - /** - * Perform template download to pool specified on downloader creation - * @return (true if successful, false if not, download file path) - */ - Pair downloadTemplate(); - - /** - * Perform checksum validation of previously downloadeed template - * @return true if successful, false if not - */ - boolean validateChecksum(); -} + @InjectMocks + protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl); + @Test + public void testCheckUrlMetalink() { + metalinkDownloader.downloader = httpsDownloader; + boolean result = metalinkDownloader.checkUrl(httpsUrl); + Assert.assertTrue(result); + } +} \ No newline at end of file diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 9b37132d07c..d3ba95061b6 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -458,72 +458,79 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throws InsufficientCapacityException { s_logger.info(String.format("allocating virtual machine from template:%s with hostname:%s and %d networks", template.getUuid(), vmInstanceName, auxiliaryNetworks.size())); + VMInstanceVO persistedVm = null; + try { + final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName); + final Account owner = _entityMgr.findById(Account.class, vm.getAccountId()); - final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName); - final Account owner = _entityMgr.findById(Account.class, vm.getAccountId()); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocating entries for VM: " + vm); + } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocating entries for VM: " + vm); - } + vm.setDataCenterId(plan.getDataCenterId()); + if (plan.getPodId() != null) { + vm.setPodIdToDeployIn(plan.getPodId()); + } + assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet"; + persistedVm = _vmDao.persist(vm); - vm.setDataCenterId(plan.getDataCenterId()); - if (plan.getPodId() != null) { - vm.setPodIdToDeployIn(plan.getPodId()); - } - assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet"; - final VMInstanceVO vmFinal = _vmDao.persist(vm); + final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(persistedVm, template, serviceOffering, null, null); - final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmFinal, template, serviceOffering, null, null); + Long rootDiskSize = rootDiskOfferingInfo.getSize(); + if (vm.getType().isUsedBySystem() && SystemVmRootDiskSize.value() != null && SystemVmRootDiskSize.value() > 0L) { + rootDiskSize = SystemVmRootDiskSize.value(); + } + final Long rootDiskSizeFinal = rootDiskSize; - Long rootDiskSize = rootDiskOfferingInfo.getSize(); - if (vm.getType().isUsedBySystem() && SystemVmRootDiskSize.value() != null && SystemVmRootDiskSize.value() > 0L) { - rootDiskSize = SystemVmRootDiskSize.value(); - } - final Long rootDiskSizeFinal = rootDiskSize; + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocating nics for " + persistedVm); + } - Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { - @Override - public void doInTransactionWithoutResult(final TransactionStatus status) throws InsufficientCapacityException { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocating nics for " + vmFinal); + try { + if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) { + _networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions); } + } catch (final ConcurrentOperationException e) { + throw new CloudRuntimeException("Concurrent operation while trying to allocate resources for the VM", e); + } - try { - if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) { - _networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions); - } - } catch (final ConcurrentOperationException e) { - throw new CloudRuntimeException("Concurrent operation while trying to allocate resources for the VM", e); - } + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocating disks for " + persistedVm); + } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocating disks for " + vmFinal); - } + allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal); - allocateRootVolume(vmFinal, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal); - - if (dataDiskOfferings != null) { - for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) { - volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vmFinal.getId(), dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), - dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), vmFinal, template, owner, null); - } - } - if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { - int diskNumber = 1; - for (Entry dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) { - DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue(); - long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); - VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey()); - volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vmFinal.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null, - vmFinal, dataDiskTemplate, owner, Long.valueOf(diskNumber)); - diskNumber++; - } + if (dataDiskOfferings != null) { + for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) { + volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId(), dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), + dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, null); + } + } + if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { + int diskNumber = 1; + for (Entry dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) { + DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue(); + long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); + VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey()); + volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null, + persistedVm, dataDiskTemplate, owner, Long.valueOf(diskNumber)); + diskNumber++; } } - }); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocation completed for VM: " + vmFinal); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocation completed for VM: " + persistedVm); + } + } catch (InsufficientCapacityException | CloudRuntimeException e) { + // Failed VM will be in Stopped. Transition it to Error, so it can be expunged by ExpungeTask or similar + try { + if (persistedVm != null) { + stateTransitTo(persistedVm, VirtualMachine.Event.OperationFailedToError, null); + } + } catch (NoTransitionException nte) { + s_logger.error(String.format("Failed to transition %s in %s state to Error state", persistedVm, persistedVm.getState().toString())); + } + throw e; } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java index 5d0edddb02e..21d5a205a09 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java @@ -26,6 +26,17 @@ public class DatabaseAccessObject { private static Logger s_logger = Logger.getLogger(DatabaseAccessObject.class); + public void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) { + String addForeignKeyStmt = String.format("ALTER TABLE `cloud`.`%s` ADD CONSTRAINT `fk_%s__%s` FOREIGN KEY `fk_%s__%s`(`%s`) REFERENCES `%s`(`%s`)", tableName, tableName, tableColumn, tableName, tableColumn, tableColumn, foreignTableName, foreignColumnName); + try(PreparedStatement pstmt = conn.prepareStatement(addForeignKeyStmt);) + { + pstmt.executeUpdate(); + s_logger.debug(String.format("Foreign key is added successfully from the table %s", tableName)); + } catch (SQLException e) { + s_logger.error("Ignored SQL Exception when trying to add foreign key on table " + tableName + " exception: " + e.getMessage()); + } + } + public void dropKey(Connection conn, String tableName, String key, boolean isForeignKey) { String alter_sql_str; diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java index 38ca5c9c272..02dad6250dc 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java @@ -23,7 +23,10 @@ public class DbUpgradeUtils { private static DatabaseAccessObject dao = new DatabaseAccessObject(); - public static void dropKeysIfExist(Connection conn, String tableName, List keys, boolean isForeignKey) { + public static void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) { + dao.addForeignKey(conn, tableName, tableColumn, foreignTableName, foreignColumnName); + } + public static void dropKeysIfExist(Connection conn, String tableName, List keys, boolean isForeignKey) { for (String key : keys) { dao.dropKey(conn, tableName, key, isForeignKey); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java index c4fbeb73eb2..ab493da8cd8 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java @@ -22,6 +22,8 @@ import org.apache.log4j.Logger; import java.io.InputStream; import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate { final static Logger LOG = Logger.getLogger(Upgrade41800to41810.class); @@ -55,6 +57,7 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate @Override public void performDataMigration(Connection conn) { + fixForeignKeyNames(conn); } @Override @@ -82,4 +85,30 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate throw new CloudRuntimeException("Failed to find / register SystemVM template(s)"); } } + + private void fixForeignKeyNames(Connection conn) { + //Alter foreign key name for user_vm table from fk_user_data_id to fk_user_vm__user_data_id (if exists) + List keys = new ArrayList(); + keys.add("fk_user_data_id"); + keys.add("fk_user_vm__user_data_id"); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.user_vm", keys, true); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.user_vm", keys, false); + DbUpgradeUtils.addForeignKey(conn, "user_vm", "user_data_id", "user_data", "id"); + + //Alter foreign key name for vm_template table from fk_user_data_id to fk_vm_template__user_data_id (if exists) + keys = new ArrayList<>(); + keys.add("fk_user_data_id"); + keys.add("fk_vm_template__user_data_id"); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.vm_template", keys, true); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.vm_template", keys, false); + DbUpgradeUtils.addForeignKey(conn, "vm_template", "user_data_id", "user_data", "id"); + + //Alter foreign key name for volumes table from fk_passphrase_id to fk_volumes__passphrase_id (if exists) + keys = new ArrayList<>(); + keys.add("fk_passphrase_id"); + keys.add("fk_volumes__passphrase_id"); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.volumes", keys, true); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.volumes", keys, false); + DbUpgradeUtils.addForeignKey(conn, "volumes", "passphrase_id","passphrase", "id"); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java index 8647e968127..a8d1af62f53 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java @@ -277,7 +277,7 @@ public class TemplateDataStoreVO implements StateObject { @@ -37,20 +36,10 @@ public class LibvirtCheckUrlCommand extends CommandWrapper result = downloader.downloadTemplate(); if (!result.first()) { diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 62214b7ea57..57d5d0e1f6e 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -52,6 +52,7 @@ import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.direct.download.DirectDownloadHelper; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -533,7 +534,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic UriUtils.validateUrl(format, url); if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access s_logger.debug("Checking url: " + url); - UriUtils.checkUrlExistence(url); + DirectDownloadHelper.checkUrlExistence(url); } // Check that the resource limit for secondary storage won't be exceeded _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index ea9d1f2da0a..a3d36d6bfb6 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -349,7 +349,6 @@ import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.db.UUIDManager; @@ -4372,161 +4371,156 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, final Map userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState, final boolean dynamicScalingEnabled, String vmType, final Long rootDiskOfferingId, String sshkeypairs) throws InsufficientCapacityException { - return Transaction.execute(new TransactionCallbackWithException() { - @Override - public UserVmVO doInTransaction(TransactionStatus status) throws InsufficientCapacityException { - UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, template.getGuestOSId(), offering.isOfferHA(), - offering.getLimitCpuUse(), owner.getDomainId(), owner.getId(), userId, offering.getId(), userData, userDataId, userDataDetails, hostName); - vm.setUuid(uuidName); - vm.setDynamicallyScalable(dynamicScalingEnabled); + UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, template.getGuestOSId(), offering.isOfferHA(), + offering.getLimitCpuUse(), owner.getDomainId(), owner.getId(), userId, offering.getId(), userData, userDataId, userDataDetails, hostName); + vm.setUuid(uuidName); + vm.setDynamicallyScalable(dynamicScalingEnabled); - Map details = template.getDetails(); - if (details != null && !details.isEmpty()) { - vm.details.putAll(details); - } + Map details = template.getDetails(); + if (details != null && !details.isEmpty()) { + vm.details.putAll(details); + } - if (StringUtils.isNotBlank(sshPublicKeys)) { - vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKeys); - } + if (StringUtils.isNotBlank(sshPublicKeys)) { + vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKeys); + } - if (StringUtils.isNotBlank(sshkeypairs)) { - vm.setDetail(VmDetailConstants.SSH_KEY_PAIR_NAMES, sshkeypairs); - } + if (StringUtils.isNotBlank(sshkeypairs)) { + vm.setDetail(VmDetailConstants.SSH_KEY_PAIR_NAMES, sshkeypairs); + } - if (keyboard != null && !keyboard.isEmpty()) { - vm.setDetail(VmDetailConstants.KEYBOARD, keyboard); - } + if (keyboard != null && !keyboard.isEmpty()) { + vm.setDetail(VmDetailConstants.KEYBOARD, keyboard); + } - if (!isImport && isIso) { - vm.setIsoId(template.getId()); - } + if (!isImport && isIso) { + vm.setIsoId(template.getId()); + } - long guestOSId = template.getGuestOSId(); - GuestOSVO guestOS = _guestOSDao.findById(guestOSId); - long guestOSCategoryId = guestOS.getCategoryId(); - GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId); - if (hypervisorType.equals(HypervisorType.VMware)) { - updateVMDiskController(vm, customParameters, guestOS); - } + long guestOSId = template.getGuestOSId(); + GuestOSVO guestOS = _guestOSDao.findById(guestOSId); + long guestOSCategoryId = guestOS.getCategoryId(); + GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId); + if (hypervisorType.equals(HypervisorType.VMware)) { + updateVMDiskController(vm, customParameters, guestOS); + } - Long rootDiskSize = null; - // custom root disk size, resizes base template to larger size - if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { - // already verified for positive number - rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE)); + Long rootDiskSize = null; + // custom root disk size, resizes base template to larger size + if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { + // already verified for positive number + rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE)); - VMTemplateVO templateVO = _templateDao.findById(template.getId()); - if (templateVO == null) { - throw new InvalidParameterValueException("Unable to look up template by id " + template.getId()); - } - - validateRootDiskResize(hypervisorType, rootDiskSize, templateVO, vm, customParameters); - } - - if (isDisplayVm != null) { - vm.setDisplayVm(isDisplayVm); - } else { - vm.setDisplayVm(true); - } - - if (isImport) { - vm.setDataCenterId(zone.getId()); - vm.setHostId(host.getId()); - if (lastHost != null) { - vm.setLastHostId(lastHost.getId()); - } - vm.setPowerState(powerState); - if (powerState == VirtualMachine.PowerState.PowerOn) { - vm.setState(State.Running); - } - } - - vm.setUserVmType(vmType); - _vmDao.persist(vm); - for (String key : customParameters.keySet()) { - // BIOS was explicitly passed as the boot type, so honour it - if (key.equalsIgnoreCase(ApiConstants.BootType.BIOS.toString())) { - vm.details.remove(ApiConstants.BootType.UEFI.toString()); - continue; - } - - // Deploy as is, Don't care about the boot type or template settings - if (key.equalsIgnoreCase(ApiConstants.BootType.UEFI.toString()) && template.isDeployAsIs()) { - vm.details.remove(ApiConstants.BootType.UEFI.toString()); - continue; - } - - if (!hypervisorType.equals(HypervisorType.KVM)) { - if (key.equalsIgnoreCase(VmDetailConstants.IOTHREADS)) { - vm.details.remove(VmDetailConstants.IOTHREADS); - continue; - } - if (key.equalsIgnoreCase(VmDetailConstants.IO_POLICY)) { - vm.details.remove(VmDetailConstants.IO_POLICY); - continue; - } - } - - if (key.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || - key.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || - key.equalsIgnoreCase(VmDetailConstants.MEMORY)) { - // handle double byte strings. - vm.setDetail(key, Integer.toString(Integer.parseInt(customParameters.get(key)))); - } else { - vm.setDetail(key, customParameters.get(key)); - } - } - vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); - - persistVMDeployAsIsProperties(vm, userVmOVFPropertiesMap); - - List hiddenDetails = new ArrayList<>(); - if (customParameters.containsKey(VmDetailConstants.NAME_ON_HYPERVISOR)) { - hiddenDetails.add(VmDetailConstants.NAME_ON_HYPERVISOR); - } - _vmDao.saveDetails(vm, hiddenDetails); - if (!isImport) { - s_logger.debug("Allocating in the DB for vm"); - DataCenterDeployment plan = new DataCenterDeployment(zone.getId()); - - List computeTags = new ArrayList(); - computeTags.add(offering.getHostTag()); - - List rootDiskTags = new ArrayList(); - DiskOfferingVO rootDiskOfferingVO = _diskOfferingDao.findById(rootDiskOfferingId); - rootDiskTags.add(rootDiskOfferingVO.getTags()); - - if (isIso) { - _orchSrvc.createVirtualMachineFromScratch(vm.getUuid(), Long.toString(owner.getAccountId()), vm.getIsoId().toString(), hostName, displayName, - hypervisorType.name(), guestOSCategory.getName(), offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, - networkNicMap, plan, extraDhcpOptionMap, rootDiskOfferingId); - } else { - _orchSrvc.createVirtualMachine(vm.getUuid(), Long.toString(owner.getAccountId()), Long.toString(template.getId()), hostName, displayName, hypervisorType.name(), - offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, networkNicMap, plan, rootDiskSize, extraDhcpOptionMap, - dataDiskTemplateToDiskOfferingMap, diskOfferingId, rootDiskOfferingId); - } - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Successfully allocated DB entry for " + vm); - } - } - CallContext.current().setEventDetails("Vm Id: " + vm.getUuid()); - - if (!isImport) { - if (!offering.isDynamic()) { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), - hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); - } else { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), - hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), customParameters, vm.isDisplayVm()); - } - - //Update Resource Count for the given account - resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize())); - } - return vm; + VMTemplateVO templateVO = _templateDao.findById(template.getId()); + if (templateVO == null) { + throw new InvalidParameterValueException("Unable to look up template by id " + template.getId()); } - }); + + validateRootDiskResize(hypervisorType, rootDiskSize, templateVO, vm, customParameters); + } + + if (isDisplayVm != null) { + vm.setDisplayVm(isDisplayVm); + } else { + vm.setDisplayVm(true); + } + + if (isImport) { + vm.setDataCenterId(zone.getId()); + vm.setHostId(host.getId()); + if (lastHost != null) { + vm.setLastHostId(lastHost.getId()); + } + vm.setPowerState(powerState); + if (powerState == VirtualMachine.PowerState.PowerOn) { + vm.setState(State.Running); + } + } + + vm.setUserVmType(vmType); + _vmDao.persist(vm); + for (String key : customParameters.keySet()) { + // BIOS was explicitly passed as the boot type, so honour it + if (key.equalsIgnoreCase(ApiConstants.BootType.BIOS.toString())) { + vm.details.remove(ApiConstants.BootType.UEFI.toString()); + continue; + } + + // Deploy as is, Don't care about the boot type or template settings + if (key.equalsIgnoreCase(ApiConstants.BootType.UEFI.toString()) && template.isDeployAsIs()) { + vm.details.remove(ApiConstants.BootType.UEFI.toString()); + continue; + } + + if (!hypervisorType.equals(HypervisorType.KVM)) { + if (key.equalsIgnoreCase(VmDetailConstants.IOTHREADS)) { + vm.details.remove(VmDetailConstants.IOTHREADS); + continue; + } + if (key.equalsIgnoreCase(VmDetailConstants.IO_POLICY)) { + vm.details.remove(VmDetailConstants.IO_POLICY); + continue; + } + } + + if (key.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || + key.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || + key.equalsIgnoreCase(VmDetailConstants.MEMORY)) { + // handle double byte strings. + vm.setDetail(key, Integer.toString(Integer.parseInt(customParameters.get(key)))); + } else { + vm.setDetail(key, customParameters.get(key)); + } + } + vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); + + persistVMDeployAsIsProperties(vm, userVmOVFPropertiesMap); + + List hiddenDetails = new ArrayList<>(); + if (customParameters.containsKey(VmDetailConstants.NAME_ON_HYPERVISOR)) { + hiddenDetails.add(VmDetailConstants.NAME_ON_HYPERVISOR); + } + _vmDao.saveDetails(vm, hiddenDetails); + if (!isImport) { + s_logger.debug("Allocating in the DB for vm"); + DataCenterDeployment plan = new DataCenterDeployment(zone.getId()); + + List computeTags = new ArrayList(); + computeTags.add(offering.getHostTag()); + + List rootDiskTags = new ArrayList(); + DiskOfferingVO rootDiskOfferingVO = _diskOfferingDao.findById(rootDiskOfferingId); + rootDiskTags.add(rootDiskOfferingVO.getTags()); + + if (isIso) { + _orchSrvc.createVirtualMachineFromScratch(vm.getUuid(), Long.toString(owner.getAccountId()), vm.getIsoId().toString(), hostName, displayName, + hypervisorType.name(), guestOSCategory.getName(), offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, + networkNicMap, plan, extraDhcpOptionMap, rootDiskOfferingId); + } else { + _orchSrvc.createVirtualMachine(vm.getUuid(), Long.toString(owner.getAccountId()), Long.toString(template.getId()), hostName, displayName, hypervisorType.name(), + offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, networkNicMap, plan, rootDiskSize, extraDhcpOptionMap, + dataDiskTemplateToDiskOfferingMap, diskOfferingId, rootDiskOfferingId); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Successfully allocated DB entry for " + vm); + } + } + CallContext.current().setEventDetails("Vm Id: " + vm.getUuid()); + + if (!isImport) { + if (!offering.isDynamic()) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), + hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); + } else { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), + hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), customParameters, vm.isDisplayVm()); + } + + //Update Resource Count for the given account + resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize())); + } + return vm; } private void updateVMDiskController(UserVmVO vm, Map customParameters, GuestOSVO guestOS) { diff --git a/ui/src/views/network/CreateIsolatedNetworkForm.vue b/ui/src/views/network/CreateIsolatedNetworkForm.vue index 3ed219eb97d..046510dacf2 100644 --- a/ui/src/views/network/CreateIsolatedNetworkForm.vue +++ b/ui/src/views/network/CreateIsolatedNetworkForm.vue @@ -646,6 +646,9 @@ export default { if (this.isValidTextValueForKey(values, 'privatemtu')) { params.privatemtu = values.privatemtu } + if ('vpcid' in values) { + params.vpcid = this.selectedVpc.id + } if ('domainid' in values && values.domainid > 0) { params.domainid = this.selectedDomain.id if (this.isValidTextValueForKey(values, 'account') && this.selectedAccount.id !== '-1') { diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index 11e62255fae..e68b5307f0e 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -46,13 +46,11 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.httpclient.methods.HeadMethod; import org.apache.commons.httpclient.util.URIUtil; import org.apache.commons.lang3.StringUtils; import org.apache.http.NameValuePair; @@ -348,32 +346,10 @@ public class UriUtils { return new HttpClient(s_httpClientManager); } - public static List getMetalinkChecksums(String url) { - HttpClient httpClient = getHttpClient(); - GetMethod getMethod = new GetMethod(url); - try { - if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) { - InputStream is = getMethod.getResponseBodyAsStream(); - Map> checksums = getMultipleValuesFromXML(is, new String[] {"hash"}); - if (checksums.containsKey("hash")) { - List listChksum = new ArrayList<>(); - for (String chk : checksums.get("hash")) { - listChksum.add(chk.replaceAll("\n", "").replaceAll(" ", "").trim()); - } - return listChksum; - } - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - getMethod.releaseConnection(); - } - return null; - } /** * Retrieve values from XML documents ordered by ascending priority for each tag name */ - protected static Map> getMultipleValuesFromXML(InputStream is, String[] tagNames) { + public static Map> getMultipleValuesFromXML(InputStream is, String[] tagNames) { Map> returnValues = new HashMap>(); try { DocumentBuilderFactory factory = ParserUtils.getSaferDocumentBuilderFactory(); @@ -400,45 +376,6 @@ public class UriUtils { return returnValues; } - /** - * Check if there is at least one existent URL defined on metalink - * @param url metalink url - * @return true if at least one existent URL defined on metalink, false if not - */ - protected static boolean checkUrlExistenceMetalink(String url) { - HttpClient httpClient = getHttpClient(); - GetMethod getMethod = new GetMethod(url); - try { - if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) { - InputStream is = getMethod.getResponseBodyAsStream(); - Map> metalinkUrls = getMultipleValuesFromXML(is, new String[] {"url"}); - if (metalinkUrls.containsKey("url")) { - List urls = metalinkUrls.get("url"); - boolean validUrl = false; - for (String u : urls) { - if (u.endsWith("torrent")) { - continue; - } - try { - UriUtils.checkUrlExistence(u); - validUrl = true; - break; - } - catch (IllegalArgumentException e) { - s_logger.warn(e.getMessage()); - } - } - return validUrl; - } - } - } catch (IOException e) { - s_logger.warn(e.getMessage()); - } finally { - getMethod.releaseConnection(); - } - return false; - } - /** * Get list of urls on metalink ordered by ascending priority (for those which priority tag is not defined, highest priority value is assumed) */ @@ -471,28 +408,6 @@ public class UriUtils { return urls; } - // use http HEAD method to validate url - public static void checkUrlExistence(String url) { - if (url.toLowerCase().startsWith("http") || url.toLowerCase().startsWith("https")) { - HttpClient httpClient = getHttpClient(); - HeadMethod httphead = new HeadMethod(url); - try { - if (httpClient.executeMethod(httphead) != HttpStatus.SC_OK) { - throw new IllegalArgumentException("Invalid URL: " + url); - } - if (url.endsWith("metalink") && !checkUrlExistenceMetalink(url)) { - throw new IllegalArgumentException("Invalid URLs defined on metalink: " + url); - } - } catch (HttpException hte) { - throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + hte.getMessage()); - } catch (IOException ioe) { - throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + ioe.getMessage()); - } finally { - httphead.releaseConnection(); - } - } - } - public static final Set COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz"); public static final Set buildExtensionSet(boolean metalink, String... baseExtensions) {