From d66f83ba2fb07d907997e12a6a89d477dcc8e1ec Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 5 Jul 2023 13:27:59 +0200 Subject: [PATCH 1/6] UI: fix wrong vpcid when create network in VPC (#7707) --- ui/src/views/network/CreateIsolatedNetworkForm.vue | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ui/src/views/network/CreateIsolatedNetworkForm.vue b/ui/src/views/network/CreateIsolatedNetworkForm.vue index d42dc90d823..21ab5e34914 100644 --- a/ui/src/views/network/CreateIsolatedNetworkForm.vue +++ b/ui/src/views/network/CreateIsolatedNetworkForm.vue @@ -596,7 +596,7 @@ export default { displayText: values.displaytext, networkOfferingId: this.selectedNetworkOffering.id } - var usefulFields = ['gateway', 'netmask', 'startip', 'endip', 'dns1', 'dns2', 'ip6dns1', 'ip6dns2', 'externalid', 'vpcid', 'vlan', 'networkdomain'] + var usefulFields = ['gateway', 'netmask', 'startip', 'endip', 'dns1', 'dns2', 'ip6dns1', 'ip6dns2', 'externalid', 'vlan', 'networkdomain'] for (var field of usefulFields) { if (this.isValidTextValueForKey(values, field)) { params[field] = values[field] @@ -608,6 +608,9 @@ export default { if (this.isValidTextValueForKey(values, 'privatemtu')) { params.privatemtu = values.privatemtu } + if ('vpcid' in values) { + params.vpcid = this.selectedVpc.id + } if ('domainid' in values && values.domainid > 0) { params.domainid = this.selectedDomain.id if (this.isValidTextValueForKey(values, 'account') && this.selectedAccount.id !== '-1') { From 0cbe77024a82cebf8734ed2a619eaa2ee8b3a602 Mon Sep 17 00:00:00 2001 From: Harikrishna Date: Thu, 6 Jul 2023 12:13:49 +0530 Subject: [PATCH 2/6] Fix foreign key constraints and the mysql procedure that is used (#7381) --- .../upgrade/dao/DatabaseAccessObject.java | 11 +++++++ .../com/cloud/upgrade/dao/DbUpgradeUtils.java | 5 +++- .../upgrade/dao/Upgrade41800to41810.java | 29 +++++++++++++++++++ .../META-INF/db/schema-41720to41800.sql | 14 --------- 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java index 5d0edddb02e..21d5a205a09 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java @@ -26,6 +26,17 @@ public class DatabaseAccessObject { private static Logger s_logger = Logger.getLogger(DatabaseAccessObject.class); + public void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) { + String addForeignKeyStmt = String.format("ALTER TABLE `cloud`.`%s` ADD CONSTRAINT `fk_%s__%s` FOREIGN KEY `fk_%s__%s`(`%s`) REFERENCES `%s`(`%s`)", tableName, tableName, tableColumn, tableName, tableColumn, tableColumn, foreignTableName, foreignColumnName); + try(PreparedStatement pstmt = conn.prepareStatement(addForeignKeyStmt);) + { + pstmt.executeUpdate(); + s_logger.debug(String.format("Foreign key is added successfully from the table %s", tableName)); + } catch (SQLException e) { + s_logger.error("Ignored SQL Exception when trying to add foreign key on table " + tableName + " exception: " + e.getMessage()); + } + } + public void dropKey(Connection conn, String tableName, String key, boolean isForeignKey) { String alter_sql_str; diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java index 38ca5c9c272..02dad6250dc 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java @@ -23,7 +23,10 @@ public class DbUpgradeUtils { private static DatabaseAccessObject dao = new DatabaseAccessObject(); - public static void dropKeysIfExist(Connection conn, String tableName, List keys, boolean isForeignKey) { + public static void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) { + dao.addForeignKey(conn, tableName, tableColumn, foreignTableName, foreignColumnName); + } + public static void dropKeysIfExist(Connection conn, String tableName, List keys, boolean isForeignKey) { for (String key : keys) { dao.dropKey(conn, tableName, key, isForeignKey); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java index c4fbeb73eb2..ab493da8cd8 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java @@ -22,6 +22,8 @@ import org.apache.log4j.Logger; import java.io.InputStream; import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate { final static Logger LOG = Logger.getLogger(Upgrade41800to41810.class); @@ -55,6 +57,7 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate @Override public void performDataMigration(Connection conn) { + fixForeignKeyNames(conn); } @Override @@ -82,4 +85,30 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate throw new CloudRuntimeException("Failed to find / register SystemVM template(s)"); } } + + private void fixForeignKeyNames(Connection conn) { + //Alter foreign key name for user_vm table from fk_user_data_id to fk_user_vm__user_data_id (if exists) + List keys = new ArrayList(); + keys.add("fk_user_data_id"); + keys.add("fk_user_vm__user_data_id"); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.user_vm", keys, true); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.user_vm", keys, false); + DbUpgradeUtils.addForeignKey(conn, "user_vm", "user_data_id", "user_data", "id"); + + //Alter foreign key name for vm_template table from fk_user_data_id to fk_vm_template__user_data_id (if exists) + keys = new ArrayList<>(); + keys.add("fk_user_data_id"); + keys.add("fk_vm_template__user_data_id"); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.vm_template", keys, true); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.vm_template", keys, false); + DbUpgradeUtils.addForeignKey(conn, "vm_template", "user_data_id", "user_data", "id"); + + //Alter foreign key name for volumes table from fk_passphrase_id to fk_volumes__passphrase_id (if exists) + keys = new ArrayList<>(); + keys.add("fk_passphrase_id"); + keys.add("fk_volumes__passphrase_id"); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.volumes", keys, true); + DbUpgradeUtils.dropKeysIfExist(conn, "cloud.volumes", keys, false); + DbUpgradeUtils.addForeignKey(conn, "volumes", "passphrase_id","passphrase", "id"); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql index b8eee33cad7..2af6723c134 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql @@ -224,17 +224,6 @@ CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` ( BEGIN DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END; - --- Add foreign key procedure to link volumes to passphrase table -DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`; -CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY` ( - IN in_table_name VARCHAR(200), - IN in_foreign_table_name VARCHAR(200), - IN in_foreign_column_name VARCHAR(200) -) -BEGIN - DECLARE CONTINUE HANDLER FOR 1005,1826 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', ' ADD CONSTRAINT '); SET @ddl = CONCAT(@ddl, 'fk_', in_foreign_table_name, '_', in_foreign_column_name); SET @ddl = CONCAT(@ddl, ' FOREIGN KEY (', in_foreign_table_name, '_', in_foreign_column_name, ')'); SET @ddl = CONCAT(@ddl, ' REFERENCES ', in_foreign_table_name, '(', in_foreign_column_name, ')'); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END; - -- Add passphrase table CREATE TABLE IF NOT EXISTS `cloud`.`passphrase` ( `id` bigint unsigned NOT NULL auto_increment, @@ -244,7 +233,6 @@ CREATE TABLE IF NOT EXISTS `cloud`.`passphrase` ( -- Add passphrase column to volumes table CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'passphrase_id', 'bigint unsigned DEFAULT NULL COMMENT "encryption passphrase id" '); -CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.volumes', 'passphrase', 'id'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'encrypt_format', 'varchar(64) DEFAULT NULL COMMENT "encryption format" '); -- Add encrypt column to disk_offering @@ -653,11 +641,9 @@ CREATE TABLE IF NOT EXISTS `cloud`.`user_data` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_vm', 'user_data_id', 'bigint unsigned DEFAULT NULL COMMENT "id of the user data" AFTER `user_data`'); -CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.user_vm', 'user_data', 'id'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_vm', 'user_data_details', 'mediumtext DEFAULT NULL COMMENT "value of the comma-separated list of parameters" AFTER `user_data_id`'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_template', 'user_data_id', 'bigint unsigned DEFAULT NULL COMMENT "id of the user data"'); -CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.vm_template', 'user_data', 'id'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_template', 'user_data_link_policy', 'varchar(255) DEFAULT NULL COMMENT "user data link policy with template"'); -- Added userdata details to template From acc6f4e725ef785ce90e616e6a013c4792d3b496 Mon Sep 17 00:00:00 2001 From: dahn Date: Thu, 6 Jul 2023 09:29:42 +0200 Subject: [PATCH 3/6] NPE guard (#7691) --- .../cloudstack/storage/datastore/db/TemplateDataStoreVO.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java index 8647e968127..a8d1af62f53 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java @@ -277,7 +277,7 @@ public class TemplateDataStoreVO implements StateObject Date: Thu, 6 Jul 2023 09:33:51 +0200 Subject: [PATCH 4/6] use `Files.createTempDirectory()` instead of `new File()` (#7713) --- .../com/cloud/storage/JavaStorageLayer.java | 9 +++- .../cloud/storage/JavaStorageLayerTest.java | 46 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/com/cloud/storage/JavaStorageLayerTest.java diff --git a/core/src/main/java/com/cloud/storage/JavaStorageLayer.java b/core/src/main/java/com/cloud/storage/JavaStorageLayer.java index b65a76b6c25..d4c2639d478 100644 --- a/core/src/main/java/com/cloud/storage/JavaStorageLayer.java +++ b/core/src/main/java/com/cloud/storage/JavaStorageLayer.java @@ -22,8 +22,11 @@ package com.cloud.storage; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -186,8 +189,12 @@ public class JavaStorageLayer implements StorageLayer { @Override public File createUniqDir() throws IOException { String dirName = System.getProperty("java.io.tmpdir"); + String subDirNamePrefix = ""; + FileAttribute> perms = PosixFilePermissions + .asFileAttribute(PosixFilePermissions.fromString("rwxrwx---")); if (dirName != null) { - File dir = new File(dirName); + Path p = Files.createTempDirectory(Path.of(dirName), subDirNamePrefix, perms); + File dir = p.toFile(); if (dir.exists()) { if (isWorldReadable(dir)) { if (STD_TMP_DIR_PATH.equals(dir.getAbsolutePath())) { diff --git a/core/src/test/java/com/cloud/storage/JavaStorageLayerTest.java b/core/src/test/java/com/cloud/storage/JavaStorageLayerTest.java new file mode 100644 index 00000000000..241afcd5a92 --- /dev/null +++ b/core/src/test/java/com/cloud/storage/JavaStorageLayerTest.java @@ -0,0 +1,46 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.storage; + +import java.io.File; +import java.io.IOException; + +import org.junit.Assert; +import org.junit.Test; + +public class JavaStorageLayerTest { + + JavaStorageLayer jsl = new JavaStorageLayer(); + + @Test + public void createUniqDir() { + + try { + File one = jsl.createUniqDir(); + Assert.assertTrue(one.isDirectory()); + Assert.assertTrue(one.canRead()); + Assert.assertTrue(one.canWrite()); + Assert.assertTrue(one.canExecute()); + } catch (IOException e) { + Assert.fail("creation of a unique dir should succeed."); + } + } +} + From c733a23c9056735c9922a701ddee7e7ef0efefa0 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 6 Jul 2023 05:17:13 -0300 Subject: [PATCH 5/6] Fix direct download URL checks (#7693) This PR fixes the URL check for direct downloads, in the case of HTTPS URLs the certificates were not loaded into the SSL context --- .../HttpsDirectTemplateDownloader.java | 131 --------- .../MetalinkDirectTemplateDownloader.java | 101 ------- .../direct/download/DirectDownloadHelper.java | 83 ++++++ .../download/DirectTemplateDownloader.java | 61 +++++ .../DirectTemplateDownloaderImpl.java | 34 ++- .../HttpDirectTemplateDownloader.java | 89 ++++++- .../HttpsDirectTemplateDownloader.java | 252 ++++++++++++++++++ .../download/HttpsMultiTrustManager.java | 102 +++++++ .../MetalinkDirectTemplateDownloader.java | 177 ++++++++++++ .../download/NfsDirectTemplateDownloader.java | 33 ++- .../BaseDirectTemplateDownloaderTest.java | 72 +++++ .../HttpsDirectTemplateDownloaderTest.java | 36 +++ .../MetalinkDirectTemplateDownloaderTest.java | 30 +-- .../wrapper/LibvirtCheckUrlCommand.java | 19 +- .../kvm/storage/KVMStorageProcessor.java | 35 +-- .../cloud/storage/VolumeApiServiceImpl.java | 3 +- .../main/java/com/cloud/utils/UriUtils.java | 87 +----- 17 files changed, 950 insertions(+), 395 deletions(-) delete mode 100644 agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java delete mode 100644 agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java create mode 100644 core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java create mode 100644 core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java rename {agent/src/main/java/com/cloud/agent => core/src/main/java/org/apache/cloudstack}/direct/download/DirectTemplateDownloaderImpl.java (80%) rename {agent/src/main/java/com/cloud/agent => core/src/main/java/org/apache/cloudstack}/direct/download/HttpDirectTemplateDownloader.java (61%) create mode 100644 core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java create mode 100644 core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java create mode 100644 core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java rename {agent/src/main/java/com/cloud/agent => core/src/main/java/org/apache/cloudstack}/direct/download/NfsDirectTemplateDownloader.java (77%) create mode 100644 core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java create mode 100644 core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java rename agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java => core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java (58%) diff --git a/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java deleted file mode 100644 index d788310f68e..00000000000 --- a/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java +++ /dev/null @@ -1,131 +0,0 @@ -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// - -package com.cloud.agent.direct.download; - -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.commons.collections.MapUtils; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContexts; - -import javax.net.ssl.SSLContext; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.util.Map; - -public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader { - - private CloseableHttpClient httpsClient; - private HttpUriRequest req; - - public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, - Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) { - super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, temporaryDownloadPath); - SSLContext sslcontext = null; - try { - sslcontext = getSSLContext(); - } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) { - throw new CloudRuntimeException("Failure getting SSL context for HTTPS downloader: " + e.getMessage()); - } - SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); - RequestConfig config = RequestConfig.custom() - .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout) - .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout) - .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build(); - httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build(); - createUriRequest(url, headers); - } - - protected void createUriRequest(String downloadUrl, Map headers) { - req = new HttpGet(downloadUrl); - if (MapUtils.isNotEmpty(headers)) { - for (String headerKey: headers.keySet()) { - req.setHeader(headerKey, headers.get(headerKey)); - } - } - } - - private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException { - KeyStore trustStore = KeyStore.getInstance("jks"); - FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks")); - try { - String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null"; - String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties"); - String privatePassword = Script.runSimpleBashScript(privatePasswordCmd); - trustStore.load(instream, privatePassword.toCharArray()); - } finally { - instream.close(); - } - return SSLContexts.custom() - .loadTrustMaterial(trustStore, new TrustSelfSignedStrategy()) - .build(); - } - - @Override - public Pair downloadTemplate() { - CloseableHttpResponse response; - try { - response = httpsClient.execute(req); - } catch (IOException e) { - throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage()); - } - return consumeResponse(response); - } - - /** - * Consume response and persist it on getDownloadedFilePath() file - */ - protected Pair consumeResponse(CloseableHttpResponse response) { - s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); - if (response.getStatusLine().getStatusCode() != 200) { - throw new CloudRuntimeException("Error on HTTPS response"); - } - try { - HttpEntity entity = response.getEntity(); - InputStream in = entity.getContent(); - OutputStream out = new FileOutputStream(getDownloadedFilePath()); - IOUtils.copy(in, out); - } catch (Exception e) { - s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage()); - return new Pair<>(false, null); - } - return new Pair<>(true, getDownloadedFilePath()); - } - -} diff --git a/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java deleted file mode 100644 index 40e77c37110..00000000000 --- a/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java +++ /dev/null @@ -1,101 +0,0 @@ -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -package com.cloud.agent.direct.download; - -import com.cloud.utils.Pair; -import com.cloud.utils.UriUtils; -import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; - -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Random; - -public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader { - - private String metalinkUrl; - private List metalinkUrls; - private List metalinkChecksums; - private Random random = new Random(); - private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName()); - - public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, - Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { - super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, downloadPath); - metalinkUrl = url; - metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl); - metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl); - if (CollectionUtils.isEmpty(metalinkUrls)) { - throw new CloudRuntimeException("No urls found on metalink file: " + metalinkUrl + ". Not possible to download template " + templateId); - } - setUrl(metalinkUrls.get(0)); - s_logger.info("Metalink downloader created, metalink url: " + metalinkUrl + " parsed - " + - metalinkUrls.size() + " urls and " + - (CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found"); - } - - @Override - public Pair downloadTemplate() { - if (StringUtils.isBlank(getUrl())) { - throw new CloudRuntimeException("Download url has not been set, aborting"); - } - boolean downloaded = false; - int i = 0; - String downloadDir = getDirectDownloadTempPath(getTemplateId()); - do { - if (!isRedownload()) { - setUrl(metalinkUrls.get(i)); - } - s_logger.info("Trying to download template from url: " + getUrl()); - try { - setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); - File f = new File(getDownloadedFilePath()); - if (f.exists()) { - f.delete(); - f.createNewFile(); - } - request = createRequest(getUrl(), reqHeaders); - Pair downloadResult = super.downloadTemplate(); - downloaded = downloadResult.first(); - if (downloaded) { - s_logger.info("Successfully downloaded template from url: " + getUrl()); - } - - } catch (Exception e) { - s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage()); - } - i++; - } - while (!downloaded && !isRedownload() && i < metalinkUrls.size()); - return new Pair<>(downloaded, getDownloadedFilePath()); - } - - @Override - public boolean validateChecksum() { - if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) { - String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size())); - setChecksum(chk); - s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk); - } - return super.validateChecksum(); - } -} diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java new file mode 100644 index 00000000000..e06483ac44d --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java @@ -0,0 +1,83 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand; +import org.apache.log4j.Logger; + +public class DirectDownloadHelper { + + public static final Logger LOGGER = Logger.getLogger(DirectDownloadHelper.class.getName()); + + /** + * Get direct template downloader from direct download command and destination pool + */ + public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd, + String destPoolLocalPath, + String temporaryDownloadPath) { + if (cmd instanceof HttpDirectDownloadCommand) { + return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath); + } else if (cmd instanceof HttpsDirectDownloadCommand) { + return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath); + } else if (cmd instanceof NfsDirectDownloadCommand) { + return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath); + } else if (cmd instanceof MetalinkDirectDownloadCommand) { + return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath); + } else { + throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink"); + } + } + + public static boolean checkUrlExistence(String url) { + try { + DirectTemplateDownloader checker = getCheckerDownloader(url); + return checker.checkUrl(url); + } catch (CloudRuntimeException e) { + LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e); + return false; + } + } + + private static DirectTemplateDownloader getCheckerDownloader(String url) { + if (url.toLowerCase().startsWith("https:")) { + return new HttpsDirectTemplateDownloader(url); + } else if (url.toLowerCase().startsWith("http:")) { + return new HttpDirectTemplateDownloader(url); + } else if (url.toLowerCase().startsWith("nfs:")) { + return new NfsDirectTemplateDownloader(url); + } else if (url.toLowerCase().endsWith(".metalink")) { + return new MetalinkDirectTemplateDownloader(url); + } else { + throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url)); + } + } + + public static Long getFileSize(String url, String format) { + DirectTemplateDownloader checker = getCheckerDownloader(url); + return checker.getRemoteFileSize(url, format); + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java new file mode 100644 index 00000000000..c9dd32f72e8 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloader.java @@ -0,0 +1,61 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.Pair; + +import java.util.List; + +public interface DirectTemplateDownloader { + + /** + * Perform template download to pool specified on downloader creation + * @return (true if successful, false if not, download file path) + */ + Pair downloadTemplate(); + + /** + * Perform checksum validation of previously downloaded template + * @return true if successful, false if not + */ + boolean validateChecksum(); + + /** + * Validate if the URL is reachable and returns HTTP.OK status code + * @return true if the URL is reachable, false if not + */ + boolean checkUrl(String url); + + /** + * Obtain the remote file size (and virtual size in case format is qcow2) + */ + Long getRemoteFileSize(String url, String format); + + /** + * Get list of urls within metalink content ordered by ascending priority + * (for those which priority tag is not defined, highest priority value is assumed) + */ + List getMetalinkUrls(String metalinkUrl); + + /** + * Get the list of checksums within a metalink content + */ + List getMetalinkChecksums(String metalinkUrl); +} \ No newline at end of file diff --git a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java similarity index 80% rename from agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java rename to core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java index eb816192288..9476dbaa5ce 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java @@ -16,8 +16,9 @@ // specific language governing permissions and limitations // under the License. // -package com.cloud.agent.direct.download; +package org.apache.cloudstack.direct.download; +import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.lang3.StringUtils; @@ -26,7 +27,11 @@ import org.apache.log4j.Logger; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader { @@ -106,6 +111,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown return redownload; } + /** + * Create download directory (if it does not exist) + */ + protected File createTemporaryDirectoryAndFile(String downloadDir) { + createFolder(downloadDir); + return new File(downloadDir + File.separator + getFileNameFromUrl()); + } + /** * Return filename from url */ @@ -160,4 +173,23 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown } } + protected void addMetalinkUrlsToListFromInputStream(InputStream inputStream, List urls) { + Map> metalinkUrlsMap = UriUtils.getMultipleValuesFromXML(inputStream, new String[] {"url"}); + if (metalinkUrlsMap.containsKey("url")) { + List metalinkUrls = metalinkUrlsMap.get("url"); + urls.addAll(metalinkUrls); + } + } + + protected List generateChecksumListFromInputStream(InputStream is) { + Map> checksums = UriUtils.getMultipleValuesFromXML(is, new String[] {"hash"}); + if (checksums.containsKey("hash")) { + List listChksum = new ArrayList<>(); + for (String chk : checksums.get("hash")) { + listChksum.add(chk.replaceAll("\n", "").replaceAll(" ", "").trim()); + } + return listChksum; + } + return null; + } } diff --git a/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java similarity index 61% rename from agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java rename to core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index fc236034404..11ba6a5aab0 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -17,23 +17,28 @@ // under the License. // -package com.cloud.agent.direct.download; +package org.apache.cloudstack.direct.download; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import com.cloud.utils.Pair; +import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.storage.QCOW2Utils; import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.HeadMethod; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; @@ -45,6 +50,10 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod request; protected Map reqHeaders = new HashMap<>(); + protected HttpDirectTemplateDownloader(String url) { + this(url, null, null, null, null, null, null, null); + } + public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { super(url, destPoolPath, templateId, checksum, downloadPath); @@ -57,15 +66,6 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { setDownloadedFilePath(tempFile.getAbsolutePath()); } - /** - * Create download directory (if it does not exist) and set the download file - * @return - */ - protected File createTemporaryDirectoryAndFile(String downloadDir) { - createFolder(downloadDir); - return new File(downloadDir + File.separator + getFileNameFromUrl()); - } - protected GetMethod createRequest(String downloadUrl, Map headers) { GetMethod request = new GetMethod(downloadUrl); request.setFollowRedirects(true); @@ -98,7 +98,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); try ( InputStream in = request.getResponseBodyAsStream(); - OutputStream out = new FileOutputStream(getDownloadedFilePath()); + OutputStream out = new FileOutputStream(getDownloadedFilePath()) ) { IOUtils.copy(in, out); } catch (IOException e) { @@ -107,4 +107,71 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { } return new Pair<>(true, getDownloadedFilePath()); } + + @Override + public boolean checkUrl(String url) { + HeadMethod httpHead = new HeadMethod(url); + try { + if (client.executeMethod(httpHead) != HttpStatus.SC_OK) { + s_logger.error(String.format("Invalid URL: %s", url)); + return false; + } + return true; + } catch (IOException e) { + s_logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e); + return false; + } finally { + httpHead.releaseConnection(); + } + } + + @Override + public Long getRemoteFileSize(String url, String format) { + if ("qcow2".equalsIgnoreCase(format)) { + return QCOW2Utils.getVirtualSize(url); + } else { + return UriUtils.getRemoteSize(url); + } + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + GetMethod getMethod = new GetMethod(metalinkUrl); + List urls = new ArrayList<>(); + int status; + try { + status = client.executeMethod(getMethod); + } catch (IOException e) { + s_logger.error("Error retrieving urls form metalink: " + metalinkUrl); + getMethod.releaseConnection(); + return null; + } + try { + InputStream is = getMethod.getResponseBodyAsStream(); + if (status == HttpStatus.SC_OK) { + addMetalinkUrlsToListFromInputStream(is, urls); + } + } catch (IOException e) { + s_logger.warn(e.getMessage()); + } finally { + getMethod.releaseConnection(); + } + return urls; + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + GetMethod getMethod = new GetMethod(metalinkUrl); + try { + if (client.executeMethod(getMethod) == HttpStatus.SC_OK) { + InputStream is = getMethod.getResponseBodyAsStream(); + return generateChecksumListFromInputStream(is); + } + } catch (IOException e) { + s_logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e); + } finally { + getMethod.releaseConnection(); + } + return null; + } } \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java new file mode 100644 index 00000000000..a274eb562e4 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java @@ -0,0 +1,252 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import com.cloud.utils.storage.QCOW2Utils; +import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.commons.collections.MapUtils; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { + + protected CloseableHttpClient httpsClient; + private HttpUriRequest req; + + protected HttpsDirectTemplateDownloader(String url) { + this(url, null, null, null, null, null, null, null, null); + } + + public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, + Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) { + super(url, destPoolPath, templateId, checksum, temporaryDownloadPath); + SSLContext sslcontext = getSSLContext(); + SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout) + .setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout) + .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build(); + httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build(); + createUriRequest(url, headers); + String downloadDir = getDirectDownloadTempPath(templateId); + File tempFile = createTemporaryDirectoryAndFile(downloadDir); + setDownloadedFilePath(tempFile.getAbsolutePath()); + } + + protected void createUriRequest(String downloadUrl, Map headers) { + req = new HttpGet(downloadUrl); + if (MapUtils.isNotEmpty(headers)) { + for (String headerKey: headers.keySet()) { + req.setHeader(headerKey, headers.get(headerKey)); + } + } + } + + private SSLContext getSSLContext() { + try { + KeyStore customKeystore = KeyStore.getInstance("jks"); + try (FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"))) { + String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null"; + String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties"); + String privatePassword = Script.runSimpleBashScript(privatePasswordCmd); + customKeystore.load(instream, privatePassword.toCharArray()); + } + KeyStore defaultKeystore = KeyStore.getInstance(KeyStore.getDefaultType()); + String relativeCacertsPath = "/lib/security/cacerts".replace("/", File.separator); + String filename = System.getProperty("java.home") + relativeCacertsPath; + try (FileInputStream is = new FileInputStream(filename)) { + String password = "changeit"; + defaultKeystore.load(is, password.toCharArray()); + } + TrustManager[] tm = HttpsMultiTrustManager.getTrustManagersFromKeyStores(customKeystore, defaultKeystore); + SSLContext sslContext = SSLUtils.getSSLContext(); + sslContext.init(null, tm, null); + return sslContext; + } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) { + s_logger.error(String.format("Failure getting SSL context for HTTPS downloader, using default SSL context: %s", e.getMessage()), e); + try { + return SSLContext.getDefault(); + } catch (NoSuchAlgorithmException ex) { + throw new CloudRuntimeException(String.format("Cannot return the default SSL context due to: %s", ex.getMessage()), e); + } + } + + } + + @Override + public Pair downloadTemplate() { + CloseableHttpResponse response; + try { + response = httpsClient.execute(req); + } catch (IOException e) { + throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage()); + } + return consumeResponse(response); + } + + /** + * Consume response and persist it on getDownloadedFilePath() file + */ + protected Pair consumeResponse(CloseableHttpResponse response) { + s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); + if (response.getStatusLine().getStatusCode() != 200) { + throw new CloudRuntimeException("Error on HTTPS response"); + } + try { + HttpEntity entity = response.getEntity(); + InputStream in = entity.getContent(); + OutputStream out = new FileOutputStream(getDownloadedFilePath()); + IOUtils.copy(in, out); + } catch (Exception e) { + s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage()); + return new Pair<>(false, null); + } + return new Pair<>(true, getDownloadedFilePath()); + } + + @Override + public boolean checkUrl(String url) { + HttpHead httpHead = new HttpHead(url); + try { + CloseableHttpResponse response = httpsClient.execute(httpHead); + if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { + s_logger.error(String.format("Invalid URL: %s", url)); + return false; + } + return true; + } catch (IOException e) { + s_logger.error(String.format("Cannot reach URL: %s due to: %s", url, e.getMessage()), e); + return false; + } finally { + httpHead.releaseConnection(); + } + } + + @Override + public Long getRemoteFileSize(String url, String format) { + if ("qcow2".equalsIgnoreCase(format)) { + try { + URL urlObj = new URL(url); + HttpsURLConnection urlConnection = (HttpsURLConnection)urlObj.openConnection(); + SSLContext context = getSSLContext(); + urlConnection.setSSLSocketFactory(context.getSocketFactory()); + urlConnection.connect(); + return QCOW2Utils.getVirtualSize(urlConnection.getInputStream()); + } catch (IOException e) { + throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e); + } + } else { + HttpHead httpHead = new HttpHead(url); + CloseableHttpResponse response = null; + try { + response = httpsClient.execute(httpHead); + Header[] headers = response.getHeaders("Content-Length"); + for (Header header : headers) { + return Long.parseLong(header.getValue()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return null; + } + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + HttpGet getMethod = new HttpGet(metalinkUrl); + List urls = new ArrayList<>(); + CloseableHttpResponse response; + try { + response = httpsClient.execute(getMethod); + if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { + String msg = String.format("Cannot access metalink content on URL %s", metalinkUrl); + s_logger.error(msg); + throw new IOException(msg); + } + } catch (IOException e) { + s_logger.error(String.format("Error retrieving urls form metalink URL %s: %s", metalinkUrl, e.getMessage()), e); + getMethod.releaseConnection(); + return null; + } + + try { + String responseStr = EntityUtils.toString(response.getEntity()); + ByteArrayInputStream inputStream = new ByteArrayInputStream(responseStr.getBytes(StandardCharsets.UTF_8)); + addMetalinkUrlsToListFromInputStream(inputStream, urls); + } catch (IOException e) { + s_logger.warn(e.getMessage(), e); + } finally { + getMethod.releaseConnection(); + } + return urls; + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + HttpGet getMethod = new HttpGet(metalinkUrl); + try { + CloseableHttpResponse response = httpsClient.execute(getMethod); + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + InputStream is = response.getEntity().getContent(); + return generateChecksumListFromInputStream(is); + } + } catch (IOException e) { + s_logger.error(String.format("Error obtaining metalink checksums on URL %s: %s", metalinkUrl, e.getMessage()), e); + } finally { + getMethod.releaseConnection(); + } + return null; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java new file mode 100644 index 00000000000..f991bfdfb3c --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.direct.download; + +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +public class HttpsMultiTrustManager implements X509TrustManager { + + private final List trustManagers; + + public HttpsMultiTrustManager(KeyStore... keystores) { + List trustManagers = new ArrayList<>(); + trustManagers.add(getTrustManager(null)); + for (KeyStore keystore : keystores) { + trustManagers.add(getTrustManager(keystore)); + } + this.trustManagers = ImmutableList.copyOf(trustManagers); + } + + public static TrustManager[] getTrustManagersFromKeyStores(KeyStore... keyStore) { + return new TrustManager[] { new HttpsMultiTrustManager(keyStore) }; + + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + for (X509TrustManager trustManager : trustManagers) { + try { + trustManager.checkClientTrusted(chain, authType); + return; + } catch (CertificateException ignored) {} + } + throw new CertificateException("None of the TrustManagers trust this certificate chain"); + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + for (X509TrustManager trustManager : trustManagers) { + try { + trustManager.checkServerTrusted(chain, authType); + return; + } catch (CertificateException ignored) {} + } + throw new CertificateException("None of the TrustManagers trust this certificate chain"); + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + ImmutableList.Builder certificates = ImmutableList.builder(); + for (X509TrustManager trustManager : trustManagers) { + for (X509Certificate cert : trustManager.getAcceptedIssuers()) { + certificates.add(cert); + } + } + return Iterables.toArray(certificates.build(), X509Certificate.class); + } + + public X509TrustManager getTrustManager(KeyStore keystore) { + return getTrustManager(TrustManagerFactory.getDefaultAlgorithm(), keystore); + } + + public X509TrustManager getTrustManager(String algorithm, KeyStore keystore) { + TrustManagerFactory factory; + try { + factory = TrustManagerFactory.getInstance(algorithm); + factory.init(keystore); + return Iterables.getFirst(Iterables.filter( + Arrays.asList(factory.getTrustManagers()), X509TrustManager.class), null); + } catch (NoSuchAlgorithmException | KeyStoreException e) { + e.printStackTrace(); + } + return null; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java new file mode 100644 index 00000000000..8051313f968 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java @@ -0,0 +1,177 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.collections.CollectionUtils; + +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Random; + +public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl { + private List metalinkUrls; + private List metalinkChecksums; + private Random random = new Random(); + protected DirectTemplateDownloader downloader; + private Map headers; + private Integer connectTimeout; + private Integer soTimeout; + + private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName()); + + protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId, + String destPoolPath, String checksum, + Map headers, + Integer connectTimeout, Integer soTimeout, + Integer connectionRequestTimeout, String temporaryDownloadPath) { + if (url.toLowerCase().startsWith("https:")) { + return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers, + connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath); + } else if (url.toLowerCase().startsWith("http:")) { + return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers, + connectTimeout, soTimeout, temporaryDownloadPath); + } else if (url.toLowerCase().startsWith("nfs:")) { + return new NfsDirectTemplateDownloader(url); + } else { + s_logger.error(String.format("Cannot find a suitable downloader to handle the metalink URL %s", url)); + return null; + } + } + + protected MetalinkDirectTemplateDownloader(String url) { + this(url, null, null, null, null, null, null, null); + } + + public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, + Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { + super(url, destPoolPath, templateId, checksum, downloadPath); + this.headers = headers; + this.connectTimeout = connectTimeout; + this.soTimeout = soTimeout; + downloader = createDownloaderForMetalinks(url, templateId, destPoolPath, checksum, headers, + connectTimeout, soTimeout, null, downloadPath); + metalinkUrls = downloader.getMetalinkUrls(url); + metalinkChecksums = downloader.getMetalinkChecksums(url); + if (CollectionUtils.isEmpty(metalinkUrls)) { + s_logger.error(String.format("No urls found on metalink file: %s. Not possible to download template %s ", url, templateId)); + } else { + setUrl(metalinkUrls.get(0)); + s_logger.info("Metalink downloader created, metalink url: " + url + " parsed - " + + metalinkUrls.size() + " urls and " + + (CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found"); + } + } + + @Override + public Pair downloadTemplate() { + if (StringUtils.isBlank(getUrl())) { + throw new CloudRuntimeException("Download url has not been set, aborting"); + } + boolean downloaded = false; + int i = 0; + String downloadDir = getDirectDownloadTempPath(getTemplateId()); + do { + if (!isRedownload()) { + setUrl(metalinkUrls.get(i)); + } + s_logger.info("Trying to download template from url: " + getUrl()); + DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(getUrl(), getTemplateId(), getDestPoolPath(), + getChecksum(), headers, connectTimeout, soTimeout, null, temporaryDownloadPath); + try { + setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); + File f = new File(getDownloadedFilePath()); + if (f.exists()) { + f.delete(); + f.createNewFile(); + } + Pair downloadResult = urlDownloader.downloadTemplate(); + downloaded = downloadResult.first(); + if (downloaded) { + s_logger.info("Successfully downloaded template from url: " + getUrl()); + } + } catch (Exception e) { + s_logger.error(String.format("Error downloading template: %s from URL: %s due to: %s", getTemplateId(), getUrl(), e.getMessage()), e); + } + i++; + } + while (!downloaded && !isRedownload() && i < metalinkUrls.size()); + return new Pair<>(downloaded, getDownloadedFilePath()); + } + + @Override + public boolean validateChecksum() { + if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) { + String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size())); + setChecksum(chk); + s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk); + } + return super.validateChecksum(); + } + + @Override + public boolean checkUrl(String metalinkUrl) { + if (!downloader.checkUrl(metalinkUrl)) { + s_logger.error(String.format("Metalink URL check failed for: %s", metalinkUrl)); + return false; + } + + List metalinkUrls = downloader.getMetalinkUrls(metalinkUrl); + for (String url : metalinkUrls) { + if (url.endsWith(".torrent")) { + continue; + } + DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(url, null, null, null, headers, connectTimeout, soTimeout, null, null); + if (!urlDownloader.checkUrl(url)) { + return false; + } + } + return true; + } + + @Override + public Long getRemoteFileSize(String metalinkUrl, String format) { + List urls = downloader.getMetalinkUrls(metalinkUrl); + for (String url : urls) { + if (url.endsWith("torrent")) { + continue; + } + if (downloader.checkUrl(url)) { + return downloader.getRemoteFileSize(url, format); + } + } + return null; + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + return downloader.getMetalinkUrls(metalinkUrl); + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + return downloader.getMetalinkChecksums(metalinkUrl); + } + +} \ No newline at end of file diff --git a/agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java similarity index 77% rename from agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java rename to core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java index 932477031d6..d606136e297 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java @@ -16,7 +16,7 @@ // specific language governing permissions and limitations // under the License. // -package com.cloud.agent.direct.download; +package org.apache.cloudstack.direct.download; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; @@ -26,6 +26,7 @@ import com.cloud.utils.script.Script; import java.io.File; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; import java.util.UUID; public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @@ -52,6 +53,10 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { } } + protected NfsDirectTemplateDownloader(String url) { + this(url, null, null, null, null); + } + public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) { super(url, destPool, templateId, checksum, downloadPath); parseUrl(); @@ -68,4 +73,30 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid); return new Pair<>(true, getDownloadedFilePath()); } + + @Override + public boolean checkUrl(String url) { + try { + parseUrl(); + return true; + } catch (CloudRuntimeException e) { + s_logger.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e); + return false; + } + } + + @Override + public Long getRemoteFileSize(String url, String format) { + return null; + } + + @Override + public List getMetalinkUrls(String metalinkUrl) { + return null; + } + + @Override + public List getMetalinkChecksums(String metalinkUrl) { + return null; + } } diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java new file mode 100644 index 00000000000..8b70d29b965 --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java @@ -0,0 +1,72 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpStatus; +import org.apache.http.HttpVersion; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicStatusLine; +import org.junit.Before; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class BaseDirectTemplateDownloaderTest { + protected static final String httpUrl = "http://server/path"; + protected static final String httpsUrl = "https://server:443/path"; + protected static final String httpMetalinkUrl = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2"; + protected static final String httpMetalinkContent = "\n" + + "\n" + + "" + httpMetalinkUrl + "\n" + + "\n" + + ""; + + @Mock + private CloseableHttpClient httpsClient; + + @Mock + private CloseableHttpResponse response; + @Mock + private HttpEntity httpEntity; + + @InjectMocks + protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl); + + @Before + public void init() throws IOException { + MockitoAnnotations.initMocks(this); + Mockito.when(httpsClient.execute(Mockito.any(HttpGet.class))).thenReturn(response); + Mockito.when(httpsClient.execute(Mockito.any(HttpHead.class))).thenReturn(response); + StatusLine statusLine = new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK"); + Mockito.when(response.getStatusLine()).thenReturn(statusLine); + Mockito.when(response.getEntity()).thenReturn(httpEntity); + ByteArrayInputStream inputStream = new ByteArrayInputStream(httpMetalinkContent.getBytes(StandardCharsets.UTF_8)); + Mockito.when(httpEntity.getContent()).thenReturn(inputStream); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java new file mode 100644 index 00000000000..589ec77902f --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloaderTest.java @@ -0,0 +1,36 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.direct.download; + +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class HttpsDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest { + + @Test + public void testGetMetalinkUrls() { + List metalinkUrls = httpsDownloader.getMetalinkUrls(httpUrl); + Assert.assertTrue(CollectionUtils.isNotEmpty(metalinkUrls)); + Assert.assertEquals(1, metalinkUrls.size()); + Assert.assertEquals(httpMetalinkUrl, metalinkUrls.get(0)); + } +} \ No newline at end of file diff --git a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java similarity index 58% rename from agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java rename to core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java index 32b84a34436..fec23c5533c 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java +++ b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java @@ -16,22 +16,20 @@ // specific language governing permissions and limitations // under the License. // +package org.apache.cloudstack.direct.download; -package com.cloud.agent.direct.download; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.InjectMocks; -import com.cloud.utils.Pair; +public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest { -public interface DirectTemplateDownloader { - - /** - * Perform template download to pool specified on downloader creation - * @return (true if successful, false if not, download file path) - */ - Pair downloadTemplate(); - - /** - * Perform checksum validation of previously downloadeed template - * @return true if successful, false if not - */ - boolean validateChecksum(); -} + @InjectMocks + protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl); + @Test + public void testCheckUrlMetalink() { + metalinkDownloader.downloader = httpsDownloader; + boolean result = metalinkDownloader.checkUrl(httpsUrl); + Assert.assertTrue(result); + } +} \ No newline at end of file diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java index 2618f20fae1..939d43086f4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java @@ -18,6 +18,7 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import org.apache.cloudstack.direct.download.DirectDownloadHelper; import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer; import org.apache.cloudstack.agent.directdownload.CheckUrlCommand; import org.apache.log4j.Logger; @@ -25,8 +26,6 @@ import org.apache.log4j.Logger; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; -import com.cloud.utils.UriUtils; -import com.cloud.utils.storage.QCOW2Utils; @ResourceWrapper(handles = CheckUrlCommand.class) public class LibvirtCheckUrlCommand extends CommandWrapper { @@ -37,20 +36,10 @@ public class LibvirtCheckUrlCommand extends CommandWrapper result = downloader.downloadTemplate(); if (!result.first()) { diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 6b1d170be9c..d6325b43134 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -52,6 +52,7 @@ import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.direct.download.DirectDownloadHelper; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -533,7 +534,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic UriUtils.validateUrl(format, url); if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access s_logger.debug("Checking url: " + url); - UriUtils.checkUrlExistence(url); + DirectDownloadHelper.checkUrlExistence(url); } // Check that the resource limit for secondary storage won't be exceeded _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index 11e62255fae..e68b5307f0e 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -46,13 +46,11 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.httpclient.methods.HeadMethod; import org.apache.commons.httpclient.util.URIUtil; import org.apache.commons.lang3.StringUtils; import org.apache.http.NameValuePair; @@ -348,32 +346,10 @@ public class UriUtils { return new HttpClient(s_httpClientManager); } - public static List getMetalinkChecksums(String url) { - HttpClient httpClient = getHttpClient(); - GetMethod getMethod = new GetMethod(url); - try { - if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) { - InputStream is = getMethod.getResponseBodyAsStream(); - Map> checksums = getMultipleValuesFromXML(is, new String[] {"hash"}); - if (checksums.containsKey("hash")) { - List listChksum = new ArrayList<>(); - for (String chk : checksums.get("hash")) { - listChksum.add(chk.replaceAll("\n", "").replaceAll(" ", "").trim()); - } - return listChksum; - } - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - getMethod.releaseConnection(); - } - return null; - } /** * Retrieve values from XML documents ordered by ascending priority for each tag name */ - protected static Map> getMultipleValuesFromXML(InputStream is, String[] tagNames) { + public static Map> getMultipleValuesFromXML(InputStream is, String[] tagNames) { Map> returnValues = new HashMap>(); try { DocumentBuilderFactory factory = ParserUtils.getSaferDocumentBuilderFactory(); @@ -400,45 +376,6 @@ public class UriUtils { return returnValues; } - /** - * Check if there is at least one existent URL defined on metalink - * @param url metalink url - * @return true if at least one existent URL defined on metalink, false if not - */ - protected static boolean checkUrlExistenceMetalink(String url) { - HttpClient httpClient = getHttpClient(); - GetMethod getMethod = new GetMethod(url); - try { - if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) { - InputStream is = getMethod.getResponseBodyAsStream(); - Map> metalinkUrls = getMultipleValuesFromXML(is, new String[] {"url"}); - if (metalinkUrls.containsKey("url")) { - List urls = metalinkUrls.get("url"); - boolean validUrl = false; - for (String u : urls) { - if (u.endsWith("torrent")) { - continue; - } - try { - UriUtils.checkUrlExistence(u); - validUrl = true; - break; - } - catch (IllegalArgumentException e) { - s_logger.warn(e.getMessage()); - } - } - return validUrl; - } - } - } catch (IOException e) { - s_logger.warn(e.getMessage()); - } finally { - getMethod.releaseConnection(); - } - return false; - } - /** * Get list of urls on metalink ordered by ascending priority (for those which priority tag is not defined, highest priority value is assumed) */ @@ -471,28 +408,6 @@ public class UriUtils { return urls; } - // use http HEAD method to validate url - public static void checkUrlExistence(String url) { - if (url.toLowerCase().startsWith("http") || url.toLowerCase().startsWith("https")) { - HttpClient httpClient = getHttpClient(); - HeadMethod httphead = new HeadMethod(url); - try { - if (httpClient.executeMethod(httphead) != HttpStatus.SC_OK) { - throw new IllegalArgumentException("Invalid URL: " + url); - } - if (url.endsWith("metalink") && !checkUrlExistenceMetalink(url)) { - throw new IllegalArgumentException("Invalid URLs defined on metalink: " + url); - } - } catch (HttpException hte) { - throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + hte.getMessage()); - } catch (IOException ioe) { - throw new IllegalArgumentException("Cannot reach URL: " + url + " due to: " + ioe.getMessage()); - } finally { - httphead.releaseConnection(); - } - } - } - public static final Set COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz"); public static final Set buildExtensionSet(boolean metalink, String... baseExtensions) { From 939ee9e1534a1e6013a74f92dbb151666b3887e3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 6 Jul 2023 14:04:38 +0530 Subject: [PATCH 6/6] server,engine-orchestration: allocate vm without transaction (#7695) When deploying a VM is failed during the allocation process it may leave the resources that have been already allocated before the failure. They will get removed from the database as the whole code block is wrapped inside a transaction twice but the server would not inform the network or storage plugins to clean up the allocated resources. This PR removes Transactions during VM allocation which results in the allocated VM and its resource records being persisted in DB even during failures. When failure is encountered VM is moved to Error state. This helps VM and its resources to be properly deallocated when it is expunged either by a server task such as ExpungeTask or during manual expunge. Signed-off-by: Abhishek Kumar --- .../cloud/vm/VirtualMachineManagerImpl.java | 113 +++---- .../java/com/cloud/vm/UserVmManagerImpl.java | 288 +++++++++--------- 2 files changed, 201 insertions(+), 200 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 9b37132d07c..d3ba95061b6 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -458,72 +458,79 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throws InsufficientCapacityException { s_logger.info(String.format("allocating virtual machine from template:%s with hostname:%s and %d networks", template.getUuid(), vmInstanceName, auxiliaryNetworks.size())); + VMInstanceVO persistedVm = null; + try { + final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName); + final Account owner = _entityMgr.findById(Account.class, vm.getAccountId()); - final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName); - final Account owner = _entityMgr.findById(Account.class, vm.getAccountId()); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocating entries for VM: " + vm); + } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocating entries for VM: " + vm); - } + vm.setDataCenterId(plan.getDataCenterId()); + if (plan.getPodId() != null) { + vm.setPodIdToDeployIn(plan.getPodId()); + } + assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet"; + persistedVm = _vmDao.persist(vm); - vm.setDataCenterId(plan.getDataCenterId()); - if (plan.getPodId() != null) { - vm.setPodIdToDeployIn(plan.getPodId()); - } - assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet"; - final VMInstanceVO vmFinal = _vmDao.persist(vm); + final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(persistedVm, template, serviceOffering, null, null); - final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmFinal, template, serviceOffering, null, null); + Long rootDiskSize = rootDiskOfferingInfo.getSize(); + if (vm.getType().isUsedBySystem() && SystemVmRootDiskSize.value() != null && SystemVmRootDiskSize.value() > 0L) { + rootDiskSize = SystemVmRootDiskSize.value(); + } + final Long rootDiskSizeFinal = rootDiskSize; - Long rootDiskSize = rootDiskOfferingInfo.getSize(); - if (vm.getType().isUsedBySystem() && SystemVmRootDiskSize.value() != null && SystemVmRootDiskSize.value() > 0L) { - rootDiskSize = SystemVmRootDiskSize.value(); - } - final Long rootDiskSizeFinal = rootDiskSize; + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocating nics for " + persistedVm); + } - Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { - @Override - public void doInTransactionWithoutResult(final TransactionStatus status) throws InsufficientCapacityException { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocating nics for " + vmFinal); + try { + if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) { + _networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions); } + } catch (final ConcurrentOperationException e) { + throw new CloudRuntimeException("Concurrent operation while trying to allocate resources for the VM", e); + } - try { - if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) { - _networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions); - } - } catch (final ConcurrentOperationException e) { - throw new CloudRuntimeException("Concurrent operation while trying to allocate resources for the VM", e); - } + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocating disks for " + persistedVm); + } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocating disks for " + vmFinal); - } + allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal); - allocateRootVolume(vmFinal, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal); - - if (dataDiskOfferings != null) { - for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) { - volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vmFinal.getId(), dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), - dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), vmFinal, template, owner, null); - } - } - if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { - int diskNumber = 1; - for (Entry dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) { - DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue(); - long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); - VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey()); - volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vmFinal.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null, - vmFinal, dataDiskTemplate, owner, Long.valueOf(diskNumber)); - diskNumber++; - } + if (dataDiskOfferings != null) { + for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) { + volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId(), dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), + dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, null); + } + } + if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { + int diskNumber = 1; + for (Entry dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) { + DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue(); + long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); + VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey()); + volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null, + persistedVm, dataDiskTemplate, owner, Long.valueOf(diskNumber)); + diskNumber++; } } - }); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocation completed for VM: " + vmFinal); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocation completed for VM: " + persistedVm); + } + } catch (InsufficientCapacityException | CloudRuntimeException e) { + // Failed VM will be in Stopped. Transition it to Error, so it can be expunged by ExpungeTask or similar + try { + if (persistedVm != null) { + stateTransitTo(persistedVm, VirtualMachine.Event.OperationFailedToError, null); + } + } catch (NoTransitionException nte) { + s_logger.error(String.format("Failed to transition %s in %s state to Error state", persistedVm, persistedVm.getState().toString())); + } + throw e; } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index f0aed627f15..b4e8e137984 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -348,7 +348,6 @@ import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.db.UUIDManager; @@ -4366,161 +4365,156 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, final Map userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState, final boolean dynamicScalingEnabled, String vmType, final Long rootDiskOfferingId, String sshkeypairs) throws InsufficientCapacityException { - return Transaction.execute(new TransactionCallbackWithException() { - @Override - public UserVmVO doInTransaction(TransactionStatus status) throws InsufficientCapacityException { - UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, template.getGuestOSId(), offering.isOfferHA(), - offering.getLimitCpuUse(), owner.getDomainId(), owner.getId(), userId, offering.getId(), userData, userDataId, userDataDetails, hostName); - vm.setUuid(uuidName); - vm.setDynamicallyScalable(dynamicScalingEnabled); + UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, template.getGuestOSId(), offering.isOfferHA(), + offering.getLimitCpuUse(), owner.getDomainId(), owner.getId(), userId, offering.getId(), userData, userDataId, userDataDetails, hostName); + vm.setUuid(uuidName); + vm.setDynamicallyScalable(dynamicScalingEnabled); - Map details = template.getDetails(); - if (details != null && !details.isEmpty()) { - vm.details.putAll(details); - } + Map details = template.getDetails(); + if (details != null && !details.isEmpty()) { + vm.details.putAll(details); + } - if (StringUtils.isNotBlank(sshPublicKeys)) { - vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKeys); - } + if (StringUtils.isNotBlank(sshPublicKeys)) { + vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKeys); + } - if (StringUtils.isNotBlank(sshkeypairs)) { - vm.setDetail(VmDetailConstants.SSH_KEY_PAIR_NAMES, sshkeypairs); - } + if (StringUtils.isNotBlank(sshkeypairs)) { + vm.setDetail(VmDetailConstants.SSH_KEY_PAIR_NAMES, sshkeypairs); + } - if (keyboard != null && !keyboard.isEmpty()) { - vm.setDetail(VmDetailConstants.KEYBOARD, keyboard); - } + if (keyboard != null && !keyboard.isEmpty()) { + vm.setDetail(VmDetailConstants.KEYBOARD, keyboard); + } - if (!isImport && isIso) { - vm.setIsoId(template.getId()); - } + if (!isImport && isIso) { + vm.setIsoId(template.getId()); + } - long guestOSId = template.getGuestOSId(); - GuestOSVO guestOS = _guestOSDao.findById(guestOSId); - long guestOSCategoryId = guestOS.getCategoryId(); - GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId); - if (hypervisorType.equals(HypervisorType.VMware)) { - updateVMDiskController(vm, customParameters, guestOS); - } + long guestOSId = template.getGuestOSId(); + GuestOSVO guestOS = _guestOSDao.findById(guestOSId); + long guestOSCategoryId = guestOS.getCategoryId(); + GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId); + if (hypervisorType.equals(HypervisorType.VMware)) { + updateVMDiskController(vm, customParameters, guestOS); + } - Long rootDiskSize = null; - // custom root disk size, resizes base template to larger size - if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { - // already verified for positive number - rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE)); + Long rootDiskSize = null; + // custom root disk size, resizes base template to larger size + if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { + // already verified for positive number + rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE)); - VMTemplateVO templateVO = _templateDao.findById(template.getId()); - if (templateVO == null) { - throw new InvalidParameterValueException("Unable to look up template by id " + template.getId()); - } - - validateRootDiskResize(hypervisorType, rootDiskSize, templateVO, vm, customParameters); - } - - if (isDisplayVm != null) { - vm.setDisplayVm(isDisplayVm); - } else { - vm.setDisplayVm(true); - } - - if (isImport) { - vm.setDataCenterId(zone.getId()); - vm.setHostId(host.getId()); - if (lastHost != null) { - vm.setLastHostId(lastHost.getId()); - } - vm.setPowerState(powerState); - if (powerState == VirtualMachine.PowerState.PowerOn) { - vm.setState(State.Running); - } - } - - vm.setUserVmType(vmType); - _vmDao.persist(vm); - for (String key : customParameters.keySet()) { - // BIOS was explicitly passed as the boot type, so honour it - if (key.equalsIgnoreCase(ApiConstants.BootType.BIOS.toString())) { - vm.details.remove(ApiConstants.BootType.UEFI.toString()); - continue; - } - - // Deploy as is, Don't care about the boot type or template settings - if (key.equalsIgnoreCase(ApiConstants.BootType.UEFI.toString()) && template.isDeployAsIs()) { - vm.details.remove(ApiConstants.BootType.UEFI.toString()); - continue; - } - - if (!hypervisorType.equals(HypervisorType.KVM)) { - if (key.equalsIgnoreCase(VmDetailConstants.IOTHREADS)) { - vm.details.remove(VmDetailConstants.IOTHREADS); - continue; - } - if (key.equalsIgnoreCase(VmDetailConstants.IO_POLICY)) { - vm.details.remove(VmDetailConstants.IO_POLICY); - continue; - } - } - - if (key.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || - key.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || - key.equalsIgnoreCase(VmDetailConstants.MEMORY)) { - // handle double byte strings. - vm.setDetail(key, Integer.toString(Integer.parseInt(customParameters.get(key)))); - } else { - vm.setDetail(key, customParameters.get(key)); - } - } - vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); - - persistVMDeployAsIsProperties(vm, userVmOVFPropertiesMap); - - List hiddenDetails = new ArrayList<>(); - if (customParameters.containsKey(VmDetailConstants.NAME_ON_HYPERVISOR)) { - hiddenDetails.add(VmDetailConstants.NAME_ON_HYPERVISOR); - } - _vmDao.saveDetails(vm, hiddenDetails); - if (!isImport) { - s_logger.debug("Allocating in the DB for vm"); - DataCenterDeployment plan = new DataCenterDeployment(zone.getId()); - - List computeTags = new ArrayList(); - computeTags.add(offering.getHostTag()); - - List rootDiskTags = new ArrayList(); - DiskOfferingVO rootDiskOfferingVO = _diskOfferingDao.findById(rootDiskOfferingId); - rootDiskTags.add(rootDiskOfferingVO.getTags()); - - if (isIso) { - _orchSrvc.createVirtualMachineFromScratch(vm.getUuid(), Long.toString(owner.getAccountId()), vm.getIsoId().toString(), hostName, displayName, - hypervisorType.name(), guestOSCategory.getName(), offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, - networkNicMap, plan, extraDhcpOptionMap, rootDiskOfferingId); - } else { - _orchSrvc.createVirtualMachine(vm.getUuid(), Long.toString(owner.getAccountId()), Long.toString(template.getId()), hostName, displayName, hypervisorType.name(), - offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, networkNicMap, plan, rootDiskSize, extraDhcpOptionMap, - dataDiskTemplateToDiskOfferingMap, diskOfferingId, rootDiskOfferingId); - } - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Successfully allocated DB entry for " + vm); - } - } - CallContext.current().setEventDetails("Vm Id: " + vm.getUuid()); - - if (!isImport) { - if (!offering.isDynamic()) { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), - hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); - } else { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), - hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), customParameters, vm.isDisplayVm()); - } - - //Update Resource Count for the given account - resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize())); - } - return vm; + VMTemplateVO templateVO = _templateDao.findById(template.getId()); + if (templateVO == null) { + throw new InvalidParameterValueException("Unable to look up template by id " + template.getId()); } - }); + + validateRootDiskResize(hypervisorType, rootDiskSize, templateVO, vm, customParameters); + } + + if (isDisplayVm != null) { + vm.setDisplayVm(isDisplayVm); + } else { + vm.setDisplayVm(true); + } + + if (isImport) { + vm.setDataCenterId(zone.getId()); + vm.setHostId(host.getId()); + if (lastHost != null) { + vm.setLastHostId(lastHost.getId()); + } + vm.setPowerState(powerState); + if (powerState == VirtualMachine.PowerState.PowerOn) { + vm.setState(State.Running); + } + } + + vm.setUserVmType(vmType); + _vmDao.persist(vm); + for (String key : customParameters.keySet()) { + // BIOS was explicitly passed as the boot type, so honour it + if (key.equalsIgnoreCase(ApiConstants.BootType.BIOS.toString())) { + vm.details.remove(ApiConstants.BootType.UEFI.toString()); + continue; + } + + // Deploy as is, Don't care about the boot type or template settings + if (key.equalsIgnoreCase(ApiConstants.BootType.UEFI.toString()) && template.isDeployAsIs()) { + vm.details.remove(ApiConstants.BootType.UEFI.toString()); + continue; + } + + if (!hypervisorType.equals(HypervisorType.KVM)) { + if (key.equalsIgnoreCase(VmDetailConstants.IOTHREADS)) { + vm.details.remove(VmDetailConstants.IOTHREADS); + continue; + } + if (key.equalsIgnoreCase(VmDetailConstants.IO_POLICY)) { + vm.details.remove(VmDetailConstants.IO_POLICY); + continue; + } + } + + if (key.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || + key.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || + key.equalsIgnoreCase(VmDetailConstants.MEMORY)) { + // handle double byte strings. + vm.setDetail(key, Integer.toString(Integer.parseInt(customParameters.get(key)))); + } else { + vm.setDetail(key, customParameters.get(key)); + } + } + vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); + + persistVMDeployAsIsProperties(vm, userVmOVFPropertiesMap); + + List hiddenDetails = new ArrayList<>(); + if (customParameters.containsKey(VmDetailConstants.NAME_ON_HYPERVISOR)) { + hiddenDetails.add(VmDetailConstants.NAME_ON_HYPERVISOR); + } + _vmDao.saveDetails(vm, hiddenDetails); + if (!isImport) { + s_logger.debug("Allocating in the DB for vm"); + DataCenterDeployment plan = new DataCenterDeployment(zone.getId()); + + List computeTags = new ArrayList(); + computeTags.add(offering.getHostTag()); + + List rootDiskTags = new ArrayList(); + DiskOfferingVO rootDiskOfferingVO = _diskOfferingDao.findById(rootDiskOfferingId); + rootDiskTags.add(rootDiskOfferingVO.getTags()); + + if (isIso) { + _orchSrvc.createVirtualMachineFromScratch(vm.getUuid(), Long.toString(owner.getAccountId()), vm.getIsoId().toString(), hostName, displayName, + hypervisorType.name(), guestOSCategory.getName(), offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, + networkNicMap, plan, extraDhcpOptionMap, rootDiskOfferingId); + } else { + _orchSrvc.createVirtualMachine(vm.getUuid(), Long.toString(owner.getAccountId()), Long.toString(template.getId()), hostName, displayName, hypervisorType.name(), + offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, networkNicMap, plan, rootDiskSize, extraDhcpOptionMap, + dataDiskTemplateToDiskOfferingMap, diskOfferingId, rootDiskOfferingId); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Successfully allocated DB entry for " + vm); + } + } + CallContext.current().setEventDetails("Vm Id: " + vm.getUuid()); + + if (!isImport) { + if (!offering.isDynamic()) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), + hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); + } else { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), + hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), customParameters, vm.isDisplayVm()); + } + + //Update Resource Count for the given account + resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize())); + } + return vm; } private void updateVMDiskController(UserVmVO vm, Map customParameters, GuestOSVO guestOS) {