diff --git a/client/conf/db.properties.in b/client/conf/db.properties.in index 5ea63e43de2..572cfbc1ff2 100644 --- a/client/conf/db.properties.in +++ b/client/conf/db.properties.in @@ -51,6 +51,7 @@ db.cloud.trustStorePassword= # Encryption Settings db.cloud.encryption.type=none db.cloud.encrypt.secret= +db.cloud.encryptor.version= # usage database settings db.usage.username=@DBUSER@ diff --git a/debian/rules b/debian/rules index 287ec4256c2..16f10ad8047 100755 --- a/debian/rules +++ b/debian/rules @@ -135,6 +135,7 @@ override_dh_auto_install: install -D systemvm/dist/* $(DESTDIR)/usr/share/$(PACKAGE)-common/vms/ # We need jasypt for cloud-install-sys-tmplt, so this is a nasty hack to get it into the right place install -D agent/target/dependencies/jasypt-1.9.3.jar $(DESTDIR)/usr/share/$(PACKAGE)-common/lib + install -D utils/target/cloud-utils-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-common/lib/$(PACKAGE)-utils.jar # cloudstack-python mkdir -p $(DESTDIR)/usr/share/pyshared diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index 728b30fc506..2d3665b149e 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -23,6 +23,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; import java.util.Date; @@ -109,6 +111,7 @@ import com.cloud.upgrade.dao.VersionDaoImpl; import com.cloud.upgrade.dao.VersionVO; import com.cloud.upgrade.dao.VersionVO.Step; import com.cloud.utils.component.SystemIntegrityChecker; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.ScriptRunner; import com.cloud.utils.db.TransactionLegacy; @@ -369,6 +372,7 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker { } try { + initializeDatabaseEncryptors(); final CloudStackVersion dbVersion = CloudStackVersion.parse(_dao.getCurrentVersion()); final String currentVersionValue = this.getClass().getPackage().getImplementationVersion(); @@ -403,6 +407,40 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker { } } + private void initializeDatabaseEncryptors() { + TransactionLegacy txn = TransactionLegacy.open("initializeDatabaseEncryptors"); + txn.start(); + String errorMessage = "Unable to get the database connections"; + try { + Connection conn = txn.getConnection(); + errorMessage = "Unable to get the 'init' value from 'configuration' table in the 'cloud' database"; + decryptInit(conn); + txn.commit(); + } catch (CloudRuntimeException e) { + s_logger.error(e.getMessage()); + errorMessage = String.format("Unable to initialize the database encryptors due to %s. " + + "Please check if database encryption key and database encryptor version are correct.", errorMessage); + s_logger.error(errorMessage); + throw new CloudRuntimeException(errorMessage, e); + } catch (SQLException e) { + s_logger.error(errorMessage, e); + throw new CloudRuntimeException(errorMessage, e); + } finally { + txn.close(); + } + } + + private void decryptInit(Connection conn) throws SQLException { + String sql = "SELECT value from configuration WHERE name = 'init'"; + try (PreparedStatement pstmt = conn.prepareStatement(sql); + ResultSet result = pstmt.executeQuery()) { + if (result.next()) { + String init = result.getString(1); + s_logger.info("init = " + DBEncryptionUtil.decrypt(init)); + } + } + } + @VisibleForTesting protected static final class NoopDbUpgrade implements DbUpgrade { diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade450to451.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade450to451.java index 71476e7dd63..015d463347a 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade450to451.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade450to451.java @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.List; import org.apache.log4j.Logger; -import org.jasypt.exceptions.EncryptionOperationNotPossibleException; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; @@ -111,7 +110,7 @@ public class Upgrade450to451 implements DbUpgrade { String preSharedKey = resultSet.getString(2); try { preSharedKey = DBEncryptionUtil.decrypt(preSharedKey); - } catch (EncryptionOperationNotPossibleException ignored) { + } catch (CloudRuntimeException ignored) { s_logger.debug("The ipsec_psk preshared key id=" + rowId + "in remote_access_vpn is not encrypted, encrypting it."); } try (PreparedStatement updateStatement = conn.prepareStatement("UPDATE `cloud`.`remote_access_vpn` SET ipsec_psk=? WHERE id=?");) { diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql index 75a70bf0bed..fe0c2f8c614 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql @@ -214,7 +214,7 @@ BEGIN -- Add passphrase table CREATE TABLE IF NOT EXISTS `cloud`.`passphrase` ( `id` bigint unsigned NOT NULL auto_increment, - `passphrase` varchar(64) DEFAULT NULL, + `passphrase` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/framework/db/src/main/java/com/cloud/utils/crypt/DBEncryptionFinderCLI.java b/framework/db/src/main/java/com/cloud/utils/crypt/DBEncryptionFinderCLI.java new file mode 100644 index 00000000000..10f51833618 --- /dev/null +++ b/framework/db/src/main/java/com/cloud/utils/crypt/DBEncryptionFinderCLI.java @@ -0,0 +1,30 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.crypt; + +import java.util.Map; +import java.util.Set; + +public class DBEncryptionFinderCLI { + public static void main(String[] args) { + Map> encryptedTableCols = EncryptionSecretKeyChanger.findEncryptedTableColumns(); + encryptedTableCols.forEach((table, cols) -> System.out.printf("Table %s has encrypted columns %s%n", table, cols)); + } +} diff --git a/framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java b/framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java index a958d4ada72..88830b3e3f9 100644 --- a/framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java +++ b/framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java @@ -22,149 +22,332 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileWriter; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; -import java.util.Iterator; -import java.util.List; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; -import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; -import org.jasypt.encryption.pbe.config.SimpleStringPBEConfig; -import org.jasypt.exceptions.EncryptionOperationNotPossibleException; -import org.jasypt.properties.EncryptableProperties; +import org.apache.commons.lang3.StringUtils; import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.ReflectUtil; +import com.cloud.utils.db.Encrypt; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; + +import javax.persistence.Column; +import javax.persistence.Table; + /* * EncryptionSecretKeyChanger updates Management Secret Key / DB Secret Key or both. * DB secret key is validated against the key in db.properties * db.properties is updated with values encrypted using new MS secret key + * server.properties is updated with values encrypted using new MS secret key * DB data migrated using new DB secret key */ public class EncryptionSecretKeyChanger { - private StandardPBEStringEncryptor oldEncryptor = new StandardPBEStringEncryptor(); - private StandardPBEStringEncryptor newEncryptor = new StandardPBEStringEncryptor(); - private static final String keyFile = "/etc/cloudstack/management/key"; + private CloudStackEncryptor oldEncryptor; + private CloudStackEncryptor newEncryptor; + private static final String KEY_FILE = "/etc/cloudstack/management/key"; + private static final String ENV_NEW_MANAGEMENT_KEY = "CLOUD_SECRET_KEY_NEW"; + private final Gson gson = new Gson(); + private static final String PASSWORD = "password"; + + private static final Options options = initializeOptions(); + private static final HelpFormatter helper = initializeHelper(); + private static final String CMD_LINE_SYNTAX = "cloudstack-migrate-databases"; + private static final int WIDTH = 100; + private static final String HEADER = "Options:"; + private static final String FOOTER = " \nExamples: \n" + + " " + CMD_LINE_SYNTAX + " -m password -d password -n newmgmtkey -v V2 \n" + + " Migrate cloudstack properties (db.properties and server.properties) \n" + + " with new management key and encryptor V2. \n" + + " " + CMD_LINE_SYNTAX + " -m password -d password -n newmgmtkey -e newdbkey \n" + + " Migrate cloudstack properties and databases with new management key and database secret key. \n" + + " " + CMD_LINE_SYNTAX + " -m password -d password -n newmgmtkey -e newdbkey -s -v V2 \n" + + " Migrate cloudstack properties with new keys and encryptor V2, but skip database migration. \n" + + " " + CMD_LINE_SYNTAX + " -m password -d password -l -f \n" + + " Migrate cloudstack properties with new management key (load from $CLOUD_SECRET_KEY_NEW), \n" + + " and migrate database with old db key. \n" + + " \nReturn codes: \n" + + " 0 - Succeed to change keys and/or migrate databases \n" + + " 1 - Fail to parse the command line arguments \n" + + " 2 - Fail to validate parameters \n" + + " 3 - Fail to migrate database"; + private static final String OLD_MS_KEY_OPTION = "oldMSKey"; + private static final String OLD_DB_KEY_OPTION = "oldDBKey"; + private static final String NEW_MS_KEY_OPTION = "newMSKey"; + private static final String NEW_DB_KEY_OPTION = "newDBKey"; + private static final String ENCRYPTOR_VERSION_OPTION = "version"; + private static final String LOAD_NEW_MS_KEY_FROM_ENV_FLAG = "load-new-management-key-from-env"; + private static final String FORCE_DATABASE_MIGRATION_FLAG = "force-database-migration"; + private static final String SKIP_DATABASE_MIGRATION_FLAG = "skip-database-migration"; + private static final String HELP_FLAG = "help"; public static void main(String[] args) { - List argsList = Arrays.asList(args); - Iterator iter = argsList.iterator(); - String oldMSKey = null; - String oldDBKey = null; - String newMSKey = null; - String newDBKey = null; - - //Parse command-line args - while (iter.hasNext()) { - String arg = iter.next(); - // Old MS Key - if (arg.equals("-m")) { - oldMSKey = iter.next(); - } - // Old DB Key - if (arg.equals("-d")) { - oldDBKey = iter.next(); - } - // New MS Key - if (arg.equals("-n")) { - newMSKey = iter.next(); - } - // New DB Key - if (arg.equals("-e")) { - newDBKey = iter.next(); - } + if (args.length == 0 || StringUtils.equalsAny(args[0], "-h", "--help")) { + helper.printHelp(WIDTH, CMD_LINE_SYNTAX, HEADER, options, FOOTER, true); + System.exit(0); } + CommandLine cmdLine = null; + CommandLineParser parser = new DefaultParser(); + try { + cmdLine = parser.parse(options, args); + } catch (ParseException e) { + System.out.println(e.getMessage()); + helper.printHelp(WIDTH, CMD_LINE_SYNTAX, HEADER, options, FOOTER, true); + System.exit(1); + } + + String oldMSKey = cmdLine.getOptionValue(OLD_MS_KEY_OPTION); + String oldDBKey = cmdLine.getOptionValue(OLD_DB_KEY_OPTION); + String newMSKey = cmdLine.getOptionValue(NEW_MS_KEY_OPTION); + String newDBKey = cmdLine.getOptionValue(NEW_DB_KEY_OPTION); + String newEncryptorVersion = cmdLine.getOptionValue(ENCRYPTOR_VERSION_OPTION); + boolean loadNewMsKeyFromEnv = cmdLine.hasOption(LOAD_NEW_MS_KEY_FROM_ENV_FLAG); + boolean forced = cmdLine.hasOption(FORCE_DATABASE_MIGRATION_FLAG); + boolean skipped = cmdLine.hasOption(SKIP_DATABASE_MIGRATION_FLAG); + + if (!validateParameters(oldMSKey, oldDBKey, newMSKey, newDBKey, newEncryptorVersion, loadNewMsKeyFromEnv)) { + helper.printHelp(WIDTH, CMD_LINE_SYNTAX, HEADER, options, FOOTER, true); + System.exit(2); + } + + System.out.println("Started database migration at " + new Date()); + if (!migratePropertiesAndDatabase(oldMSKey, oldDBKey, newMSKey, newDBKey, newEncryptorVersion, loadNewMsKeyFromEnv, forced, skipped)) { + System.out.println("Got error during database migration at " + new Date()); + System.exit(3); + } + System.out.println("Finished database migration at " + new Date()); + } + + private static Options initializeOptions() { + Options options = new Options(); + + Option oldMSKey = Option.builder("m").longOpt(OLD_MS_KEY_OPTION).argName(OLD_MS_KEY_OPTION).required(true).hasArg().desc("(required) Current Mgmt Secret Key").build(); + Option oldDBKey = Option.builder("d").longOpt(OLD_DB_KEY_OPTION).argName(OLD_DB_KEY_OPTION).required(true).hasArg().desc("(required) Current DB Secret Key").build(); + Option newMSKey = Option.builder("n").longOpt(NEW_MS_KEY_OPTION).argName(NEW_MS_KEY_OPTION).required(false).hasArg().desc("New Mgmt Secret Key").build(); + Option newDBKey = Option.builder("e").longOpt(NEW_DB_KEY_OPTION).argName(NEW_DB_KEY_OPTION).required(false).hasArg().desc("New DB Secret Key").build(); + Option encryptorVersion = Option.builder("v").longOpt(ENCRYPTOR_VERSION_OPTION).argName(ENCRYPTOR_VERSION_OPTION).required(false).hasArg().desc("New DB Encryptor Version. Options are V1, V2.").build(); + + Option loadNewMsKeyFromEnv = Option.builder("l").longOpt(LOAD_NEW_MS_KEY_FROM_ENV_FLAG).desc("Load new management key from environment variable " + ENV_NEW_MANAGEMENT_KEY).build(); + Option forceDatabaseMigration = Option.builder("f").longOpt(FORCE_DATABASE_MIGRATION_FLAG).desc("Force database migration even if DB Secret key is not changed").build(); + Option skipDatabaseMigration = Option.builder("s").longOpt(SKIP_DATABASE_MIGRATION_FLAG).desc("Skip database migration even if DB Secret key is changed").build(); + Option help = Option.builder("h").longOpt(HELP_FLAG).desc("Show help message").build(); + + options.addOption(oldMSKey); + options.addOption(oldDBKey); + options.addOption(newMSKey); + options.addOption(newDBKey); + options.addOption(encryptorVersion); + options.addOption(loadNewMsKeyFromEnv); + options.addOption(forceDatabaseMigration); + options.addOption(skipDatabaseMigration); + options.addOption(help); + + return options; + } + + private static HelpFormatter initializeHelper() { + HelpFormatter helper = new HelpFormatter(); + + helper.setOptionComparator((o1, o2) -> { + if (o1.isRequired() && !o2.isRequired()) { + return -1; + } + if (!o1.isRequired() && o2.isRequired()) { + return 1; + } + if (o1.hasArg() && !o2.hasArg()) { + return -1; + } + if (!o1.hasArg() && o2.hasArg()) { + return 1; + } + return o1.getOpt().compareTo(o2.getOpt()); + }); + + return helper; + } + + private static boolean validateParameters(String oldMSKey, String oldDBKey, String newMSKey, String newDBKey, + String newEncryptorVersion, boolean loadNewMsKeyFromEnv) { + if (oldMSKey == null || oldDBKey == null) { - System.out.println("Existing MS secret key or DB secret key is not provided"); - usage(); - return; + System.out.println("Existing Management secret key or DB secret key is not provided"); + return false; + } + + if (loadNewMsKeyFromEnv) { + if (StringUtils.isNotEmpty(newMSKey)) { + System.out.println("The new management key has already been set. Please check if it is set twice."); + return false; + } + newMSKey = System.getenv(ENV_NEW_MANAGEMENT_KEY); + if (StringUtils.isEmpty(newMSKey)) { + System.out.println("Environment variable " + ENV_NEW_MANAGEMENT_KEY + " is not set or empty"); + return false; + } } if (newMSKey == null && newDBKey == null) { - System.out.println("New MS secret key and DB secret are both not provided"); - usage(); - return; + System.out.println("New Management secret key and DB secret are both not provided"); + return false; } + if (newEncryptorVersion != null) { + try { + CloudStackEncryptor.EncryptorVersion.fromString(newEncryptorVersion); + } catch (CloudRuntimeException ex) { + System.out.println(ex.getMessage()); + return false; + } + } + + return true; + } + + private static boolean migratePropertiesAndDatabase(String oldMSKey, String oldDBKey, String newMSKey, String newDBKey, + String newEncryptorVersion, boolean loadNewMsKeyFromEnv, + boolean forced, boolean skipped) { + final File dbPropsFile = PropertiesUtil.findConfigFile("db.properties"); - final Properties dbProps; + final Properties dbProps = new Properties(); EncryptionSecretKeyChanger keyChanger = new EncryptionSecretKeyChanger(); - StandardPBEStringEncryptor encryptor = new StandardPBEStringEncryptor(); - keyChanger.initEncryptor(encryptor, oldMSKey); - dbProps = new EncryptableProperties(encryptor); PropertiesConfiguration backupDBProps = null; System.out.println("Parsing db.properties file"); - try(FileInputStream db_prop_fstream = new FileInputStream(dbPropsFile);) { - dbProps.load(db_prop_fstream); + try(FileInputStream dbPropFstream = new FileInputStream(dbPropsFile)) { + dbProps.load(dbPropFstream); backupDBProps = new PropertiesConfiguration(dbPropsFile); } catch (FileNotFoundException e) { - System.out.println("db.properties file not found while reading DB secret key" + e.getMessage()); + System.out.println("db.properties file not found while reading DB secret key: " + e.getMessage()); + return false; } catch (IOException e) { - System.out.println("Error while reading DB secret key from db.properties" + e.getMessage()); + System.out.println("Error while reading DB secret key from db.properties: " + e.getMessage()); + return false; } catch (ConfigurationException e) { - e.printStackTrace(); + System.out.println("Error while getting configurations from db.properties: " + e.getMessage()); + return false; } - String dbSecretKey = null; try { - dbSecretKey = dbProps.getProperty("db.cloud.encrypt.secret"); - } catch (EncryptionOperationNotPossibleException e) { - System.out.println("Failed to decrypt existing DB secret key from db.properties. " + e.getMessage()); - return; + EncryptionSecretKeyChecker.initEncryptor(oldMSKey); + EncryptionSecretKeyChecker.decryptAnyProperties(dbProps); + } catch (CloudRuntimeException e) { + System.out.println("Error: Incorrect Management Secret Key"); + return false; } + String dbSecretKey = dbProps.getProperty("db.cloud.encrypt.secret"); if (!oldDBKey.equals(dbSecretKey)) { - System.out.println("Incorrect MS Secret Key or DB Secret Key"); - return; + System.out.println("Error: Incorrect DB Secret Key"); + return false; } - System.out.println("Secret key provided matched the key in db.properties"); + System.out.println("DB Secret key provided matched the key in db.properties"); final String encryptionType = dbProps.getProperty("db.cloud.encryption.type"); + final String oldEncryptorVersion = dbProps.getProperty("db.cloud.encryptor.version"); + // validate old and new encryptor versions + try { + CloudStackEncryptor.EncryptorVersion.fromString(oldEncryptorVersion); + CloudStackEncryptor.EncryptorVersion.fromString(newEncryptorVersion); + } catch (CloudRuntimeException ex) { + System.out.println(ex.getMessage()); + return false; + } + + if (loadNewMsKeyFromEnv) { + newMSKey = System.getenv(ENV_NEW_MANAGEMENT_KEY); + } if (newMSKey == null) { - System.out.println("No change in MS Key. Skipping migrating db.properties"); - } else { - if (!keyChanger.migrateProperties(dbPropsFile, dbProps, newMSKey, newDBKey)) { - System.out.println("Failed to update db.properties"); - return; + newMSKey = oldMSKey; + } + if (newDBKey == null) { + newDBKey = oldDBKey; + } + boolean isMSKeyChanged = !newMSKey.equals(oldMSKey); + boolean isDBKeyChanged = !newDBKey.equals(oldDBKey); + if (newEncryptorVersion == null && (isDBKeyChanged || forced) && !skipped) { + if (StringUtils.isNotEmpty(oldEncryptorVersion)) { + newEncryptorVersion = oldEncryptorVersion; } else { - //db.properties updated successfully - if (encryptionType.equals("file")) { - //update key file with new MS key - try (FileWriter fwriter = new FileWriter(keyFile); - BufferedWriter bwriter = new BufferedWriter(fwriter);) - { - bwriter.write(newMSKey); - } catch (IOException e) { - System.out.println(String.format("Please update the file %s manually. Failed to write new secret to file with error %s", keyFile, e.getMessage())); - } + newEncryptorVersion = CloudStackEncryptor.EncryptorVersion.defaultVersion().name(); + } + } + boolean isEncryptorVersionChanged = false; + if (newEncryptorVersion != null) { + isEncryptorVersionChanged = !newEncryptorVersion.equalsIgnoreCase(oldEncryptorVersion); + } + + if (isMSKeyChanged || isDBKeyChanged || isEncryptorVersionChanged) { + System.out.println("INFO: Migrate properties with DB encryptor version: " + newEncryptorVersion); + if (!keyChanger.migrateProperties(dbPropsFile, dbProps, newMSKey, newDBKey, newEncryptorVersion)) { + System.out.println("Failed to update db.properties"); + return false; + } + if (!keyChanger.migrateServerProperties(newMSKey)) { + System.out.println("Failed to update server.properties"); + return false; + } + //db.properties updated successfully + if (encryptionType.equals("file")) { + //update key file with new MS key + try (FileWriter fwriter = new FileWriter(KEY_FILE); + BufferedWriter bwriter = new BufferedWriter(fwriter)) + { + bwriter.write(newMSKey); + } catch (IOException e) { + System.out.printf("Please update the file %s manually. Failed to write new secret to file with error %s %n", KEY_FILE, e.getMessage()); + return false; } } + } else { + System.out.println("No changes with Management Secret Key, DB Secret Key and DB encryptor version. Skipping migrating db.properties"); } boolean success = false; - if (newDBKey == null || newDBKey.equals(oldDBKey)) { - System.out.println("No change in DB Secret Key. Skipping Data Migration"); - } else { - EncryptionSecretKeyChecker.initEncryptorForMigration(oldMSKey); + if (isDBKeyChanged || isEncryptorVersionChanged || forced) { + if (skipped) { + System.out.println("Skipping Data Migration as '-s' or '--skip-database-migration' is passed"); + return true; + } + EncryptionSecretKeyChecker.initEncryptor(newMSKey); try { - success = keyChanger.migrateData(oldDBKey, newDBKey); + success = keyChanger.migrateData(oldDBKey, newDBKey, oldEncryptorVersion, newEncryptorVersion); } catch (Exception e) { System.out.println("Error during data migration"); e.printStackTrace(); - success = false; } + } else { + System.out.println("No changes with DB Secret Key and DB encryptor version. Skipping Data Migration"); + return true; } if (success) { @@ -179,36 +362,88 @@ public class EncryptionSecretKeyChanger { } if (encryptionType.equals("file")) { //revert secret key in file - try (FileWriter fwriter = new FileWriter(keyFile); - BufferedWriter bwriter = new BufferedWriter(fwriter);) + try (FileWriter fwriter = new FileWriter(KEY_FILE); + BufferedWriter bwriter = new BufferedWriter(fwriter)) { bwriter.write(oldMSKey); } catch (IOException e) { System.out.println("Failed to revert to old secret to file. Please update the file manually"); } } + return false; } + + return true; } - private boolean migrateProperties(File dbPropsFile, Properties dbProps, String newMSKey, String newDBKey) { + private boolean migrateServerProperties(String newMSKey) { + System.out.println("Migrating server.properties.."); + final File serverPropsFile = PropertiesUtil.findConfigFile("server.properties"); + final Properties serverProps = new Properties(); + PropertiesConfiguration newServerProps; + + try(FileInputStream serverPropFstream = new FileInputStream(serverPropsFile)) { + serverProps.load(serverPropFstream); + newServerProps = new PropertiesConfiguration(serverPropsFile); + } catch (FileNotFoundException e) { + System.out.println("server.properties file not found: " + e.getMessage()); + return false; + } catch (IOException e) { + System.out.println("Error while reading server.properties: " + e.getMessage()); + return false; + } catch (ConfigurationException e) { + System.out.println("Error while getting configurations from server.properties: " + e.getMessage()); + return false; + } + + try { + EncryptionSecretKeyChecker.decryptAnyProperties(serverProps); + } catch (CloudRuntimeException e) { + System.out.println(e.getMessage()); + return false; + } + + CloudStackEncryptor msEncryptor = new CloudStackEncryptor(newMSKey, null, getClass()); + + try { + String encryptionType = serverProps.getProperty("password.encryption.type"); + if (StringUtils.isEmpty(encryptionType) || encryptionType.equalsIgnoreCase("none")) { + System.out.println("Skipping server.properties as password.encryption.type is " + encryptionType); + return true; + } + String keystorePassword = serverProps.getProperty("https.keystore.password"); + if (StringUtils.isNotEmpty(keystorePassword)) { + newServerProps.setProperty("https.keystore.password", "ENC(" + msEncryptor.encrypt(keystorePassword) + ")"); + } + newServerProps.save(serverPropsFile.getAbsolutePath()); + } catch (Exception e) { + System.out.println(e.getMessage()); + return false; + } + System.out.println("Migrating server.properties Done."); + return true; + } + + private boolean migrateProperties(File dbPropsFile, Properties dbProps, String newMSKey, String newDBKey, String newEncryptorVersion) { System.out.println("Migrating db.properties.."); - StandardPBEStringEncryptor msEncryptor = new StandardPBEStringEncryptor(); - ; - initEncryptor(msEncryptor, newMSKey); + CloudStackEncryptor msEncryptor = new CloudStackEncryptor(newMSKey, null, getClass()); try { PropertiesConfiguration newDBProps = new PropertiesConfiguration(dbPropsFile); - if (newDBKey != null && !newDBKey.isEmpty()) { + if (StringUtils.isNotEmpty(newDBKey)) { newDBProps.setProperty("db.cloud.encrypt.secret", "ENC(" + msEncryptor.encrypt(newDBKey) + ")"); } String prop = dbProps.getProperty("db.cloud.password"); - if (prop != null && !prop.isEmpty()) { + if (StringUtils.isNotEmpty(prop)) { newDBProps.setProperty("db.cloud.password", "ENC(" + msEncryptor.encrypt(prop) + ")"); } prop = dbProps.getProperty("db.usage.password"); - if (prop != null && !prop.isEmpty()) { + if (StringUtils.isNotEmpty(prop)) { newDBProps.setProperty("db.usage.password", "ENC(" + msEncryptor.encrypt(prop) + ")"); } + if (newEncryptorVersion != null) { + newDBProps.setProperty("db.cloud.encryptor.version", newEncryptorVersion); + } newDBProps.save(dbPropsFile.getAbsolutePath()); } catch (Exception e) { e.printStackTrace(); @@ -218,10 +453,10 @@ public class EncryptionSecretKeyChanger { return true; } - private boolean migrateData(String oldDBKey, String newDBKey) { + private boolean migrateData(String oldDBKey, String newDBKey, String oldEncryptorVersion, String newEncryptorVersion) throws SQLException { System.out.println("Begin Data migration"); - initEncryptor(oldEncryptor, oldDBKey); - initEncryptor(newEncryptor, newDBKey); + oldEncryptor = new CloudStackEncryptor(oldDBKey, oldEncryptorVersion, getClass()); + newEncryptor = new CloudStackEncryptor(newDBKey, newEncryptorVersion, getClass()); System.out.println("Initialised Encryptors"); TransactionLegacy txn = TransactionLegacy.open("Migrate"); @@ -231,13 +466,28 @@ public class EncryptionSecretKeyChanger { try { conn = txn.getConnection(); } catch (SQLException e) { + System.out.println("Unable to migrate encrypted data in the database due to: " + e.getMessage()); throw new CloudRuntimeException("Unable to migrate encrypted data in the database", e); } + // migrate values in configuration migrateConfigValues(conn); + + // migrate resource details values migrateHostDetails(conn); - migrateVNCPassword(conn); - migrateUserCredentials(conn); + migrateClusterDetails(conn); + migrateImageStoreDetails(conn); + migrateStoragePoolDetails(conn); + migrateScaleIOStoragePoolDetails(conn); + migrateUserVmDetails(conn); + + // migrate other encrypted fields + migrateTemplateDeployAsIsDetails(conn); + migrateImageStoreUrlForCifs(conn); + migrateStoragePoolPathForSMB(conn); + + // migrate columns with annotation @Encrypt + migrateEncryptedTableColumns(conn); txn.commit(); } finally { @@ -247,125 +497,300 @@ public class EncryptionSecretKeyChanger { return true; } - private void initEncryptor(StandardPBEStringEncryptor encryptor, String secretKey) { - encryptor.setAlgorithm("PBEWithMD5AndDES"); - SimpleStringPBEConfig stringConfig = new SimpleStringPBEConfig(); - stringConfig.setPassword(secretKey); - encryptor.setConfig(stringConfig); - } - - private String migrateValue(String value) { - if (value == null || value.isEmpty()) { + protected String migrateValue(String value) { + if (StringUtils.isEmpty(value)) { return value; } String decryptVal = oldEncryptor.decrypt(value); return newEncryptor.encrypt(decryptVal); } + protected String migrateUrlOrPath(String urlOrPath) { + if (StringUtils.isEmpty(urlOrPath)) { + return urlOrPath; + } + String[] properties = urlOrPath.split("&"); + for (String property : properties) { + if (property.startsWith("password=")) { + String password = property.substring(property.indexOf("=") + 1); + password = migrateValue(password); + return urlOrPath.replaceAll(property, "password=" + password); + } + } + return urlOrPath; + } + private void migrateConfigValues(Connection conn) { System.out.println("Begin migrate config values"); - try(PreparedStatement select_pstmt = conn.prepareStatement("select name, value from configuration where category in ('Hidden', 'Secure')"); - ResultSet rs = select_pstmt.executeQuery(); - PreparedStatement update_pstmt = conn.prepareStatement("update configuration set value=? where name=?"); + + String tableName = "configuration"; + String selectSql = "SELECT name, value FROM configuration WHERE category IN ('Hidden', 'Secure')"; + String updateSql = "UPDATE configuration SET value=? WHERE name=?"; + migrateValueAndUpdateDatabaseByName(conn, tableName, selectSql, updateSql); + + System.out.println("End migrate config values"); + } + + + private void migrateValueAndUpdateDatabaseById(Connection conn, String tableName, String selectSql, String updateSql, boolean isUrlOrPath) { + try( PreparedStatement selectPstmt = conn.prepareStatement(selectSql); + ResultSet rs = selectPstmt.executeQuery(); + PreparedStatement updatePstmt = conn.prepareStatement(updateSql) + ) { + while (rs.next()) { + long id = rs.getLong(1); + String value = rs.getString(2); + if (StringUtils.isEmpty(value)) { + continue; + } + String encryptedValue = isUrlOrPath ? migrateUrlOrPath(value) : migrateValue(value); + updatePstmt.setBytes(1, encryptedValue.getBytes(StandardCharsets.UTF_8)); + updatePstmt.setLong(2, id); + updatePstmt.executeUpdate(); + } + } catch (SQLException e) { + throwCloudRuntimeException(String.format("Unable to update %s values", tableName), e); + } + } + + private void migrateValueAndUpdateDatabaseByName(Connection conn, String tableName, String selectSql, String updateSql) { + try(PreparedStatement selectPstmt = conn.prepareStatement(selectSql); + ResultSet rs = selectPstmt.executeQuery(); + PreparedStatement updatePstmt = conn.prepareStatement(updateSql) ) { while (rs.next()) { String name = rs.getString(1); String value = rs.getString(2); - if (value == null || value.isEmpty()) { + if (StringUtils.isEmpty(value)) { continue; } String encryptedValue = migrateValue(value); - update_pstmt.setBytes(1, encryptedValue.getBytes("UTF-8")); - update_pstmt.setString(2, name); - update_pstmt.executeUpdate(); + updatePstmt.setBytes(1, encryptedValue.getBytes(StandardCharsets.UTF_8)); + updatePstmt.setString(2, name); + updatePstmt.executeUpdate(); } } catch (SQLException e) { - throw new CloudRuntimeException("Unable to update configuration values ", e); - } catch (UnsupportedEncodingException e) { - throw new CloudRuntimeException("Unable to update configuration values ", e); + throwCloudRuntimeException(String.format("Unable to update %s values", tableName), e); } - System.out.println("End migrate config values"); } private void migrateHostDetails(Connection conn) { System.out.println("Begin migrate host details"); - - try( PreparedStatement sel_pstmt = conn.prepareStatement("select id, value from host_details where name = 'password'"); - ResultSet rs = sel_pstmt.executeQuery(); - PreparedStatement pstmt = conn.prepareStatement("update host_details set value=? where id=?"); - ) { - while (rs.next()) { - long id = rs.getLong(1); - String value = rs.getString(2); - if (value == null || value.isEmpty()) { - continue; - } - String encryptedValue = migrateValue(value); - pstmt.setBytes(1, encryptedValue.getBytes("UTF-8")); - pstmt.setLong(2, id); - pstmt.executeUpdate(); - } - } catch (SQLException e) { - throw new CloudRuntimeException("Unable update host_details values ", e); - } catch (UnsupportedEncodingException e) { - throw new CloudRuntimeException("Unable update host_details values ", e); - } + migrateDetails(conn, "host_details", PASSWORD); System.out.println("End migrate host details"); } - private void migrateVNCPassword(Connection conn) { - System.out.println("Begin migrate VNC password"); - try(PreparedStatement select_pstmt = conn.prepareStatement("select id, vnc_password from vm_instance"); - ResultSet rs = select_pstmt.executeQuery(); - PreparedStatement pstmt = conn.prepareStatement("update vm_instance set vnc_password=? where id=?"); + private void migrateClusterDetails(Connection conn) { + System.out.println("Begin migrate cluster details"); + migrateDetails(conn, "cluster_details", PASSWORD); + System.out.println("End migrate cluster details"); + } + + private void migrateImageStoreDetails(Connection conn) { + System.out.println("Begin migrate image store details"); + migrateDetails(conn, "image_store_details", "key", "secretkey"); + System.out.println("End migrate image store details"); + } + + private void migrateStoragePoolDetails(Connection conn) { + System.out.println("Begin migrate storage pool details"); + migrateDetails(conn, "storage_pool_details", PASSWORD); + System.out.println("End migrate storage pool details"); + } + + private void migrateScaleIOStoragePoolDetails(Connection conn) { + System.out.println("Begin migrate storage pool details for ScaleIO"); + migrateDetails(conn, "storage_pool_details", "powerflex.gw.username", "powerflex.gw.password"); + System.out.println("End migrate storage pool details for ScaleIO"); + } + + private void migrateUserVmDetails(Connection conn) { + System.out.println("Begin migrate user vm details"); + migrateDetails(conn, "user_vm_details", PASSWORD); + System.out.println("End migrate user vm details"); + } + + private void migrateDetails(Connection conn, String tableName, String... detailNames) { + String convertedDetails = Arrays.stream(detailNames).map(detail -> "'" + detail + "'").collect(Collectors.joining(", ")); + String selectSql = String.format("SELECT id, value FROM %s WHERE name IN (%s)", tableName, convertedDetails); + String updateSql = String.format("UPDATE %s SET value=? WHERE id=?", tableName); + migrateValueAndUpdateDatabaseById(conn, tableName, selectSql, updateSql, false); + } + + private void migrateTemplateDeployAsIsDetails(Connection conn) throws SQLException { + System.out.println("Begin migrate user vm deploy_as_is details"); + if (!ifTableExists(conn.getMetaData(), "user_vm_deploy_as_is_details")) { + System.out.printf("Skipped as table %s does not exist %n", "user_vm_deploy_as_is_details"); + return; + } + if (!ifTableExists(conn.getMetaData(), "template_deploy_as_is_details")) { + System.out.printf("Skipped as table %s does not exist %n", "template_deploy_as_is_details"); + return; + } + String sqlTemplateDeployAsIsDetails = "SELECT template_deploy_as_is_details.value " + + "FROM template_deploy_as_is_details JOIN vm_instance " + + "WHERE template_deploy_as_is_details.template_id = vm_instance.vm_template_id " + + "vm_instance.id = %s AND template_deploy_as_is_details.name = '%s' LIMIT 1"; + try (PreparedStatement selectPstmt = conn.prepareStatement("SELECT id, vm_id, name, value FROM user_vm_deploy_as_is_details"); + ResultSet rs = selectPstmt.executeQuery(); + PreparedStatement updatePstmt = conn.prepareStatement("UPDATE user_vm_deploy_as_is_details SET value=? WHERE id=?") ) { while (rs.next()) { long id = rs.getLong(1); - String value = rs.getString(2); - if (value == null || value.isEmpty()) { + long vmId = rs.getLong(2); + String name = rs.getString(3); + String value = rs.getString(4); + if (StringUtils.isEmpty(value)) { continue; } - String encryptedValue = migrateValue(value); + String key = name.startsWith("property-") ? name : "property-" + name; - pstmt.setBytes(1, encryptedValue.getBytes("UTF-8")); - pstmt.setLong(2, id); - pstmt.executeUpdate(); - } - } catch (SQLException e) { - throw new CloudRuntimeException("Unable update vm_instance vnc_password ", e); - } catch (UnsupportedEncodingException e) { - throw new CloudRuntimeException("Unable update vm_instance vnc_password ", e); - } - System.out.println("End migrate VNC password"); - } - - private void migrateUserCredentials(Connection conn) { - System.out.println("Begin migrate user credentials"); - try(PreparedStatement select_pstmt = conn.prepareStatement("select id, secret_key from user"); - ResultSet rs = select_pstmt.executeQuery(); - PreparedStatement pstmt = conn.prepareStatement("update user set secret_key=? where id=?"); - ) { - while (rs.next()) { - long id = rs.getLong(1); - String secretKey = rs.getString(2); - if (secretKey == null || secretKey.isEmpty()) { - continue; + try (PreparedStatement pstmtTemplateDeployAsIs = conn.prepareStatement(String.format(sqlTemplateDeployAsIsDetails, vmId, key)); + ResultSet rsTemplateDeployAsIs = pstmtTemplateDeployAsIs.executeQuery()) { + if (rsTemplateDeployAsIs.next()) { + String templateDeployAsIsDetailValue = rsTemplateDeployAsIs.getString(1); + OVFPropertyTO property = gson.fromJson(templateDeployAsIsDetailValue, OVFPropertyTO.class); + if (property != null && property.isPassword()) { + String encryptedValue = migrateValue(value); + updatePstmt.setBytes(1, encryptedValue.getBytes(StandardCharsets.UTF_8)); + updatePstmt.setLong(2, id); + updatePstmt.executeUpdate(); + } + } } - String encryptedSecretKey = migrateValue(secretKey); - pstmt.setBytes(1, encryptedSecretKey.getBytes("UTF-8")); - pstmt.setLong(2, id); - pstmt.executeUpdate(); } - } catch (SQLException e) { - throw new CloudRuntimeException("Unable update user secret key ", e); - } catch (UnsupportedEncodingException e) { - throw new CloudRuntimeException("Unable update user secret key ", e); + } catch (SQLException | JsonSyntaxException e) { + throwCloudRuntimeException("Unable to update user_vm_deploy_as_is_details values", e); } - System.out.println("End migrate user credentials"); + System.out.println("End migrate user vm deploy_as_is details"); } - private static void usage() { - System.out.println("Usage: \tEncryptionSecretKeyChanger \n" + "\t\t-m \n" + "\t\t-d \n" + "\t\t-n [New Mgmt Secret Key] \n" - + "\t\t-e [New DB Secret Key]"); + private void migrateImageStoreUrlForCifs(Connection conn) { + System.out.println("Begin migrate image store url if protocol is cifs"); + + String tableName = "image_store"; + String fieldName = "url"; + if (getCountOfTable(conn, tableName) == 0) { + System.out.printf("Skipped table %s as there is no image store with protocol is cifs in the table %n", tableName); + return; + } + String selectSql = String.format("SELECT id, `%s` FROM %s WHERE protocol = 'cifs'", fieldName, tableName); + String updateSql = String.format("UPDATE %s SET `%s`=? WHERE id=?", tableName, fieldName); + migrateValueAndUpdateDatabaseById(conn, tableName, selectSql, updateSql, true); + + System.out.println("End migrate image store url if protocol is cifs"); + } + + private void migrateStoragePoolPathForSMB(Connection conn) { + System.out.println("Begin migrate storage pool path if pool type is SMB"); + + String tableName = "storage_pool"; + String fieldName = "path"; + if (getCountOfTable(conn, tableName) == 0) { + System.out.printf("Skipped table %s as there is no storage pool with pool type is SMB in the table %n", tableName); + return; + } + String selectSql = String.format("SELECT id, `%s` FROM %s WHERE pool_type = 'SMB'", fieldName, tableName); + String updateSql = String.format("UPDATE %s SET `%s`=? WHERE id=?", tableName, fieldName); + migrateValueAndUpdateDatabaseById(conn, tableName, selectSql, updateSql, true); + + System.out.println("End migrate storage pool path if pool type is SMB"); + } + + private void migrateDatabaseField(Connection conn, String tableName, String fieldName) { + System.out.printf("Begin migrate table %s field %s %n", tableName, fieldName); + + String selectSql = String.format("SELECT id, `%s` FROM %s", fieldName, tableName); + String updateSql = String.format("UPDATE %s SET `%s`=? WHERE id=?", tableName, fieldName); + migrateValueAndUpdateDatabaseById(conn, tableName, selectSql, updateSql, false); + + System.out.printf("Done migrating database field %s.%s %n", tableName, fieldName); + } + + protected static Map> findEncryptedTableColumns() { + Map> tableCols = new HashMap<>(); + Set> vos = ReflectUtil.getClassesWithAnnotation(Table.class, new String[]{"com", "org"}); + vos.forEach( vo -> { + Table tableAnnotation = vo.getAnnotation(Table.class); + if (tableAnnotation == null || (tableAnnotation.name() != null && tableAnnotation.name().endsWith("_view"))) { + return; + } + for (Field field : vo.getDeclaredFields()) { + if (field.isAnnotationPresent(Encrypt.class)) { + Set encryptedColumns = tableCols.getOrDefault(tableAnnotation.name(), new HashSet<>()); + String columnName = field.getName(); + if (field.isAnnotationPresent(Column.class)) { + Column columnAnnotation = field.getAnnotation(Column.class); + columnName = columnAnnotation.name(); + } + encryptedColumns.add(columnName); + tableCols.put(tableAnnotation.name(), encryptedColumns); + } + } + }); + return tableCols; + } + + private void migrateEncryptedTableColumns(Connection conn) throws SQLException { + Map> encryptedTableCols = findEncryptedTableColumns(); + DatabaseMetaData metadata = conn.getMetaData(); + encryptedTableCols.forEach((table, columns) -> { + if (!ifTableExists(metadata, table)) { + System.out.printf("Skipped table %s as it does not exist %n", table); + return; + } + if (getCountOfTable(conn, table) == 0) { + System.out.printf("Skipped table %s as there is no data in the table %n", table); + return; + } + columns.forEach(column -> { + if (!ifTableColumnExists(metadata, table, column)) { + System.out.printf("Skipped column %s in table %s as it does not exist %n", column, table); + return; + } + migrateDatabaseField(conn, table, column); + }); + }); + } + + private boolean ifTableExists(DatabaseMetaData metadata, String table) { + try { + ResultSet rs = metadata.getTables(null, null, table, null); + if (rs.next()) { + return true; + } + } catch (SQLException e) { + throwCloudRuntimeException(String.format("Unable to get table %s", table), e); + } + return false; + } + + private boolean ifTableColumnExists(DatabaseMetaData metadata, String table, String column) { + try { + ResultSet rs = metadata.getColumns(null, null, table, column); + if (rs.next()) { + return true; + } + } catch (SQLException e) { + throwCloudRuntimeException(String.format("Unable to get column %s in table %s", column, table), e); + } + return false; + } + + private int getCountOfTable(Connection conn, String table) { + try (PreparedStatement pstmt = conn.prepareStatement(String.format("SELECT count(*) FROM %s", table)); + ResultSet rs = pstmt.executeQuery()) { + if (rs.next()) { + return rs.getInt(1); + } + } catch (SQLException e) { + throwCloudRuntimeException(String.format("Unable to get count of records in table %s", table), e); + } + return 0; + } + + private static void throwCloudRuntimeException(String msg, Exception e) { + System.out.println(msg + " due to: " + e.getMessage()); + throw new CloudRuntimeException(msg, e); } } diff --git a/framework/db/src/main/java/com/cloud/utils/crypt/OVFPropertyTO.java b/framework/db/src/main/java/com/cloud/utils/crypt/OVFPropertyTO.java new file mode 100644 index 00000000000..1f7a2744d31 --- /dev/null +++ b/framework/db/src/main/java/com/cloud/utils/crypt/OVFPropertyTO.java @@ -0,0 +1,130 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.crypt; + +/** + * This is a copy of ./api/src/main/java/com/cloud/agent/api/to/deployasis/OVFPropertyTO.java + */ +public class OVFPropertyTO { + + private String key; + private String type; + private String value; + private String qualifiers; + private Boolean userConfigurable; + private String label; + private String description; + private Boolean password; + private int index; + private String category; + + public OVFPropertyTO() { + } + + public OVFPropertyTO(String key, String type, String value, String qualifiers, boolean userConfigurable, + String label, String description, boolean password, int index, String category) { + this.key = key; + this.type = type; + this.value = value; + this.qualifiers = qualifiers; + this.userConfigurable = userConfigurable; + this.label = label; + this.description = description; + this.password = password; + this.index = index; + this.category = category; + } + + public Long getTemplateId() { + return null; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getQualifiers() { + return qualifiers; + } + + public void setQualifiers(String qualifiers) { + this.qualifiers = qualifiers; + } + + public Boolean isUserConfigurable() { + return userConfigurable; + } + + public void setUserConfigurable(Boolean userConfigurable) { + this.userConfigurable = userConfigurable; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Boolean isPassword() { + return password; + } + + public void setPassword(Boolean password) { + this.password = password; + } + + public String getCategory() { + return category; + } + + public int getIndex() { + return index; + } +} diff --git a/framework/db/src/test/java/com/cloud/utils/crypt/EncryptionSecretKeyChangerTest.java b/framework/db/src/test/java/com/cloud/utils/crypt/EncryptionSecretKeyChangerTest.java new file mode 100644 index 00000000000..239628160d2 --- /dev/null +++ b/framework/db/src/test/java/com/cloud/utils/crypt/EncryptionSecretKeyChangerTest.java @@ -0,0 +1,87 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.crypt; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.springframework.test.util.ReflectionTestUtils; + +public class EncryptionSecretKeyChangerTest { + @Spy + EncryptionSecretKeyChanger changer = new EncryptionSecretKeyChanger(); + @Mock + CloudStackEncryptor oldEncryptor; + @Mock + CloudStackEncryptor newEncryptor; + + private static final String emtpyString = ""; + private static final String encryptedValue = "encryptedValue"; + private static final String plainText = "plaintext"; + private static final String newEncryptedValue = "newEncryptedValue"; + + @Before + public void setUp() { + oldEncryptor = Mockito.mock(CloudStackEncryptor.class); + newEncryptor = Mockito.mock(CloudStackEncryptor.class); + + ReflectionTestUtils.setField(changer, "oldEncryptor", oldEncryptor); + ReflectionTestUtils.setField(changer, "newEncryptor", newEncryptor); + + Mockito.when(oldEncryptor.decrypt(encryptedValue)).thenReturn(plainText); + Mockito.when(newEncryptor.encrypt(plainText)).thenReturn(newEncryptedValue); + } + + @Test + public void migrateValueTest() { + String value = changer.migrateValue(encryptedValue); + Assert.assertEquals(newEncryptedValue, value); + + Mockito.verify(oldEncryptor).decrypt(encryptedValue); + Mockito.verify(newEncryptor).encrypt(plainText); + } + + @Test + public void migrateValueTest2() { + String value = changer.migrateValue(emtpyString); + Assert.assertEquals(emtpyString, value); + } + + @Test + public void migrateUrlOrPathTest() { + String path = emtpyString; + Assert.assertEquals(path, changer.migrateUrlOrPath(path)); + + path = String.format("password=%s", encryptedValue); + Assert.assertEquals(path.replaceAll("password=" + encryptedValue, "password=" + newEncryptedValue), changer.migrateUrlOrPath(path)); + + path = String.format("username=user&password=%s", encryptedValue); + Assert.assertEquals(path.replaceAll("password=" + encryptedValue, "password=" + newEncryptedValue), changer.migrateUrlOrPath(path)); + + path = String.format("username=user&password2=%s", encryptedValue); + Assert.assertEquals(path, changer.migrateUrlOrPath(path)); + + path = String.format("username=user&password=%s&add=false", encryptedValue); + Assert.assertEquals(path.replaceAll("password=" + encryptedValue, "password=" + newEncryptedValue), changer.migrateUrlOrPath(path)); + } +} \ No newline at end of file diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index 1cbd4235b58..44dc06d1849 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -299,6 +299,7 @@ ln -sf log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/log4j install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/%{name}-external-ipallocator.py install -D client/target/pythonlibs/jasypt-1.9.3.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar +install -D utils/target/cloud-utils-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar install -D packaging/centos7/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator install -D packaging/centos7/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud @@ -648,6 +649,7 @@ pip3 install --upgrade urllib3 %attr(0644,root,root) %{python_sitearch}/__pycache__/* %attr(0644,root,root) %{python_sitearch}/cloudutils/* %attr(0644, root, root) %{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar +%attr(0644, root, root) %{_datadir}/%{name}-common/lib/%{name}-utils.jar %{_defaultdocdir}/%{name}-common-%{version}/LICENSE %{_defaultdocdir}/%{name}-common-%{version}/NOTICE diff --git a/packaging/centos8/cloud.spec b/packaging/centos8/cloud.spec index a851a61067d..abaf492c450 100644 --- a/packaging/centos8/cloud.spec +++ b/packaging/centos8/cloud.spec @@ -281,6 +281,7 @@ ln -sf log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/log4j install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/%{name}-external-ipallocator.py install -D client/target/pythonlibs/jasypt-1.9.3.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar +install -D utils/target/cloud-utils-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar install -D packaging/centos8/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator install -D packaging/centos8/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud @@ -626,6 +627,7 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz %attr(0644,root,root) %{_datadir}/%{name}-common/python-site/__pycache__/* %attr(0644,root,root) %{_datadir}/%{name}-common/python-site/cloudutils/* %attr(0644, root, root) %{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar +%attr(0644, root, root) %{_datadir}/%{name}-common/lib/%{name}-utils.jar %{_defaultdocdir}/%{name}-common-%{version}/LICENSE %{_defaultdocdir}/%{name}-common-%{version}/NOTICE diff --git a/pom.xml b/pom.xml index caa73d25544..2026de00c91 100644 --- a/pom.xml +++ b/pom.xml @@ -86,6 +86,7 @@ 1.15 + 1.5.0 4.4 1.21 1.3 @@ -171,6 +172,7 @@ 0.9.12 3.4.4_1 4.0.1 + 1.7.0 10.0.22 build-217-jenkins-27 8.0 diff --git a/scripts/storage/secondary/cloud-install-sys-tmplt b/scripts/storage/secondary/cloud-install-sys-tmplt index 7ff05b11613..ad976c502c6 100755 --- a/scripts/storage/secondary/cloud-install-sys-tmplt +++ b/scripts/storage/secondary/cloud-install-sys-tmplt @@ -55,7 +55,7 @@ dbHost="localhost" dbUser="root" dbPassword= dbPort=3306 -jasypt='/usr/share/cloudstack-common/lib/jasypt-1.9.3.jar' +jarfile='/usr/share/cloudstack-common/lib/cloudstack-utils.jar' # check if first parameter is not a dash (-) then print the usage block if [[ ! $@ =~ ^\-.+ ]]; then @@ -149,7 +149,7 @@ if [[ -f /etc/cloudstack/management/db.properties ]]; then if [[ "$encType" == "file" || "$encType" == "web" ]]; then encPassword=$(sed '/^\#/d' /etc/cloudstack/management/db.properties | grep 'db.cloud.password' | tail -n 1 | cut -d "=" -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'i | sed 's/^ENC(\(.*\))/\1/') if [[ ! $encPassword == "" ]]; then - dbPassword=(`java -classpath $jasypt org.jasypt.intf.cli.JasyptPBEStringDecryptionCLI decrypt.sh input=$encPassword password=$msKey verbose=false`) + dbPassword=(`java -classpath $jarfile com.cloud.utils.crypt.EncryptionCLI -d -i "$encPassword" -p "$msKey"`) if [[ ! $dbPassword ]]; then failed 2 "Failed to decrypt DB password from db.properties" fi diff --git a/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java b/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java index 56c34834458..3f9447812a7 100644 --- a/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java +++ b/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java @@ -162,8 +162,9 @@ public class ConfigurationServerImpl extends ManagerBase implements Configuratio try { persistDefaultValues(); _configDepotAdmin.populateConfigurations(); - } catch (InternalErrorException e) { - throw new RuntimeException("Unhandled configuration exception", e); + } catch (InternalErrorException | CloudRuntimeException e) { + s_logger.error("Unhandled configuration exception: " + e.getMessage()); + throw new CloudRuntimeException("Unhandled configuration exception", e); } return true; } diff --git a/setup/bindir/cloud-migrate-databases.in b/setup/bindir/cloud-migrate-databases.in index e9a4df37fff..d9a124f963c 100644 --- a/setup/bindir/cloud-migrate-databases.in +++ b/setup/bindir/cloud-migrate-databases.in @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -17,271 +17,32 @@ # specific language governing permissions and limitations # under the License. +LOGFILE=/tmp/cloudstack-migrate-databases.log -import os,logging,sys -from optparse import OptionParser -import mysql.connector -import subprocess -import glob +check_if_svc_active() { + svc_name=$1 + systemctl is-active $svc_name -q + if [ $? -eq 0 ];then + echo "service $svc_name is still active. Please stop it and retry." |tee -a ${LOGFILE} + exit 1 + fi +} -# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ---- -# ---- We do this so cloud_utils can be looked up in the following order: -# ---- 1) Sources directory -# ---- 2) waf configured PYTHONDIR -# ---- 3) System Python path -for pythonpath in ( - "@PYTHONDIR@", - os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"), - ): - if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath) -# ---- End snippet of code ---- -from cloud_utils import check_selinux, CheckFailed, resolves_to_ipv6 -import cloud_utils +if [ "$1" != "" ] && [ "$1" != "-h" ] && [ "$1" != "--help" ];then + check_if_svc_active "cloudstack-management" + check_if_svc_active "cloudstack-usage" +fi -# RUN ME LIKE THIS -# python setup/bindir/cloud-migrate-databases.in --config=client/conf/override/db.properties --resourcedir=setup/db --dry-run -# --dry-run makes it so the changes to the database in the context of the migrator are rolled back +java -classpath /etc/cloudstack/management:/usr/share/cloudstack-management/lib/* \ + com.cloud.utils.crypt.EncryptionSecretKeyChanger \ + "$@" \ + > >(tee -a ${LOGFILE}) 2> >(tee -a ${LOGFILE} >/dev/null) -# This program / library breaks down as follows: -# high-level breakdown: -# the module calls main() -# main processes command-line options -# main() instantiates a migrator with a a list of possible migration steps -# migrator discovers and topologically sorts migration steps from the given list -# main() run()s the migrator -# for each one of the migration steps: -# the migrator instantiates the migration step with the context as first parameter -# the instantiated migration step saves the context onto itself as self.context -# the migrator run()s the instantiated migration step. within run(), self.context is the context -# the migrator commits the migration context to the database (or rollsback if --dry-run is specified) -# that is it +res=$? +if [ $res -eq 0 ];then + rm -f $LOGFILE +else + echo "Failed to migrate databases. You may find more logs in $LOGFILE" +fi -# The specific library code is in cloud_utils.py -# What needs to be implemented is MigrationSteps -# Specifically in the FromInitialTo21 evolver. -# What Db20to21MigrationUtil.java does, needs to be done within run() of that class -# refer to the class docstring to find out how -# implement them below - -class CloudContext(cloud_utils.MigrationContext): - def __init__(self,host,port,username,password,database,configdir,resourcedir): - self.host = host - self.port = port - self.username = username - self.password = password - self.database = database - self.configdir = configdir - self.resourcedir = resourcedir - self.conn = mysql.connector.connect(host=self.host, - user=self.username, - password=self.password, - database=self.database, - port=self.port) - self.conn.autocommit(False) - self.db = self.conn.cursor() - def wrapex(func): - sqlogger = logging.getLogger("SQL") - def f(stmt,parms=None): - if parms: sqlogger.debug("%s | with parms %s",stmt,parms) - else: sqlogger.debug("%s",stmt) - return func(stmt,parms) - return f - self.db.execute = wrapex(self.db.execute) - - def __str__(self): - return "CloudStack %s database at %s"%(self.database,self.host) - - def get_schema_level(self): - return self.get_config_value('schema.level') or cloud_utils.INITIAL_LEVEL - - def set_schema_level(self,l): - self.db.execute( - "INSERT INTO configuration (category,instance,component,name,value,description) VALUES ('Hidden', 'DEFAULT', 'database', 'schema.level', %s, 'The schema level of this database') ON DUPLICATE KEY UPDATE value = %s", (l,l) - ) - self.commit() - - def commit(self): - self.conn.commit() - #self.conn.close() - - def close(self): - self.conn.close() - - def get_config_value(self,name): - self.db.execute("select value from configuration where name = %s",(name,)) - try: return self.db.fetchall()[0][0] - except IndexError: return - - def run_sql_resource(self,resource): - sqlfiletext = file(os.path.join(self.resourcedir,resource)).read(-1) - sqlstatements = sqlfiletext.split(";") - for stmt in sqlstatements: - if not stmt.strip(): continue # skip empty statements - self.db.execute(stmt) - - -class FromInitialTo21NewSchema(cloud_utils.MigrationStep): - def __str__(self): return "Altering the database schema" - from_level = cloud_utils.INITIAL_LEVEL - to_level = "2.1-01" - def run(self): self.context.run_sql_resource("schema-20to21.sql") - -class From21NewSchemaTo21NewSchemaPlusIndex(cloud_utils.MigrationStep): - def __str__(self): return "Altering indexes" - from_level = "2.1-01" - to_level = "2.1-02" - def run(self): self.context.run_sql_resource("index-20to21.sql") - -class From21NewSchemaPlusIndexTo21DataMigratedPart1(cloud_utils.MigrationStep): - def __str__(self): return "Performing data migration, stage 1" - from_level = "2.1-02" - to_level = "2.1-03" - def run(self): self.context.run_sql_resource("data-20to21.sql") - -class From21step1toTo21datamigrated(cloud_utils.MigrationStep): - def __str__(self): return "Performing data migration, stage 2" - from_level = "2.1-03" - to_level = "2.1-04" - - def run(self): - systemjars = "@SYSTEMJARS@".split() - pipe = subprocess.Popen(["build-classpath"]+systemjars,stdout=subprocess.PIPE) - systemcp,throwaway = pipe.communicate() - systemcp = systemcp.strip() - if pipe.wait(): # this means that build-classpath failed miserably - systemcp = "@SYSTEMCLASSPATH@" - pcp = os.path.pathsep.join( glob.glob( os.path.join ( "@PREMIUMJAVADIR@" , "*" ) ) ) - mscp = "@MSCLASSPATH@" - depscp = "@DEPSCLASSPATH@" - migrationxml = "@SERVERSYSCONFDIR@" - conf = self.context.configdir - cp = os.path.pathsep.join([pcp,systemcp,depscp,mscp,migrationxml,conf]) - cmd = ["java"] - cmd += ["-cp",cp] - cmd += ["com.cloud.migration.Db20to21MigrationUtil"] - logging.debug("Running command: %s"," ".join(cmd)) - subprocess.check_call(cmd) - -class From21datamigratedTo21postprocessed(cloud_utils.MigrationStep): - def __str__(self): return "Postprocessing migrated data" - from_level = "2.1-04" - to_level = "2.1" - def run(self): self.context.run_sql_resource("postprocess-20to21.sql") - -class From21To213(cloud_utils.MigrationStep): - def __str__(self): return "Dropping obsolete indexes" - from_level = "2.1" - to_level = "2.1.3" - def run(self): self.context.run_sql_resource("index-212to213.sql") - -class From213To22data(cloud_utils.MigrationStep): - def __str__(self): return "Migrating data" - from_level = "2.1.3" - to_level = "2.2-01" - def run(self): self.context.run_sql_resource("data-21to22.sql") - -class From22dataTo22(cloud_utils.MigrationStep): - def __str__(self): return "Migrating indexes" - from_level = "2.2-01" - to_level = "2.2" - def run(self): self.context.run_sql_resource("index-21to22.sql") - -# command line harness functions - -def setup_logging(level): - l = logging.getLogger() - l.setLevel(level) - h = logging.StreamHandler(sys.stderr) - l.addHandler(h) - - -def setup_optparse(): - usage = \ -"""%prog [ options ... ] - -This command migrates the CloudStack database.""" - parser = OptionParser(usage=usage) - parser.add_option("-c", "--config", action="store", type="string",dest='configdir', - default=os.path.join("@MSCONF@"), - help="Configuration directory with a db.properties file, pointing to the CloudStack database") - parser.add_option("-r", "--resourcedir", action="store", type="string",dest='resourcedir', - default="@SETUPDATADIR@", - help="Resource directory with database SQL files used by the migration process") - parser.add_option("-d", "--debug", action="store_true", dest='debug', - default=False, - help="Increase log level from INFO to DEBUG") - parser.add_option("-e", "--dump-evolvers", action="store_true", dest='dumpevolvers', - default=False, - help="Dump evolvers in the order they would be executed, but do not run them") - #parser.add_option("-n", "--dry-run", action="store_true", dest='dryrun', - #default=False, - #help="Run the process as it would normally run, but do not commit the final transaction, so database changes are never saved") - parser.add_option("-f", "--start-at-level", action="store", type="string",dest='fromlevel', - default=None, - help="Rather than discovering the database schema level to start from, start migration from this level. The special value '-' (a dash without quotes) represents the earliest schema level") - parser.add_option("-t", "--end-at-level", action="store", type="string",dest='tolevel', - default=None, - help="Rather than evolving the database to the most up-to-date level, end migration at this level") - return parser - - -def main(*args): - """The entry point of this program""" - - parser = setup_optparse() - opts, args = parser.parse_args(*args) - if args: parser.error("This command accepts no parameters") - - if opts.debug: loglevel = logging.DEBUG - else: loglevel = logging.INFO - setup_logging(loglevel) - - # FIXME implement - opts.dryrun = False - - configdir = opts.configdir - resourcedir = opts.resourcedir - - try: - props = cloud_utils.read_properties(os.path.join(configdir,'db.properties')) - except (IOError,OSError) as e: - logging.error("Cannot read from config file: %s",e) - logging.error("You may want to point to a specific config directory with the --config= option") - return 2 - - if not os.path.isdir(resourcedir): - logging.error("Cannot find directory with SQL files %s",resourcedir) - logging.error("You may want to point to a specific resource directory with the --resourcedir= option") - return 2 - - host = props["db.cloud.host"] - port = int(props["db.cloud.port"]) - username = props["db.cloud.username"] - password = props["db.cloud.password"] - database = props["db.cloud.name"] - - # tell the migrator to load its steps from the globals list - migrator = cloud_utils.Migrator(list(globals().values())) - - if opts.dumpevolvers: - print("Evolution steps:") - print(" %s %s %s"%("From","To","Evolver in charge")) - for f,t,e in migrator.get_evolver_chain(): - print(" %s %s %s"%(f,t,e)) - return - - #initialize a context with the read configuration - context = CloudContext(host=host,port=port,username=username,password=password,database=database,configdir=configdir,resourcedir=resourcedir) - try: - try: - migrator.run(context,dryrun=opts.dryrun,starting_level=opts.fromlevel,ending_level=opts.tolevel) - finally: - context.close() - except (cloud_utils.NoMigrationPath,cloud_utils.NoMigrator) as e: - logging.error("%s",e) - return 4 - -if __name__ == "__main__": - retval = main() - if retval: sys.exit(retval) - else: sys.exit() +exit $res diff --git a/setup/bindir/cloud-setup-databases.in b/setup/bindir/cloud-setup-databases.in index 0532613dd81..57bfbcbc35b 100755 --- a/setup/bindir/cloud-setup-databases.in +++ b/setup/bindir/cloud-setup-databases.in @@ -67,7 +67,7 @@ class DBDeployer(object): dbDotProperties = {} dbDotPropertiesIndex = 0 encryptionKeyFile = '@MSCONF@/key' - encryptionJarPath = '@COMMONLIBDIR@/lib/jasypt-1.9.3.jar' + encryptionJarPath = '@COMMONLIBDIR@/lib/cloudstack-utils.jar' success = False magicString = 'This_is_a_magic_string_i_think_no_one_will_duplicate' tmpMysqlFile = os.path.join(os.path.expanduser('~/'), 'cloudstackmysql.tmp.sql') @@ -391,8 +391,8 @@ for example: checkSELinux() def processEncryptionStuff(self): - def encrypt(input): - cmd = ['java','-Djava.security.egd=file:/dev/urandom','-classpath','"' + self.encryptionJarPath + '"','org.jasypt.intf.cli.JasyptPBEStringEncryptionCLI', 'encrypt.sh', 'input=\'%s\''%input, 'password=\'%s\''%self.mgmtsecretkey,'verbose=false'] + def encrypt(value): + cmd = ['java','-classpath','"' + self.encryptionJarPath + '"','com.cloud.utils.crypt.EncryptionCLI','-i','"' + value + '"', '-p', '"' + self.mgmtsecretkey + '"', self.encryptorVersion] return str(runCmd(cmd)).strip('\r\n') def saveMgmtServerSecretKey(): @@ -408,6 +408,7 @@ for example: def encryptDBSecretKey(): self.putDbProperty('db.cloud.encrypt.secret', formatEncryptResult(encrypt(self.dbsecretkey))) + self.putDbProperty("db.cloud.encryptor.version", self.options.encryptorVersion) def encryptDBPassword(): dbPassword = self.getDbProperty('db.cloud.password') @@ -450,7 +451,12 @@ for example: self.info("Using specified cluster management server node IP %s" % self.options.mshostip, True) self.encryptiontype = self.options.encryptiontype - self.mgmtsecretkey = self.options.mgmtsecretkey + if self.encryptiontype == "env": + self.mgmtsecretkey = os.getenv("CLOUD_SECRET_KEY") + if not self.mgmtsecretkey: + self.errorAndExit("Please set environment variable CLOUD_SECRET_KEY if the encryption type is 'env'") + else: + self.mgmtsecretkey = self.options.mgmtsecretkey self.dbsecretkey = self.options.dbsecretkey self.isDebug = self.options.debug if self.options.dbConfPath: @@ -464,6 +470,11 @@ for example: if self.options.mysqlbinpath: self.mysqlBinPath = self.options.mysqlbinpath + if self.options.encryptorVersion: + self.encryptorVersion = "--encryptorversion %s" % self.options.encryptorVersion + else: + self.encryptorVersion = "" + def parseUserAndPassword(cred): stuff = cred.split(':') if len(stuff) != 1 and len(stuff) != 2: @@ -524,11 +535,11 @@ for example: def validateParameters(): if self.options.schemaonly and self.rootuser != None: self.errorAndExit("--schema-only and --deploy-as cannot be passed together\n") - if self.encryptiontype != 'file' and self.encryptiontype != 'web': - self.errorAndExit('Wrong encryption type %s, --encrypt-type can only be "file" or "web'%self.encryptiontype) + if self.encryptiontype != 'file' and self.encryptiontype != 'web' and self.encryptiontype != 'env': + self.errorAndExit('Wrong encryption type %s, --encrypt-type can only be "file" or "web" or "env"' % self.encryptiontype) #---------------------- option parsing and command line checks ------------------------ - usage = """%prog user:[password]@mysqlhost:[port] [--deploy-as=rootuser:[rootpassword]] [--auto=/path/to/server-setup.xml] [-e ENCRYPTIONTYPE] [-m MGMTSECRETKEY] [-k DBSECRETKEY] [--debug] + usage = """%prog user:[password]@mysqlhost:[port] [--deploy-as=rootuser:[rootpassword]] [--auto=/path/to/server-setup.xml] [-e ENCRYPTIONTYPE] [-m MGMTSECRETKEY] [-k DBSECRETKEY] [-g ENCRYPTORVERSION] [--debug] This command sets up the CloudStack Management Server and CloudStack Usage Server database configuration (connection credentials and host information) based on the first argument. @@ -538,6 +549,8 @@ for example: The port and the password are optional and can be left out.. If host is omitted altogether, it will default to localhost. + The encryptor version is optional. The options are V1 and V2. If it is not set, the default encryptor will be used. + Examples: %prog cloud:secret @@ -553,10 +566,13 @@ for example: with password 'nonsense', and recreates the databases, creating the user alex with password 'founder' as necessary - %prog alex:founder@1.2.3.4 --deploy-as=root:nonsense -e file -m password -k dbpassword + %prog alex:founder@1.2.3.4 --deploy-as=root:nonsense -e file -m password -k dbpassword -g V2 In addition actions performing in above example, using 'password' as management server encryption key and 'dbpassword' as database encryption key, saving management server encryption key to a file as the encryption type specified by -e is file. + The credentials in @MSCONF@/db.properties are encrypted by encryptor V2 (AeadBase64Encryptor). + The db.cloud.encryptor.version is also set to V2. Sensitive values in cloudstack databases will be + encrypted by the encryptor V2 using the database encryption key. %prog alena:tests@5.6.7.8 --deploy-as=root:nonsense --auto=/root/server-setup.xml sets alena up as the MySQL user, then connects as the root user @@ -577,7 +593,7 @@ for example: self.parser.add_option("-a", "--auto", action="store", type="string", dest="serversetup", default="", help="Path to an XML file describing an automated unattended cloud setup") self.parser.add_option("-e", "--encrypt-type", action="store", type="string", dest="encryptiontype", default="file", - help="Encryption method used for db password encryption. Valid values are file, web. Default is file.") + help="Encryption method used for db password encryption. Valid values are file, web and env. Default is file.") self.parser.add_option("-m", "--managementserver-secretkey", action="store", type="string", dest="mgmtsecretkey", default="password", help="Secret key used to encrypt confidential parameters in db.properties. A string, default is password") self.parser.add_option("-k", "--database-secretkey", action="store", type="string", dest="dbsecretkey", default="password", @@ -588,8 +604,10 @@ for example: help="Region Id for the management server cluster") self.parser.add_option("-c", "--db-conf-path", action="store", dest="dbConfPath", help="The path to find db.properties which hold db properties") self.parser.add_option("-f", "--db-files-path", action="store", dest="dbFilesPath", help="The path to find sql files to create initial database(s)") - self.parser.add_option("-j", "--encryption-jar-path", action="store", dest="encryptionJarPath", help="The path to the jasypt library to be used to encrypt the values in db.properties") + self.parser.add_option("-j", "--encryption-jar-path", action="store", dest="encryptionJarPath", help="The cloudstack jar to be used to encrypt the values in db.properties") self.parser.add_option("-n", "--encryption-key-file", action="store", dest="encryptionKeyFile", help="The name of the file in which encryption key to be generated") + self.parser.add_option("-g", "--encryptor-version", action="store", dest="encryptorVersion", default="V2", + help="The encryptor version to be used to encrypt the values in db.properties") self.parser.add_option("-b", "--mysql-bin-path", action="store", dest="mysqlbinpath", help="The mysql installed bin path") (self.options, self.args) = self.parser.parse_args() parseCasualCredit() diff --git a/setup/bindir/cloud-setup-encryption.in b/setup/bindir/cloud-setup-encryption.in index cd9212a119b..880edee8882 100755 --- a/setup/bindir/cloud-setup-encryption.in +++ b/setup/bindir/cloud-setup-encryption.in @@ -63,7 +63,7 @@ class DBDeployer(object): dbDotProperties = {} dbDotPropertiesIndex = 0 encryptionKeyFile = '@MSCONF@/key' - encryptionJarPath = '@COMMONLIBDIR@/lib/jasypt-1.9.3.jar' + encryptionJarPath = '@COMMONLIBDIR@/lib/cloudstack-utils.jar' success = False magicString = 'This_is_a_magic_string_i_think_no_one_will_duplicate' @@ -183,8 +183,8 @@ for example: self.success = True # At here, we have done successfully and nothing more after this flag is set def processEncryptionStuff(self): - def encrypt(input): - cmd = ['java','-classpath',self.encryptionJarPath,'org.jasypt.intf.cli.JasyptPBEStringEncryptionCLI', 'encrypt.sh', 'input=\'%s\''%input, 'password=%s'%self.mgmtsecretkey,'verbose=false'] + def encrypt(value): + cmd = ['java','-classpath','"' + self.encryptionJarPath + '"','com.cloud.utils.crypt.EncryptionCLI','-i','"' + value + '"', '-p', '"' + self.mgmtsecretkey + '"'] return runCmd(cmd).strip('\n') def saveMgmtServerSecretKey(): diff --git a/test/integration/smoke/test_primary_storage.py b/test/integration/smoke/test_primary_storage.py index 4e630b70a78..477d3317ad6 100644 --- a/test/integration/smoke/test_primary_storage.py +++ b/test/integration/smoke/test_primary_storage.py @@ -27,7 +27,7 @@ from marvin.lib.decoratorGenerators import skipTestIf from marvin.lib.utils import * from nose.plugins.attrib import attr -_multiprocess_shared_ = True +_multiprocess_shared_ = False class TestPrimaryStorageServices(cloudstackTestCase): @@ -51,6 +51,12 @@ class TestPrimaryStorageServices(cloudstackTestCase): if self.template == FAILED: assert False, "get_suitable_test_template() failed to return template with description %s" % self.services["ostype"] + self.excluded_pools = [] + storage_pool_list = StoragePool.list(self.apiclient, zoneid=self.zone.id) + for pool in storage_pool_list: + if pool.state != 'Up': + self.excluded_pools.append(pool.id) + return def tearDown(self): @@ -301,6 +307,8 @@ class TestPrimaryStorageServices(cloudstackTestCase): for pool in storage_pool_list: if (pool.id == storage_pool_2.id): continue + if pool.id in self.excluded_pools: + continue StoragePool.update(self.apiclient, id=pool.id, enabled=False) # deployvm @@ -334,6 +342,8 @@ class TestPrimaryStorageServices(cloudstackTestCase): for pool in storage_pool_list: if (pool.id == storage_pool_2.id): continue + if pool.id in self.excluded_pools: + continue StoragePool.update(self.apiclient, id=pool.id, enabled=True) # Enable all hosts for host in list_hosts_response: @@ -582,6 +592,8 @@ class TestStorageTags(cloudstackTestCase): ) self.debug("VM-1 Volumes: %s" % vm_1_volumes) self.assertEqual(vm_1_volumes[0].id, self.volume_1.id, "Check that volume V-1 has been attached to VM-1") + + time.sleep(30) self.virtual_machine_1.detach_volume(self.apiclient, self.volume_1) return diff --git a/tools/devcloud4/common/development-installation/files/default/cloud-install-sys-tmplt b/tools/devcloud4/common/development-installation/files/default/cloud-install-sys-tmplt index 313b5999e85..eb1f193a9cf 100755 --- a/tools/devcloud4/common/development-installation/files/default/cloud-install-sys-tmplt +++ b/tools/devcloud4/common/development-installation/files/default/cloud-install-sys-tmplt @@ -41,7 +41,8 @@ dbHost= dbUser= dbPassword= name= -jasypt='/usr/share/cloudstack-common/lib/jasypt-1.9.0.jar' +jarfile='/usr/share/cloudstack-common/lib/cloudstack-utils.jar' + while getopts 'm:h:f:u:Ft:e:s:o:r:d:n:' OPTION do case $OPTION in @@ -134,7 +135,7 @@ then encPassword=$(sed '/^\#/d' /etc/cloudstack/management/db.properties | grep 'db.cloud.password' | tail -n 1 | cut -d "=" -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'i | sed 's/^ENC(\(.*\))/\1/') if [ ! $encPassword == "" ] then - dbPassword=(`java -classpath $jasypt org.jasypt.intf.cli.JasyptPBEStringDecryptionCLI decrypt.sh input=$encPassword password=$msKey verbose=false`) + dbPassword=(`java -classpath $jarfile com.cloud.utils.crypt.EncryptionCLI -d -i "$encPassword" -p "$msKey"`) if [ ! $dbPassword ] then echo "Failed to decrypt DB password from db.properties" diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 9cf1e30865c..21a81ad5c02 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -95,6 +95,7 @@ import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -208,10 +209,17 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna s_logger.info("Implementation Version is " + _version); } - Map configs = _configDao.getConfiguration(params); + Map configs; + try { + configs = _configDao.getConfiguration(params); - if (params != null) { - mergeConfigs(configs, params); + if (params != null) { + mergeConfigs(configs, params); + s_logger.info("configs = " + configs); + } + } catch (CloudRuntimeException e) { + s_logger.error("Unhandled configuration exception: " + e.getMessage()); + throw new CloudRuntimeException("Unhandled configuration exception", e); } String execTime = configs.get("usage.stats.job.exec.time"); diff --git a/utils/pom.xml b/utils/pom.xml index db219b080f7..1cc46ebac8f 100755 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -210,6 +210,16 @@ nashorn-core 15.3 + + commons-cli + commons-cli + ${cs.commons-cli.version} + + + com.google.crypto.tink + tink + ${cs.tink.version} + @@ -234,6 +244,34 @@ + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + rebuild-war + package + + shade + + + false + + + ch.qos.reload4j + com.google.crypto.tink:tink + com.google.protobuf:protobuf-java + commons-cli:commons-cli + commons-codec:commons-codec + org.apache.commons:commons-lang3 + org.jasypt:jasypt + + + + + + diff --git a/utils/src/main/java/com/cloud/utils/EncryptionUtil.java b/utils/src/main/java/com/cloud/utils/EncryptionUtil.java index 37cea3499d9..4baa58b739b 100644 --- a/utils/src/main/java/com/cloud/utils/EncryptionUtil.java +++ b/utils/src/main/java/com/cloud/utils/EncryptionUtil.java @@ -18,11 +18,10 @@ */ package com.cloud.utils; +import com.cloud.utils.crypt.CloudStackEncryptor; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; -import org.jasypt.encryption.pbe.PBEStringEncryptor; -import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -32,13 +31,10 @@ import java.security.NoSuchAlgorithmException; public class EncryptionUtil { public static final Logger s_logger = Logger.getLogger(EncryptionUtil.class.getName()); - private static PBEStringEncryptor encryptor; + private static CloudStackEncryptor encryptor; private static void initialize(String key) { - StandardPBEStringEncryptor standardPBEStringEncryptor = new StandardPBEStringEncryptor(); - standardPBEStringEncryptor.setAlgorithm("PBEWITHSHA1ANDDESEDE"); - standardPBEStringEncryptor.setPassword(key); - encryptor = standardPBEStringEncryptor; + encryptor = new CloudStackEncryptor(key, null, EncryptionUtil.class); } public static String encodeData(String data, String key) { diff --git a/utils/src/main/java/com/cloud/utils/SerialVersionUID.java b/utils/src/main/java/com/cloud/utils/SerialVersionUID.java index 363248c99a9..e8587262423 100644 --- a/utils/src/main/java/com/cloud/utils/SerialVersionUID.java +++ b/utils/src/main/java/com/cloud/utils/SerialVersionUID.java @@ -71,4 +71,5 @@ public interface SerialVersionUID { public static final long UnavailableCommandException = Base | 0x2f; public static final long OriginDeniedException = Base | 0x30; public static final long StorageAccessException = Base | 0x31; + public static final long EncryptionException = Base | 0x32; } diff --git a/utils/src/main/java/com/cloud/utils/crypt/AeadBase64Encryptor.java b/utils/src/main/java/com/cloud/utils/crypt/AeadBase64Encryptor.java new file mode 100644 index 00000000000..f62dff7c6fe --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/crypt/AeadBase64Encryptor.java @@ -0,0 +1,63 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.crypt; + +import com.google.crypto.tink.Aead; +import com.google.crypto.tink.aead.AeadConfig; +import com.google.crypto.tink.subtle.AesGcmJce; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; + +public class AeadBase64Encryptor implements Base64Encryptor { + Aead aead = null; + private final byte[] aad = new byte[]{}; + + public AeadBase64Encryptor(byte[] key) { + try { + AeadConfig.register(); + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(key); + this.aead = new AesGcmJce(hash); + } catch (Exception e) { + throw new EncryptionException("Failed to initialize AeadBase64Encryptor"); + } + } + + @Override + public String encrypt(String plain) { + try { + return Base64.getEncoder().encodeToString(aead.encrypt(plain.getBytes(StandardCharsets.UTF_8), aad)); + } catch (Exception ex) { + throw new EncryptionException("Failed to encrypt " + plain + ". Error: " + ex.getMessage()); + } + } + + @Override + public String decrypt(String encrypted) { + try { + return new String(aead.decrypt(Base64.getDecoder().decode(encrypted), aad)); + } catch (Exception ex) { + throw new EncryptionException("Failed to decrypt " + CloudStackEncryptor.hideValueWithAsterisks(encrypted) + ". Error: " + ex.getMessage()); + } + } + +} diff --git a/utils/src/main/java/com/cloud/utils/crypt/Base64Encryptor.java b/utils/src/main/java/com/cloud/utils/crypt/Base64Encryptor.java new file mode 100644 index 00000000000..a0b70084e3b --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/crypt/Base64Encryptor.java @@ -0,0 +1,27 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.crypt; + +public interface Base64Encryptor { + + String encrypt(String plain); + + String decrypt(String encrypted); +} diff --git a/utils/src/main/java/com/cloud/utils/crypt/CloudStackEncryptor.java b/utils/src/main/java/com/cloud/utils/crypt/CloudStackEncryptor.java new file mode 100644 index 00000000000..91a6cd558c4 --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/crypt/CloudStackEncryptor.java @@ -0,0 +1,147 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.crypt; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import com.cloud.utils.exception.CloudRuntimeException; + +public class CloudStackEncryptor { + public static final Logger s_logger = Logger.getLogger(CloudStackEncryptor.class); + private Base64Encryptor encryptor = null; + private LegacyBase64Encryptor encryptorV1; + private AeadBase64Encryptor encryptorV2; + String password; + EncryptorVersion version; + Class callerClass; + + enum EncryptorVersion { + V1, V2; + + public static EncryptorVersion fromString(String version) { + if (version != null && version.equalsIgnoreCase("v1")) { + return V1; + } + if (version != null && version.equalsIgnoreCase("v2")) { + return V2; + } + if (StringUtils.isNotEmpty(version)) { + throw new CloudRuntimeException(String.format("Invalid encryptor version: %s, valid options are: %s", version, + Arrays.stream(EncryptorVersion.values()).map(EncryptorVersion::name).collect(Collectors.joining(",")))); + } + return null; + } + + public static EncryptorVersion defaultVersion() { + return V2; + } + } + + public CloudStackEncryptor(String password, String version, Class callerClass) { + this.password = password; + this.version = EncryptorVersion.fromString(version); + this.callerClass = callerClass; + initialize(); + } + + public String encrypt(String plain) { + if (StringUtils.isEmpty(plain)) { + return plain; + } + try { + if (encryptor == null) { + encryptor = encryptorV2; + s_logger.debug(String.format("CloudStack will encrypt and decrypt values using default encryptor : %s for class %s", + encryptor.getClass().getSimpleName(), callerClass.getSimpleName())); + } + return encryptor.encrypt(plain); + } catch (EncryptionException e) { + throw new CloudRuntimeException("Error encrypting value: ", e); + } + } + + public String decrypt(String encrypted) { + if (StringUtils.isEmpty(encrypted)) { + return encrypted; + } + if (encryptor != null) { + try { + return encryptor.decrypt(encrypted); + } catch (EncryptionException e) { + throw new CloudRuntimeException("Error decrypting value: " + hideValueWithAsterisks(encrypted), e); + } + } else { + String result = decrypt(encryptorV2, encrypted); + if (result != null) { + return result; + } + result = decrypt(encryptorV1, encrypted); + if (result != null) { + return result; + } + throw new CloudRuntimeException("Failed to decrypt value: " + hideValueWithAsterisks(encrypted)); + } + } + + private String decrypt(Base64Encryptor encryptorToUse, String encrypted) { + try { + String result = encryptorToUse.decrypt(encrypted); + s_logger.debug(String.format("CloudStack will encrypt and decrypt values using encryptor : %s for class %s", + encryptorToUse.getClass().getSimpleName(), callerClass.getSimpleName())); + encryptor = encryptorToUse; + return result; + } catch (EncryptionException ex) { + s_logger.warn(String.format("Failed to decrypt value using %s: %s", encryptorToUse.getClass().getSimpleName(), hideValueWithAsterisks(encrypted))); + } + return null; + } + + protected static String hideValueWithAsterisks(String value) { + if (StringUtils.isEmpty(value)) { + return value; + } + int numChars = value.length() >= 10 ? 5: 1; + int numAsterisks = 10 - numChars; + return value.substring(0, numChars) + "*".repeat(numAsterisks); + } + + protected void initialize() { + s_logger.debug("Calling to initialize for class " + callerClass.getName()); + encryptor = null; + if (EncryptorVersion.V1.equals(version)) { + encryptorV1 = new LegacyBase64Encryptor(password); + encryptor = encryptorV1; + s_logger.debug("Initialized with encryptor : " + encryptorV1.getClass().getSimpleName()); + } else if (EncryptorVersion.V2.equals(version)) { + encryptorV2 = new AeadBase64Encryptor(password.getBytes(StandardCharsets.UTF_8)); + encryptor = encryptorV2; + s_logger.debug("Initialized with encryptor : " + encryptorV2.getClass().getSimpleName()); + } else { + encryptorV1 = new LegacyBase64Encryptor(password); + encryptorV2 = new AeadBase64Encryptor(password.getBytes(StandardCharsets.UTF_8)); + s_logger.debug("Initialized with all possible encryptors"); + } + } +} diff --git a/utils/src/main/java/com/cloud/utils/crypt/DBEncryptionUtil.java b/utils/src/main/java/com/cloud/utils/crypt/DBEncryptionUtil.java index 52f2034c0ac..571e1449522 100644 --- a/utils/src/main/java/com/cloud/utils/crypt/DBEncryptionUtil.java +++ b/utils/src/main/java/com/cloud/utils/crypt/DBEncryptionUtil.java @@ -22,16 +22,13 @@ package com.cloud.utils.crypt; import java.util.Properties; import org.apache.log4j.Logger; -import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; -import org.jasypt.exceptions.EncryptionOperationNotPossibleException; import com.cloud.utils.db.DbProperties; import com.cloud.utils.exception.CloudRuntimeException; public class DBEncryptionUtil { - public static final Logger s_logger = Logger.getLogger(DBEncryptionUtil.class); - private static StandardPBEStringEncryptor s_encryptor = null; + private static CloudStackEncryptor s_encryptor = null; public static String encrypt(String plain) { if (!EncryptionSecretKeyChecker.useEncryption() || (plain == null) || plain.isEmpty()) { @@ -40,14 +37,7 @@ public class DBEncryptionUtil { if (s_encryptor == null) { initialize(); } - String encryptedString = null; - try { - encryptedString = s_encryptor.encrypt(plain); - } catch (EncryptionOperationNotPossibleException e) { - s_logger.debug("Error while encrypting: " + plain); - throw e; - } - return encryptedString; + return s_encryptor.encrypt(plain); } public static String decrypt(String encrypted) { @@ -58,17 +48,11 @@ public class DBEncryptionUtil { initialize(); } - String plain = null; - try { - plain = s_encryptor.decrypt(encrypted); - } catch (EncryptionOperationNotPossibleException e) { - s_logger.debug("Error while decrypting: " + encrypted); - throw e; - } - return plain; + return s_encryptor.decrypt(encrypted); } - private static void initialize() { + protected static void initialize() { + s_logger.debug("Calling to initialize"); final Properties dbProps = DbProperties.getDbProperties(); if (EncryptionSecretKeyChecker.useEncryption()) { @@ -76,12 +60,12 @@ public class DBEncryptionUtil { if (dbSecretKey == null || dbSecretKey.isEmpty()) { throw new CloudRuntimeException("Empty DB secret key in db.properties"); } + String dbEncryptorVersion = dbProps.getProperty("db.cloud.encryptor.version"); - s_encryptor = new StandardPBEStringEncryptor(); - s_encryptor.setAlgorithm("PBEWithMD5AndDES"); - s_encryptor.setPassword(dbSecretKey); + s_encryptor = new CloudStackEncryptor(dbSecretKey, dbEncryptorVersion, DBEncryptionUtil.class); } else { - throw new CloudRuntimeException("Trying to encrypt db values when encrytion is not enabled"); + throw new CloudRuntimeException("Trying to encrypt db values when encryption is not enabled"); } + s_logger.debug("initialized"); } } diff --git a/utils/src/main/java/com/cloud/utils/crypt/EncryptionCLI.java b/utils/src/main/java/com/cloud/utils/crypt/EncryptionCLI.java new file mode 100644 index 00000000000..af9717c8102 --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/crypt/EncryptionCLI.java @@ -0,0 +1,80 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.crypt; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +public class EncryptionCLI { + private static final String VERBOSE_OPTION = "verbose"; + private static final String DECRYPT_OPTION = "decrypt"; + private static final String INPUT_OPTION = "input"; + private static final String PASSWORD_OPTION = "password"; + private static final String ENCRYPTOR_VERSION_OPTION = "encryptorversion"; + + public static void main(String[] args) throws ParseException { + Options options = new Options(); + Option verbose = Option.builder("v").longOpt(VERBOSE_OPTION).argName(VERBOSE_OPTION).required(false).desc("Verbose output").hasArg(false).build(); + Option decrypt = Option.builder("d").longOpt(DECRYPT_OPTION).argName(DECRYPT_OPTION).required(false).desc("Decrypt instead of encrypt").hasArg(false).build(); + Option input = Option.builder("i").longOpt(INPUT_OPTION).argName(INPUT_OPTION).required(true).hasArg().desc("The input string to process").build(); + Option password = Option.builder("p").longOpt(PASSWORD_OPTION).argName(PASSWORD_OPTION).required(true).hasArg().desc("The encryption password").build(); + Option encryptorVersion = Option.builder("e").longOpt(ENCRYPTOR_VERSION_OPTION).argName(ENCRYPTOR_VERSION_OPTION).required(false).hasArg().desc("The encryptor version").build(); + + options.addOption(verbose); + options.addOption(decrypt); + options.addOption(input); + options.addOption(password); + options.addOption(encryptorVersion); + + CommandLine cmdLine = null; + CommandLineParser parser = new DefaultParser(); + HelpFormatter helper = new HelpFormatter(); + try { + cmdLine = parser.parse(options, args); + } catch (ParseException ex) { + System.out.println(ex.getMessage()); + helper.printHelp("Usage:", options); + System.exit(1); + } + + CloudStackEncryptor encryptor = new CloudStackEncryptor(cmdLine.getOptionValue(PASSWORD_OPTION), + cmdLine.getOptionValue(encryptorVersion), EncryptionCLI.class); + + String result; + String inString = cmdLine.getOptionValue(INPUT_OPTION); + if (cmdLine.hasOption(DECRYPT_OPTION)) { + result = encryptor.decrypt(inString); + } else { + result = encryptor.encrypt(inString); + } + + if (cmdLine.hasOption(VERBOSE_OPTION)) { + System.out.printf("Input: %s%n", inString); + System.out.printf("Encrypted: %s%n", result); + } else { + System.out.printf("%s%n", result); + } + } +} diff --git a/utils/src/main/java/com/cloud/utils/crypt/EncryptionException.java b/utils/src/main/java/com/cloud/utils/crypt/EncryptionException.java new file mode 100644 index 00000000000..fb13d48e3b5 --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/crypt/EncryptionException.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.crypt; + +import com.cloud.utils.SerialVersionUID; + +public class EncryptionException extends RuntimeException { + private static final long serialVersionUID = SerialVersionUID.EncryptionException; + + public EncryptionException(String message) { + super(message); + } + + public EncryptionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/utils/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChecker.java b/utils/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChecker.java index ef17f7b5e75..44cf52ce59d 100644 --- a/utils/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChecker.java +++ b/utils/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChecker.java @@ -26,13 +26,12 @@ import java.io.InputStreamReader; import java.io.PrintWriter; import java.net.ServerSocket; import java.net.Socket; +import java.util.Map; import java.util.Properties; import javax.annotation.PostConstruct; import org.apache.log4j.Logger; -import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; -import org.jasypt.encryption.pbe.config.SimpleStringPBEConfig; import com.cloud.utils.db.DbProperties; import com.cloud.utils.exception.CloudRuntimeException; @@ -45,7 +44,7 @@ public class EncryptionSecretKeyChecker { private static final String s_altKeyFile = "key"; private static final String s_keyFile = "key"; private static final String s_envKey = "CLOUD_SECRET_KEY"; - private static StandardPBEStringEncryptor s_encryptor = new StandardPBEStringEncryptor(); + private static CloudStackEncryptor s_encryptor = null; private static boolean s_useEncryption = false; @PostConstruct @@ -69,11 +68,8 @@ public class EncryptionSecretKeyChecker { return; } - s_encryptor.setAlgorithm("PBEWithMD5AndDES"); String secretKey = null; - SimpleStringPBEConfig stringConfig = new SimpleStringPBEConfig(); - if (encryptionType.equals("file")) { InputStream is = this.getClass().getClassLoader().getResourceAsStream(s_keyFile); if (is == null) { @@ -122,12 +118,14 @@ public class EncryptionSecretKeyChecker { throw new CloudRuntimeException("Invalid encryption type: " + encryptionType); } - stringConfig.setPassword(secretKey); - s_encryptor.setConfig(stringConfig); - s_useEncryption = true; + if (secretKey == null) { + throw new CloudRuntimeException("null secret key is found when setting up server encryption"); + } + + initEncryptor(secretKey); } - public static StandardPBEStringEncryptor getEncryptor() { + public static CloudStackEncryptor getEncryptor() { return s_encryptor; } @@ -135,12 +133,36 @@ public class EncryptionSecretKeyChecker { return s_useEncryption; } - //Initialize encryptor for migration during secret key change - public static void initEncryptorForMigration(String secretKey) { - s_encryptor.setAlgorithm("PBEWithMD5AndDES"); - SimpleStringPBEConfig stringConfig = new SimpleStringPBEConfig(); - stringConfig.setPassword(secretKey); - s_encryptor.setConfig(stringConfig); + public static void initEncryptor(String secretKey) { + s_encryptor = new CloudStackEncryptor(secretKey, null, EncryptionSecretKeyChecker.class); s_useEncryption = true; } + + public static void resetEncryptor() { + s_encryptor = null; + s_useEncryption = false; + } + + protected static String decryptPropertyIfNeeded(String value) { + if (s_encryptor == null) { + throw new CloudRuntimeException("encryptor not initialized"); + } + + if (value.startsWith("ENC(") && value.endsWith(")")) { + String inner = value.substring("ENC(".length(), value.length() - ")".length()); + return s_encryptor.decrypt(inner); + } + return value; + } + + public static void decryptAnyProperties(Properties properties) { + if (s_encryptor == null) { + throw new CloudRuntimeException("encryptor not initialized"); + } + + for (Map.Entry entry : properties.entrySet()) { + String value = (String) entry.getValue(); + properties.replace(entry.getKey(), decryptPropertyIfNeeded(value)); + } + } } diff --git a/utils/src/main/java/com/cloud/utils/crypt/LegacyBase64Encryptor.java b/utils/src/main/java/com/cloud/utils/crypt/LegacyBase64Encryptor.java new file mode 100644 index 00000000000..fb3dfbf549b --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/crypt/LegacyBase64Encryptor.java @@ -0,0 +1,58 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.crypt; + +import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; +import org.jasypt.encryption.pbe.config.SimpleStringPBEConfig; + +public class LegacyBase64Encryptor implements Base64Encryptor { + StandardPBEStringEncryptor encryptor; + + public LegacyBase64Encryptor(String password) { + try { + encryptor = new StandardPBEStringEncryptor(); + encryptor.setAlgorithm("PBEWithMD5AndDES"); + encryptor.setPassword(password); + SimpleStringPBEConfig stringConfig = new SimpleStringPBEConfig(); + encryptor.setConfig(stringConfig); + } catch (Exception e) { + throw new EncryptionException("Failed to initialize LegacyBase64Encryptor"); + } + } + + @Override + public String encrypt(String plain) { + try { + return encryptor.encrypt(plain); + } catch (Exception ex) { + throw new EncryptionException("Failed to encrypt " + plain + ". Error: " + ex.getMessage()); + } + } + + @Override + public String decrypt(String encrypted) { + try { + return encryptor.decrypt(encrypted); + } catch (Exception ex) { + throw new EncryptionException("Failed to decrypt " + CloudStackEncryptor.hideValueWithAsterisks(encrypted) + ". Error: " + ex.getMessage()); + } + } + +} diff --git a/utils/src/main/java/com/cloud/utils/db/DbProperties.java b/utils/src/main/java/com/cloud/utils/db/DbProperties.java index d99e6c011be..3851501e746 100644 --- a/utils/src/main/java/com/cloud/utils/db/DbProperties.java +++ b/utils/src/main/java/com/cloud/utils/db/DbProperties.java @@ -27,14 +27,11 @@ import java.util.Properties; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; -import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; -import org.jasypt.properties.EncryptableProperties; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.crypt.EncryptionSecretKeyChecker; public class DbProperties { - private static final Logger log = Logger.getLogger(DbProperties.class); private static Properties properties = new Properties(); @@ -46,11 +43,12 @@ public class DbProperties { checker.check(dbProps, dbEncryptionType); if (EncryptionSecretKeyChecker.useEncryption()) { + log.debug("encryptionsecretkeychecker using encryption"); + EncryptionSecretKeyChecker.decryptAnyProperties(dbProps); return dbProps; } else { - EncryptableProperties encrProps = new EncryptableProperties(EncryptionSecretKeyChecker.getEncryptor()); - encrProps.putAll(dbProps); - return encrProps; + log.debug("encryptionsecretkeychecker not using encryption"); + return dbProps; } } @@ -81,12 +79,10 @@ public class DbProperties { checker.check(dbProps, dbEncryptionType); if (EncryptionSecretKeyChecker.useEncryption()) { - StandardPBEStringEncryptor encryptor = EncryptionSecretKeyChecker.getEncryptor(); - EncryptableProperties encrDbProps = new EncryptableProperties(encryptor); - encrDbProps.putAll(dbProps); - dbProps = encrDbProps; + EncryptionSecretKeyChecker.decryptAnyProperties(dbProps); } } catch (IOException e) { + log.error(String.format("Failed to load DB properties: %s", e.getMessage()), e); throw new IllegalStateException("Failed to load db.properties", e); } finally { IOUtils.closeQuietly(is); @@ -94,6 +90,8 @@ public class DbProperties { properties = dbProps; loaded = true; + } else { + log.debug("DB properties were already loaded"); } return properties; diff --git a/utils/src/main/java/com/cloud/utils/server/ServerProperties.java b/utils/src/main/java/com/cloud/utils/server/ServerProperties.java index 4eabc5f99f7..b1a845cc501 100644 --- a/utils/src/main/java/com/cloud/utils/server/ServerProperties.java +++ b/utils/src/main/java/com/cloud/utils/server/ServerProperties.java @@ -19,8 +19,6 @@ package com.cloud.utils.server; import com.cloud.utils.crypt.EncryptionSecretKeyChecker; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; -import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; -import org.jasypt.properties.EncryptableProperties; import java.io.IOException; import java.io.InputStream; @@ -43,10 +41,7 @@ public class ServerProperties { checker.check(serverProps, passwordEncryptionType); if (EncryptionSecretKeyChecker.useEncryption()) { - StandardPBEStringEncryptor encryptor = EncryptionSecretKeyChecker.getEncryptor(); - EncryptableProperties encrServerProps = new EncryptableProperties(encryptor); - encrServerProps.putAll(serverProps); - serverProps = encrServerProps; + EncryptionSecretKeyChecker.decryptAnyProperties(serverProps); } } catch (IOException e) { throw new IllegalStateException("Failed to load server.properties", e); diff --git a/utils/src/test/java/com/cloud/utils/crypt/EncryptionSecretKeyCheckerTest.java b/utils/src/test/java/com/cloud/utils/crypt/EncryptionSecretKeyCheckerTest.java new file mode 100644 index 00000000000..7ac6a3ee71d --- /dev/null +++ b/utils/src/test/java/com/cloud/utils/crypt/EncryptionSecretKeyCheckerTest.java @@ -0,0 +1,58 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.crypt; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Properties; + +public class EncryptionSecretKeyCheckerTest { + @Before + public void setup() { + EncryptionSecretKeyChecker.initEncryptor("managementkey"); + } + + @After + public void tearDown() { + EncryptionSecretKeyChecker.resetEncryptor(); + } + + @Test + public void decryptPropertyIfNeededTest() { + String rawValue = "ENC(iYVsCZXiGiC6SzZLMNBvBL93hoUpntxkuRjyaqC8L+JYKXw=)"; + String result = EncryptionSecretKeyChecker.decryptPropertyIfNeeded(rawValue); + Assert.assertEquals("encthis", result); + } + + @Test + public void decryptAnyPropertiesTest() { + Properties props = new Properties(); + props.setProperty("db.cloud.encrypt.secret", "ENC(iYVsCZXiGiC6SzZLMNBvBL93hoUpntxkuRjyaqC8L+JYKXw=)"); + props.setProperty("other.unencrypted", "somevalue"); + + EncryptionSecretKeyChecker.decryptAnyProperties(props); + + Assert.assertEquals("encthis", props.getProperty("db.cloud.encrypt.secret")); + Assert.assertEquals("somevalue", props.getProperty("other.unencrypted")); + } +}