diff --git a/client/conf/db.properties.in b/client/conf/db.properties.in index 572cfbc1ff2..8f31aff17e6 100644 --- a/client/conf/db.properties.in +++ b/client/conf/db.properties.in @@ -29,6 +29,10 @@ db.cloud.driver=@DBDRIVER@ db.cloud.port=3306 db.cloud.name=cloud +# Connection URI to the database "cloud". When this property is set, only the following properties will be used along with it: db.cloud.maxActive, db.cloud.maxIdle, db.cloud.maxWait, db.cloud.username, db.cloud.password, db.cloud.driver, db.cloud.validationQuery, db.cloud.isolation.level. Other properties will be ignored. +db.cloud.uri= + + # CloudStack database tuning parameters db.cloud.maxActive=250 db.cloud.maxIdle=30 @@ -61,6 +65,10 @@ db.usage.driver=@DBDRIVER@ db.usage.port=3306 db.usage.name=cloud_usage +# Connection URI to the database "usage". When this property is set, only the following properties will be used along with it: db.usage.maxActive, db.cloud.maxIdle, db.cloud.maxWait, db.usage.username, db.usage.password, db.usage.driver, db.usage.validationQuery, db.usage.isolation.level. Other properties will be ignored. +db.usage.uri= + + # usage database tuning parameters db.usage.maxActive=100 db.usage.maxIdle=30 @@ -79,6 +87,9 @@ db.simulator.maxIdle=30 db.simulator.maxWait=10000 db.simulator.autoReconnect=true +# Connection URI to the database "simulator". When this property is set, only the following properties will be used along with it: db.simulator.host, db.simulator.port, db.simulator.name, db.simulator.autoReconnect. Other properties will be ignored. +db.simulator.uri= + # High Availability And Cluster Properties db.ha.enabled=false diff --git a/framework/db/pom.xml b/framework/db/pom.xml index c3f99ae61c0..846c9007708 100644 --- a/framework/db/pom.xml +++ b/framework/db/pom.xml @@ -53,6 +53,11 @@ cloud-utils ${project.version} + + org.mariadb.jdbc + mariadb-java-client + 3.1.4 + diff --git a/framework/db/src/main/java/com/cloud/utils/db/DriverLoader.java b/framework/db/src/main/java/com/cloud/utils/db/DriverLoader.java index 34af85fbcbd..55fc1dbb6ed 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/DriverLoader.java +++ b/framework/db/src/main/java/com/cloud/utils/db/DriverLoader.java @@ -38,6 +38,7 @@ public class DriverLoader { DRIVERS.put("jdbc:mysql", "com.mysql.cj.jdbc.Driver"); DRIVERS.put("jdbc:postgresql", "org.postgresql.Driver"); DRIVERS.put("jdbc:h2", "org.h2.Driver"); + DRIVERS.put("jdbc:mariadb", "org.mariadb.jdbc.Driver"); LOADED_DRIVERS = new ArrayList(); } diff --git a/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java b/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java index eb6b09c31f3..df0df60f519 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java +++ b/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java @@ -38,6 +38,7 @@ import org.apache.commons.dbcp2.DriverManagerConnectionFactory; import org.apache.commons.dbcp2.PoolableConnection; import org.apache.commons.dbcp2.PoolableConnectionFactory; import org.apache.commons.dbcp2.PoolingDataSource; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; @@ -1001,7 +1002,7 @@ public class TransactionLegacy implements Closeable { private static DataSource s_ds; private static DataSource s_usageDS; private static DataSource s_simulatorDS; - private static boolean s_dbHAEnabled; + protected static boolean s_dbHAEnabled; static { // Initialize with assumed db.properties file @@ -1032,11 +1033,6 @@ public class TransactionLegacy implements Closeable { final long cloudMaxWait = Long.parseLong(dbProps.getProperty("db.cloud.maxWait")); final String cloudUsername = dbProps.getProperty("db.cloud.username"); final String cloudPassword = dbProps.getProperty("db.cloud.password"); - final String cloudHost = dbProps.getProperty("db.cloud.host"); - final String cloudDriver = dbProps.getProperty("db.cloud.driver"); - final int cloudPort = Integer.parseInt(dbProps.getProperty("db.cloud.port")); - final String cloudDbName = dbProps.getProperty("db.cloud.name"); - final boolean cloudAutoReconnect = Boolean.parseBoolean(dbProps.getProperty("db.cloud.autoReconnect")); final String cloudValidationQuery = dbProps.getProperty("db.cloud.validationQuery"); final String cloudIsolationLevel = dbProps.getProperty("db.cloud.isolation.level"); @@ -1059,16 +1055,6 @@ public class TransactionLegacy implements Closeable { final boolean cloudTestWhileIdle = Boolean.parseBoolean(dbProps.getProperty("db.cloud.testWhileIdle")); final long cloudTimeBtwEvictionRunsMillis = Long.parseLong(dbProps.getProperty("db.cloud.timeBetweenEvictionRunsMillis")); final long cloudMinEvcitableIdleTimeMillis = Long.parseLong(dbProps.getProperty("db.cloud.minEvictableIdleTimeMillis")); - final boolean cloudPoolPreparedStatements = Boolean.parseBoolean(dbProps.getProperty("db.cloud.poolPreparedStatements")); - final String url = dbProps.getProperty("db.cloud.url.params"); - - String cloudDbHAParams = null; - String cloudReplicas = null; - if (s_dbHAEnabled) { - cloudDbHAParams = getDBHAParams("cloud", dbProps); - cloudReplicas = dbProps.getProperty("db.cloud.replicas"); - s_logger.info("The replicas configured for Cloud Data base is/are : " + cloudReplicas); - } final boolean useSSL = Boolean.parseBoolean(dbProps.getProperty("db.cloud.useSSL")); if (useSSL) { @@ -1078,13 +1064,12 @@ public class TransactionLegacy implements Closeable { System.setProperty("javax.net.ssl.trustStorePassword", dbProps.getProperty("db.cloud.trustStorePassword")); } - final String cloudConnectionUri = cloudDriver + "://" + cloudHost + (s_dbHAEnabled ? "," + cloudReplicas : "") + ":" + cloudPort + "/" + cloudDbName + - "?autoReconnect=" + cloudAutoReconnect + (url != null ? "&" + url : "") + (useSSL ? "&useSSL=true" : "") + - (s_dbHAEnabled ? "&" + cloudDbHAParams : "") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : ""); - DriverLoader.loadDriver(cloudDriver); + Pair cloudUriAndDriver = getConnectionUriAndDriver(dbProps, loadBalanceStrategy, useSSL, "cloud"); + + DriverLoader.loadDriver(cloudUriAndDriver.second()); // Default Data Source for CloudStack - s_ds = createDataSource(cloudConnectionUri, cloudUsername, cloudPassword, cloudMaxActive, cloudMaxIdle, cloudMaxWait, + s_ds = createDataSource(cloudUriAndDriver.first(), cloudUsername, cloudPassword, cloudMaxActive, cloudMaxIdle, cloudMaxWait, cloudTimeBtwEvictionRunsMillis, cloudMinEvcitableIdleTimeMillis, cloudTestWhileIdle, cloudTestOnBorrow, cloudValidationQuery, isolationLevel); @@ -1094,20 +1079,13 @@ public class TransactionLegacy implements Closeable { final long usageMaxWait = Long.parseLong(dbProps.getProperty("db.usage.maxWait")); final String usageUsername = dbProps.getProperty("db.usage.username"); final String usagePassword = dbProps.getProperty("db.usage.password"); - final String usageHost = dbProps.getProperty("db.usage.host"); - final String usageDriver = dbProps.getProperty("db.usage.driver"); - final int usagePort = Integer.parseInt(dbProps.getProperty("db.usage.port")); - final String usageDbName = dbProps.getProperty("db.usage.name"); - final boolean usageAutoReconnect = Boolean.parseBoolean(dbProps.getProperty("db.usage.autoReconnect")); - final String usageUrl = dbProps.getProperty("db.usage.url.params"); - final String usageConnectionUri = usageDriver + "://" + usageHost + (s_dbHAEnabled ? "," + dbProps.getProperty("db.cloud.replicas") : "") + ":" + usagePort + - "/" + usageDbName + "?autoReconnect=" + usageAutoReconnect + (usageUrl != null ? "&" + usageUrl : "") + - (s_dbHAEnabled ? "&" + getDBHAParams("usage", dbProps) : "") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : ""); - DriverLoader.loadDriver(usageDriver); + Pair usageUriAndDriver = getConnectionUriAndDriver(dbProps, loadBalanceStrategy, useSSL, "usage"); + + DriverLoader.loadDriver(usageUriAndDriver.second()); // Data Source for usage server - s_usageDS = createDataSource(usageConnectionUri, usageUsername, usagePassword, + s_usageDS = createDataSource(usageUriAndDriver.first(), usageUsername, usagePassword, usageMaxActive, usageMaxIdle, usageMaxWait, null, null, null, null, null, isolationLevel); @@ -1118,14 +1096,28 @@ public class TransactionLegacy implements Closeable { final long simulatorMaxWait = Long.parseLong(dbProps.getProperty("db.simulator.maxWait")); final String simulatorUsername = dbProps.getProperty("db.simulator.username"); final String simulatorPassword = dbProps.getProperty("db.simulator.password"); - final String simulatorHost = dbProps.getProperty("db.simulator.host"); - final String simulatorDriver = dbProps.getProperty("db.simulator.driver"); - final int simulatorPort = Integer.parseInt(dbProps.getProperty("db.simulator.port")); - final String simulatorDbName = dbProps.getProperty("db.simulator.name"); - final boolean simulatorAutoReconnect = Boolean.parseBoolean(dbProps.getProperty("db.simulator.autoReconnect")); - final String simulatorConnectionUri = simulatorDriver + "://" + simulatorHost + ":" + simulatorPort + "/" + simulatorDbName + "?autoReconnect=" + - simulatorAutoReconnect; + String simulatorDriver; + String simulatorConnectionUri; + String simulatorUri = dbProps.getProperty("db.simulator.uri"); + + if (StringUtils.isEmpty(simulatorUri)) { + simulatorDriver = dbProps.getProperty("db.simulator.driver"); + final int simulatorPort = Integer.parseInt(dbProps.getProperty("db.simulator.port")); + final String simulatorDbName = dbProps.getProperty("db.simulator.name"); + final boolean simulatorAutoReconnect = Boolean.parseBoolean(dbProps.getProperty("db.simulator.autoReconnect")); + final String simulatorHost = dbProps.getProperty("db.simulator.host"); + + simulatorConnectionUri = simulatorDriver + "://" + simulatorHost + ":" + simulatorPort + "/" + simulatorDbName + "?autoReconnect=" + + simulatorAutoReconnect; + } else { + s_logger.warn("db.simulator.uri was set, ignoring the following properties on db.properties: [db.simulator.driver, db.simulator.host, db.simulator.port, " + + "db.simulator.name, db.simulator.autoReconnect]."); + String[] splitUri = simulatorUri.split(":"); + simulatorDriver = String.format("%s:%s", splitUri[0], splitUri[1]); + simulatorConnectionUri = simulatorUri; + } + DriverLoader.loadDriver(simulatorDriver); s_simulatorDS = createDataSource(simulatorConnectionUri, simulatorUsername, simulatorPassword, @@ -1143,6 +1135,85 @@ public class TransactionLegacy implements Closeable { } } + protected static Pair getConnectionUriAndDriver(Properties dbProps, String loadBalanceStrategy, boolean useSSL, String schema) { + String connectionUri; + String driver; + String propertyUri = dbProps.getProperty(String.format("db.%s.uri", schema)); + + if (StringUtils.isEmpty(propertyUri)) { + driver = dbProps.getProperty(String.format("db.%s.driver", schema)); + connectionUri = getPropertiesAndBuildConnectionUri(dbProps, loadBalanceStrategy, driver, useSSL, schema); + } else { + s_logger.warn(String.format("db.%s.uri was set, ignoring the following properties for schema %s of db.properties: [host, port, name, driver, autoReconnect, url.params," + + " replicas, ha.loadBalanceStrategy, ha.enable, failOverReadOnly, reconnectAtTxEnd, autoReconnectForPools, secondsBeforeRetrySource, queriesBeforeRetrySource, " + + "initialTimeout].", schema, schema)); + + String[] splitUri = propertyUri.split(":"); + driver = String.format("%s:%s", splitUri[0], splitUri[1]); + + connectionUri = propertyUri; + } + s_logger.info(String.format("Using the following URI to connect to %s database [%s].", schema, connectionUri)); + return new Pair<>(connectionUri, driver); + } + + protected static String getPropertiesAndBuildConnectionUri(Properties dbProps, String loadBalanceStrategy, String driver, boolean useSSL, String schema) { + String host = dbProps.getProperty(String.format("db.%s.host", schema)); + int port = Integer.parseInt(dbProps.getProperty(String.format("db.%s.port", schema))); + String dbName = dbProps.getProperty(String.format("db.%s.name", schema)); + boolean autoReconnect = Boolean.parseBoolean(dbProps.getProperty(String.format("db.%s.autoReconnect", schema))); + String urlParams = dbProps.getProperty(String.format("db.%s.url.params", schema)); + + String replicas = null; + String dbHaParams = null; + if (s_dbHAEnabled) { + dbHaParams = getDBHAParams(schema, dbProps); + replicas = dbProps.getProperty(String.format("db.%s.replicas", schema)); + s_logger.info(String.format("The replicas configured for %s data base are %s.", schema, replicas)); + } + + return buildConnectionUri(loadBalanceStrategy, driver, useSSL, host, replicas, port, dbName, autoReconnect, urlParams, dbHaParams); + } + + protected static String buildConnectionUri(String loadBalanceStrategy, String driver, boolean useSSL, String host, String replicas, int port, String dbName, boolean autoReconnect, + String urlParams, String dbHaParams) { + + StringBuilder connectionUri = new StringBuilder(); + connectionUri.append(driver); + connectionUri.append("://"); + connectionUri.append(host); + + if (s_dbHAEnabled) { + connectionUri.append(","); + connectionUri.append(replicas); + } + + connectionUri.append(":"); + connectionUri.append(port); + connectionUri.append("/"); + connectionUri.append(dbName); + connectionUri.append("?autoReconnect="); + connectionUri.append(autoReconnect); + + if (urlParams != null) { + connectionUri.append("&"); + connectionUri.append(urlParams); + } + + if (useSSL) { + connectionUri.append("&useSSL=true"); + } + + if (s_dbHAEnabled) { + connectionUri.append("&"); + connectionUri.append(dbHaParams); + connectionUri.append("&loadBalanceStrategy="); + connectionUri.append(loadBalanceStrategy); + } + + return connectionUri.toString(); + } + /** * Creates a data source */ diff --git a/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java b/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java new file mode 100644 index 00000000000..2e0af6fa186 --- /dev/null +++ b/framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.utils.db; + +import com.cloud.utils.Pair; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Properties; + +@RunWith(MockitoJUnitRunner.class) +public class TransactionLegacyTest { + + Properties properties; + + @Before + public void setup(){ + properties = new Properties(); + properties.setProperty("db.cloud.host", "host"); + properties.setProperty("db.cloud.port", "5555"); + properties.setProperty("db.cloud.name", "name"); + properties.setProperty("db.cloud.autoReconnect", "false"); + properties.setProperty("db.cloud.url.params", "someParams"); + TransactionLegacy.s_dbHAEnabled = false; + } + @Test + public void getConnectionUriAndDriverTestWithoutUri() { + properties.setProperty("db.cloud.uri", ""); + properties.setProperty("db.cloud.driver", "driver"); + + Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); + + Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams", result.first()); + Assert.assertEquals("driver", result.second()); + } + + @Test + public void getConnectionUriAndDriverTestWithUri() { + properties.setProperty("db.cloud.uri", "jdbc:driver:myFavoriteUri"); + + Pair result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud"); + + Assert.assertEquals("jdbc:driver:myFavoriteUri", result.first()); + Assert.assertEquals("jdbc:driver", result.second()); + } + + @Test + public void getPropertiesAndBuildConnectionUriTestDbHaDisabled() { + String result = TransactionLegacy.getPropertiesAndBuildConnectionUri(properties, "strat", "driver", true, "cloud"); + + Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams&useSSL=true", result); + } + + @Test + public void getPropertiesAndBuildConnectionUriTestDbHaEnabled() { + TransactionLegacy.s_dbHAEnabled = true; + properties.setProperty("db.cloud.failOverReadOnly", "true"); + properties.setProperty("db.cloud.reconnectAtTxEnd", "false"); + properties.setProperty("db.cloud.autoReconnectForPools", "true"); + properties.setProperty("db.cloud.secondsBeforeRetrySource", "25"); + properties.setProperty("db.cloud.queriesBeforeRetrySource", "105"); + properties.setProperty("db.cloud.initialTimeout", "1000"); + properties.setProperty("db.cloud.replicas", "second_host"); + + String result = TransactionLegacy.getPropertiesAndBuildConnectionUri(properties, "strat", "driver", true, "cloud"); + + Assert.assertEquals("driver://host,second_host:5555/name?autoReconnect=false&someParams&useSSL=true&failOverReadOnly=true&reconnectAtTxEnd=false&autoReconnectFor" + + "Pools=true&secondsBeforeRetrySource=25&queriesBeforeRetrySource=105&initialTimeout=1000&loadBalanceStrategy=strat", result); + } + + @Test + public void buildConnectionUriTestDbHaDisabled() { + String result = TransactionLegacy.buildConnectionUri(null, "driver", false, "host", null, 5555, "cloud", false, null, null); + + Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false", result); + } + + @Test + public void buildConnectionUriTestDbHaEnabled() { + TransactionLegacy.s_dbHAEnabled = true; + + String result = TransactionLegacy.buildConnectionUri("strat", "driver", false, "host", "second_host", 5555, "cloud", false, null, "dbHaParams"); + + Assert.assertEquals("driver://host,second_host:5555/cloud?autoReconnect=false&dbHaParams&loadBalanceStrategy=strat", result); + } + + @Test + public void buildConnectionUriTestUrlParamsNotNull() { + String result = TransactionLegacy.buildConnectionUri(null, "driver", false, "host", null, 5555, "cloud", false, "urlParams", null); + + Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&urlParams", result); + } + + @Test + public void buildConnectionUriTestUseSslTrue() { + String result = TransactionLegacy.buildConnectionUri(null, "driver", true, "host", null, 5555, "cloud", false, null, null); + + Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&useSSL=true", result); + } +}