Flexible URI for connection with DB and new MariaDB driver (#7895)

Co-authored-by: João Jandre <joao@scclouds.com.br>

client/conf/db.properties.in

@@ -29,6 +29,10 @@ db.cloud.driver=@DBDRIVER@
 db.cloud.port=3306
 db.cloud.name=cloud
 
+# Connection URI to the database "cloud". When this property is set, only the following properties will be used along with it: db.cloud.maxActive, db.cloud.maxIdle, db.cloud.maxWait, db.cloud.username, db.cloud.password, db.cloud.driver, db.cloud.validationQuery, db.cloud.isolation.level. Other properties will be ignored.
+db.cloud.uri=
+
+
 # CloudStack database tuning parameters
 db.cloud.maxActive=250
 db.cloud.maxIdle=30
@@ -61,6 +65,10 @@ db.usage.driver=@DBDRIVER@
 db.usage.port=3306
 db.usage.name=cloud_usage
 
+# Connection URI to the database "usage". When this property is set, only the following properties will be used along with it: db.usage.maxActive, db.cloud.maxIdle, db.cloud.maxWait, db.usage.username, db.usage.password, db.usage.driver, db.usage.validationQuery, db.usage.isolation.level. Other properties will be ignored.
+db.usage.uri=
+
+
 # usage database tuning parameters
 db.usage.maxActive=100
 db.usage.maxIdle=30
@@ -79,6 +87,9 @@ db.simulator.maxIdle=30
 db.simulator.maxWait=10000
 db.simulator.autoReconnect=true
 
+# Connection URI to the database "simulator". When this property is set, only the following properties will be used along with it: db.simulator.host, db.simulator.port, db.simulator.name, db.simulator.autoReconnect. Other properties will be ignored.
+db.simulator.uri=
+
 
 # High Availability And Cluster Properties
 db.ha.enabled=false

framework/db/pom.xml

@@ -53,6 +53,11 @@
             <artifactId>cloud-utils</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.mariadb.jdbc</groupId>
+            <artifactId>mariadb-java-client</artifactId>
+            <version>3.1.4</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

framework/db/src/main/java/com/cloud/utils/db/DriverLoader.java

@@ -38,6 +38,7 @@ public class DriverLoader {
         DRIVERS.put("jdbc:mysql", "com.mysql.cj.jdbc.Driver");
         DRIVERS.put("jdbc:postgresql", "org.postgresql.Driver");
         DRIVERS.put("jdbc:h2", "org.h2.Driver");
+        DRIVERS.put("jdbc:mariadb", "org.mariadb.jdbc.Driver");
 
         LOADED_DRIVERS = new ArrayList<String>();
     }

framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java

@@ -38,6 +38,7 @@ import org.apache.commons.dbcp2.DriverManagerConnectionFactory;
 import org.apache.commons.dbcp2.PoolableConnection;
 import org.apache.commons.dbcp2.PoolableConnectionFactory;
 import org.apache.commons.dbcp2.PoolingDataSource;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.pool2.ObjectPool;
 import org.apache.commons.pool2.impl.GenericObjectPool;
 import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
@@ -1001,7 +1002,7 @@ public class TransactionLegacy implements Closeable {
     private static DataSource s_ds;
     private static DataSource s_usageDS;
     private static DataSource s_simulatorDS;
-    private static boolean s_dbHAEnabled;
+    protected static boolean s_dbHAEnabled;
 
     static {
         // Initialize with assumed db.properties file
@@ -1032,11 +1033,6 @@ public class TransactionLegacy implements Closeable {
             final long cloudMaxWait = Long.parseLong(dbProps.getProperty("db.cloud.maxWait"));
             final String cloudUsername = dbProps.getProperty("db.cloud.username");
             final String cloudPassword = dbProps.getProperty("db.cloud.password");
-            final String cloudHost = dbProps.getProperty("db.cloud.host");
-            final String cloudDriver = dbProps.getProperty("db.cloud.driver");
-            final int cloudPort = Integer.parseInt(dbProps.getProperty("db.cloud.port"));
-            final String cloudDbName = dbProps.getProperty("db.cloud.name");
-            final boolean cloudAutoReconnect = Boolean.parseBoolean(dbProps.getProperty("db.cloud.autoReconnect"));
             final String cloudValidationQuery = dbProps.getProperty("db.cloud.validationQuery");
             final String cloudIsolationLevel = dbProps.getProperty("db.cloud.isolation.level");
 
@@ -1059,16 +1055,6 @@ public class TransactionLegacy implements Closeable {
             final boolean cloudTestWhileIdle = Boolean.parseBoolean(dbProps.getProperty("db.cloud.testWhileIdle"));
             final long cloudTimeBtwEvictionRunsMillis = Long.parseLong(dbProps.getProperty("db.cloud.timeBetweenEvictionRunsMillis"));
             final long cloudMinEvcitableIdleTimeMillis = Long.parseLong(dbProps.getProperty("db.cloud.minEvictableIdleTimeMillis"));
-            final boolean cloudPoolPreparedStatements = Boolean.parseBoolean(dbProps.getProperty("db.cloud.poolPreparedStatements"));
-            final String url = dbProps.getProperty("db.cloud.url.params");
-
-            String cloudDbHAParams = null;
-            String cloudReplicas = null;
-            if (s_dbHAEnabled) {
-                cloudDbHAParams = getDBHAParams("cloud", dbProps);
-                cloudReplicas = dbProps.getProperty("db.cloud.replicas");
-                s_logger.info("The replicas configured for Cloud Data base is/are : " + cloudReplicas);
-            }
 
             final boolean useSSL = Boolean.parseBoolean(dbProps.getProperty("db.cloud.useSSL"));
             if (useSSL) {
@@ -1078,13 +1064,12 @@ public class TransactionLegacy implements Closeable {
                 System.setProperty("javax.net.ssl.trustStorePassword", dbProps.getProperty("db.cloud.trustStorePassword"));
             }
 
-            final String cloudConnectionUri = cloudDriver + "://" + cloudHost + (s_dbHAEnabled ? "," + cloudReplicas : "") + ":" + cloudPort + "/" + cloudDbName +
+            Pair<String, String> cloudUriAndDriver = getConnectionUriAndDriver(dbProps, loadBalanceStrategy, useSSL, "cloud");
-                    "?autoReconnect=" + cloudAutoReconnect + (url != null ? "&" + url : "") + (useSSL ? "&useSSL=true" : "") +
+
-                    (s_dbHAEnabled ? "&" + cloudDbHAParams : "") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : "");
+            DriverLoader.loadDriver(cloudUriAndDriver.second());
-            DriverLoader.loadDriver(cloudDriver);
 
             // Default Data Source for CloudStack
-            s_ds = createDataSource(cloudConnectionUri, cloudUsername, cloudPassword, cloudMaxActive, cloudMaxIdle, cloudMaxWait,
+            s_ds = createDataSource(cloudUriAndDriver.first(), cloudUsername, cloudPassword, cloudMaxActive, cloudMaxIdle, cloudMaxWait,
                     cloudTimeBtwEvictionRunsMillis, cloudMinEvcitableIdleTimeMillis, cloudTestWhileIdle, cloudTestOnBorrow,
                     cloudValidationQuery, isolationLevel);
 
@@ -1094,20 +1079,13 @@ public class TransactionLegacy implements Closeable {
             final long usageMaxWait = Long.parseLong(dbProps.getProperty("db.usage.maxWait"));
             final String usageUsername = dbProps.getProperty("db.usage.username");
             final String usagePassword = dbProps.getProperty("db.usage.password");
-            final String usageHost = dbProps.getProperty("db.usage.host");
-            final String usageDriver = dbProps.getProperty("db.usage.driver");
-            final int usagePort = Integer.parseInt(dbProps.getProperty("db.usage.port"));
-            final String usageDbName = dbProps.getProperty("db.usage.name");
-            final boolean usageAutoReconnect = Boolean.parseBoolean(dbProps.getProperty("db.usage.autoReconnect"));
-            final String usageUrl = dbProps.getProperty("db.usage.url.params");
 
-            final String usageConnectionUri = usageDriver + "://" + usageHost + (s_dbHAEnabled ? "," + dbProps.getProperty("db.cloud.replicas") : "") + ":" + usagePort +
+            Pair<String, String> usageUriAndDriver = getConnectionUriAndDriver(dbProps, loadBalanceStrategy, useSSL, "usage");
-                    "/" + usageDbName + "?autoReconnect=" + usageAutoReconnect + (usageUrl != null ? "&" + usageUrl : "") +
+
-                    (s_dbHAEnabled ? "&" + getDBHAParams("usage", dbProps) : "") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : "");
+            DriverLoader.loadDriver(usageUriAndDriver.second());
-            DriverLoader.loadDriver(usageDriver);
 
             // Data Source for usage server
-            s_usageDS = createDataSource(usageConnectionUri, usageUsername, usagePassword,
+            s_usageDS = createDataSource(usageUriAndDriver.first(), usageUsername, usagePassword,
                     usageMaxActive, usageMaxIdle, usageMaxWait, null, null, null, null,
                     null, isolationLevel);
 
@@ -1118,14 +1096,28 @@ public class TransactionLegacy implements Closeable {
                 final long simulatorMaxWait = Long.parseLong(dbProps.getProperty("db.simulator.maxWait"));
                 final String simulatorUsername = dbProps.getProperty("db.simulator.username");
                 final String simulatorPassword = dbProps.getProperty("db.simulator.password");
-                final String simulatorHost = dbProps.getProperty("db.simulator.host");
-                final String simulatorDriver = dbProps.getProperty("db.simulator.driver");
-                final int simulatorPort = Integer.parseInt(dbProps.getProperty("db.simulator.port"));
-                final String simulatorDbName = dbProps.getProperty("db.simulator.name");
-                final boolean simulatorAutoReconnect = Boolean.parseBoolean(dbProps.getProperty("db.simulator.autoReconnect"));
 
-                final String simulatorConnectionUri = simulatorDriver + "://" + simulatorHost + ":" + simulatorPort + "/" + simulatorDbName + "?autoReconnect=" +
+                String simulatorDriver;
-                        simulatorAutoReconnect;
+                String simulatorConnectionUri;
+                String simulatorUri = dbProps.getProperty("db.simulator.uri");
+
+                if (StringUtils.isEmpty(simulatorUri)) {
+                    simulatorDriver = dbProps.getProperty("db.simulator.driver");
+                    final int simulatorPort = Integer.parseInt(dbProps.getProperty("db.simulator.port"));
+                    final String simulatorDbName = dbProps.getProperty("db.simulator.name");
+                    final boolean simulatorAutoReconnect = Boolean.parseBoolean(dbProps.getProperty("db.simulator.autoReconnect"));
+                    final String simulatorHost = dbProps.getProperty("db.simulator.host");
+
+                    simulatorConnectionUri = simulatorDriver + "://" + simulatorHost + ":" + simulatorPort + "/" + simulatorDbName + "?autoReconnect=" +
+                            simulatorAutoReconnect;
+                } else {
+                    s_logger.warn("db.simulator.uri was set, ignoring the following properties on db.properties: [db.simulator.driver, db.simulator.host, db.simulator.port, "
+                            + "db.simulator.name, db.simulator.autoReconnect].");
+                    String[] splitUri = simulatorUri.split(":");
+                    simulatorDriver = String.format("%s:%s", splitUri[0], splitUri[1]);
+                    simulatorConnectionUri = simulatorUri;
+                }
+
                 DriverLoader.loadDriver(simulatorDriver);
 
                 s_simulatorDS = createDataSource(simulatorConnectionUri, simulatorUsername, simulatorPassword,
@@ -1143,6 +1135,85 @@ public class TransactionLegacy implements Closeable {
         }
     }
 
+    protected static Pair<String, String> getConnectionUriAndDriver(Properties dbProps, String loadBalanceStrategy, boolean useSSL, String schema) {
+        String connectionUri;
+        String driver;
+        String propertyUri = dbProps.getProperty(String.format("db.%s.uri", schema));
+
+        if (StringUtils.isEmpty(propertyUri)) {
+            driver = dbProps.getProperty(String.format("db.%s.driver", schema));
+            connectionUri = getPropertiesAndBuildConnectionUri(dbProps, loadBalanceStrategy, driver, useSSL, schema);
+        } else {
+            s_logger.warn(String.format("db.%s.uri was set, ignoring the following properties for schema %s of db.properties: [host, port, name, driver, autoReconnect, url.params,"
+                    + " replicas, ha.loadBalanceStrategy, ha.enable, failOverReadOnly, reconnectAtTxEnd, autoReconnectForPools, secondsBeforeRetrySource, queriesBeforeRetrySource, "
+                    + "initialTimeout].", schema, schema));
+
+            String[] splitUri = propertyUri.split(":");
+            driver = String.format("%s:%s", splitUri[0], splitUri[1]);
+
+            connectionUri = propertyUri;
+        }
+        s_logger.info(String.format("Using the following URI to connect to %s database [%s].", schema, connectionUri));
+        return new Pair<>(connectionUri, driver);
+    }
+
+    protected static String getPropertiesAndBuildConnectionUri(Properties dbProps, String loadBalanceStrategy, String driver, boolean useSSL, String schema) {
+        String host = dbProps.getProperty(String.format("db.%s.host", schema));
+        int port = Integer.parseInt(dbProps.getProperty(String.format("db.%s.port", schema)));
+        String dbName = dbProps.getProperty(String.format("db.%s.name", schema));
+        boolean autoReconnect = Boolean.parseBoolean(dbProps.getProperty(String.format("db.%s.autoReconnect", schema)));
+        String urlParams = dbProps.getProperty(String.format("db.%s.url.params", schema));
+
+        String replicas = null;
+        String dbHaParams = null;
+        if (s_dbHAEnabled) {
+            dbHaParams = getDBHAParams(schema, dbProps);
+            replicas = dbProps.getProperty(String.format("db.%s.replicas", schema));
+            s_logger.info(String.format("The replicas configured for %s data base are %s.", schema, replicas));
+        }
+
+        return buildConnectionUri(loadBalanceStrategy, driver, useSSL, host, replicas, port, dbName, autoReconnect, urlParams, dbHaParams);
+    }
+
+    protected static String buildConnectionUri(String loadBalanceStrategy, String driver, boolean useSSL, String host, String replicas, int port, String dbName, boolean autoReconnect,
+            String urlParams, String dbHaParams) {
+
+        StringBuilder connectionUri = new StringBuilder();
+        connectionUri.append(driver);
+        connectionUri.append("://");
+        connectionUri.append(host);
+
+        if (s_dbHAEnabled) {
+            connectionUri.append(",");
+            connectionUri.append(replicas);
+        }
+
+        connectionUri.append(":");
+        connectionUri.append(port);
+        connectionUri.append("/");
+        connectionUri.append(dbName);
+        connectionUri.append("?autoReconnect=");
+        connectionUri.append(autoReconnect);
+
+        if (urlParams != null) {
+            connectionUri.append("&");
+            connectionUri.append(urlParams);
+        }
+
+        if (useSSL) {
+            connectionUri.append("&useSSL=true");
+        }
+
+        if (s_dbHAEnabled) {
+            connectionUri.append("&");
+            connectionUri.append(dbHaParams);
+            connectionUri.append("&loadBalanceStrategy=");
+            connectionUri.append(loadBalanceStrategy);
+        }
+
+        return connectionUri.toString();
+    }
+
     /**
      * Creates a data source
      */

framework/db/src/test/java/com/cloud/utils/db/TransactionLegacyTest.java

@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.utils.db;
+
+import com.cloud.utils.Pair;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.Properties;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TransactionLegacyTest {
+
+    Properties properties;
+
+    @Before
+    public void setup(){
+        properties = new Properties();
+        properties.setProperty("db.cloud.host", "host");
+        properties.setProperty("db.cloud.port", "5555");
+        properties.setProperty("db.cloud.name", "name");
+        properties.setProperty("db.cloud.autoReconnect", "false");
+        properties.setProperty("db.cloud.url.params", "someParams");
+        TransactionLegacy.s_dbHAEnabled = false;
+    }
+    @Test
+    public void getConnectionUriAndDriverTestWithoutUri() {
+        properties.setProperty("db.cloud.uri", "");
+        properties.setProperty("db.cloud.driver", "driver");
+
+        Pair<String, String> result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+        Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams", result.first());
+        Assert.assertEquals("driver", result.second());
+    }
+
+    @Test
+    public void getConnectionUriAndDriverTestWithUri() {
+        properties.setProperty("db.cloud.uri", "jdbc:driver:myFavoriteUri");
+
+        Pair<String, String> result = TransactionLegacy.getConnectionUriAndDriver(properties, null, false, "cloud");
+
+        Assert.assertEquals("jdbc:driver:myFavoriteUri", result.first());
+        Assert.assertEquals("jdbc:driver", result.second());
+    }
+
+    @Test
+    public void getPropertiesAndBuildConnectionUriTestDbHaDisabled() {
+        String result = TransactionLegacy.getPropertiesAndBuildConnectionUri(properties, "strat", "driver", true, "cloud");
+
+        Assert.assertEquals("driver://host:5555/name?autoReconnect=false&someParams&useSSL=true", result);
+    }
+
+    @Test
+    public void getPropertiesAndBuildConnectionUriTestDbHaEnabled() {
+        TransactionLegacy.s_dbHAEnabled = true;
+        properties.setProperty("db.cloud.failOverReadOnly", "true");
+        properties.setProperty("db.cloud.reconnectAtTxEnd", "false");
+        properties.setProperty("db.cloud.autoReconnectForPools", "true");
+        properties.setProperty("db.cloud.secondsBeforeRetrySource", "25");
+        properties.setProperty("db.cloud.queriesBeforeRetrySource", "105");
+        properties.setProperty("db.cloud.initialTimeout", "1000");
+        properties.setProperty("db.cloud.replicas", "second_host");
+
+        String result = TransactionLegacy.getPropertiesAndBuildConnectionUri(properties, "strat", "driver", true, "cloud");
+
+        Assert.assertEquals("driver://host,second_host:5555/name?autoReconnect=false&someParams&useSSL=true&failOverReadOnly=true&reconnectAtTxEnd=false&autoReconnectFor"
+                + "Pools=true&secondsBeforeRetrySource=25&queriesBeforeRetrySource=105&initialTimeout=1000&loadBalanceStrategy=strat", result);
+    }
+
+    @Test
+    public void buildConnectionUriTestDbHaDisabled() {
+        String result = TransactionLegacy.buildConnectionUri(null, "driver", false, "host", null, 5555, "cloud", false, null, null);
+
+        Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false", result);
+    }
+
+    @Test
+    public void buildConnectionUriTestDbHaEnabled() {
+        TransactionLegacy.s_dbHAEnabled = true;
+
+        String result = TransactionLegacy.buildConnectionUri("strat", "driver", false, "host", "second_host", 5555, "cloud", false, null, "dbHaParams");
+
+        Assert.assertEquals("driver://host,second_host:5555/cloud?autoReconnect=false&dbHaParams&loadBalanceStrategy=strat", result);
+    }
+
+    @Test
+    public void buildConnectionUriTestUrlParamsNotNull() {
+        String result = TransactionLegacy.buildConnectionUri(null, "driver", false, "host", null, 5555, "cloud", false, "urlParams", null);
+
+        Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&urlParams", result);
+    }
+
+    @Test
+    public void buildConnectionUriTestUseSslTrue() {
+        String result = TransactionLegacy.buildConnectionUri(null, "driver", true, "host", null, 5555, "cloud", false, null, null);
+
+        Assert.assertEquals("driver://host:5555/cloud?autoReconnect=false&useSSL=true", result);
+    }
+}