CLOUDSTACK-4916: Fixing the issue with DB HA when there are more than 2 db nodes (Issue was mysql DriverManager was returning salve db nodes on random basis which was causing issue)

Signed-off-by: Abhinandan Prateek <aprateek@apache.org>

client/tomcatconf/db.properties.in

@@ -85,6 +85,7 @@ db.simulator.autoReconnect=true
 
 # High Availability And Cluster Properties
 db.ha.enabled=false
+db.ha.loadBalanceStrategy=com.cloud.utils.db.StaticStrategy
 # cloud stack Database
 db.cloud.slaves=localhost,localhost
 db.cloud.autoReconnect=true

framework/db/pom.xml

@@ -44,6 +44,11 @@
       <artifactId>cloud-utils</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>mysql</groupId>
+      <artifactId>mysql-connector-java</artifactId>
+      <scope>compile</scope>
+    </dependency>
   </dependencies>
   <build>
     <plugins>

framework/db/src/com/cloud/utils/db/StaticStrategy.java

@@ -0,0 +1,130 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.utils.db;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import com.mysql.jdbc.BalanceStrategy;
+import com.mysql.jdbc.Connection;
+import com.mysql.jdbc.ConnectionImpl;
+import com.mysql.jdbc.LoadBalancingConnectionProxy;
+import com.mysql.jdbc.SQLError;
+
+public class StaticStrategy implements BalanceStrategy {
+
+    public StaticStrategy() {
+    }
+
+    public void destroy() {
+        // we don't have anything to clean up
+    }
+
+    public void init(Connection conn, Properties props) throws SQLException {
+        // we don't have anything to initialize
+    }
+
+    public ConnectionImpl pickConnection(LoadBalancingConnectionProxy proxy,
+            List<String> configuredHosts, Map<String, ConnectionImpl> liveConnections, long[] responseTimes,
+            int numRetries) throws SQLException {
+        int numHosts = configuredHosts.size();
+
+        SQLException ex = null;
+
+        List<String> whiteList = new ArrayList<String>(numHosts);
+        whiteList.addAll(configuredHosts);
+        
+        Map<String, Long> blackList = proxy.getGlobalBlacklist();
+
+        whiteList.removeAll(blackList.keySet());
+        
+        Map<String, Integer> whiteListMap = this.getArrayIndexMap(whiteList);
+        
+
+        for (int attempts = 0; attempts < numRetries;) {
+            if(whiteList.size() == 0){
+                throw SQLError.createSQLException("No hosts configured", null);
+            }
+
+            String hostPortSpec = whiteList.get(0);     //Always take the first host
+
+            ConnectionImpl conn = liveConnections.get(hostPortSpec);
+
+            if (conn == null) {
+                try {
+                    conn = proxy.createConnectionForHost(hostPortSpec);
+                } catch (SQLException sqlEx) {
+                    ex = sqlEx;
+
+                    if (proxy.shouldExceptionTriggerFailover(sqlEx)) {
+
+                        Integer whiteListIndex = whiteListMap.get(hostPortSpec);
+
+                        // exclude this host from being picked again
+                        if (whiteListIndex != null) {
+                            whiteList.remove(whiteListIndex.intValue());
+                            whiteListMap = this.getArrayIndexMap(whiteList);
+                        }
+                        proxy.addToGlobalBlacklist( hostPortSpec );
+
+                        if (whiteList.size() == 0) {
+                            attempts++;
+                            try {
+                                Thread.sleep(250);
+                            } catch (InterruptedException e) {
+                            }
+
+                            // start fresh
+                            whiteListMap = new HashMap<String, Integer>(numHosts);
+                            whiteList.addAll(configuredHosts);
+                            blackList = proxy.getGlobalBlacklist();
+
+                            whiteList.removeAll(blackList.keySet());
+                            whiteListMap = this.getArrayIndexMap(whiteList);
+                        }
+
+                        continue;
+                    }
+
+                    throw sqlEx;
+                }
+            }
+            
+            return conn;
+        }
+
+        if (ex != null) {
+            throw ex;
+        }
+
+        return null; // we won't get here, compiler can't tell
+    }
+    
+    private Map<String, Integer> getArrayIndexMap(List<String> l) {
+        Map<String, Integer> m = new HashMap<String, Integer>(l.size());
+        for (int i = 0; i < l.size(); i++) {
+            m.put(l.get(i), Integer.valueOf(i));
+        }
+        return m;
+        
+    }
+
+}

framework/db/src/com/cloud/utils/db/TransactionLegacy.java

@@ -1034,7 +1034,7 @@ public class TransactionLegacy {
             
             s_dbHAEnabled = Boolean.valueOf(dbProps.getProperty("db.ha.enabled"));
             s_logger.info("Is Data Base High Availiability enabled? Ans : " + s_dbHAEnabled);
-            
+            String loadBalanceStrategy = dbProps.getProperty("db.ha.loadBalanceStrategy");
             // FIXME:  If params are missing...default them????
             final int cloudMaxActive = Integer.parseInt(dbProps.getProperty("db.cloud.maxActive"));
             final int cloudMaxIdle = Integer.parseInt(dbProps.getProperty("db.cloud.maxIdle"));
@@ -1090,7 +1090,7 @@ public class TransactionLegacy {
                     cloudMaxWait, cloudMaxIdle, cloudTestOnBorrow, false, cloudTimeBtwEvictionRunsMillis, 1, cloudMinEvcitableIdleTimeMillis, cloudTestWhileIdle);
 
             final ConnectionFactory cloudConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + cloudHost + (s_dbHAEnabled ? "," + cloudSlaves : "") + ":" + cloudPort + "/" + cloudDbName +
-                    "?autoReconnect=" + cloudAutoReconnect + (url != null ? "&" + url : "") + (useSSL ? "&useSSL=true" : "") + (s_dbHAEnabled ? "&" + cloudDbHAParams : ""), cloudUsername, cloudPassword);
+                    "?autoReconnect=" + cloudAutoReconnect + (url != null ? "&" + url : "") + (useSSL ? "&useSSL=true" : "") + (s_dbHAEnabled ? "&" + cloudDbHAParams : "") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : ""), cloudUsername, cloudPassword);
 
             final KeyedObjectPoolFactory poolableObjFactory = (cloudPoolPreparedStatements ? new StackKeyedObjectPoolFactory() : null);
 
@@ -1116,7 +1116,7 @@ public class TransactionLegacy {
                     usageMaxWait, usageMaxIdle);
 
             final ConnectionFactory usageConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + usageHost + (s_dbHAEnabled ? "," + dbProps.getProperty("db.cloud.slaves") : "") + ":" + usagePort + "/" + usageDbName +
-                    "?autoReconnect=" + usageAutoReconnect + (usageUrl != null ? "&" + usageUrl : "") + (s_dbHAEnabled ? "&" + getDBHAParams("usage", dbProps) : ""), usageUsername, usagePassword);
+                    "?autoReconnect=" + usageAutoReconnect + (usageUrl != null ? "&" + usageUrl : "") + (s_dbHAEnabled ? "&" + getDBHAParams("usage", dbProps) : "") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : ""), usageUsername, usagePassword);
 
             final PoolableConnectionFactory usagePoolableConnectionFactory = new PoolableConnectionFactory(usageConnectionFactory, usageConnectionPool,
                     new StackKeyedObjectPoolFactory(), null, false, false);
@@ -1129,7 +1129,7 @@ public class TransactionLegacy {
             final GenericObjectPool awsapiConnectionPool = new GenericObjectPool(null, usageMaxActive, GenericObjectPool.DEFAULT_WHEN_EXHAUSTED_ACTION,
                     usageMaxWait, usageMaxIdle);
             final ConnectionFactory awsapiConnectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://" + cloudHost + (s_dbHAEnabled ? "," + cloudSlaves : "") + ":" + cloudPort + "/" + awsapiDbName +
-                    "?autoReconnect=" + cloudAutoReconnect + (s_dbHAEnabled ? "&" + cloudDbHAParams : ""), cloudUsername, cloudPassword);
+                    "?autoReconnect=" + cloudAutoReconnect + (s_dbHAEnabled ? "&" + cloudDbHAParams : "") + (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : ""), cloudUsername, cloudPassword);
             final PoolableConnectionFactory awsapiPoolableConnectionFactory = new PoolableConnectionFactory(awsapiConnectionFactory, awsapiConnectionPool,
                     new StackKeyedObjectPoolFactory(), null, false, false);