CLOUDSTACK-9348: Optimize NioTest and NioConnection main loop

- Reduces SSL handshake timeout to 15s, previously this was only 10s in
  commit debfcdef788ce0d51be06db0ef10f6815f9b563b
- Adds an aggresive explicit wakeup to save the Nio main IO loop/handler from
  getting blocked
- Fix NioTest to fail/succeed in about 60s, previously this was 300s
- Due to aggresive wakeup usage, NioTest should complete in less than 5s on most
  systems. On virtualized environment this may slightly increase due to thread,
  CPU burst/scheduling delays.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

utils/src/main/java/com/cloud/utils/nio/Link.java

@@ -596,8 +596,8 @@ public class Link {
         while (handshakeStatus != SSLEngineResult.HandshakeStatus.FINISHED
                 && handshakeStatus != SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING) {
             final long timeTaken = System.currentTimeMillis() - startTimeMills;
-            if (timeTaken > 60000L) {
+            if (timeTaken > 15000L) {
-                s_logger.warn("SSL Handshake has taken more than 60s to connect to: " + socketChannel.getRemoteAddress() +
+                s_logger.warn("SSL Handshake has taken more than 15s to connect to: " + socketChannel.getRemoteAddress() +
                         ". Please investigate this connection.");
                 return false;
             }

utils/src/main/java/com/cloud/utils/nio/NioConnection.java

@@ -171,6 +171,8 @@ public abstract class NioConnection implements Callable<Boolean> {
             } catch (final IOException e) {
                 s_logger.error("Agent will die due to this IOException!", e);
                 throw new NioConnectionException(e.getMessage(), e);
+            } finally {
+                _selector.wakeup();
             }
         }
         _isStartup = false;

utils/src/test/java/com/cloud/utils/testcase/NioTest.java

@@ -61,9 +61,9 @@ public class NioTest {
     private static final Logger LOGGER = Logger.getLogger(NioTest.class);
 
     // Test should fail in due time instead of looping forever
-    private static final int TESTTIMEOUT = 300000;
+    private static final int TESTTIMEOUT = 60000;
 
-    final private int totalTestCount = 5;
+    final private int totalTestCount = 4;
     private int completedTestCount = 0;
 
     private NioServer server;
@@ -71,7 +71,7 @@ public class NioTest {
     private List<NioClient> maliciousClients = new ArrayList<>();
 
     private ExecutorService clientExecutor = Executors.newFixedThreadPool(totalTestCount, new NamedThreadFactory("NioClientHandler"));;
-    private ExecutorService maliciousExecutor = Executors.newFixedThreadPool(5*totalTestCount, new NamedThreadFactory("MaliciousNioClientHandler"));;
+    private ExecutorService maliciousExecutor = Executors.newFixedThreadPool(totalTestCount, new NamedThreadFactory("MaliciousNioClientHandler"));;
 
     private Random randomGenerator = new Random();
     private byte[] testBytes;
@@ -105,12 +105,18 @@ public class NioTest {
             Assert.fail(e.getMessage());
         }
 
+        /**
+         * The malicious client(s) tries to block NioServer's main IO loop
+         * thread until SSL handshake timeout value (from Link class, 15s) after
+         * which the valid NioClient(s) get the opportunity to make connection(s)
+         */
+        for (int i = 0; i < totalTestCount; i++) {
+            final NioClient maliciousClient = new NioMaliciousClient("NioMaliciousTestClient-" + i, "127.0.0.1", server.getPort(), 1, new NioMaliciousTestClient());
+            maliciousClients.add(maliciousClient);
+            maliciousExecutor.submit(new ThreadedNioClient(maliciousClient));
+        }
+
         for (int i = 0; i < totalTestCount; i++) {
-            for (int j = 0; j < 4; j++) {
-                final NioClient maliciousClient = new NioMaliciousClient("NioMaliciousTestClient-" + i, "127.0.0.1", server.getPort(), 1, new NioMaliciousTestClient());
-                maliciousClients.add(maliciousClient);
-                maliciousExecutor.submit(new ThreadedNioClient(maliciousClient));
-            }
             final NioClient client = new NioClient("NioTestClient-" + i, "127.0.0.1", server.getPort(), 1, new NioTestClient());
             clients.add(client);
             clientExecutor.submit(new ThreadedNioClient(client));
@@ -286,7 +292,6 @@ public class NioTest {
                     LOGGER.info("Server: Received OTHER task");
                 }
             }
-
         }
     }
 }