console: optimise buffer sizes for faster console performance (#11221)

* console-proxy: fix stream buffer sizes to improve console performance

This bumps the input and output stream buffers to 64KiB and uses them
consistent across TLS and non-TLS based VNC connections.

This fixes #10650

Co-authored-by: Vishesh Jindal <vishesh.jindal@shapeblue.com>
Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

* Make buffer size configurable & other improvements for CPU & memory utilisation

* Setup batching of data for TLS connections to the VNC server

* Apply suggestions from code review

* Fix buffer size for xenserver

---------

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
Co-authored-by: Vishesh Jindal <vishesh.jindal@shapeblue.com>
Co-authored-by: vishesh92 <vishesh92@gmail.com>

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java

@@ -76,6 +76,7 @@ public class ConsoleProxy {
     static int httpCmdListenPort = 8001;
     static int reconnectMaxRetry = 5;
     static int readTimeoutSeconds = 90;
+    public static int defaultBufferSize = 64 * 1024;
     static int keyboardType = KEYBOARD_RAW;
     static String factoryClzName;
     static boolean standaloneStart = false;
@@ -160,6 +161,12 @@ public class ConsoleProxy {
             readTimeoutSeconds = Integer.parseInt(s);
             LOGGER.info("Setting readTimeoutSeconds=" + readTimeoutSeconds);
         }
+
+        s = conf.getProperty("consoleproxy.defaultBufferSize");
+        if (s != null) {
+            defaultBufferSize = Integer.parseInt(s);
+            LOGGER.info("Setting defaultBufferSize=" + defaultBufferSize);
+        }
     }
 
     public static ConsoleProxyServerFactory getHttpServerFactory() {

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java

@@ -52,6 +52,9 @@ public class ConsoleProxyNoVncClient implements ConsoleProxyClient {
     private ConsoleProxyClientParam clientParam;
     private String sessionUuid;
 
+    private ByteBuffer readBuffer = null;
+    private int flushThreshold = -1;
+
     public ConsoleProxyNoVncClient(Session session) {
         this.session = session;
     }
@@ -109,8 +112,9 @@ public class ConsoleProxyNoVncClient implements ConsoleProxyClient {
                     connectClientToVNCServer(tunnelUrl, tunnelSession, websocketUrl);
                     authenticateToVNCServer(clientSourceIp);
 
-                    int readBytes;
+                    // Track consecutive iterations with no data and sleep accordingly. Only used for NIO socket connections.
-                    byte[] b;
+                    int consecutiveZeroReads = 0;
+                    int sleepTime = 1;
                     while (connectionAlive) {
                         logger.trace("Connection with client [{}] [IP: {}] is alive.", clientId, clientSourceIp);
                         if (client.isVncOverWebSocketConnection()) {
@@ -118,30 +122,53 @@ public class ConsoleProxyNoVncClient implements ConsoleProxyClient {
                                 updateFrontEndActivityTime();
                             }
                             connectionAlive = session.isOpen();
+                            sleepTime = 1;
                         } else if (client.isVncOverNioSocket()) {
-                            byte[] bytesArr;
+                            ByteBuffer buffer = getOrCreateReadBuffer();
-                            int nextBytes = client.getNextBytes();
+                            int bytesRead = client.readAvailableDataIntoBuffer(buffer, buffer.remaining());
-                            bytesArr = new byte[nextBytes];
+
-                            client.readBytes(bytesArr, nextBytes);
+                            if (bytesRead > 0) {
-                            logger.trace("Read [{}] bytes from client [{}].", nextBytes, clientId);
-                            if (nextBytes > 0) {
-                                session.getRemote().sendBytes(ByteBuffer.wrap(bytesArr));
                                 updateFrontEndActivityTime();
+                                consecutiveZeroReads = 0; // Reset counter on successful read
+
+                                sleepTime = 0; // Still no sleep to catch any remaining data quickly
                             } else {
                                 connectionAlive = session.isOpen();
+                                consecutiveZeroReads++;
+                                // Use adaptive sleep time to prevent excessive busy waiting
+                                sleepTime = Math.min(consecutiveZeroReads, 10); // Cap at 10ms max
+                            }
+
+                            final boolean bufferHasData = buffer.position() > 0;
+                            if (bufferHasData && (bytesRead == 0 || buffer.remaining() <= flushThreshold)) {
+                                buffer.flip();
+                                logger.trace("Flushing buffer with [{}] bytes for client [{}]", buffer.remaining(), clientId);
+                                session.getRemote().sendBytes(buffer);
+                                buffer.compact();
                             }
                         } else {
-                            b = new byte[100];
+                            ByteBuffer buffer = getOrCreateReadBuffer();
-                            readBytes = client.read(b);
+                            buffer.clear();
+                            int readBytes = client.read(buffer.array());
                             logger.trace("Read [{}] bytes from client [{}].", readBytes, clientId);
-                            if (readBytes == -1 || (readBytes > 0 && !sendReadBytesToNoVNC(b, readBytes))) {
+                            if (readBytes > 0) {
+                                // Update buffer position to reflect bytes read and flip for reading
+                                buffer.position(readBytes);
+                                buffer.flip();
+                                if (!sendReadBytesToNoVNC(buffer)) {
+                                    connectionAlive = false;
+                                }
+                            } else if (readBytes == -1) {
                                 connectionAlive = false;
                             }
+                            sleepTime = 1;
                         }
-                        try {
+                        if (sleepTime > 0 && connectionAlive) {
-                            Thread.sleep(1);
+                            try {
-                        } catch (InterruptedException e) {
+                                Thread.sleep(sleepTime);
-                            logger.error("Error on sleep for vnc sessions", e);
+                            } catch (InterruptedException e) {
+                                logger.error("Error on sleep for vnc sessions", e);
+                            }
                         }
                     }
                     logger.info("Connection with client [{}] [IP: {}] is dead.", clientId, clientSourceIp);
@@ -154,9 +181,10 @@ public class ConsoleProxyNoVncClient implements ConsoleProxyClient {
         worker.start();
     }
 
-    private boolean sendReadBytesToNoVNC(byte[] b, int readBytes) {
+    private boolean sendReadBytesToNoVNC(ByteBuffer buffer) {
         try {
-            session.getRemote().sendBytes(ByteBuffer.wrap(b, 0, readBytes));
+            // Buffer is already prepared for reading by flip()
+            session.getRemote().sendBytes(buffer);
             updateFrontEndActivityTime();
         } catch (WebSocketException | IOException e) {
             logger.error("VNC server connection exception.", e);
@@ -316,9 +344,29 @@ public class ConsoleProxyNoVncClient implements ConsoleProxyClient {
         this.clientParam = param;
     }
 
+    private ByteBuffer getOrCreateReadBuffer() {
+        if (readBuffer == null) {
+            readBuffer = ByteBuffer.allocate(ConsoleProxy.defaultBufferSize);
+            logger.debug("Allocated {} KB read buffer for client [{}]", ConsoleProxy.defaultBufferSize / 1024 , clientId);
+
+            // Only apply batching logic for NIO TLS connections to work around 16KB record limitation
+            // For non-TLS or non-NIO connections, use immediate flush for better responsiveness
+            if (client != null && client.isVncOverNioSocket() && client.isTLSConnectionEstablished()) {
+                flushThreshold = Math.min(ConsoleProxy.defaultBufferSize / 4, 2048);
+                logger.debug("NIO TLS connection detected - using batching with threshold {} for client [{}]", flushThreshold, clientId);
+            } else {
+                flushThreshold = ConsoleProxy.defaultBufferSize + 1; // Always flush immediately
+                logger.debug("Non-TLS or non-NIO connection - using immediate flush for client [{}]", clientId);
+            }
+        }
+        return readBuffer;
+    }
+
     @Override
     public void closeClient() {
         this.connectionAlive = false;
+        // Clear buffer reference to allow GC when client disconnects
+        this.readBuffer = null;
         ConsoleProxy.removeViewer(this);
     }
 

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/NoVncClient.java

@@ -502,18 +502,14 @@ public class NoVncClient {
         return nioSocketConnection.readServerInit();
     }
 
-    public int getNextBytes() {
+    public int readAvailableDataIntoBuffer(ByteBuffer buffer, int maxSize) {
-        return nioSocketConnection.readNextBytes();
+        return nioSocketConnection.readAvailableDataIntoBuffer(buffer, maxSize);
     }
 
     public boolean isTLSConnectionEstablished() {
         return nioSocketConnection.isTLSConnection();
     }
 
-    public void readBytes(byte[] arr, int len) {
-        nioSocketConnection.readNextByteArray(arr, len);
-    }
-
     public void processHandshakeSecurityType(int secType, String vmPassword, String host, int port) {
         waitForNoVNCReply();
 

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocket.java

@@ -41,6 +41,8 @@ public class NioSocket {
             socketChannel = SocketChannel.open();
             socketChannel.configureBlocking(false);
             socketChannel.socket().setSoTimeout(5000);
+            socketChannel.socket().setKeepAlive(true);
+            socketChannel.socket().setTcpNoDelay(true);
             writeSelector = Selector.open();
             readSelector = Selector.open();
             socketChannel.register(writeSelector, SelectionKey.OP_WRITE);
@@ -77,7 +79,6 @@ public class NioSocket {
             socketChannel.register(selector, SelectionKey.OP_CONNECT);
 
             waitForSocketSelectorConnected(selector);
-            socketChannel.socket().setTcpNoDelay(false);
         } catch (IOException e) {
             logger.error(String.format("Error creating NioSocket to %s:%s: %s", host, port, e.getMessage()), e);
         }

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandler.java

@@ -29,8 +29,7 @@ public interface NioSocketHandler {
     void readBytes(ByteBuffer data, int length);
     String readString();
     byte[] readServerInit();
-    int readNextBytes();
+    int readAvailableDataIntoBuffer(ByteBuffer buffer, int maxSize);
-    void readNextByteArray(byte[] arr, int len);
 
     // Write operations
     void writeUnsignedInteger(int sizeInBits, int value);

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandlerImpl.java

@@ -17,6 +17,7 @@
 package com.cloud.consoleproxy.vnc.network;
 
 
+import com.cloud.consoleproxy.ConsoleProxy;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -28,13 +29,11 @@ public class NioSocketHandlerImpl implements NioSocketHandler {
     private NioSocketOutputStream outputStream;
     private boolean isTLS = false;
 
-    private static final int DEFAULT_BUF_SIZE = 16384;
-
     protected Logger logger = LogManager.getLogger(getClass());
 
     public NioSocketHandlerImpl(NioSocket socket) {
-        this.inputStream = new NioSocketInputStream(DEFAULT_BUF_SIZE, socket);
+        this.inputStream = new NioSocketInputStream(ConsoleProxy.defaultBufferSize, socket);
-        this.outputStream = new NioSocketOutputStream(DEFAULT_BUF_SIZE, socket);
+        this.outputStream = new NioSocketOutputStream(ConsoleProxy.defaultBufferSize, socket);
     }
 
     @Override
@@ -97,13 +96,8 @@ public class NioSocketHandlerImpl implements NioSocketHandler {
     }
 
     @Override
-    public int readNextBytes() {
+    public int readAvailableDataIntoBuffer(ByteBuffer buffer, int maxSize) {
-        return inputStream.getNextBytes();
+        return inputStream.readAvailableDataIntoBuffer(buffer, maxSize);
-    }
-
-    @Override
-    public void readNextByteArray(byte[] arr, int len) {
-        inputStream.readNextByteArrayFromReadBuffer(arr, len);
     }
 
     @Override

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketInputStream.java

@@ -175,28 +175,38 @@ public class NioSocketInputStream extends NioSocketStream {
         return ArrayUtils.addAll(ret, (byte) 0, (byte) 0, (byte) 0);
     }
 
-    protected int getNextBytes() {
+    /**
-        int size = 200;
+     * This method checks what data is immediately available and returns a reasonable amount.
-        while (size > 0) {
+     *
-            if (checkForSizeWithoutWait(size)) {
+     * @param maxSize Maximum number of bytes to attempt to read
-                break;
+     * @return Number of bytes available to read (0 if none available)
-            }
+     */
-            size--;
+    protected int getAvailableBytes(int maxSize) {
+        // First check if we have data already in our buffer
+        int bufferedData = endPosition - currentPosition;
+        if (bufferedData > 0) {
+            return Math.min(bufferedData, maxSize);
         }
-        return size;
+
+        // Try to read more data with non-blocking call
+        // This determines how much data is available
+        return getReadBytesAvailableToFitSize(1, maxSize, false);
     }
 
-    protected void readNextByteArrayFromReadBuffer(byte[] arr, int len) {
+    /**
-        copyBytesFromReadBuffer(len, arr);
+     * Read available data directly into a ByteBuffer.
-    }
+     *
-
+     * @param buffer ByteBuffer to read data into
-    protected void copyBytesFromReadBuffer(int length, byte[] arr) {
+     * @param maxSize Maximum number of bytes to read
-        int ptr = 0;
+     * @return Number of bytes actually read (0 if none available)
-        while (length > 0) {
+     */
-            int n = getReadBytesAvailableToFitSize(1, length, true);
+    protected int readAvailableDataIntoBuffer(ByteBuffer buffer, int maxSize) {
-            readBytes(ByteBuffer.wrap(arr, ptr, n), n);
+        // Get the amount of data available to read
-            ptr += n;
+        int available = getAvailableBytes(maxSize);
-            length -= n;
+        if (available > 0) {
+            // Read directly into the ByteBuffer
+            readBytes(buffer, available);
         }
+        return available;
     }
 }

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketSSLEngineManager.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.consoleproxy.vnc.network;
 
+import com.cloud.consoleproxy.ConsoleProxy;
+
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLException;
@@ -43,9 +45,9 @@ public class NioSocketSSLEngineManager {
 
         executor = Executors.newSingleThreadExecutor();
 
-        int pktBufSize = engine.getSession().getPacketBufferSize();
+        int networkBufSize = Math.max(engine.getSession().getPacketBufferSize(), ConsoleProxy.defaultBufferSize);
-        myNetData = ByteBuffer.allocate(pktBufSize);
+        myNetData = ByteBuffer.allocate(networkBufSize);
-        peerNetData = ByteBuffer.allocate(pktBufSize);
+        peerNetData = ByteBuffer.allocate(networkBufSize);
     }
 
     private void handshakeNeedUnwrap(ByteBuffer peerAppData) throws SSLException {
@@ -155,22 +157,25 @@ public class NioSocketSSLEngineManager {
     }
 
     public int write(ByteBuffer data) throws IOException {
-        int n = 0;
+        int totalBytesConsumed = 0;
+        int sessionAppBufSize = engine.getSession().getApplicationBufferSize();
+        boolean shouldBatch = ConsoleProxy.defaultBufferSize > sessionAppBufSize;
+
         while (data.hasRemaining()) {
             SSLEngineResult result = engine.wrap(data, myNetData);
-            n += result.bytesConsumed();
+            totalBytesConsumed += result.bytesConsumed();
             switch (result.getStatus()) {
                 case OK:
-                    myNetData.flip();
+                    // Flush immediately if: batching is disabled, small data, or last chunk
-                    outputStream.writeBytes(myNetData, myNetData.remaining());
+                    if (!shouldBatch || result.bytesConsumed() < sessionAppBufSize || !data.hasRemaining()) {
-                    outputStream.flushWriteBuffer();
+                        flush();
-                    myNetData.compact();
+                    }
+                    // Otherwise accumulate for batching (large chunk with more data coming)
                     break;
 
                 case BUFFER_OVERFLOW:
-                    myNetData.flip();
+                    // Flush when buffer is full
-                    outputStream.writeBytes(myNetData, myNetData.remaining());
+                    flush();
-                    myNetData.compact();
                     break;
 
                 case CLOSED:
@@ -181,7 +186,16 @@ public class NioSocketSSLEngineManager {
                     break;
             }
         }
-        return n;
+        return totalBytesConsumed;
+    }
+
+    public void flush() {
+        if (myNetData.position() > 0) {
+            myNetData.flip();
+            outputStream.writeBytes(myNetData, myNetData.remaining());
+            outputStream.flushWriteBuffer();
+            myNetData.compact();
+        }
     }
 
     public SSLSession getSession() {

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketTLSInputStream.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.consoleproxy.vnc.network;
 
+import com.cloud.consoleproxy.ConsoleProxy;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 import java.io.IOException;
@@ -26,7 +27,7 @@ public class NioSocketTLSInputStream extends NioSocketInputStream {
     private final NioSocketSSLEngineManager sslEngineManager;
 
     public NioSocketTLSInputStream(NioSocketSSLEngineManager sslEngineManager, NioSocket socket) {
-        super(sslEngineManager.getSession().getApplicationBufferSize(), socket);
+        super(ConsoleProxy.defaultBufferSize, socket);
         this.sslEngineManager = sslEngineManager;
     }
 

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketTLSOutputStream.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.consoleproxy.vnc.network;
 
+import com.cloud.consoleproxy.ConsoleProxy;
+
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
@@ -24,7 +26,7 @@ public class NioSocketTLSOutputStream extends NioSocketOutputStream {
     private final NioSocketSSLEngineManager sslEngineManager;
 
     public NioSocketTLSOutputStream(NioSocketSSLEngineManager sslEngineManager, NioSocket socket) {
-        super(sslEngineManager.getSession().getApplicationBufferSize(), socket);
+        super(ConsoleProxy.defaultBufferSize, socket);
         this.sslEngineManager = sslEngineManager;
     }
 
@@ -38,6 +40,7 @@ public class NioSocketTLSOutputStream extends NioSocketOutputStream {
         }
 
         currentPosition = start;
+        sslEngineManager.flush();
     }
 
     protected int writeThroughSSLEngineManager(byte[] data, int startPos, int length) {

systemvm/agent/conf/consoleproxy.properties

@@ -21,3 +21,4 @@ consoleproxy.httpCmdListenPort=8001
 consoleproxy.jarDir=./applet/
 consoleproxy.viewerLinger=180
 consoleproxy.reconnectMaxRetry=5
+consoleproxy.defaultBufferSize=65536