From e4b28e2e6d21d34fbbbbb130f574a8cfb597f76e Mon Sep 17 00:00:00 2001
From: Uri Yagelnik <uriy@amazon.com>
Date: Thu, 7 Nov 2024 13:18:26 +0000
Subject: [PATCH] Improvements for TLS with I/O threads

---
 src/tls.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/tls.c b/src/tls.c
index f1c82d35e4..e7adc12006 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -446,6 +446,7 @@ typedef enum {
 #define TLS_CONN_FLAG_WRITE_WANT_READ (1 << 1)
 #define TLS_CONN_FLAG_FD_SET (1 << 2)
 #define TLS_CONN_FLAG_POSTPONE_UPDATE_STATE (1 << 3)
+#define TLS_CONN_FLAG_HAS_PENDING (1 << 4)
 
 typedef struct tls_connection {
     connection c;
@@ -614,7 +615,7 @@ static void updatePendingData(tls_connection *conn) {
 
     /* If SSL has pending data, already read from the socket, we're at risk of not calling the read handler again, make
      * sure to add it to a list of pending connection that should be handled anyway. */
-    if (SSL_pending(conn->ssl) > 0) {
+    if (conn->flags & TLS_CONN_FLAG_HAS_PENDING) {
         if (!conn->pending_list_node) {
             listAddNodeTail(pending_list, conn);
             conn->pending_list_node = listLast(pending_list);
@@ -653,8 +654,6 @@ static void tlsHandleEvent(tls_connection *conn, int mask) {
     TLSCONN_DEBUG("tlsEventHandler(): fd=%d, state=%d, mask=%d, r=%d, w=%d, flags=%d", fd, conn->c.state, mask,
                   conn->c.read_handler != NULL, conn->c.write_handler != NULL, conn->flags);
 
-    ERR_clear_error();
-
     switch (conn->c.state) {
     case CONN_STATE_CONNECTING:
         conn_error = anetGetError(conn->c.fd);
@@ -662,6 +661,7 @@ static void tlsHandleEvent(tls_connection *conn, int mask) {
             conn->c.last_errno = conn_error;
             conn->c.state = CONN_STATE_ERROR;
         } else {
+            ERR_clear_error();
             if (!(conn->flags & TLS_CONN_FLAG_FD_SET)) {
                 SSL_set_fd(conn->ssl, conn->c.fd);
                 conn->flags |= TLS_CONN_FLAG_FD_SET;
@@ -690,6 +690,7 @@ static void tlsHandleEvent(tls_connection *conn, int mask) {
         conn->c.conn_handler = NULL;
         break;
     case CONN_STATE_ACCEPTING:
+        ERR_clear_error();
         ret = SSL_accept(conn->ssl);
         if (ret <= 0) {
             WantIOType want = 0;
@@ -747,10 +748,7 @@ static void tlsHandleEvent(tls_connection *conn, int mask) {
             conn->flags &= ~TLS_CONN_FLAG_READ_WANT_WRITE;
             if (!callHandler((connection *)conn, conn->c.read_handler)) return;
         }
-
-        if (mask & AE_READABLE) {
-            updatePendingData(conn);
-        }
+        updatePendingData(conn);
 
         break;
     }
@@ -941,6 +939,11 @@ static int connTLSRead(connection *conn_, void *buf, size_t buf_len) {
     if (conn->c.state != CONN_STATE_CONNECTED) return -1;
     ERR_clear_error();
     ret = SSL_read(conn->ssl, buf, buf_len);
+    if (SSL_pending(conn->ssl) > 0) {
+        conn->flags |= TLS_CONN_FLAG_HAS_PENDING;
+    } else {
+        conn->flags &= ~TLS_CONN_FLAG_HAS_PENDING;
+    }
     return updateStateAfterSSLIO(conn, ret, 1);
 }
 
@@ -992,7 +995,7 @@ static int connTLSBlockingConnect(connection *conn_, const char *addr, int port,
      * which means the specified timeout will not be enforced accurately. */
     SSL_set_fd(conn->ssl, conn->c.fd);
     setBlockingTimeout(conn, timeout);
-
+    ERR_clear_error();
     if ((ret = SSL_connect(conn->ssl)) <= 0) {
         conn->c.state = CONN_STATE_ERROR;
         return C_ERR;