Merge pull request #268 from cconlon/sslSessionBufferPool

Use static pool of direct ByteBuffers for WolfSSLSession read/write()

README.md

@@ -431,8 +431,15 @@ used as the default descriptor monitoring function.
 wolfJSSE allows for some customization through the `java.security` file
 and use of Security properties.
 
+#### Pre-Existing Java Security Properties
+
 Support is included for the following pre-existing Java Security properties.
 
+| System Property | Default | To Enable | Description |
+| --- | --- | --- | --- |
+| keystore.type | JKS | String | Specifies the default KeyStore type |
+| jdk.tls.disabledAlgorithms | | String | Disables algorithms, TLS protocol versions, and key lengths |
+
 **keystore.type (String)** - Specifies the default KeyStore type. This defaults
 to JKS, but could be set to something else if desired.
 
@@ -446,12 +453,60 @@ minimum RSA/ECC/DH key sizes. An example of potential use:
 jdk.tls.disabledAlgorithms=SSLv3, TLSv1.1, DH keySize < 1024, EC keySize < 224, RSA keySize < 1024
 ```
 
-The following custom wolfJSSE-specific Security property settings are supported.
+#### wolfSSL JNI/JSSE Specific Security Properties
-These can be placed into the `java.security` file and will be parsed and used
+
-by wolfJSSE.
+The following custom wolfSSL JNI/JSSE specific Security property settings are
+supported. These can be placed into the `java.security` file and will be parsed
+and used by wolfSSL JNI/JSSE.
+
+| System Property | Default | To Enable | Description |
+| --- | --- | --- | --- |
+| wolfssl.readWriteByteBufferPool.disabled | "false" | "true" | Disables the read/write ByteBuffer pool |
+| wolfssl.readWriteByteBufferPool.size | 16 | Integer | Sets the read/write per-thread ByteBuffer pool size |
+| wolfssl.readWriteByteBufferPool.bufferSize | 17408 | String | Sets the read/write per-thread ByteBuffer size |
+| wolfjsse.enabledCipherSuites | | String | Restricts enabled cipher suites |
+| wolfjsse.enabledSupportedCurves | | String | Restricts enabled ECC curves |
+| wolfjsse.enabledSignatureAlgorithms | | String | Restricts enabled signature algorithms |
+| wolfjsse.keystore.type.required | | String | Restricts KeyStore type |
+| wolfjsse.clientSessionCache.disabled | | "true" | Disables client session cache |
+
+**wolfssl.readWriteByteBufferPool.disabled (String)** - Can be used to disable
+the static per-thread ByteBuffer pool used in com.wolfssl.WolfSSLSession
+for native JNI wolfSS\_read() and wolfSSL\_write() calls. This pool is in place
+to prevent unaligned memory access at the JNI level when using byte array
+offsets. This pool is enabled by default unless explicitly disabled by setting
+this property to "true":
+
+```
+wolfssl.readWriteByteBufferPool.disabled=true
+```
+
+**wolfssl.readWriteByteBufferPool.size (Integer)** - Can be used to set the
+maximum per-thread ByteBuffer pool size. This is the maximum number of
+direct ByteBuffer objects that will be allocated and added to the pool. The
+pool starts at size 0, then grows as needed up to this maximum size. The
+default is 16. This should be set to a positive integer value:
+
+```
+wolfssl.readWriteByteBufferPool.size=16
+```
+
+**wolfssl.readWriteByteBufferPool.bufferSize (String)** - Can be used to set
+the size of each direct ByteBuffer in the static per-thread WolfSSLSession
+pool. This is set to 17k (17 * 1024) by default which allows for the maximum
+SSL/TLS record size of 2^14 (16k) plus some extra space for the record header
+overhead. This should be set to a positive integer value. This can be used
+to optimize performance if the size of data an application is reading/writing
+is known. If sized properly, fewer read/write loops will need to be done
+when calling native `wolfSSL_read()` and `wolfSSL_write()` inside
+com.wolfssl.WolfSSLSession read() and write() methods.
+
+```
+wolfssl.readWriteByteBufferPool.bufferSize=17408
+```
 
 **wolfjsse.enabledCipherSuites (String)** - Allows restriction of the enabled
-cipher suiets to those listed in this Security property. When set, applications
+cipher suites to those listed in this Security property. When set, applications
 wil not be able to override or add additional suites at runtime without
 changing this property. This should be a comma-delimited String. Example use:
 

native/com_wolfssl_WolfSSLSession.c

@@ -1111,18 +1111,113 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_connect
     return ret;
 }
 
-JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_write
+/**
-  (JNIEnv* jenv, jobject jcl, jlong sslPtr, jbyteArray raw, jint offset,
+ * Write len bytes with wolfSSL_write() from provided input data buffer.
-   jint length, jint timeout)
+ *
+ * Internal function called by WolfSSLSession.write() calls.
+ *
+ * If wolfSSL_get_fd(ssl) returns a socket descriptor, try to wait writability
+ * with select()/poll() up to provided timeout.
+ *
+ * Returns number of bytes written on success, or negative on error.
+ */
+static int SSLWriteNonblockingWithSelectPoll(WOLFSSL* ssl, byte* data,
+    int length, int timeout)
 {
-    byte* data = NULL;
+    int ret, err, sockfd;
-    int ret = SSL_FAILURE, err, sockfd;
     int pollRx = 0;
 #if !defined(WOLFJNI_USE_IO_SELECT) && !defined(USE_WINDOWS_API)
     int pollTx = 0;
 #endif
     wolfSSL_Mutex* jniSessLock = NULL;
     SSLAppData* appData = NULL;
+
+    if (ssl == NULL || data == NULL) {
+        return BAD_FUNC_ARG;
+    }
+
+    /* get session mutex from SSL app data */
+    appData = (SSLAppData*)wolfSSL_get_app_data(ssl);
+    if (appData == NULL) {
+        return WOLFSSL_FAILURE;
+    }
+
+    jniSessLock = appData->jniSessLock;
+    if (jniSessLock == NULL) {
+        return SSL_FAILURE;
+    }
+
+    do {
+        /* lock mutex around session I/O before write attempt */
+        if (wc_LockMutex(jniSessLock) != 0) {
+            ret = WOLFSSL_FAILURE;
+            break;
+        }
+
+        ret = wolfSSL_write(ssl, data, length);
+        err = wolfSSL_get_error(ssl, ret);
+
+        /* unlock mutex around session I/O after write attempt */
+        if (wc_UnLockMutex(jniSessLock) != 0) {
+            ret = WOLFSSL_FAILURE;
+            break;
+        }
+
+        if ((ret < 0) &&
+            ((err == SSL_ERROR_WANT_READ) || (err == SSL_ERROR_WANT_WRITE))) {
+
+            sockfd = wolfSSL_get_fd(ssl);
+            if (sockfd == -1) {
+                /* For I/O that does not use sockets, sockfd may be -1,
+                 * skip try to call select() */
+                break;
+            }
+
+            if (err == SSL_ERROR_WANT_READ) {
+                pollRx = 1;
+            }
+        #if !defined(WOLFJNI_USE_IO_SELECT) && !defined(USE_WINDOWS_API)
+            else if (err == SSL_ERROR_WANT_WRITE) {
+                pollTx = 1;
+            }
+        #endif
+
+        #if defined(WOLFJNI_USE_IO_SELECT) || defined(USE_WINDOWS_API)
+            ret = socketSelect(appData, sockfd, (int)timeout, pollRx, 0);
+        #else
+            ret = socketPoll(appData, sockfd, (int)timeout, pollRx,
+                pollTx, 0);
+        #endif
+            if ((ret == WOLFJNI_IO_EVENT_RECV_READY) ||
+                (ret == WOLFJNI_IO_EVENT_SEND_READY)) {
+                /* loop around and try wolfSSL_write() again */
+                continue;
+            } else if (ret == WOLFJNI_IO_EVENT_TIMEOUT ||
+                       ret == WOLFJNI_IO_EVENT_FD_CLOSED ||
+                       ret == WOLFJNI_IO_EVENT_ERROR ||
+                       ret == WOLFJNI_IO_EVENT_POLLHUP ||
+                       ret == WOLFJNI_IO_EVENT_FAIL) {
+                /* Java will throw SocketTimeoutException or
+                 * SocketException */
+                break;
+            } else {
+                /* error */
+                ret = WOLFSSL_FAILURE;
+                break;
+            }
+        }
+
+    } while (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ);
+
+    return ret;
+}
+
+JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_write__J_3BIII
+  (JNIEnv* jenv, jobject jcl, jlong sslPtr, jbyteArray raw, jint offset,
+   jint length, jint timeout)
+{
+    byte* data = NULL;
+    int ret;
     WOLFSSL* ssl = (WOLFSSL*)(uintptr_t)sslPtr;
     (void)jcl;
 
@@ -1138,85 +1233,8 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_write
             return SSL_FAILURE;
         }
 
-        /* get session mutex from SSL app data */
+        ret = SSLWriteNonblockingWithSelectPoll(ssl, data + offset,
-        appData = (SSLAppData*)wolfSSL_get_app_data(ssl);
+            (int)length, (int)timeout);
-        if (appData == NULL) {
-            (*jenv)->ReleaseByteArrayElements(jenv, raw, (jbyte*)data,
-                    JNI_ABORT);
-            return WOLFSSL_FAILURE;
-        }
-
-        jniSessLock = appData->jniSessLock;
-        if (jniSessLock == NULL) {
-            (*jenv)->ReleaseByteArrayElements(jenv, raw, (jbyte*)data,
-                    JNI_ABORT);
-            return SSL_FAILURE;
-        }
-
-        do {
-
-            /* lock mutex around session I/O before write attempt */
-            if (wc_LockMutex(jniSessLock) != 0) {
-                ret = WOLFSSL_FAILURE;
-                break;
-            }
-
-            ret = wolfSSL_write(ssl, data + offset, length);
-            err = wolfSSL_get_error(ssl, ret);
-
-            /* unlock mutex around session I/O after write attempt */
-            if (wc_UnLockMutex(jniSessLock) != 0) {
-                ret = WOLFSSL_FAILURE;
-                break;
-            }
-
-            if (ret >= 0) /* return if it is success */
-                break;
-
-            if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) {
-
-                sockfd = wolfSSL_get_fd(ssl);
-                if (sockfd == -1) {
-                    /* For I/O that does not use sockets, sockfd may be -1,
-                     * skip try to call select() */
-                    break;
-                }
-
-                if (err == SSL_ERROR_WANT_READ) {
-                    pollRx = 1;
-                }
-            #if !defined(WOLFJNI_USE_IO_SELECT) && !defined(USE_WINDOWS_API)
-                else if (err == SSL_ERROR_WANT_WRITE) {
-                    pollTx = 1;
-                }
-            #endif
-
-            #if defined(WOLFJNI_USE_IO_SELECT) || defined(USE_WINDOWS_API)
-                ret = socketSelect(appData, sockfd, (int)timeout, pollRx, 0);
-            #else
-                ret = socketPoll(appData, sockfd, (int)timeout, pollRx,
-                    pollTx, 0);
-            #endif
-                if ((ret == WOLFJNI_IO_EVENT_RECV_READY) ||
-                    (ret == WOLFJNI_IO_EVENT_SEND_READY)) {
-                    /* loop around and try wolfSSL_write() again */
-                    continue;
-                } else if (ret == WOLFJNI_IO_EVENT_TIMEOUT ||
-                           ret == WOLFJNI_IO_EVENT_FD_CLOSED ||
-                           ret == WOLFJNI_IO_EVENT_ERROR ||
-                           ret == WOLFJNI_IO_EVENT_POLLHUP ||
-                           ret == WOLFJNI_IO_EVENT_FAIL) {
-                    /* Java will throw SocketTimeoutException or
-                     * SocketException */
-                    break;
-                } else {
-                    /* error */
-                    ret = WOLFSSL_FAILURE;
-                    break;
-                }
-            }
-
-        } while (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ);
 
         (*jenv)->ReleaseByteArrayElements(jenv, raw, (jbyte*)data, JNI_ABORT);
 
@@ -1227,6 +1245,86 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_write
     }
 }
 
+JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_write__JLjava_nio_ByteBuffer_2IIZII
+  (JNIEnv* jenv, jobject jcl, jlong sslPtr, jobject buf, jint position,
+   jint limit, jboolean hasArray, jint length, jint timeout)
+{
+    int ret;
+    int maxInputSz;
+    int inSz = length;
+    byte* data = NULL;
+    WOLFSSL* ssl = (WOLFSSL*)(uintptr_t)sslPtr;
+    jbyteArray bufArr = NULL;
+
+    (void)jcl;
+
+    if (jenv == NULL || ssl == NULL || buf == NULL) {
+        return BAD_FUNC_ARG;
+    }
+
+    if (length > 0) {
+
+        /* Only write up to maximum space we have in this ByteBuffer */
+        maxInputSz = (limit - position);
+        if (inSz > maxInputSz) {
+            inSz = maxInputSz;
+        }
+
+        if (inSz <= 0) {
+            return BAD_FUNC_ARG;
+        }
+
+        if (hasArray) {
+            /* Get reference to underlying byte[] from ByteBuffer */
+            bufArr = (jbyteArray)(*jenv)->CallObjectMethod(jenv, buf,
+                g_bufferArrayMethodId);
+            if ((*jenv)->ExceptionCheck(jenv)) {
+                return SSL_FAILURE;
+            }
+
+            /* Get array elements */
+            data = (byte *)(*jenv)->GetByteArrayElements(jenv, bufArr, NULL);
+            if (data == NULL) {
+                /* Handle any pending exception, we'll throw another below
+                 * anyways so just clear it */
+                if ((*jenv)->ExceptionOccurred(jenv)) {
+                    (*jenv)->ExceptionDescribe(jenv);
+                    (*jenv)->ExceptionClear(jenv);
+                }
+                throwWolfSSLJNIException(jenv,
+                    "Failed to get byte[] from ByteBuffer in native write()");
+                return BAD_FUNC_ARG;
+            }
+        }
+        else {
+            data = (byte *)(*jenv)->GetDirectBufferAddress(jenv, buf);
+            if (data == NULL) {
+                throwWolfSSLJNIException(jenv,
+                    "Failed to get DirectBuffer address in native write()");
+                return BAD_FUNC_ARG;
+            }
+        }
+
+        ret = SSLWriteNonblockingWithSelectPoll(ssl, data + position,
+            (int)inSz, (int)timeout);
+
+        /* release memory if using array mode */
+        if (hasArray) {
+            (*jenv)->ReleaseByteArrayElements(jenv, bufArr,
+                (jbyte*)data, JNI_ABORT);
+        }
+    }
+
+    /* check for Java exceptions before returning */
+    if ((*jenv)->ExceptionCheck(jenv)) {
+        (*jenv)->ExceptionDescribe(jenv);
+        (*jenv)->ExceptionClear(jenv);
+        return SSL_FAILURE;
+    }
+
+    return ret;
+}
+
 /**
  * Read len bytes from wolfSSL_read() back into provided output buffer.
  *

native/com_wolfssl_WolfSSLSession.h

@@ -92,9 +92,17 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_connect
  * Method:    write
  * Signature: (J[BIII)I
  */
-JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_write
+JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_write__J_3BIII
   (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jint);
 
+/*
+ * Class:     com_wolfssl_WolfSSLSession
+ * Method:    write
+ * Signature: (JLjava/nio/ByteBuffer;IIZII)I
+ */
+JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_write__JLjava_nio_ByteBuffer_2IIZII
+  (JNIEnv *, jobject, jlong, jobject, jint, jint, jboolean, jint, jint);
+
 /*
  * Class:     com_wolfssl_WolfSSLSession
  * Method:    read

src/java/com/wolfssl/WolfSSLSession.java

@@ -30,6 +30,8 @@ import java.net.SocketTimeoutException;
 import java.lang.StringBuilder;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.security.Security;
 
 /**
  * Wraps a native WolfSSL session object and contains methods directly related
@@ -108,6 +110,198 @@ public class WolfSSLSession {
     /* lock around native WOLFSSL pointer use */
     private final Object sslLock = new Object();
 
+    /* Is static direct ByteBuffer pool enabled for read/write() calls */
+    private boolean byteBufferPoolEnabled = true;
+
+    /* Maximum direct ByteBuffer pool size */
+    private static int MAX_POOL_SIZE = 16;
+
+    /* Size of each direct ByteBuffer in the pool. This is set to 17KB, which
+     * is slightly larger than the maximum SSL record size (16KB). This
+     * allows for some overhead (SSL record header, etc) */
+    private static int BUFFER_SIZE = 17 * 1024;
+
+    /* Thread-local direct ByteBuffer pool for optimized JNI direct memory
+     * access. Passing byte[] and offset down to JNI, on some systems this
+     * will cause unaligned memory access, with pointer addition
+     * (buffer + offset). Unaligned memory access can be considerably slower
+     * (ex: Aarch64). To avoid this, we use a thread-local pool of ByteBuffers
+     * here so native JNI does not do unaligned memory access and to eliminate
+     * cross-thread contention. */
+    private static final ThreadLocal<ConcurrentLinkedQueue<ByteBuffer>> directBufferPool =
+        ThreadLocal.withInitial(() -> new ConcurrentLinkedQueue<>());
+
+    /**
+     * Check if static direct ByteBuffer pool has been disabled for
+     * use in read/write() methods.
+     *
+     * The pool is enabled by default, unless explicitly disabled by setting
+     * the "wolfssl.readWriteByteBufferPool.disabled" property to "true".
+     *
+     * @return true if disabled, otherwise false
+     */
+    private boolean readWritePoolDisabled() {
+
+        String disabled =
+            Security.getProperty("wolfssl.readWriteByteBufferPool.disabled");
+
+        if (disabled == null || disabled.isEmpty()) {
+            return false;
+        }
+
+        if (disabled.equalsIgnoreCase("true")) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Check if the maximum size of the static per-thread direct
+     * ByteBuffer pool has been adjusted by setting of the
+     * "wolfssl.readWriteByteBufferPool.size" Security property.
+     *
+     * @return the size set, or the current default
+     * (MAX_POOL_SIZE) if not.
+     */
+    private int readWritePoolGetMaxSizeFromProperty() {
+
+        int maxSize = MAX_POOL_SIZE;
+
+        String sizeProp =
+            Security.getProperty("wolfssl.readWriteByteBufferPool.size");
+
+        if (sizeProp == null || sizeProp.isEmpty()) {
+            return maxSize;
+        }
+
+        try {
+            int size = Integer.parseInt(sizeProp);
+            if (size > 0) {
+                maxSize = size;
+            }
+        } catch (NumberFormatException e) {
+            WolfSSLDebug.log(getClass(),
+                WolfSSLDebug.Component.JNI, WolfSSLDebug.ERROR, 0,
+                () -> "Invalid value for " +
+                "wolfssl.readWriteByteBufferPool.size: " + sizeProp);
+        }
+
+        return maxSize;
+    }
+
+    /**
+     * Check if the size of the ByteBuffers in the static per-thread
+     * pool has been adjusted by setting of the
+     * "wolfssl.readWriteByteBufferPool.bufferSize" Security property.
+     *
+     * @return the size set, or the current default (BUFFER_SIZE) if not.
+     */
+    private int readWritePoolGetBufferSizeFromProperty() {
+
+        int bufferSize = BUFFER_SIZE;
+
+        String sizeProp =
+            Security.getProperty("wolfssl.readWriteByteBufferPool.bufferSize");
+
+        if (sizeProp == null || sizeProp.isEmpty()) {
+            return bufferSize;
+        }
+
+        try {
+            int size = Integer.parseInt(sizeProp);
+            if (size > 0) {
+                bufferSize = size;
+            }
+        } catch (NumberFormatException e) {
+            WolfSSLDebug.log(getClass(),
+                WolfSSLDebug.Component.JNI, WolfSSLDebug.ERROR, 0,
+                () -> "Invalid value for " +
+                "wolfssl.readWriteByteBufferPool.bufferSize: " + sizeProp);
+        }
+
+        return bufferSize;
+    }
+
+    /**
+     * Read current values of relevant Security properties and set
+     * internal behavior.
+     */
+    private void detectSecurityPropertySettings() {
+
+        /* Re-use sslLock for synchronization here */
+        synchronized (sslLock) {
+            /* Check if static direct ByteBuffer pool has been disabled
+             * with the "wolfssl.readWriteByteBufferPool.disabled"
+             * Security property. */
+            if (readWritePoolDisabled()) {
+                this.byteBufferPoolEnabled = false;
+            }
+
+            /* Check if the maximum size of the static per-thread direct
+             * ByteBuffer pool has been adjusted by setting of the
+             * "wolfssl.readWriteByteBufferPool.size" Security property. */
+            WolfSSLSession.MAX_POOL_SIZE =
+                readWritePoolGetMaxSizeFromProperty();
+
+            /* Check if the size of the ByteBuffers in the static per-thread
+             * pool has been adjusted by setting of the
+             * "wolfssl.readWriteByteBufferPool.bufferSize" Security property. */
+            WolfSSLSession.BUFFER_SIZE =
+                readWritePoolGetBufferSizeFromProperty();
+        }
+    }
+
+    /**
+     * Get a DirectByteBuffer from the thread-local pool or allocate a new one
+     * if the pool is empty.
+     *
+     * @return a direct ByteBuffer ready to use
+     */
+    private static synchronized ByteBuffer acquireDirectBuffer() {
+        ConcurrentLinkedQueue<ByteBuffer> threadPool = directBufferPool.get();
+        ByteBuffer buffer = threadPool.poll();
+        if (buffer == null) {
+            WolfSSLDebug.log(WolfSSLSession.class, WolfSSLDebug.Component.JNI,
+                WolfSSLDebug.INFO, 0,
+                () -> "Thread-local DirectByteBuffer pool empty, " +
+                "allocating new buffer");
+            buffer = ByteBuffer.allocateDirect(BUFFER_SIZE);
+        } else {
+            WolfSSLDebug.log(WolfSSLSession.class, WolfSSLDebug.Component.JNI,
+                WolfSSLDebug.INFO, 0,
+                () -> "Reusing DirectByteBuffer from thread-local pool, " +
+                "pool size: " + threadPool.size());
+            buffer.clear();
+        }
+        return buffer;
+    }
+
+    /**
+     * Return a DirectByteBuffer to the thread-local pool for reuse.
+     *
+     * If the pool is full, the ByteBuffer will be garbage collected.
+     *
+     * @param buffer the buffer to return to the pool
+     */
+    private static synchronized void releaseDirectBuffer(ByteBuffer buffer) {
+
+        if (buffer != null && buffer.isDirect()) {
+
+            buffer.clear();
+            ConcurrentLinkedQueue<ByteBuffer> threadPool =
+                directBufferPool.get();
+
+            if (threadPool.size() < MAX_POOL_SIZE) {
+                WolfSSLDebug.log(WolfSSLSession.class,
+                    WolfSSLDebug.Component.JNI, WolfSSLDebug.INFO, 0,
+                    () -> "Returning DirectByteBuffer to thread-local pool, " +
+                    "pool size: " + threadPool.size());
+                threadPool.offer(buffer);
+            }
+        }
+    }
+
     /* SNI requested by this WolfSSLSession if client side and useSNI()
      * was called successfully. */
     private byte[] clientSNIRequested = null;
@@ -138,6 +332,8 @@ public class WolfSSLSession {
             WolfSSLDebug.INFO, sslPtr,
             () -> "creating new WolfSSLSession (with I/O pipe)");
 
+        detectSecurityPropertySettings();
+
         synchronized (stateLock) {
             this.active = true;
         }
@@ -185,6 +381,8 @@ public class WolfSSLSession {
                 () -> "creating new WolfSSLSession (without I/O pipe)");
         }
 
+        detectSecurityPropertySettings();
+
         synchronized (stateLock) {
             this.active = true;
         }
@@ -377,6 +575,9 @@ public class WolfSSLSession {
     private native int connect(long ssl, int timeout);
     private native int write(long ssl, byte[] data, int offset, int length,
         int timeout);
+    private native int write(long ssl, ByteBuffer data, final int position,
+        final int limit, boolean hasArray, int sz, int timeout)
+        throws WolfSSLException;
     private native int read(long ssl, byte[] data, int offset, int sz,
         int timeout);
     private native int read(long ssl, ByteBuffer data, final int position,
@@ -931,37 +1132,7 @@ public class WolfSSLSession {
     public int write(byte[] data, int length)
         throws IllegalStateException, SocketTimeoutException, SocketException {
 
-        final int ret;
+        return write(data, 0, length, 0);
-        final int err;
-        long localPtr;
-
-        confirmObjectIsActive();
-
-        /* Fix for Infer scan, since not synchronizing on sslLock for
-         * access to this.sslPtr, see note below */
-        synchronized (sslLock) {
-            localPtr = this.sslPtr;
-        }
-
-        WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
-            WolfSSLDebug.INFO, localPtr,
-            () -> "entered write(length: " + length + ")");
-
-        /* not synchronizing on sslLock here since JNI write() locks
-         * session mutex around native wolfSSL_write() call. If sslLock
-         * is locked here, since we call select() inside native JNI we
-         * could timeout waiting for corresponding read() operation to
-         * occur if needed */
-        ret = write(localPtr, data, 0, length, 0);
-        err = getError(ret);
-
-        WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
-            WolfSSLDebug.INFO, localPtr,
-            () -> "write() ret: " + ret + ", err: " + err);
-
-        throwExceptionFromIOReturnValue(ret, "wolfSSL_write()");
-
-        return ret;
     }
 
     /**
@@ -1044,9 +1215,12 @@ public class WolfSSLSession {
     public int write(byte[] data, int offset, int length, int timeout)
         throws IllegalStateException, SocketTimeoutException, SocketException {
 
-        final int ret;
+        int ret = 0;
-        final int err;
+        int err = 0;
+        int totalWritten = 0;
+        int remaining = length;
         long localPtr;
+        ByteBuffer directBuffer = null;
 
         confirmObjectIsActive();
 
@@ -1061,21 +1235,82 @@ public class WolfSSLSession {
             () -> "entered write(offset: " + offset + ", length: " +
             length + ", timeout: " + timeout + ")");
 
-        /* not synchronizing on sslLock here since JNI write() locks
+        /* Use a direct ByteBuffer from the pool to avoid unaligned
-         * session mutex around native wolfSSL_write() call. If sslLock
+         * memory access. Otherwise our native JNI code may need to do
-         * is locked here, since we call select() inside native JNI we
+         * "buffer + offset" and end up with unaligned memory which
-         * could timeout waiting for corresponding read() operation to
+         * can be slow on some targets (ex: ARM/Aarch64) */
-         * occur if needed */
+        try {
-        ret = write(localPtr, data, offset, length, timeout);
+            if (!this.byteBufferPoolEnabled) {
-        err = getError(ret);
+                /* Throw exception so we fall back to using byte[] in the
+                 * catch block below */
+                throw new Exception("ByteBuffer pool not enabled");
+            }
+
+            /* Get a buffer from the pool */
+            directBuffer = acquireDirectBuffer();
+
+            WolfSSLDebug.log(getClass(),
+                WolfSSLDebug.Component.JNI, WolfSSLDebug.INFO, localPtr,
+                () -> "write() using thread-local ByteBuffer pool: pool size: " +
+                directBufferPool.get().size());
+
+            /* Write in chunks until all data is written or an error occurs.
+             * The DirectByteBuffer size might be smaller than the data length,
+             * so we need to loop to handle all the data */
+            while (remaining > 0) {
+                /* Calculate size for current chunk */
+                int writeSize = Math.min(remaining, directBuffer.capacity());
+
+                /* Copy data from user array to direct buffer */
+                directBuffer.clear();
+                directBuffer.put(data, offset + totalWritten, writeSize);
+                directBuffer.flip();
+
+                /* Call native write with DirectByteBuffer */
+                ret = write(localPtr, directBuffer, directBuffer.position(),
+                    directBuffer.limit(), false, writeSize, timeout);
+
+                if (ret <= 0) {
+                    /* Error occurred, break out of loop */
+                    err = getError(ret);
+                    break;
+                }
+
+                /* Update tracking variables */
+                totalWritten += ret;
+                remaining -= ret;
+            }
+
+        } catch (Exception e) {
+
+            WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
+                WolfSSLDebug.ERROR, localPtr,
+                () -> "write() falling back to use byte[]");
+
+            /* Fall back to original implementation on exception (write not
+             * done yet at this point in JNI call above) */
+            totalWritten = write(localPtr, data, offset, length, timeout);
+
+        } finally {
+            /* Return buffer to pool */
+            if (directBuffer != null) {
+                releaseDirectBuffer(directBuffer);
+            }
+        }
+
+        /* Return total bytes written, or last error code */
+        final int finalRet = (totalWritten > 0) ? totalWritten : ret;
+        final int finalErr = err;
+        final int finalTotal = totalWritten;
 
         WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
             WolfSSLDebug.INFO, localPtr,
-            () -> "write() ret: " + ret + ", err: " + err);
+            () -> "write() ret: " + finalRet + ", err: " + finalErr +
+                  ", totalWritten: " + finalTotal);
 
-        throwExceptionFromIOReturnValue(ret, "wolfSSL_write()");
+        throwExceptionFromIOReturnValue(finalRet, "wolfSSL_write()");
 
-        return ret;
+        return finalRet;
     }
 
     /**
@@ -1118,38 +1353,7 @@ public class WolfSSLSession {
     public int read(byte[] data, int sz)
         throws IllegalStateException, SocketTimeoutException, SocketException {
 
-        final int ret;
+        return read(data, 0, sz, 0);
-        final int err;
-        final int readSz = sz;
-        long localPtr;
-
-        confirmObjectIsActive();
-
-        /* Fix for Infer scan, since not synchronizing on sslLock for
-         * access to this.sslPtr, see note below */
-        synchronized (sslLock) {
-            localPtr = this.sslPtr;
-        }
-
-        WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
-            WolfSSLDebug.INFO, localPtr, () -> "entered read(sz: " +
-            readSz + ")");
-
-        /* not synchronizing on sslLock here since JNI read() locks
-         * session mutex around native wolfSSL_read() call. If sslLock
-         * is locked here, since we call select() inside native JNI we
-         * could timeout waiting for corresponding write() operation to
-         * occur if needed */
-        ret = read(localPtr, data, 0, readSz, 0);
-        err = getError(ret);
-
-        WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
-            WolfSSLDebug.INFO, localPtr,
-            () -> "read() ret: " + ret + ", err: " + err);
-
-        throwExceptionFromIOReturnValue(ret, "wolfSSL_read()");
-
-        return ret;
     }
 
     /**
@@ -1236,12 +1440,14 @@ public class WolfSSLSession {
     public int read(byte[] data, int offset, int sz, int timeout)
         throws IllegalStateException, SocketTimeoutException, SocketException {
 
-        final int ret;
+        int ret;
-        final int err;
+        int err;
+        int readSz = sz;
         final int readOff = offset;
-        final int readSz = sz;
+        final int tmpReadSz = sz;
         final int readTimeout = timeout;
         long localPtr;
+        ByteBuffer directBuffer = null;
 
         confirmObjectIsActive();
 
@@ -1253,20 +1459,70 @@ public class WolfSSLSession {
 
         WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
             WolfSSLDebug.INFO, localPtr,
-            () -> "entered read(offset: " + readOff + ", sz: " + readSz +
+            () -> "entered read(offset: " + readOff + ", sz: " + tmpReadSz +
             ", timeout: " + readTimeout + ")");
 
-        /* not synchronizing on sslLock here since JNI read() locks
+        /* Use a DirectByteBuffer from the pool to avoid unaligned
-         * session mutex around native wolfSSL_read() call. If sslLock
+         * memory access. Otherwise our native JNI code may need to
-         * is locked here, since we call select() inside native JNI we
+         * do "buffer + offset" and end up with unaligned memory which
-         * could timeout waiting for corresponding write() operation to
+         * can be slow on some targets (ex: ARM/Aarch64) */
-         * occur if needed */
+        try {
-        ret = read(localPtr, data, readOff, readSz, readTimeout);
+            if (!this.byteBufferPoolEnabled) {
-        err = getError(ret);
+                /* Throw exception so we fall back to using byte[] in the
+                 * catch block below */
+                throw new Exception("ByteBuffer pool not enabled");
+            }
 
+            /* Get a buffer from the pool */
+            directBuffer = acquireDirectBuffer();
+
+            WolfSSLDebug.log(getClass(),
+                WolfSSLDebug.Component.JNI, WolfSSLDebug.INFO, localPtr,
+                () -> "read() using thread-local ByteBuffer pool: pool size: " +
+                directBufferPool.get().size());
+
+            /* Only read up to the size of the buffer or readSz,
+             * whichever is smaller. */
+            readSz = Math.min(readSz, directBuffer.capacity());
+
+            /* Use direct buffer for JNI call */
+            directBuffer.limit(readSz);
+
+            /* Call native read with DirectByteBuffer */
+            ret = read(localPtr, directBuffer, 0, readSz, false,
+                readSz, readTimeout);
+
+            if (ret > 0) {
+                /* Copy data from direct buffer to user array */
+                directBuffer.flip();
+                directBuffer.get(data, offset, ret);
+            }
+
+            err = getError(ret);
+
+        } catch (Exception e) {
+
+            WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
+                WolfSSLDebug.INFO, localPtr,
+                () -> "read() falling back to use byte[]");
+
+            /* Fall back to original implementation on errors */
+            ret = read(localPtr, data, readOff, readSz, readTimeout);
+            err = getError(ret);
+
+        } finally {
+
+            /* Return buffer to pool */
+            if (directBuffer != null) {
+                releaseDirectBuffer(directBuffer);
+            }
+        }
+
+        final int finalRet = ret;
+        final int finalErr = err;
         WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
             WolfSSLDebug.INFO, localPtr,
-            () -> "read() ret: " + ret + ", err: " + err);
+            () -> "read() ret: " + finalRet + ", err: " + finalErr);
 
         throwExceptionFromIOReturnValue(ret, "wolfSSL_read()");
 

src/java/com/wolfssl/provider/jsse/WolfSSLSocket.java

@@ -2895,8 +2895,9 @@ public class WolfSSLSocket extends SSLSocket {
                 if (ret < 0) {
                     /* print error description string */
                     String errStr = WolfSSL.getErrorString(err);
-                    throw new IOException("Native wolfSSL_write() error: "
+                    throw new IOException("Native wolfSSL_write() error: " +
-                            + errStr + " (error code: " + err + ")");
+                        errStr + " (ret: " + ret + ", error code: " +
+                        err + ")");
                 }
 
             } catch (IllegalStateException e) {

src/test/com/wolfssl/test/WolfSSLSessionTest.java

@@ -40,6 +40,7 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.CountDownLatch;
 import java.nio.ByteBuffer;
+import java.security.Security;
 
 import com.wolfssl.WolfSSL;
 import com.wolfssl.WolfSSLDebug;
@@ -70,6 +71,10 @@ public class WolfSSLSessionTest {
 
     private static WolfSSLContext ctx = null;
 
+    /* Lock around WolfSSLSession static per-thread ByteBuffer pool
+     * Security property use in this test class */
+    private static final Object byteBufferPoolPropertyLock = new Object();
+
     @BeforeClass
     public static void loadLibrary()
         throws WolfSSLException{
@@ -1412,6 +1417,356 @@ public class WolfSSLSessionTest {
         System.out.println("\t... passed");
     }
 
+    /**
+     * Internal method that connects a client to a server and does
+     * one resumption.
+     *
+     * @throws Exception on error
+     */
+    private void runClientServerOneResumption() throws Exception {
+
+        int ret = 0;
+        int err = 0;
+        long sessionPtr = 0;
+        long sesDup = 0;
+        Socket cliSock = null;
+        WolfSSLSession cliSes = null;
+
+        /* Create client/server WolfSSLContext objects, Server context
+         * must be final since used inside inner class. */
+        final WolfSSLContext srvCtx;
+        WolfSSLContext cliCtx;
+
+        /* Create ServerSocket first to get ephemeral port */
+        final ServerSocket srvSocket = new ServerSocket(0);
+
+        srvCtx = createAndSetupWolfSSLContext(srvCert, srvKey,
+            WolfSSL.SSL_FILETYPE_PEM, cliCert,
+            WolfSSL.SSLv23_ServerMethod());
+        cliCtx = createAndSetupWolfSSLContext(cliCert, cliKey,
+            WolfSSL.SSL_FILETYPE_PEM, caCert,
+            WolfSSL.SSLv23_ClientMethod());
+
+        /* Start server, handles 1 resumption */
+        try {
+            ExecutorService es = Executors.newSingleThreadExecutor();
+            es.submit(new Callable<Void>() {
+                @Override
+                public Void call() throws Exception {
+                    int ret;
+                    int err;
+                    Socket server = null;
+                    WolfSSLSession srvSes = null;
+
+                    try {
+                        /* Loop twice to allow handle one resumption */
+                        for (int i = 0; i < 2; i++) {
+                            server = srvSocket.accept();
+                            srvSes = new WolfSSLSession(srvCtx);
+
+                            ret = srvSes.setFd(server);
+                            if (ret != WolfSSL.SSL_SUCCESS) {
+                                throw new Exception(
+                                    "WolfSSLSession.setFd() failed: " +
+                                    ret);
+                            }
+
+                            do {
+                                ret = srvSes.accept();
+                                err = srvSes.getError(ret);
+                            } while (ret != WolfSSL.SSL_SUCCESS &&
+                                     (err == WolfSSL.SSL_ERROR_WANT_READ ||
+                                      err == WolfSSL.SSL_ERROR_WANT_WRITE));
+
+                            if (ret != WolfSSL.SSL_SUCCESS) {
+                                throw new Exception(
+                                    "WolfSSLSession.accept() failed: " +
+                                    ret);
+                            }
+
+                            srvSes.shutdownSSL();
+                            srvSes.freeSSL();
+                            srvSes = null;
+                        }
+
+                    } finally {
+                        if (srvSes != null) {
+                            srvSes.freeSSL();
+                        }
+                        if (server != null) {
+                            server.close();
+                        }
+                    }
+
+                    return null;
+                }
+            });
+
+        } catch (Exception e) {
+            System.out.println("\t... failed");
+            e.printStackTrace();
+            fail();
+        }
+
+        try {
+            /* ------------------------------------------------------ */
+            /* Client connection #1 */
+            /* ------------------------------------------------------ */
+            cliSock = new Socket(InetAddress.getLocalHost(),
+                srvSocket.getLocalPort());
+
+            cliSes = new WolfSSLSession(cliCtx);
+
+            ret = cliSes.setFd(cliSock);
+            if (ret != WolfSSL.SSL_SUCCESS) {
+                throw new Exception(
+                    "WolfSSLSession.setFd() failed, ret = " + ret);
+            }
+
+            do {
+                ret = cliSes.connect();
+                err = cliSes.getError(ret);
+            } while (ret != WolfSSL.SSL_SUCCESS &&
+                   (err == WolfSSL.SSL_ERROR_WANT_READ ||
+                    err == WolfSSL.SSL_ERROR_WANT_WRITE));
+
+            if (ret != WolfSSL.SSL_SUCCESS) {
+                throw new Exception(
+                    "WolfSSLSession.connect() failed: " + err);
+            }
+
+            /* Get WOLFSSL_SESSION pointer */
+            sessionPtr = cliSes.getSession();
+            if (sessionPtr == 0) {
+                throw new Exception(
+                    "WolfSSLSession.getSession() failed, ptr == 0");
+            }
+
+            /* wolfSSL_SessionIsSetup() may not be available, don't
+             * treat NOT_COMPILED_IN as an error */
+            ret = WolfSSLSession.sessionIsSetup(sessionPtr);
+            if ((ret != 1) && (ret != WolfSSL.NOT_COMPILED_IN)) {
+                throw new Exception(
+                    "WolfSSLSession.sessionIsSetup() did not " +
+                    "return 1: " + ret);
+            }
+
+            /* Test duplicateSession(), wraps wolfSSL_SESSION_dup() */
+            sesDup = WolfSSLSession.duplicateSession(sessionPtr);
+            if (sesDup == 0) {
+                throw new Exception(
+                    "WolfSSLSession.duplicateSession() returned 0");
+            }
+            if (sesDup == sessionPtr) {
+                throw new Exception(
+                    "WolfSSLSession.duplicateSession() returned " +
+                    "same pointer");
+            }
+            WolfSSLSession.freeSession(sesDup);
+            sesDup = 0;
+
+            cliSes.shutdownSSL();
+            cliSes.freeSSL();
+            cliSes = null;
+            cliSock.close();
+            cliSock = null;
+
+            /* ------------------------------------------------------ */
+            /* Client connection #2, set session and try resumption */
+            /* ------------------------------------------------------ */
+            cliSock = new Socket(InetAddress.getLocalHost(),
+                srvSocket.getLocalPort());
+            cliSes = new WolfSSLSession(cliCtx);
+
+            ret = cliSes.setFd(cliSock);
+            if (ret != WolfSSL.SSL_SUCCESS) {
+                throw new Exception(
+                    "WolfSSLSession.setFd() failed, ret = " + ret);
+            }
+
+            /* Set session pointer from original connection */
+            ret = cliSes.setSession(sessionPtr);
+            if (ret != WolfSSL.SSL_SUCCESS) {
+                throw new Exception(
+                    "WolfSSLSession.setSession() failed: " + ret);
+            }
+
+            do {
+                ret = cliSes.connect();
+                err = cliSes.getError(ret);
+            } while (ret != WolfSSL.SSL_SUCCESS &&
+                   (err == WolfSSL.SSL_ERROR_WANT_READ ||
+                    err == WolfSSL.SSL_ERROR_WANT_WRITE));
+
+            if (ret != WolfSSL.SSL_SUCCESS) {
+                throw new Exception(
+                    "WolfSSLSession.connect() failed: " + err);
+            }
+
+            /* Get WOLFSSL_SESSION pointer, free original one first */
+            WolfSSLSession.freeSession(sessionPtr);
+            sessionPtr = cliSes.getSession();
+            if (sessionPtr == 0) {
+                throw new Exception(
+                    "WolfSSLSession.getSession() failed, ptr == 0");
+            }
+
+            /* Free WOLFSSL_SESSION pointer */
+            WolfSSLSession.freeSession(sessionPtr);
+            sessionPtr = 0;
+
+            /* Session should be marked as resumed */
+            if (cliSes.sessionReused() == 0) {
+                throw new Exception(
+                    "Second connection not resumed");
+            }
+
+            cliSes.shutdownSSL();
+            cliSes.freeSSL();
+            cliSes = null;
+            cliSock.close();
+            cliSock = null;
+
+        } catch (Exception e) {
+            System.out.println("\t... failed");
+            e.printStackTrace();
+            fail();
+
+        } finally {
+            /* Free resources */
+            if (sessionPtr != 0) {
+                WolfSSLSession.freeSession(sessionPtr);
+            }
+            if (sesDup != 0) {
+                WolfSSLSession.freeSession(sesDup);
+            }
+            if (cliSes != null) {
+                cliSes.freeSSL();
+            }
+            if (cliSock != null) {
+                cliSock.close();
+            }
+            if (srvSocket != null) {
+                srvSocket.close();
+            }
+            if (srvCtx != null) {
+                srvCtx.free();
+            }
+        }
+    }
+
+    @Test
+    public void test_WolfSSLSession_disableByteBufferPool() throws Exception {
+
+        System.out.print("\tByteBuffer pool disabled");
+
+        synchronized (byteBufferPoolPropertyLock) {
+
+            String originalProp =
+                Security.getProperty("wolfssl.readWriteByteBufferPool.disabled");
+
+            try {
+                /* Disable WolfSSLSession internal direct ByteBuffer pool
+                 * for use with read/write() calls */
+                Security.setProperty("wolfssl.readWriteByteBufferPool.disabled",
+                    "true");
+
+                runClientServerOneResumption();
+
+            } finally {
+                if (originalProp == null) {
+                    originalProp = "";
+                }
+                /* restore system property */
+                Security.setProperty("wolfssl.readWriteByteBufferPool.disabled",
+                    originalProp);
+            }
+        }
+
+        System.out.println("\t... passed");
+    }
+
+    @Test
+    public void test_WolfSSLSession_byteBufferPoolSize() throws Exception {
+
+        System.out.print("\tByteBuffer pool size changes");
+
+        synchronized (byteBufferPoolPropertyLock) {
+
+            String originalProp =
+                Security.getProperty("wolfssl.readWriteByteBufferPool.size");
+
+            try {
+
+                /* Pool size of 0 */
+                Security.setProperty("wolfssl.readWriteByteBufferPool.size",
+                    "0");
+                runClientServerOneResumption();
+
+                /* Pool size of 1 */
+                Security.setProperty("wolfssl.readWriteByteBufferPool.size",
+                    "1");
+                runClientServerOneResumption();
+
+                /* Pool size of 100 */
+                Security.setProperty("wolfssl.readWriteByteBufferPool.size",
+                    "100");
+                runClientServerOneResumption();
+
+            } finally {
+                if (originalProp == null) {
+                    originalProp = "";
+                }
+                /* restore system property */
+                Security.setProperty("wolfssl.readWriteByteBufferPool.size",
+                    originalProp);
+            }
+        }
+
+        System.out.println("\t... passed");
+    }
+
+    @Test
+    public void test_WolfSSLSession_byteBufferPoolBufferSize() throws Exception {
+
+        System.out.print("\tByteBuffer pool buffer sizes");
+
+        synchronized (byteBufferPoolPropertyLock) {
+
+            String originalProp =
+                Security.getProperty(
+                    "wolfssl.readWriteByteBufferPool.bufferSize");
+
+            try {
+
+                /* Tiny buffer size of 128 bytes, lots of looping  */
+                Security.setProperty(
+                    "wolfssl.readWriteByteBufferPool.bufferSize", "128");
+                runClientServerOneResumption();
+
+                /* Bigger buffer size than default (17k), try 32k */
+                Security.setProperty(
+                    "wolfssl.readWriteByteBufferPool.bufferSize", "32768");
+                runClientServerOneResumption();
+
+                /* Bigger buffer size than default (17k), try 64k */
+                Security.setProperty(
+                    "wolfssl.readWriteByteBufferPool.bufferSize", "65536");
+                runClientServerOneResumption();
+
+            } finally {
+                if (originalProp == null) {
+                    originalProp = "";
+                }
+                /* restore system property */
+                Security.setProperty(
+                    "wolfssl.readWriteByteBufferPool.bufferSize", originalProp);
+            }
+        }
+
+        System.out.println("\t... passed");
+    }
+
     /**
      * wolfSSL I/O context, is passed to I/O callbacks when called
      * by native wolfSSL.