JSSE: adjust SSLSession.getPacketBufferSize() for tls-channel compatibility, wrap native wolfSSL_GetMaxOutputSize()

native/com_wolfssl_WolfSSLSession.c

@@ -2836,6 +2836,28 @@ JNIEXPORT jstring JNICALL Java_com_wolfssl_WolfSSLSession_stateStringLong
 #endif
 }
 
+JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_getMaxOutputSize
+  (JNIEnv* jenv, jobject jcl, jlong sslPtr)
+{
+#ifndef NO_TLS
+    int ret;
+    WOLFSSL* ssl = (WOLFSSL*)(uintptr_t)sslPtr;
+
+    if (jenv == NULL || ssl == NULL) {
+        return 0;
+    }
+
+    ret = wolfSSL_GetMaxOutputSize(ssl);
+
+    return (jint)ret;
+#else
+    (void)jenv;
+    (void)jcl;
+    (void)sslPtr;
+    return (jint)NOT_COMPILED_IN;
+#endif
+}
+
 JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_sessionReused
   (JNIEnv* jenv, jobject jcl, jlong sslPtr)
 {

native/com_wolfssl_WolfSSLSession.h

@@ -967,6 +967,14 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_setMTU
 JNIEXPORT jstring JNICALL Java_com_wolfssl_WolfSSLSession_stateStringLong
   (JNIEnv *, jobject, jlong);
 
+/*
+ * Class:     com_wolfssl_WolfSSLSession
+ * Method:    getMaxOutputSize
+ * Signature: (J)I
+ */
+JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_getMaxOutputSize
+  (JNIEnv *, jobject, jlong);
+
 #ifdef __cplusplus
 }
 #endif

src/java/com/wolfssl/WolfSSLSession.java

@@ -440,6 +440,7 @@ public class WolfSSLSession {
     private native int getThreadsBlockedInPoll(long ssl);
     private native int setMTU(long ssl, int mtu);
     private native String stateStringLong(long ssl);
+    private native int getMaxOutputSize(long ssl);
 
     /* ------------------- session-specific methods --------------------- */
 
@@ -2556,6 +2557,35 @@ public class WolfSSLSession {
         return state;
     }
 
+    /**
+     * Return max record layer size plaintext input size
+     *
+     * @return max record layer size plaintext input size in bytes, or
+     *         negative value on error.
+     *
+     * @throws IllegalStateException WolfSSLContext has been freed
+     */
+    public int getMaxOutputSize() throws IllegalStateException {
+
+        int ret;
+
+        confirmObjectIsActive();
+
+        synchronized (sslLock) {
+            WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
+                WolfSSLDebug.INFO, this.sslPtr,
+                "entered getOutputSize()");
+
+            ret = getMaxOutputSize(this.sslPtr);
+
+            WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
+                WolfSSLDebug.INFO, this.sslPtr,
+                "max output size: " + ret);
+        }
+
+        return ret;
+    }
+
     /**
      * Determine if a reused session was negotiated during the SSL
      * handshake.

src/java/com/wolfssl/provider/jsse/WolfSSLEngine.java

@@ -756,6 +756,10 @@ public class WolfSSLEngine extends SSLEngine {
         /* Force out buffer to be large enough to hold max packet size */
         if (out.remaining() <
             this.engineHelper.getSession().getPacketBufferSize()) {
+            WolfSSLDebug.log(getClass(), WolfSSLDebug.INFO,
+                "out.remaining() too small (" +
+                out.remaining() + "), need at least: " +
+                this.engineHelper.getSession().getPacketBufferSize());
             return new SSLEngineResult(Status.BUFFER_OVERFLOW, hs, 0, 0);
         }
 
@@ -1064,7 +1068,7 @@ public class WolfSSLEngine extends SSLEngine {
     public synchronized SSLEngineResult unwrap(ByteBuffer in, ByteBuffer[] out,
             int ofst, int length) throws SSLException {
 
-        int i, ret = 0, sz = 0, err = 0;
+        int i, ret = 0, err = 0;
         int inPosition = 0;
         int inRemaining = 0;
         int consumed = 0;
@@ -1072,7 +1076,6 @@ public class WolfSSLEngine extends SSLEngine {
         long dtlsPrevDropCount = 0;
         long dtlsCurrDropCount = 0;
         int prevSessionTicketCount = 0;
-        byte[] tmp;
 
         /* Set initial status for SSLEngineResult return */
         Status status = SSLEngineResult.Status.OK;

src/java/com/wolfssl/provider/jsse/WolfSSLImplementSSLSession.java

@@ -760,15 +760,52 @@ public class WolfSSLImplementSSLSession extends ExtendedSSLSession
         return this.port;
     }
 
+    /**
+     * Can be called by SSLEngine consumers to determine maximum
+     * output buffer size, to be used for buffer allocations.
+     *
+     * @return maximum output buffer size
+     */
     @Override
     public int getPacketBufferSize() {
-        /* Match conscrypt's calculations here for maximum potential
-         * SSL/TLS record length. Used by SSLEngine consumers to allocate
-         * output buffer size.
+
+        /* JSSE implementations seem to set the SSL/TLS maximum packet size
+         * (record size) differently.
+         *
+         * Conscrypt calculates this as 18437 bytes:
          *
          * type(1) + version(2) + length(2) + 2^14 plaintext +
-         * max compression overhead (1024) + max AEAD overhead (1024) */
-        return 18437;
+         * max compression overhead (1024) + max AEAD overhead (1024)
+         *
+         * wolfJSSE originally matched the Conscrypt value, but it conflicts
+         * with the maximum allowed/used by the tls-channel project, which
+         * we have a few customers using. tls-channel uses a maximum size of
+         * 17k.
+         *
+         * Native wolfSSL has the wolfSSL_GetMaxOutputSize() function, but
+         * it only works post-handshake. getPacketBufferSize() can be called
+         * pre-handshake, so we set a default value here that matches
+         * the upper limit for tls-channel, which is 17k, then if
+         * wolfSSL_GetMaxOutputSize() is larger, we increase to that value.
+         */
+        int nativeMax;
+        int ret = 17 * 1024;
+
+        /* Try to get maximum TLS record size from native wolfSSL */
+        if (ssl != null) {
+            nativeMax = ssl.getMaxOutputSize();
+            WolfSSLDebug.log(getClass(), WolfSSLDebug.INFO,
+                "ssl.getMaxOutputSize() returned: " + nativeMax);
+
+            if ((nativeMax > 0) && (nativeMax > ret)) {
+                ret = nativeMax;
+            }
+        }
+
+        WolfSSLDebug.log(getClass(), WolfSSLDebug.INFO,
+            "getPacketBufferSize() returning: " + ret);
+
+        return ret;
     }
 
     @Override

src/test/com/wolfssl/provider/jsse/test/WolfSSLEngineTest.java

@@ -1662,8 +1662,8 @@ public class WolfSSLEngineTest {
                 session = engine.getSession();
                 packetBufSz = session.getPacketBufferSize();
 
-                /* expected to be 18437 */
-                if (packetBufSz != 18437) {
+                /* expected to be 17k */
+                if (packetBufSz != (17 * 1024)) {
                     error("\t\t... failed");
                     fail("got incorrect packet buffer size (" +
                         enabledProtocols.get(i) + ")");

src/test/com/wolfssl/provider/jsse/test/WolfSSLSocketTest.java

@@ -3393,7 +3393,7 @@ public class WolfSSLSocketTest {
                 throw cliException;
             }
 
-            System.out.println("\t\t... passed");
+            System.out.println("\t... passed");
 
         } finally {
             /* Restore original property value */