WolfSSL
/
wolfssljni
mirror of https://github.com/wolfSSL/wolfssljni.git
1
0
Fork 0
Code Issues Releases Wiki Activity

F-15699: reject RSA verify, sign check and decrypt callback returns larger than the inline buffer

pull/445/head
Chris Conlon 2026-10-22 14:18:58 -06:00
parent d8acb6b44c
commit 7643607cff
2 changed files with 170 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
native/com_wolfssl_WolfSSLContext.c
View File

@ -5537,6 +5537,11 @@ int NativeRsaVerifyCb(WOLFSSL* ssl, unsigned char* sig, unsigned int sigSz,
return -1;
}
if ((retval > 0) && ((unsigned int)retval > sigSz)) {
/* wolfSSL uses positive return as the output length */
retval = -1;
}
/* point out* to the beginning of our decrypted buffer */
if (retval > 0)
*out = sig;
@ -5833,6 +5838,11 @@ int NativeRsaSignCheckCb(WOLFSSL* ssl, unsigned char* sig, unsigned int sigSz,
return -1;
}
if ((retval > 0) && ((unsigned int)retval > sigSz)) {
/* wolfSSL uses positive return as the output length */
retval = -1;
}
/* point out* to the beginning of decrypted buffer */
if (retval > 0) {
*out = sig;
@ -6070,6 +6080,11 @@ int NativeRsaPssSignCheckCb(WOLFSSL* ssl, unsigned char* sig,
return -1;
}
if ((retval > 0) && ((unsigned int)retval > sigSz)) {
/* wolfSSL uses positive return as the output length */
retval = -1;
}
/* point out* to the beginning of decrypted buffer */
if (retval > 0) {
*out = sig;
@ -6648,6 +6663,11 @@ int NativeRsaDecCb(WOLFSSL* ssl, unsigned char* in, unsigned int inSz,
retval = -1;
}
if ((retval > 0) && ((unsigned int)retval > inSz)) {
/* wolfSSL uses positive return as the output length */
retval = -1;
}
/* point out* to the beginning of our decrypted buffer */
if (retval > 0)
*out = in;

158
src/test/com/wolfssl/test/WolfSSLContextTest.java
View File

@ -1476,9 +1476,10 @@ public class WolfSSLContextTest {
}
}
/* Accept or connect until the handshake finishes, returns final ret */
private static int pkCbHandshakeLoop(WolfSSLSession ses, boolean client)
throws Exception {
/* Accept or connect until the handshake finishes, returns
* { final ret, error code } */
private static int[] pkCbHandshakeLoop(WolfSSLSession ses,
boolean client) throws Exception {
int ret;
int err;
@ -1488,23 +1489,24 @@ public class WolfSSLContextTest {
} while (ret != WolfSSL.SSL_SUCCESS &&
(err == WolfSSL.SSL_ERROR_WANT_READ ||
err == WolfSSL.SSL_ERROR_WANT_WRITE));
return ret;
return new int[] { ret, err };
}
/* Run one localhost handshake, returns { accept() ret, connect() ret } */
/* Run one localhost handshake, returns { accept() ret, connect() ret,
* server error, client error } */
private int[] runPkCbHandshake(final WolfSSLContext srvCtx,
WolfSSLContext cliCtx) throws Exception {
ServerSocket srvSocket = null;
ExecutorService es = null;
int cliRet;
int[] cliRet;
try {
srvSocket = new ServerSocket(0);
srvSocket.setSoTimeout(10000);
final ServerSocket fSrvSock = srvSocket;
es = Executors.newSingleThreadExecutor();
Future<Integer> srv = es.submit(() -> {
Future<int[]> srv = es.submit(() -> {
try (Socket sock = fSrvSock.accept()) {
sock.setSoTimeout(10000);
WolfSSLSession ses = new WolfSSLSession(srvCtx);
@ -1534,7 +1536,8 @@ public class WolfSSLContextTest {
ses.freeSSL();
}
}
return new int[] { srv.get(10, TimeUnit.SECONDS), cliRet };
int[] srvRet = srv.get(10, TimeUnit.SECONDS);
return new int[] { srvRet[0], cliRet[0], srvRet[1], cliRet[1] };
} finally {
if (es != null) {
es.shutdownNow();
@ -1739,6 +1742,145 @@ public class WolfSSLContextTest {
}
}
/* RSA verify callback returning a length larger than the signature
* buffer must be rejected in JNI. wolfSSL keeps only the low 16 bits of
* the length, so 65536 plus the real length would otherwise pass */
@Test
public void test_WolfSSLContext_rsaVerifyCbOversizedReturn()
throws Exception {
Assume.assumeTrue(WolfSSL.RsaEnabled() && WolfSSL.FileSystemEnabled());
final boolean[] called = new boolean[1];
final int[] verifyRet = { -1 };
WolfSSLContext srvCtx = null;
WolfSSLContext cliCtx = null;
try {
srvCtx = createCtx(svrCert, svrKey, caCert,
WolfSSL.TLSv1_2_ServerMethod());
cliCtx = createCtx(cliCert, cliKey, caCert,
WolfSSL.TLSv1_2_ClientMethod());
if (WolfSSL.RsaPssEnabled()) {
/* Server must sign PKCS#1 v1.5 to reach this callback */
Assume.assumeTrue("set1SigAlgsList() not compiled in",
srvCtx.set1SigAlgsList("RSA+SHA256") ==
WolfSSL.SSL_SUCCESS);
}
try {
cliCtx.setRsaVerifyCb(new WolfSSLRsaVerifyCallback() {
public int rsaVerifyCallback(WolfSSLSession ssl,
ByteBuffer sig, long sigSz, ByteBuffer out,
long outSz, ByteBuffer keyDer, long keySz,
Object ctx) {
called[0] = true;
verifyRet[0] = new WolfCryptRSA().doVerify(sig,
sigSz, out, outSz, keyDer, keySz);
if (verifyRet[0] <= 0) {
return verifyRet[0];
}
return (1 << 16) + verifyRet[0];
}
});
} catch (WolfSSLJNIException e) {
assumePkCallbacks(e);
}
/* Skip cert check so only ServerKeyExchange calls verify cb */
cliCtx.setVerify(WolfSSL.SSL_VERIFY_NONE, null);
int[] ret = runPkCbHandshake(srvCtx, cliCtx);
assertTrue("RSA verify cb not called", called[0]);
assertTrue("RSA verify failed: " + verifyRet[0], verifyRet[0] > 0);
assertTrue("client connect() should fail",
ret[1] != WolfSSL.SSL_SUCCESS);
assertEquals("oversized return reached wolfSSL",
WolfSSL.SSL_FATAL_ERROR, ret[3]);
} finally {
if (cliCtx != null) {
cliCtx.free();
}
if (srvCtx != null) {
srvCtx.free();
}
}
}
/* RSA sign check callbacks returning a length larger than the signature
* buffer must be rejected in JNI, not as a signature fault in wolfSSL */
@Test
public void test_WolfSSLContext_rsaSignCheckCbOversizedReturn()
throws Exception {
Assume.assumeTrue(WolfSSL.RsaEnabled() && WolfSSL.FileSystemEnabled());
for (final boolean pss : new boolean[] { false, true }) {
if (pss && !WolfSSL.RsaPssEnabled()) {
continue;
}
final boolean[] called = new boolean[1];
WolfSSLContext srvCtx = null;
WolfSSLContext cliCtx = null;
try {
srvCtx = createCtx(svrCert, svrKey, caCert,
WolfSSL.TLSv1_2_ServerMethod());
cliCtx = createCtx(cliCert, cliKey, caCert,
WolfSSL.TLSv1_2_ClientMethod());
if (!pss && WolfSSL.RsaPssEnabled() &&
srvCtx.set1SigAlgsList("RSA+SHA256") !=
WolfSSL.SSL_SUCCESS) {
/* cannot force PKCS#1 v1.5, the PSS pass still runs */
continue;
}
try {
if (pss) {
srvCtx.setRsaPssSignCheckCb(
new WolfSSLRsaPssVerifyCallback() {
public int rsaPssVerifyCallback(
WolfSSLSession ssl, ByteBuffer sig,
long sigSz, ByteBuffer out, long outSz,
int hash, int mgf, ByteBuffer keyDer,
long keySz, Object ctx) {
called[0] = true;
return (int)sigSz + 1;
}
});
}
else {
srvCtx.setRsaSignCheckCb(
new WolfSSLRsaVerifyCallback() {
public int rsaVerifyCallback(WolfSSLSession ssl,
ByteBuffer sig, long sigSz, ByteBuffer out,
long outSz, ByteBuffer keyDer, long keySz,
Object ctx) {
called[0] = true;
return (int)sigSz + 1;
}
});
}
} catch (WolfSSLJNIException e) {
assumePkCallbacks(e);
}
int[] ret = runPkCbHandshake(srvCtx, cliCtx);
assertTrue("sign check cb not called, pss=" + pss, called[0]);
assertTrue("server accept() should fail, pss=" + pss,
ret[0] != WolfSSL.SSL_SUCCESS);
assertEquals("oversized return reached wolfSSL, pss=" + pss,
WolfSSL.SSL_FATAL_ERROR, ret[2]);
} finally {
if (cliCtx != null) {
cliCtx.free();
}
if (srvCtx != null) {
srvCtx.free();
}
}
}
}
/* ECC sign callback reporting a signature longer than the out buffer
* must fail before wolfSSL sends it to the peer */
@Test