Enforce the RSA key size floor on host keys

ParseRSAPubKey() and ParseRSAPubKeyCert() run the server's host key
through CheckRsaKeyBits(), so a client rejects an RSA host key under
WOLFSSH_RSA_MIN_KEY_BITS the same way a server rejects a short user
auth key. The failure ends the key exchange in DoKexDhReply().

- move CheckRsaKeyBits() above the host key parsers
- ParseRSAPubKey() maps only the decode failure to WS_RSA_E, so a
  parse error or a short key keeps its own code
- test 1024 and 2047 bit host keys in test_ParseRSAPubKey()

Issue: F-14687

src/internal.c

@@ -7595,6 +7595,30 @@ struct wolfSSH_sigKeyBlock {
 static int InitPubKey(struct wolfSSH_sigKeyBlock *p, WOLFSSH *ssh);
 
 
+#ifndef WOLFSSH_NO_RSA
+/* Utility for the RSA host key and user auth paths. */
+/* returns WS_SUCCESS when the key clears WOLFSSH_RSA_MIN_KEY_BITS. */
+static int CheckRsaKeyBits(RsaKey* key)
+{
+    int ret = WS_SUCCESS;
+    int keyBits;
+
+    /* The encrypt size rounds up to a byte; offload builds leave n empty. */
+    keyBits = mp_count_bits(&key->n);
+    if (keyBits == 0) {
+        keyBits = wc_RsaEncryptSize(key) * 8;
+    }
+
+    if (keyBits < WOLFSSH_RSA_MIN_KEY_BITS) {
+        WLOG(WS_LOG_DEBUG, "RSA key too small (%d bits)", keyBits);
+        ret = WS_CERT_KEY_SIZE_E;
+    }
+
+    return ret;
+}
+#endif
+
+
 /* Parse out a RAW RSA public key from buffer */
 static int ParseRSAPubKey(WOLFSSH *ssh,
     struct wolfSSH_sigKeyBlock *sigKeyBlock_ptr, byte *pubKey, word32 pubKeySz)
@@ -7627,13 +7651,15 @@ static int ParseRSAPubKey(WOLFSSH *ssh,
         n = pubKey + pubKeyIdx;
         ret = wc_RsaPublicKeyDecodeRaw(n, nSz, e, eSz,
                                        &sigKeyBlock_ptr->sk.rsa.key);
+        if (ret != 0)
+            ret = WS_RSA_E;
     }
+    if (ret == WS_SUCCESS)
+        ret = CheckRsaKeyBits(&sigKeyBlock_ptr->sk.rsa.key);
 
-    if (ret == 0) {
+    if (ret == WS_SUCCESS) {
         sigKeyBlock_ptr->keySz = (word32)sizeof(sigKeyBlock_ptr->sk.rsa.key);
     }
-    else
-        ret = WS_RSA_E;
 #else
     WOLFSSH_UNUSED(ssh);
     WOLFSSH_UNUSED(sigKeyBlock_ptr);
@@ -8102,6 +8128,8 @@ static int ParseRSAPubKeyCert(WOLFSSH *ssh,
         if (error == 0)
             error = wc_RsaPublicKeyDecode(der, &idx,
                                           &sigKeyBlock_ptr->sk.rsa.key, derSz);
+        if (error == 0)
+            error = CheckRsaKeyBits(&sigKeyBlock_ptr->sk.rsa.key);
         if (error == 0) {
             sigKeyBlock_ptr->keySz =
                 (word32)sizeof(sigKeyBlock_ptr->sk.rsa.key);
@@ -10437,28 +10465,6 @@ static int DoUserAuthRequestPassword(WOLFSSH* ssh, WS_UserAuthData* authData,
 }
 
 #ifndef WOLFSSH_NO_RSA
-/* Utility for the RSA user auth paths. */
-/* returns WS_SUCCESS when the key clears WOLFSSH_RSA_MIN_KEY_BITS. */
-static int CheckRsaKeyBits(RsaKey* key)
-{
-    int ret = WS_SUCCESS;
-    int keyBits;
-
-    /* The encrypt size rounds up to a byte; offload builds leave n empty. */
-    keyBits = mp_count_bits(&key->n);
-    if (keyBits == 0) {
-        keyBits = wc_RsaEncryptSize(key) * 8;
-    }
-
-    if (keyBits < WOLFSSH_RSA_MIN_KEY_BITS) {
-        WLOG(WS_LOG_DEBUG, "RSA auth key too small (%d bits)", keyBits);
-        ret = WS_CERT_KEY_SIZE_E;
-    }
-
-    return ret;
-}
-
-
 /* Utility for DoUserAuthRequestPublicKey() */
 /* returns negative for error, positive is size of digest. */
 static int DoUserAuthRequestRsa(WOLFSSH* ssh, WS_UserAuthData_PublicKey* pk,

tests/unit.c

@@ -22857,9 +22857,29 @@ static int test_ParseRSAPubKey(void)
 
     /* truncated blob: fails after the key is initialized */
     ret = wolfSSH_TestParseRSAPubKey(ssh, blob, (word32)(sizeof(blob) / 2));
-    if (ret != WS_RSA_E) {
+    if (ret != WS_BUFFER_E) {
         fprintf(stderr, "\t\"truncated\" FAIL: got %d, expected %d\n",
-                ret, WS_RSA_E);
+                ret, WS_BUFFER_E);
+        failures++;
+    }
+
+    /* host key under WOLFSSH_RSA_MIN_KEY_BITS */
+    WMEMCPY(blob + RSA_PUB_BLOB_NLEN, userAuthRsaPubKeyNLen1024,
+            sizeof(userAuthRsaPubKeyNLen1024));
+    ret = wolfSSH_TestParseRSAPubKey(ssh, blob, (word32)sizeof(blob));
+    if (ret != WS_CERT_KEY_SIZE_E) {
+        fprintf(stderr, "\t\"1024 bit\" FAIL: got %d, expected %d\n",
+                ret, WS_CERT_KEY_SIZE_E);
+        failures++;
+    }
+
+    WMEMCPY(blob, userAuthRsaPubKeyBlob, sizeof(blob));
+    WMEMCPY(blob + RSA_PUB_BLOB_N, userAuthRsaPubKeyMsb2047,
+            sizeof(userAuthRsaPubKeyMsb2047));
+    ret = wolfSSH_TestParseRSAPubKey(ssh, blob, (word32)sizeof(blob));
+    if (ret != WS_CERT_KEY_SIZE_E) {
+        fprintf(stderr, "\t\"2047 bit\" FAIL: got %d, expected %d\n",
+                ret, WS_CERT_KEY_SIZE_E);
         failures++;
     }
 

wolfssh/internal.h

@@ -719,7 +719,8 @@ enum NameIdType {
 #endif
 
 #ifndef WOLFSSH_RSA_MIN_KEY_BITS
-    /* Minimum accepted RSA public-key size (bits) for user authentication.
+    /* Minimum accepted RSA public-key size (bits) for host keys and user
+     * authentication.
      * Per NIST SP 800-131A; override at build time if a smaller key must be
      * accepted. */
     #define WOLFSSH_RSA_MIN_KEY_BITS 2048