diff --git a/wolfcrypt/src/tfm.c b/wolfcrypt/src/tfm.c index 2147275dc..07d8b0db6 100644 --- a/wolfcrypt/src/tfm.c +++ b/wolfcrypt/src/tfm.c @@ -2952,19 +2952,22 @@ int mp_prime_is_prime_ex(mp_int* a, int t, int* result, WC_RNG* rng) * give a (1/4)^t chance of a false prime. */ if (ret == FP_YES) { fp_int b, c; - /* FP_MAX_BITS is 2 times the modulus size. The modulus size is - * 2 times the prime size. */ word32 baseSz; #ifndef WOLFSSL_SMALL_STACK - byte base[FP_MAX_BITS/32]; + byte base[FP_MAX_PRIME_SIZE]; #else byte* base; #endif baseSz = fp_count_bits(a); + /* The base size is the number of bits / 8. One is added if the number + * of bits isn't an even 8. */ baseSz = (baseSz / 8) + ((baseSz % 8) ? 1 : 0); - #ifdef WOLFSSL_SMALL_STACK + #ifndef WOLFSSL_SMALL_STACK + if (baseSz > sizeof(base)) + return FP_MEM; + #else base = (byte*)XMALLOC(baseSz, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (base == NULL) return FP_MEM; diff --git a/wolfssl/wolfcrypt/tfm.h b/wolfssl/wolfcrypt/tfm.h index b693c97d7..cc269c44f 100644 --- a/wolfssl/wolfcrypt/tfm.h +++ b/wolfssl/wolfcrypt/tfm.h @@ -279,6 +279,12 @@ #define FP_DIGIT_MAX FP_MASK #define FP_SIZE (FP_MAX_SIZE/DIGIT_BIT) +#define FP_MAX_PRIME_SIZE (FP_MAX_BITS/(2*CHAR_BIT)) +/* In terms of FP_MAX_BITS, it is double the size possible for a number + * to allow for multiplication, divide that 2 out. Also divide by CHAR_BIT + * to convert from bits to bytes. (Note, FP_PRIME_SIZE is the number of + * values in the canned prime number list.) */ + /* signs */ #define FP_ZPOS 0 #define FP_NEG 1