From 29cf3eb84e338c5d6d948e3e3e0aa7a878034372 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Thu, 5 Jun 2025 09:18:18 +0400 Subject: [PATCH 1/3] linuxkm/lkcapi_sha_glue.c: refactor DRBG wrapper to instantiate one DRBG per core, to relieve contention. --- linuxkm/lkcapi_sha_glue.c | 101 +++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/linuxkm/lkcapi_sha_glue.c b/linuxkm/lkcapi_sha_glue.c index 647fb627c..0108bd66f 100644 --- a/linuxkm/lkcapi_sha_glue.c +++ b/linuxkm/lkcapi_sha_glue.c @@ -902,38 +902,65 @@ struct wc_swallow_the_semicolon #include struct wc_linuxkm_drbg_ctx { - wolfSSL_Mutex lock; - WC_RNG rng; + struct wc_rng_inst { + wolfSSL_Mutex lock; + WC_RNG rng; + } *rngs; /* one per CPU ID */ }; static int wc_linuxkm_drbg_init_tfm(struct crypto_tfm *tfm) { struct wc_linuxkm_drbg_ctx *ctx = (struct wc_linuxkm_drbg_ctx *)crypto_tfm_ctx(tfm); + unsigned int i; int ret; - ret = wc_InitMutex(&ctx->lock); - if (ret != 0) - return -EINVAL; + ctx->rngs = (struct wc_rng_inst *)malloc(sizeof(*ctx->rngs) * nr_cpu_ids); + if (! ctx->rngs) + return -ENOMEM; + XMEMSET(ctx->rngs, 0, sizeof(*ctx->rngs) * nr_cpu_ids); - /* Note the new DRBG instance is seeded, and later reseeded, from system - * get_random_bytes() via wc_GenerateSeed(). - */ - ret = wc_InitRng(&ctx->rng); - if (ret != 0) { - (void)wc_FreeMutex(&ctx->lock); - return -EINVAL; + for (i = 0; i < nr_cpu_ids; ++i) { + ret = wc_InitMutex(&ctx->rngs[i].lock); + if (ret != 0) { + ret = -EINVAL; + break; + } + + /* Note the new DRBG instance is seeded, and later reseeded, from system + * get_random_bytes() via wc_GenerateSeed(). + */ + ret = wc_InitRng(&ctx->rngs[i].rng); + if (ret != 0) { + ret = -EINVAL; + break; + } } - return 0; + if (ret != 0) { + for (i = 0; i < nr_cpu_ids; ++i) { + (void)wc_FreeMutex(&ctx->rngs[i].lock); + wc_FreeRng(&ctx->rngs[i].rng); + } + free(ctx->rngs); + ctx->rngs = NULL; + } + + return ret; } static void wc_linuxkm_drbg_exit_tfm(struct crypto_tfm *tfm) { struct wc_linuxkm_drbg_ctx *ctx = (struct wc_linuxkm_drbg_ctx *)crypto_tfm_ctx(tfm); + unsigned int i; - wc_FreeRng(&ctx->rng); - - (void)wc_FreeMutex(&ctx->lock); + if (ctx->rngs) { + for (i = 0; i < nr_cpu_ids; ++i) { + (void)wc_FreeMutex(&ctx->rngs[i].lock); + wc_FreeRng(&ctx->rngs[i].rng); + } + free(ctx->rngs); + ctx->rngs = NULL; + } return; } @@ -944,24 +971,34 @@ static int wc_linuxkm_drbg_generate(struct crypto_rng *tfm, { struct wc_linuxkm_drbg_ctx *ctx = (struct wc_linuxkm_drbg_ctx *)crypto_rng_ctx(tfm); int ret; + int my_cpu = + raw_smp_processor_id(); /* Note, core is not locked, so the actual core + * ID may change while executing, hence the + * mutex. + * The mutex is also needed to coordinate with + * wc_linuxkm_drbg_seed(), which seeds all + * instances. + */ + wolfSSL_Mutex *lock = &ctx->rngs[my_cpu].lock; + WC_RNG *rng = &ctx->rngs[my_cpu].rng; - wc_LockMutex(&ctx->lock); + wc_LockMutex(lock); if (slen > 0) { - ret = wc_RNG_DRBG_Reseed(&ctx->rng, src, slen); + ret = wc_RNG_DRBG_Reseed(rng, src, slen); if (ret != 0) { ret = -EINVAL; goto out; } } - ret = wc_RNG_GenerateBlock(&ctx->rng, dst, dlen); + ret = wc_RNG_GenerateBlock(rng, dst, dlen); if (ret != 0) ret = -EINVAL; out: - wc_UnLockMutex(&ctx->lock); + wc_UnLockMutex(lock); return ret; } @@ -971,22 +1008,28 @@ static int wc_linuxkm_drbg_seed(struct crypto_rng *tfm, { struct wc_linuxkm_drbg_ctx *ctx = (struct wc_linuxkm_drbg_ctx *)crypto_rng_ctx(tfm); int ret; + unsigned int i; if (slen == 0) return 0; - wc_LockMutex(&ctx->lock); + for (i = 0; i < nr_cpu_ids; ++i) { + wolfSSL_Mutex *lock = &ctx->rngs[i].lock; + WC_RNG *rng = &ctx->rngs[i].rng; - ret = wc_RNG_DRBG_Reseed(&ctx->rng, seed, slen); - if (ret != 0) { - ret = -EINVAL; - goto out; + wc_LockMutex(lock); + + ret = wc_RNG_DRBG_Reseed(rng, seed, slen); + if (ret != 0) { + ret = -EINVAL; + } + + wc_UnLockMutex(lock); + + if (ret != 0) + break; } -out: - - wc_UnLockMutex(&ctx->lock); - return ret; } From dbc34352c70debf39a0138996122e3f05fd68a06 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Thu, 5 Jun 2025 16:31:46 +0400 Subject: [PATCH 2/3] linuxkm/lkcapi_sha_glue.c: in wc_linuxkm_drbg_seed(), prefix the supplied seed with the CPU ID of each DRBG, to avoid duplicate states; wolfcrypt/src/random.c: in Hash_DRBG_Generate(), always put digest[] on the stack even in WOLFSSL_SMALL_STACK configuration (it's only 32 bytes); configure.ac: default smallstackcache on when linuxkm-defaults. --- configure.ac | 8 +++++++- linuxkm/lkcapi_sha_glue.c | 16 +++++++++++++++- wolfcrypt/src/random.c | 10 ---------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/configure.ac b/configure.ac index e0b6d769c..e5cf76e83 100644 --- a/configure.ac +++ b/configure.ac @@ -7181,10 +7181,16 @@ then fi # Small Stack - Cache on object +if test "$ENABLED_LINUXKM_DEFAULTS" = "yes" +then + ENABLED_SMALL_STACK_CACHE_DEFAULT=yes +else + ENABLED_SMALL_STACK_CACHE_DEFAULT=no +fi AC_ARG_ENABLE([smallstackcache], [AS_HELP_STRING([--enable-smallstackcache],[Enable Small Stack Usage Caching (default: disabled)])], [ ENABLED_SMALL_STACK_CACHE=$enableval ], - [ ENABLED_SMALL_STACK_CACHE=no ] + [ ENABLED_SMALL_STACK_CACHE=$ENABLED_SMALL_STACK_CACHE_DEFAULT ] ) if test "x$ENABLED_SMALL_STACK_CACHE" = "xyes" diff --git a/linuxkm/lkcapi_sha_glue.c b/linuxkm/lkcapi_sha_glue.c index 0108bd66f..dadc7846a 100644 --- a/linuxkm/lkcapi_sha_glue.c +++ b/linuxkm/lkcapi_sha_glue.c @@ -1007,19 +1007,31 @@ static int wc_linuxkm_drbg_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) { struct wc_linuxkm_drbg_ctx *ctx = (struct wc_linuxkm_drbg_ctx *)crypto_rng_ctx(tfm); + u8 *seed_copy = NULL; int ret; unsigned int i; if (slen == 0) return 0; + seed_copy = (u8 *)malloc(slen + 2); + if (! seed_copy) + return -ENOMEM; + XMEMCPY(seed_copy + 2, seed, slen); + for (i = 0; i < nr_cpu_ids; ++i) { wolfSSL_Mutex *lock = &ctx->rngs[i].lock; WC_RNG *rng = &ctx->rngs[i].rng; + /* perturb the seed with the CPU ID, so that no DRBG has the exact same + * seed. + */ + seed_copy[0] = (u8)(i >> 8); + seed_copy[1] = (u8)i; + wc_LockMutex(lock); - ret = wc_RNG_DRBG_Reseed(rng, seed, slen); + ret = wc_RNG_DRBG_Reseed(rng, seed_copy, slen + 2); if (ret != 0) { ret = -EINVAL; } @@ -1030,6 +1042,8 @@ static int wc_linuxkm_drbg_seed(struct crypto_rng *tfm, break; } + free(seed_copy); + return ret; } diff --git a/wolfcrypt/src/random.c b/wolfcrypt/src/random.c index 311d5a71c..f22a8ed6e 100644 --- a/wolfcrypt/src/random.c +++ b/wolfcrypt/src/random.c @@ -647,14 +647,7 @@ static int Hash_DRBG_Generate(DRBG_internal* drbg, byte* out, word32 outSz) return DRBG_NEED_RESEED; } else { - #ifndef WOLFSSL_SMALL_STACK byte digest[WC_SHA256_DIGEST_SIZE]; - #else - byte* digest = (byte*)XMALLOC(WC_SHA256_DIGEST_SIZE, drbg->heap, - DYNAMIC_TYPE_DIGEST); - if (digest == NULL) - return DRBG_FAILURE; - #endif type = drbgGenerateH; reseedCtr = drbg->reseedCtr; @@ -692,9 +685,6 @@ static int Hash_DRBG_Generate(DRBG_internal* drbg, byte* out, word32 outSz) drbg->reseedCtr++; } ForceZero(digest, WC_SHA256_DIGEST_SIZE); - #ifdef WOLFSSL_SMALL_STACK - XFREE(digest, drbg->heap, DYNAMIC_TYPE_DIGEST); - #endif } return (ret == 0) ? DRBG_SUCCESS : DRBG_FAILURE; From ae15693fa878d610a393942016f118ad8dfc7516 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Sat, 7 Jun 2025 07:07:20 +0400 Subject: [PATCH 3/3] linuxkm/lkcapi_sha_glue.c: in wc_linuxkm_drbg_generate() and wc_linuxkm_drbg_seed(), check retval from wc_LockMutex(). wolfcrypt/src/random.c: in Hash_DRBG_Generate(), restore smallstack path for digest[], but use non-smallstack path for WOLFSSL_LINUXKM. --- linuxkm/lkcapi_sha_glue.c | 20 ++++++++++---------- wolfcrypt/src/random.c | 10 ++++++++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/linuxkm/lkcapi_sha_glue.c b/linuxkm/lkcapi_sha_glue.c index dadc7846a..ca6b722e6 100644 --- a/linuxkm/lkcapi_sha_glue.c +++ b/linuxkm/lkcapi_sha_glue.c @@ -971,18 +971,17 @@ static int wc_linuxkm_drbg_generate(struct crypto_rng *tfm, { struct wc_linuxkm_drbg_ctx *ctx = (struct wc_linuxkm_drbg_ctx *)crypto_rng_ctx(tfm); int ret; - int my_cpu = - raw_smp_processor_id(); /* Note, core is not locked, so the actual core - * ID may change while executing, hence the - * mutex. - * The mutex is also needed to coordinate with - * wc_linuxkm_drbg_seed(), which seeds all - * instances. - */ + /* Note, core is not locked, so the actual core ID may change while + * executing, hence the mutex. + * The mutex is also needed to coordinate with wc_linuxkm_drbg_seed(), which + * seeds all instances. + */ + int my_cpu = raw_smp_processor_id(); wolfSSL_Mutex *lock = &ctx->rngs[my_cpu].lock; WC_RNG *rng = &ctx->rngs[my_cpu].rng; - wc_LockMutex(lock); + if (wc_LockMutex(lock) != 0) + return -EINVAL; if (slen > 0) { ret = wc_RNG_DRBG_Reseed(rng, src, slen); @@ -1029,7 +1028,8 @@ static int wc_linuxkm_drbg_seed(struct crypto_rng *tfm, seed_copy[0] = (u8)(i >> 8); seed_copy[1] = (u8)i; - wc_LockMutex(lock); + if (wc_LockMutex(lock) != 0) + return -EINVAL; ret = wc_RNG_DRBG_Reseed(rng, seed_copy, slen + 2); if (ret != 0) { diff --git a/wolfcrypt/src/random.c b/wolfcrypt/src/random.c index f22a8ed6e..cb62a3dcc 100644 --- a/wolfcrypt/src/random.c +++ b/wolfcrypt/src/random.c @@ -647,7 +647,14 @@ static int Hash_DRBG_Generate(DRBG_internal* drbg, byte* out, word32 outSz) return DRBG_NEED_RESEED; } else { + #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_LINUXKM) + byte* digest = (byte*)XMALLOC(WC_SHA256_DIGEST_SIZE, drbg->heap, + DYNAMIC_TYPE_DIGEST); + if (digest == NULL) + return DRBG_FAILURE; + #else byte digest[WC_SHA256_DIGEST_SIZE]; + #endif type = drbgGenerateH; reseedCtr = drbg->reseedCtr; @@ -685,6 +692,9 @@ static int Hash_DRBG_Generate(DRBG_internal* drbg, byte* out, word32 outSz) drbg->reseedCtr++; } ForceZero(digest, WC_SHA256_DIGEST_SIZE); + #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_LINUXKM) + XFREE(digest, drbg->heap, DYNAMIC_TYPE_DIGEST); + #endif } return (ret == 0) ? DRBG_SUCCESS : DRBG_FAILURE;