From e187a74b1cdcc94883be3fb792d1bc6c74ccf7bb Mon Sep 17 00:00:00 2001
From: Sean Parkinson <sean@wolfssl.com>
Date: Mon, 15 Feb 2021 09:48:18 +1000
Subject: [PATCH] sp_lcm small stack: fix size of temporary

Temporary sp_int needs to be allocated to be 1 digit larger than a or b
for the div operation.
Change sp_div to check sizes of r and rem when passed in.
Fix sp_invmod, sp_gcm, sp_submod to use temporary sp_int sizes that work
with calls to sp_div().
---
 wolfcrypt/src/sp_int.c | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/wolfcrypt/src/sp_int.c b/wolfcrypt/src/sp_int.c
index d51ab9a57..e0293441d 100644
--- a/wolfcrypt/src/sp_int.c
+++ b/wolfcrypt/src/sp_int.c
@@ -4014,7 +4014,11 @@ int sp_submod(sp_int* a, sp_int* b, sp_int* m, sp_int* r)
 {
 #ifndef WOLFSSL_SP_INT_NEGATIVE
     int err = MP_OKAY;
-    DECL_SP_INT_ARRAY(t, (m == NULL) ? 1 : m->used + 1, 2);
+    int used = ((a == NULL) || (b == NULL) || (m == NULL)) ? 1 :
+                   ((a->used >= m->used) ?
+                       ((a->used >= b->used) ? (a->used + 1) : (b->used + 1)) :
+                   ((b->used >= m->used)) ? (b->used + 1) : (m->used + 1));
+    DECL_SP_INT_ARRAY(t, used, 2);
 
     if ((a == NULL) || (b == NULL) || (m == NULL) || (r == NULL)) {
         err = MP_VAL;
@@ -4026,7 +4030,7 @@ int sp_submod(sp_int* a, sp_int* b, sp_int* m, sp_int* r)
         sp_print(m, "m");
     }
 
-    ALLOC_SP_INT_ARRAY(t, m->used + 1, 2, err, NULL);
+    ALLOC_SP_INT_ARRAY(t, used, 2, err, NULL);
     if (err == MP_OKAY) {
         if (_sp_cmp(a, m) == MP_GT) {
             err = sp_mod(a, m, t[0]);
@@ -4436,6 +4440,12 @@ int sp_div(sp_int* a, sp_int* d, sp_int* r, sp_int* rem)
     if ((err == MP_OKAY) && sp_iszero(d)) {
         err = MP_VAL;
     }
+    if ((err == MP_OKAY) && (r != NULL) && (r->size < a->used - d->used + 2)) {
+        err = MP_VAL;
+    }
+    if ((err == MP_OKAY) && (rem != NULL) && (rem->size < a->used + 1)) {
+        err = MP_VAL;
+    }
 
     if (0 && (err == MP_OKAY)) {
         sp_print(a, "a");
@@ -7721,7 +7731,9 @@ int sp_invmod(sp_int* a, sp_int* m, sp_int* r)
     sp_int* v;
     sp_int* b;
     sp_int* c;
-    DECL_SP_INT_ARRAY(t, (m == NULL) ? 0 : m->used + 1, 4);
+    int used = ((m == NULL) || (a == NULL)) ? 1 :
+                   ((m->used >= a->used) ? m->used + 1 : a->used + 1);
+    DECL_SP_INT_ARRAY(t, used, 4);
 
     if ((a == NULL) || (m == NULL) || (r == NULL)) {
         err = MP_VAL;
@@ -7739,7 +7751,7 @@ int sp_invmod(sp_int* a, sp_int* m, sp_int* r)
         v = t[1];
         b = t[2];
         c = t[3];
-        sp_init_size(v, m->used + 1);
+        sp_init_size(v, used + 1);
 
         if (_sp_cmp_abs(a, m) != MP_LT) {
             err = sp_mod(a, m, v);
@@ -13245,7 +13257,7 @@ int sp_gcd(sp_int* a, sp_int* b, sp_int* r)
         sp_int* u = NULL;
         sp_int* v = NULL;
         sp_int* t = NULL;
-        int used = (a->used >= b->used) ? a->used : b->used;
+        int used = (a->used >= b->used) ? a->used + 1 : b->used + 1;
         DECL_SP_INT_ARRAY(d, used, 3);
 
         ALLOC_SP_INT_ARRAY(d, used, 3, err, NULL);
@@ -13336,7 +13348,7 @@ int sp_lcm(sp_int* a, sp_int* b, sp_int* r)
 {
     int err = MP_OKAY;
     int used = ((a == NULL) || (b == NULL)) ? 1 :
-                   (a->used >= b->used ? a->used : b->used);
+                   (a->used >= b->used ? a->used + 1: b->used + 1);
     DECL_SP_INT_ARRAY(t, used, 2);
 
     if ((a == NULL) || (b == NULL) || (r == NULL)) {