camellia_setup128 and camellia_setup256 refactory to reduce stack usage:

--- subL and subR variables moved to the heap;
--- return type changed to int, returning 0 for success;
--- chain of dependency updated to propagate the error in CamelliaSetKey function.

ctaocrypt/benchmark/benchmark.c

@@ -405,9 +405,13 @@ void bench_camellia(void)
 {
     Camellia cam;
     double start, total, persec;
-    int    i;
+    int    i, ret;
 
-    CamelliaSetKey(&cam, key, 16, iv);
+    ret = CamelliaSetKey(&cam, key, 16, iv);
+    if (ret != 0) {
+        printf("CamelliaSetKey failed, ret = %d\n", ret);
+        return;
+    }
     start = current_time(1);
 
     for(i = 0; i < numBlocks; i++)

ctaocrypt/src/camellia.c

@@ -486,13 +486,21 @@ static const u32 camellia_sp4404[256] = {
 #define subl(x) subL[(x)]
 #define subr(x) subR[(x)]
 
-static void camellia_setup128(const unsigned char *key, u32 *subkey)
+static int camellia_setup128(const unsigned char *key, u32 *subkey)
 {
     u32 kll, klr, krl, krr;
     u32 il, ir, t0, t1, w0, w1;
     u32 kw4l, kw4r, dw, tl, tr;
-    u32 subL[26];
+    u32* subL;
-    u32 subR[26];
+    u32* subR;
+
+    subL = (u32*) XMALLOC(sizeof(u32) * 26, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    if (subL == NULL)
+        return MEMORY_E;
+
+    subR = (u32*) XMALLOC(sizeof(u32) * 26, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    if (subR == NULL)
+        return MEMORY_E;
 
     /**
      *  k == kll || klr || krl || krr (|| is concatination)
@@ -694,17 +702,28 @@ static void camellia_setup128(const unsigned char *key, u32 *subkey)
     dw = CamelliaSubkeyL(23) ^ CamelliaSubkeyR(23), dw = CAMELLIA_RL8(dw);
     CamelliaSubkeyR(23) = CamelliaSubkeyL(23) ^ dw, CamelliaSubkeyL(23) = dw;
 
-    return;
+    XFREE(subL, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(subR, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+
+    return 0;
 }
 
-static void camellia_setup256(const unsigned char *key, u32 *subkey)
+static int camellia_setup256(const unsigned char *key, u32 *subkey)
 {
     u32 kll,klr,krl,krr;           /* left half of key */
     u32 krll,krlr,krrl,krrr;       /* right half of key */
     u32 il, ir, t0, t1, w0, w1;    /* temporary variables */
     u32 kw4l, kw4r, dw, tl, tr;
-    u32 subL[34];
+    u32* subL;
-    u32 subR[34];
+    u32* subR;
+
+    subL = (u32*) XMALLOC(sizeof(u32) * 34, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    if (subL == NULL)
+        return MEMORY_E;
+
+    subR = (u32*) XMALLOC(sizeof(u32) * 34, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    if (subR == NULL)
+        return MEMORY_E;
 
     /**
      *  key = (kll || klr || krl || krr || krll || krlr || krrl || krrr)
@@ -980,10 +999,13 @@ static void camellia_setup256(const unsigned char *key, u32 *subkey)
     dw = CamelliaSubkeyL(31) ^ CamelliaSubkeyR(31), dw = CAMELLIA_RL8(dw);
     CamelliaSubkeyR(31) = CamelliaSubkeyL(31) ^ dw,CamelliaSubkeyL(31) = dw;
     
-    return;
+    XFREE(subL, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(subR, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+
+    return 0;
 }
 
-static void camellia_setup192(const unsigned char *key, u32 *subkey)
+static int camellia_setup192(const unsigned char *key, u32 *subkey)
 {
     unsigned char kk[32];
     u32 krll, krlr, krrl,krrr;
@@ -995,8 +1017,8 @@ static void camellia_setup192(const unsigned char *key, u32 *subkey)
     krrr = ~krlr;
     memcpy(kk+24, (unsigned char *)&krrl, 4);
     memcpy(kk+28, (unsigned char *)&krrr, 4);
-    camellia_setup256(kk, subkey);
+
-    return;
+    return camellia_setup256(kk, subkey);
 }
 
 
@@ -1488,22 +1510,29 @@ static void Camellia_DecryptBlock(const int keyBitLength,
 
 int CamelliaSetKey(Camellia* cam, const byte* key, word32 len, const byte* iv)
 {
+    int ret = 0;
+
     if (cam == NULL) return BAD_FUNC_ARG;
 
     XMEMSET(cam->key, 0, sizeof(KEY_TABLE_TYPE));
+
     switch (len) {
         case 16:
-        	camellia_setup128(key, cam->key);
+        	ret = camellia_setup128(key, cam->key);
             break;
         case 24:
-        	camellia_setup192(key, cam->key);
+        	ret = camellia_setup192(key, cam->key);
             break;
         case 32:
-	        camellia_setup256(key, cam->key);
+            ret = camellia_setup256(key, cam->key);
             break;
         default:
             return BAD_FUNC_ARG;
     }
+
+    if (ret != 0)
+        return ret;
+
     cam->keySz = len * 8;
 
     return CamelliaSetIV(cam, iv);

ctaocrypt/test/test.c

@@ -2476,8 +2476,9 @@ int camellia_test(void)
 
     testsSz = sizeof(testVectors)/sizeof(test_vector_t);
     for (i = 0; i < testsSz; i++) {
-        CamelliaSetKey(&cam, testVectors[i].key, testVectors[i].keySz,
+        if (CamelliaSetKey(&cam, testVectors[i].key, testVectors[i].keySz,
-                                                             testVectors[i].iv);
+                                                        testVectors[i].iv) != 0)
+            return testVectors[i].errorCode;
 
         switch (testVectors[i].type) {
             case CAM_ECB_ENC:

src/keys.c

@@ -1748,27 +1748,41 @@ static int SetKeys(Ciphers* enc, Ciphers* dec, Keys* keys, CipherSpecs* specs,
 
 #ifdef HAVE_CAMELLIA
     if (specs->bulk_cipher_algorithm == cyassl_camellia) {
+        int camRet;
+
         if (enc->cam == NULL)
             enc->cam =
                 (Camellia*)XMALLOC(sizeof(Camellia), heap, DYNAMIC_TYPE_CIPHER);
         if (enc->cam == NULL)
             return MEMORY_E;
+
         if (dec->cam == NULL)
             dec->cam =
                 (Camellia*)XMALLOC(sizeof(Camellia), heap, DYNAMIC_TYPE_CIPHER);
         if (dec->cam == NULL)
             return MEMORY_E;
+
         if (side == CYASSL_CLIENT_END) {
-            CamelliaSetKey(enc->cam, keys->client_write_key,
+            camRet = CamelliaSetKey(enc->cam, keys->client_write_key,
                       specs->key_size, keys->client_write_IV);
-            CamelliaSetKey(dec->cam, keys->server_write_key,
+            if (camRet != 0)
+                return camRet;
+
+            camRet = CamelliaSetKey(dec->cam, keys->server_write_key,
                       specs->key_size, keys->server_write_IV);
+            if (camRet != 0)
+                return camRet;
         }
         else {
-            CamelliaSetKey(enc->cam, keys->server_write_key,
+            camRet = CamelliaSetKey(enc->cam, keys->server_write_key,
                       specs->key_size, keys->server_write_IV);
-            CamelliaSetKey(dec->cam, keys->client_write_key,
+            if (camRet != 0)
+                return camRet;
+
+            camRet = CamelliaSetKey(dec->cam, keys->client_write_key,
                       specs->key_size, keys->client_write_IV);
+            if (camRet != 0)
+                return camRet;
         }
         enc->setup = 1;
         dec->setup = 1;