Improve the sniffer asynchronous test case to support multiple concurrent streams.

configure.ac

@@ -5230,7 +5230,7 @@ fi
 
 # Encrypt-Then-Mac
 AC_ARG_ENABLE([enc-then-mac],
-    [AS_HELP_STRING([--enable-enc-then-mac],[Enable Encryptr-Then-Mac extension (default: enabled)])],
+    [AS_HELP_STRING([--enable-enc-then-mac],[Enable Encrypt-Then-Mac extension (default: enabled)])],
     [ ENABLED_ENCRYPT_THEN_MAC=$enableval ],
     [ ENABLED_ENCRYPT_THEN_MAC=yes ]
     )

src/sniffer.c

@@ -362,6 +362,7 @@ static const char* const msgTable[] =
     "Loading chain input",
     "Got encrypted extension",
     "Got Hello Retry Request",
+    "Setting up keys",
 };
 
 
@@ -546,6 +547,7 @@ typedef struct SnifferSession {
 #endif
 #ifdef WOLFSSL_ASYNC_CRYPT
     void*          userCtx;
+    word32         pendSeq; /* when WC_PENDING_E is returned capture sequence */
 #endif
 } SnifferSession;
 
@@ -951,7 +953,7 @@ typedef struct IpInfo {
 /* TCP Info from TCP Header */
 typedef struct TcpInfo {
     int    srcPort;       /* source port */
-    int    dstPort;       /* source port */
+    int    dstPort;       /* destination port */
     int    length;        /* length of this header */
     word32 sequence;      /* sequence number */
     word32 ackNumber;     /* ack number */
@@ -2337,6 +2339,8 @@ static int SetupKeys(const byte* input, int* sslBytes, SnifferSession* session,
     SetupKeysArgs  args[1];
 #endif
 
+    Trace(SNIFFER_KEY_SETUP_STR);
+
     if (session->sslServer->arrays == NULL ||
         session->sslClient->arrays == NULL) {
         /* Secret's have already been established and released.
@@ -4487,7 +4491,6 @@ static int DoHandShake(const byte* input, int* sslBytes,
             }
 #endif
             if (ret == 0) {
-                /* TODO: Add async reentry support here */
                 ret = ProcessClientKeyExchange(input, sslBytes, session, error);
             #ifdef WOLFSSL_ASYNC_CRYPT
                 if (ret == WC_PENDING_E)
@@ -4717,7 +4720,7 @@ static int DecryptTls(WOLFSSL* ssl, byte* plain, const byte* input,
             ssl->decrypt.state = CIPHER_STATE_END;
 
         #ifdef WOLFSSL_ASYNC_CRYPT
-            /* If pending, leave and return below */
+            /* If pending, return now */
             if (ret == WC_PENDING_E) {
                 return ret;
             }
@@ -5698,6 +5701,15 @@ static int CheckSequence(IpInfo* ipInfo, TcpInfo* tcpInfo,
         *sslBytes = actualLen;
     }
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* check if this session is pending */
+    if (session->sslServer->error == WC_PENDING_E &&
+        session->pendSeq != tcpInfo->sequence) {
+        /* this stream is processing, queue packet */
+        return WC_HW_WAIT_E;
+    }
+#endif
+
     TraceSequence(tcpInfo->sequence, *sslBytes);
     if (CheckAck(tcpInfo, session) < 0) {
         if (!RecoveryEnabled) {
@@ -5789,8 +5801,7 @@ static int CheckPreRecord(IpInfo* ipInfo, TcpInfo* tcpInfo,
             *sslFrame = ssl->buffers.inputBuffer.buffer;
             *end = *sslFrame + *sslBytes;
         }
-
+        else {
-        if (vChain != NULL) {
     #ifdef WOLFSSL_SNIFFER_CHAIN_INPUT
             struct iovec* chain = (struct iovec*)vChain;
             word32 i, offset, headerSz, qty, remainder;
@@ -6316,8 +6327,9 @@ static int ssl_DecodePacketInternal(const byte* packet, int length, int isChain,
             NOLOCK_ADD_TO_STAT(SnifferStats.sslEncryptedBytes, sslBytes);
             UNLOCK_STAT();
         }
-        else
+        else {
             INC_STAT(SnifferStats.sslDecryptedPackets);
+        }
 #endif
          return 0; /* done for now */
     }
@@ -6336,6 +6348,10 @@ static int ssl_DecodePacketInternal(const byte* packet, int length, int isChain,
 #endif
         return 0; /* done for now */
     }
+    else if (ret != 0) {
+        /* return specific error case */
+        return ret;
+    }
 
     ret = CheckPreRecord(&ipInfo, &tcpInfo, &sslFrame, &session, &sslBytes,
                          &end, vChain, chainSz, error);
@@ -6350,14 +6366,20 @@ static int ssl_DecodePacketInternal(const byte* packet, int length, int isChain,
     }
 
 #ifdef WOLFSSL_SNIFFER_STATS
+    #ifdef WOLFSSL_ASYNC_CRYPT
+    if (session->sslServer->error != WC_PENDING_E)
+    #endif
+    {
         if (sslBytes > 0) {
             LOCK_STAT();
             NOLOCK_INC_STAT(SnifferStats.sslEncryptedPackets);
             NOLOCK_ADD_TO_STAT(SnifferStats.sslEncryptedBytes, sslBytes);
             UNLOCK_STAT();
         }
-    else
+        else {
             INC_STAT(SnifferStats.sslDecryptedPackets);
+        }
+    }
 #endif
 
 #ifdef WOLFSSL_ASYNC_CRYPT
@@ -6366,6 +6388,9 @@ static int ssl_DecodePacketInternal(const byte* packet, int length, int isChain,
         ret = ProcessMessage(sslFrame, session, sslBytes, data, end, ctx, error);
         session->sslServer->error = ret;
 #ifdef WOLFSSL_ASYNC_CRYPT
+        /* capture the seq pending for this session */
+        session->pendSeq = tcpInfo.sequence;
+
         if (ret == WC_PENDING_E) {
             if (!asyncOkay || CryptoDeviceId == INVALID_DEVID) {
                 /* If devId has not been set then we need to block here by

sslSniffer/sslSnifferTest/snifftest.c

@@ -469,6 +469,105 @@ static void show_usage(void)
     printf("\t./snifftest dump pemKey [server] [port] [password]\n");
 }
 
+
+#ifdef WOLFSSL_ASYNC_CRYPT
+
+typedef struct SnifferPacket {
+    byte* packet;
+    int   length;
+    int   lastRet;
+    int   packetNumber;
+} SnifferPacket;
+
+static SnifferPacket asyncQueue[WOLF_ASYNC_MAX_PENDING];
+
+/* returns index to queue */
+static int SnifferAsyncQueueAdd(int lastRet, void* chain, int chainSz,
+    int isChain, int packetNumber)
+{
+    int ret = MEMORY_E, i, length;
+    byte* packet;
+
+#ifdef WOLFSSL_SNIFFER_CHAIN_INPUT
+    if (isChain) {
+        struct iovec* vchain = (struct iovec*)chain;
+        length = 0;
+        for (i = 0; i < chainSz; i++)
+            length += vchain[i].iov_len;
+        packet = (byte*)vchain[0].iov_base;
+    }
+    else
+#endif
+    {
+        packet = (byte*)chain;
+        length = chainSz;
+    }
+
+    /* find first free idx */
+    for (i=0; i<WOLF_ASYNC_MAX_PENDING; i++) {
+        if (asyncQueue[i].packet == NULL) {
+            if (ret == MEMORY_E) {
+                ret = i;
+                break;
+            }
+        }
+    }
+    if (ret != MEMORY_E) {
+        asyncQueue[ret].packet = XMALLOC(length, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        if (asyncQueue[ret].packet == NULL) {
+            return MEMORY_E;
+        }
+        XMEMCPY(asyncQueue[ret].packet, packet, length);
+        asyncQueue[ret].length = length;
+        asyncQueue[ret].lastRet = lastRet;
+        asyncQueue[ret].packetNumber = packetNumber;
+    }
+
+    return ret;
+}
+
+static int SnifferAsyncPollQueue(byte** data, char* err, SSLInfo* sslInfo,
+    int* queueSz)
+{
+    int ret = 0, i;
+    WOLF_EVENT* events[WOLF_ASYNC_MAX_PENDING];
+    int eventCount = 0;
+
+    /* try to process existing items in queue */
+    for (i=0; i<WOLF_ASYNC_MAX_PENDING; i++) {
+        if (asyncQueue[i].packet != NULL) {
+            (*queueSz)++;
+
+            /* do poll for events on hardware */
+            ret = ssl_PollSniffer(events, WOLF_ASYNC_MAX_PENDING,
+                WOLF_POLL_FLAG_CHECK_HW, &eventCount);
+            if (ret == 0) {
+                /* attempt to reprocess pending packet */
+            #ifdef DEBUG_SNIFFER
+                printf("Retrying packet %d\n", asyncQueue[i].packetNumber);
+            #endif
+                ret = ssl_DecodePacketAsync(asyncQueue[i].packet,
+                    asyncQueue[i].length, 0, data, err, sslInfo, NULL);
+                asyncQueue[i].lastRet = ret;
+                if (ret >= 0) {
+                    /* done, so free and break to process below */
+                    XFREE(asyncQueue[i].packet, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+                    asyncQueue[i].packet = NULL;
+                    if (ret > 0) {
+                        /* decrypted some data, so return */
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    if (ret == WC_PENDING_E) {
+        ret = 0; /* nothing new */
+    }
+    return ret;
+}
+#endif /* WOLFSSL_ASYNC_CRYPT */
+
 int main(int argc, char** argv)
 {
     int          ret = 0;
@@ -493,6 +592,7 @@ int main(int argc, char** argv)
     struct iovec chains[CHAIN_INPUT_COUNT];
     unsigned int remainder;
 #endif
+    int packetNumber = 0;
 
     show_appinfo();
 
@@ -702,23 +802,44 @@ int main(int argc, char** argv)
         frame = NULL_IF_FRAME_LEN;
 
     while (1) {
-        static int packetNumber = 0;
         struct pcap_pkthdr header;
-        const unsigned char* packet = pcap_next(pcap, &header);
+        const unsigned char* packet = NULL;
         SSLInfo sslInfo;
         void* chain = NULL;
         int   chainSz = 0;
+        byte* data = NULL; /* pointer to decrypted data */
+#ifdef WOLFSSL_ASYNC_CRYPT
+        int queueSz = 0;
+#endif
 
+#ifndef WOLFSSL_ASYNC_CRYPT
+        ret = 0; /* reset status */
+#else
+        /* poll hardware and attempt to process items in queue. If returns > 0
+         * then data pointer has decrypted something */
+        ret = SnifferAsyncPollQueue(&data, err, &sslInfo, &queueSz);
+        if (queueSz >= WOLF_ASYNC_MAX_PENDING) {
+            /* queue full, poll again */
+            continue;
+        }
+#endif
+        if (data == NULL) {
+            /* grab next pcap packet */
             packetNumber++;
+            packet = pcap_next(pcap, &header);
+        #ifdef QAT_DEBUG
+            printf("Packet Number: %d\n", packetNumber);
+        #endif
+        }
         if (packet) {
-            byte* data = NULL;
-
             if (header.caplen > 40)  { /* min ip(20) + min tcp(20) */
                 packet        += frame;
                 header.caplen -= frame;
             }
-            else
+            else {
+                /* packet doesn't contain minimum ip/tcp header */
                 continue;
+            }
 
 #ifdef WOLFSSL_SNIFFER_CHAIN_INPUT
             isChain = 1;
@@ -740,23 +861,26 @@ int main(int argc, char** argv)
 #endif
 
 #ifdef WOLFSSL_ASYNC_CRYPT
-            do {
-                WOLF_EVENT* events[WOLF_ASYNC_MAX_PENDING];
-                int eventCount = 0;
-
             /* For async call the original API again with same data,
              * or call with different sessions for multiple concurrent
              * stream processing */
             ret = ssl_DecodePacketAsync(chain, chainSz, isChain, &data, err,
                 &sslInfo, NULL);
 
-                if (ret == WC_PENDING_E) {
+            /* WC_PENDING_E: Hardware is processing */
-                    if (ssl_PollSniffer(events, 1, WOLF_POLL_FLAG_CHECK_HW,
+            /* WC_HW_WAIT_E: Hardware is already processing stream */
-                        &eventCount) != 0) {
+            if (ret == WC_PENDING_E || ret == WC_HW_WAIT_E) {
-                        break;
+                /* add to queue, for later processing */
+            #ifdef DEBUG_SNIFFER
+                printf("Steam is pending, queue packet %d\n", packetNumber);
+            #endif
+                ret = SnifferAsyncQueueAdd(ret, chain, chainSz, isChain,
+                    packetNumber);
+                if (ret >= 0) {
+                    ret = 0; /* mark event just added */
                 }
             }
-            } while (ret == WC_PENDING_E);
+
 #elif defined(WOLFSSL_SNIFFER_CHAIN_INPUT) && \
       defined(WOLFSSL_SNIFFER_STORE_DATA_CB)
             ret = ssl_DecodePacketWithChainSessionInfoStoreData(chain, chainSz,
@@ -775,12 +899,18 @@ int main(int argc, char** argv)
             (void)chain;
             (void)chainSz;
 #endif
+        }
+        
+        /* check if we are done reading file */
+        if (packet == NULL && data == NULL && saveFile) {
+            break;
+        }
 
         if (ret < 0) {
             printf("ssl_Decode ret = %d, %s\n", ret, err);
             hadBadPacket = 1;
         }
-            if (ret > 0) {
+        if (data != NULL && ret > 0) {
             /* Convert non-printable data to periods. */
             for (j = 0; j < ret; j++) {
                 if (isprint(data[j]) || isspace(data[j])) continue;
@@ -791,9 +921,6 @@ int main(int argc, char** argv)
             ssl_FreeZeroDecodeBuffer(&data, ret, err);
         }
     }
-        else if (saveFile)
-            break;      /* we're done reading file */
-    }
     FreeAll();
     (void)isChain;
 

wolfssl/sniffer_error.h

@@ -138,6 +138,7 @@
 #define CHAIN_INPUT_STR 93
 #define GOT_ENC_EXT_STR 94
 #define GOT_HELLO_RETRY_REQ_STR 95
+#define SNIFFER_KEY_SETUP_STR 96
 /* !!!! also add to msgTable in sniffer.c and .rc file !!!! */
 
 

wolfssl/sniffer_error.rc

@@ -114,4 +114,5 @@ STRINGTABLE
     93, "Loading chain input"
     94, "Got encrypted extension"
     95, "Got Hello Retry Request"
+    96, "Setting up keys"
 }