diff --git a/configure.ac b/configure.ac index 9e828c48d..5b6a455e7 100644 --- a/configure.ac +++ b/configure.ac @@ -3872,59 +3872,57 @@ AC_ARG_WITH([cavium-v], # Intel QuickAssist QAT_DIR="" BUILD_INTEL_QAT_VERSION=2 + AC_ARG_WITH([intelqa], - [ --with-intelqa=PATH PATH to Intel QuickAssist (QAT) driver dir ], - [ - AC_MSG_CHECKING([for intelqa]) - CPPFLAGS="$CPPFLAGS -DHAVE_INTEL_QA -DDO_CRYPTO -DUSER_SPACE" - OLD_LIBS="$LIBS" + [AS_HELP_STRING([--with-intelqa=PATH],[PATH to Intel QuickAssist (QAT) driver dir])], + [ENABLED_INTEL_QA=yes; QAT_DIR=$withval], + [ENABLED_INTEL_QA=no]) - if test "x$withval" = "xyes" ; then - AC_MSG_ERROR([need a PATH for --with-intelqa]) - fi - if test "x$withval" != "xno" ; then - QAT_DIR=$withval - fi +AC_ARG_WITH([intelqa-sync], + [AS_HELP_STRING([--with-intelqa-sync=PATH],[PATH to Intel QuickAssist (QAT) driver dir (sync)])], + [ENABLED_INTEL_QA_SYNC=yes; QAT_DIR=$withval], + [ENABLED_INTEL_QA_SYNC=no]) - CPPFLAGS="$CPPFLAGS -I$QAT_DIR/quickassist/include -I$QAT_DIR/quickassist/include/lac -I$QAT_DIR/quickassist/utilities/osal/include \ - -I$QAT_DIR/quickassist/utilities/osal/src/linux/user_space/include -I$QAT_DIR/quickassist/lookaside/access_layer/include \ - -I$QAT_DIR/quickassist/lookaside/access_layer/src/common/include -I$srcdir/wolfssl -I$srcdir/wolfssl/wolfcrypt/port/intel \ - -I$QAT_DIR/quickassist/utilities/libusdm_drv" +AS_IF([test "x$ENABLED_INTEL_QA" = "xyes" && test "x$ENABLED_INTEL_QA_SYNC" = "xyes"], + [AC_MSG_ERROR([Both Intel QA Async and Sync are selected, only select one.])]) - LDFLAGS="$LDFLAGS -L$QAT_DIR/build" - LIBS="$LIBS -lqat_s -lusdm_drv_s" +AS_IF([test "x$ENABLED_INTEL_QA" = "xyes" || test "x$ENABLED_INTEL_QA_SYNC" = "xyes"], + [AC_MSG_CHECKING([for intelqa]) + AS_IF([test "x$ENABLED_INTEL_QA" = "xyes"], + [AM_CPPFLAGS="$AM_CPPFLAGS -DHAVE_INTEL_QA -DDO_CRYPTO -DUSER_SPACE"; intelqa_opt=""], + [AM_CPPFLAGS="$AM_CPPFLAGS -DHAVE_INTEL_QA_SYNC -DQAT_USE_POLLING_THREAD -DO_CRYPTO -DUSER_SPACE"; intelqa_opt="-sync"]) + OLD_LIBS="$LIBS" + OLD_CPPFLAGS="$CPPFLAGS" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include "cpa_cy_common.h"]], [[ Cpa16U count = 0; cpaCyGetNumInstances(&count); ]])], - [ intelqa_linked=yes ],[ intelqa_linked=no ]) - if test "x$intelqa_linked" = "xno" ; then - # Try old QAT driver libraries + AS_IF([test "x$QAT_DIR" = "xyes"],[AC_MSG_ERROR([need a PATH for --with-intelqa$intelqa_opt])]) + + AM_CPPFLAGS="$AM_CPPFLAGS -I$QAT_DIR/quickassist/include -I$QAT_DIR/quickassist/include/lac -I$QAT_DIR/quickassist/utilities/osal/include \ + -I$QAT_DIR/quickassist/utilities/osal/src/linux/user_space/include -I$QAT_DIR/quickassist/lookaside/access_layer/include \ + -I$QAT_DIR/quickassist/lookaside/access_layer/src/common/include -I$srcdir/wolfssl -I$srcdir/wolfssl/wolfcrypt/port/intel \ + -I$QAT_DIR/quickassist/utilities/libusdm_drv" + CPPFLAGS="$AM_CPPFLAGS" + + LDFLAGS="$LDFLAGS -L$QAT_DIR/build" + LIBS="$LIBS -lqat_s -lusdm_drv_s" + + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include "cpa_cy_common.h"]],[[Cpa16U count = 0; cpaCyGetNumInstances(&count);]])],[intelqa_linked=yes],[intelqa_linked=no]) + + AS_IF([test "x$intelqa_linked" = "xno"], + [# Try old QAT driver libraries LIBS="$OLD_LIBS -licp_qa_al_s" + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include "cpa_cy_common.h"]],[[Cpa16U count = 0; cpaCyGetNumInstances(&count);]])],[intelqa_linked=yes],[intelqa_linked=no]) + AS_IF([test "x$intelqa_linked" = "xno"], + [AC_MSG_ERROR([Intel QuickAssist not found. If it's already installed, specify its path using --with-intelqa$intelqa_opt=/dir/])], + [BUILD_INTEL_QAT_VERSION=1]) + ]) - AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include "cpa_cy_common.h"]], [[ Cpa16U count = 0; cpaCyGetNumInstances(&count); ]])], - [ intelqa_linked=yes ],[ intelqa_linked=no ]) - if test "x$intelqa_linked" = "xno" ; then - AC_MSG_ERROR([Intel QuickAssist not found. - If it's already installed, specify its path using --with-intelqa=/dir/]) - else - BUILD_INTEL_QAT_VERSION=1 - fi - else - AM_CFLAGS="$AM_CFLAGS -DHAVE_INTEL_QA -DDO_CRYPTO -DUSER_SPACE" - fi - AC_MSG_RESULT([yes]) + AC_MSG_RESULT([yes]) - if test "x$BUILD_INTEL_QAT_VERSION" == "x1" ; then - LIB_ADD="-ladf_proxy -losal -lrt $LIB_ADD" - else - LIB_ADD="-losal -lrt $LIB_ADD" - fi - - ENABLED_INTEL_QA=yes - ], - [ - ENABLED_INTEL_QA=no - ] -) + AS_IF([test "x$BUILD_INTEL_QAT_VERSION" == "x1"], + [LIB_ADD="-ladf_proxy -losal -lrt $LIB_ADD"], + [LIB_ADD="-losal -lrt $LIB_ADD"]) + CPPFLAGS="$OLD_CPPFLAGS" +]) # Single Precision maths implementation @@ -4354,7 +4352,7 @@ fi # check for async if using Intel QuckAssist or Cavium if test "x$ENABLED_INTEL_QA" = "xyes" || test "x$ENABLED_CAVIUM" = "xyes" ; then if test "x$ENABLED_ASYNCCRYPT" = "xno" ; then - AC_MSG_ERROR([Please enable enable asynchronous support using --enable-asynccrypt]) + AC_MSG_ERROR([Please enable asynchronous support using --enable-asynccrypt]) fi fi @@ -4569,6 +4567,10 @@ AS_IF([test "x$ENABLED_OCSP" = "xyes" && \ test "x$ENABLED_ECC" = "xno"], [AC_MSG_ERROR([please enable rsa or ecc if enabling ocsp.])]) +# Sync Intel QA requires the crypto callback +AS_IF([test "x$ENABLED_CRYPTOCB" = "xno" && test "x$ENABLED_INTEL_QA_SYNC" = "xyes"], + [AC_MSG_ERROR([please enable the crypto callback support using --enable-cryptocb])]) + # checks for pkcs7 needed enables AS_IF([test "x$ENABLED_PKCS7" = "xyes" && \ test "x$ENABLED_RSA" = "xno" && \ @@ -4794,7 +4796,7 @@ if test "$ENABLED_DISTRO" = "no" then CFLAGS="$CFLAGS $USER_C_EXTRA_FLAGS" fi -OPTION_FLAGS="$USER_CFLAGS $USER_C_EXTRA_FLAGS $CPPFLAGS $AM_CFLAGS" +OPTION_FLAGS="$USER_CFLAGS $USER_C_EXTRA_FLAGS $CPPFLAGS $AM_CPPFLAGS $CFLAGS $AM_CFLAGS" # The following AM_CONDITIONAL statements set flags for use in the Makefiles. @@ -4886,6 +4888,7 @@ AM_CONDITIONAL([BUILD_PKCS11],[test "x$ENABLED_PKCS11" = "xyes"]) AM_CONDITIONAL([BUILD_CAVIUM],[test "x$ENABLED_CAVIUM" = "xyes"]) AM_CONDITIONAL([BUILD_CAVIUM_V],[test "x$ENABLED_CAVIUM_V" = "xyes"]) AM_CONDITIONAL([BUILD_INTEL_QA],[test "x$ENABLED_INTEL_QA" = "xyes"]) +AM_CONDITIONAL([BUILD_INTEL_QA_SYNC],[test "x$ENABLED_INTEL_QA_SYNC" = "xyes"]) AM_CONDITIONAL([BUILD_SP],[test "x$ENABLED_SP" = "xyes"]) AM_CONDITIONAL([BUILD_SP_C],[test "x$ENABLED_SP" = "xyes" && test "x$ENABLED_SP_ASM" = "xno" ]) AM_CONDITIONAL([BUILD_SP_ARM64],[test "x$ENABLED_SP_ARM64_ASM" = "xyes" ]) diff --git a/src/sniffer.c b/src/sniffer.c index 65c3e825c..b7d08471f 100644 --- a/src/sniffer.c +++ b/src/sniffer.c @@ -54,6 +54,13 @@ #include #endif +#ifdef WOLF_CRYPTO_CB + #include + #ifdef HAVE_INTEL_QA_SYNC + #include + #endif +#endif + #ifndef WOLFSSL_SNIFFER_TIMEOUT #define WOLFSSL_SNIFFER_TIMEOUT 900 @@ -449,6 +456,14 @@ static void UpdateMissedDataSessions(void) #endif +#ifdef WOLF_CRYPTO_CB + static int CryptoDeviceId = INVALID_DEVID; + #ifdef HAVE_INTEL_QA_SYNC + static IntelQaDev CryptoDevice; + #endif +#endif + + /* Initialize overall Sniffer */ void ssl_InitSniffer(void) { @@ -460,6 +475,26 @@ void ssl_InitSniffer(void) XMEMSET(&SnifferStats, 0, sizeof(SSLStats)); wc_InitMutex(&StatsMutex); #endif +#ifdef WOLF_CRYPTO_CB + #ifdef HAVE_INTEL_QA_SYNC + { + int rc; + CryptoDeviceId = IntelQaInit(NULL); + if (CryptoDeviceId == INVALID_DEVID) { + WOLFSSL_MSG("Couldn't init the Intel QA"); + } + rc = IntelQaOpen(&CryptoDevice, CryptoDeviceId); + if (rc != 0) { + WOLFSSL_MSG("Couldn't open the device"); + } + rc = wc_CryptoCb_RegisterDevice(CryptoDeviceId, + IntelQaSymSync_CryptoDevCb, &CryptoDevice); + if (rc != 0) { + WOLFSSL_MSG("Couldn't register the device"); + } + } + #endif +#endif } @@ -1335,6 +1370,9 @@ static int CreateWatchSnifferServer(char* error) FreeSnifferServer(sniffer); return -1; } +#ifdef WOLF_CRYPTO_CB + wolfSSL_CTX_SetDevId(sniffer->ctx, CryptoDevId); +#endif ServerList = sniffer; return 0; @@ -1433,6 +1471,9 @@ static int SetNamedPrivateKey(const char* name, const char* address, int port, FreeSnifferServer(sniffer); return -1; } + #ifdef WOLF_CRYPTO_CB + wolfSSL_CTX_SetDevId(sniffer->ctx, CryptoDeviceId); + #endif } #ifdef HAVE_SNI else { diff --git a/wolfcrypt/src/des3.c b/wolfcrypt/src/des3.c index fdd18e0ed..f8575750c 100644 --- a/wolfcrypt/src/des3.c +++ b/wolfcrypt/src/des3.c @@ -1457,6 +1457,12 @@ } #endif /* WOLFSSL_ASYNC_CRYPT */ + #ifdef WOLF_CRYPTO_CB + if (des->devId != INVALID_DEVID) { + XMEMCPY(des->devKey, key, DES3_KEYLEN); + } + #endif + ret = DesSetKey(key + (dir == DES_ENCRYPTION ? 0:16), dir, des->key[0]); if (ret != 0) return ret; diff --git a/wolfcrypt/src/include.am b/wolfcrypt/src/include.am index 06bf677fe..8287ab5b9 100644 --- a/wolfcrypt/src/include.am +++ b/wolfcrypt/src/include.am @@ -102,15 +102,17 @@ endif if BUILD_CAVIUM src_libwolfssl_la_SOURCES += wolfcrypt/src/port/cavium/cavium_nitrox.c - -EXTRA_DIST += wolfcrypt/src/port/cavium/README.md endif +EXTRA_DIST += wolfcrypt/src/port/cavium/README.md if BUILD_INTEL_QA src_libwolfssl_la_SOURCES += wolfcrypt/src/port/intel/quickassist.c src_libwolfssl_la_SOURCES += wolfcrypt/src/port/intel/quickassist_mem.c - +endif EXTRA_DIST += wolfcrypt/src/port/intel/README.md + +if BUILD_INTEL_QA_SYNC +src_libwolfssl_la_SOURCES += wolfcrypt/src/port/intel/quickassist_sync.c endif if BUILD_CRYPTOAUTHLIB diff --git a/wolfcrypt/src/port/intel/quickassist_sync.c b/wolfcrypt/src/port/intel/quickassist_sync.c new file mode 100644 index 000000000..79e250f76 --- /dev/null +++ b/wolfcrypt/src/port/intel/quickassist_sync.c @@ -0,0 +1,1763 @@ +/* quickassist_sync.c + * + * Copyright (C) 2006-2019 wolfSSL Inc. + * + * This file is part of wolfSSL. (formerly known as CyaSSL) + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + */ + +#ifdef HAVE_CONFIG_H + #include +#endif + +#include + +#ifdef HAVE_INTEL_QA_SYNC + +#ifdef QAT_DEMO_MAIN + #define QAT_DEBUG +#endif + + +#include +#include +#include +#ifndef NO_AES + #include +#endif + +#include +#include + +#include "icp_sal_user.h" +#include "icp_sal_poll.h" + + +#ifdef NO_INLINE + #include +#else + #define WOLFSSL_MISC_INCLUDED + #include +#endif + + +#ifdef QAT_DEBUG + #define QLOG(...) do { printf(__VA_ARGS__); } while (0) +#else + #define QLOG(...) +#endif + + +#define OS_HOST_TO_NW_32(uData) ByteReverseWord32(uData) + + +static CpaInstanceHandle* g_cyInstances = NULL; +static CpaInstanceInfo2* g_cyInstanceInfo = NULL; +static Cpa32U* g_cyInstMap = NULL; +static Cpa16U g_numInstances = 0; +static Cpa16U g_instCounter = 0; +static CpaBoolean g_cyServiceStarted = CPA_FALSE; +#ifdef QAT_USE_POLLING_CHECK + static CpaBoolean* g_cyPolling = NULL; + static pthread_mutex_t* g_PollLock; +#endif +static volatile int g_initCount = 0; +static pthread_mutex_t g_Hwlock = PTHREAD_MUTEX_INITIALIZER; + + +typedef struct qatCapabilities { + /* capabilities */ + word32 supPartial:1; + word32 supSha3:1; +} qatCapabilities_t; +static qatCapabilities_t g_qatCapabilities = {0}; + + +#if defined(QAT_ENABLE_CRYPTO) + static int IntelQaSymClose(IntelQaDev* dev, int doFree); +#endif + + +extern Cpa32U osalLogLevelSet(Cpa32U level); + + +/* -------------------------------------------------------------------------- */ +/* Polling */ +/* -------------------------------------------------------------------------- */ + +static WC_INLINE int SyncSleep(word32 ms) +{ + int ret = 0; + struct timespec resTime, remTime; + resTime.tv_sec = ms/1000; + resTime.tv_nsec = (ms%1000)*1000000; + do { + ret = nanosleep(&resTime, &remTime); + resTime = remTime; + } while ((ret!=0) && (errno == EINTR)); + + if (ret != 0) { + QLOG("nanoSleep failed with code %d\n", ret); + return BAD_FUNC_ARG; + } + + return ret; +} + +#ifdef QAT_USE_POLLING_THREAD +static void* IntelQaPollingThread(void* context) +{ + IntelQaDev* dev = (IntelQaDev*)context; + + QLOG("Polling Thread Start\n"); + while (dev->pollingCy) { + icp_sal_CyPollInstance(dev->handle, QAT_POLL_RESP_QUOTA); + SyncSleep(10); + } + QLOG("Polling Thread Exit\n"); + pthread_exit(NULL); +} + +static CpaStatus IntelQaStartPollingThread(IntelQaDev* dev) +{ + if (dev->pollingCy == 0) { + dev->pollingCy = 1; + + QLOG("Polling Thread Created\n"); + + if (pthread_create(&dev->pollingThread, NULL, IntelQaPollingThread, + (void*)dev) != 0) { + QLOG("Failed create polling thread!\n"); + return CPA_STATUS_FAIL; + } + } + return CPA_STATUS_SUCCESS; +} + +static void IntelQaStopPollingThread(IntelQaDev* dev) +{ + dev->pollingCy = 0; + pthread_join(dev->pollingThread, 0); +} +#endif /* QAT_USE_POLLING_THREAD */ + + +/* -------------------------------------------------------------------------- */ +/* Device */ +/* -------------------------------------------------------------------------- */ +void IntelQaHardwareStop(void) +{ + int i; + CpaStatus status; + + g_initCount--; /* track de-init count */ + if (g_initCount != 0) { + return; + } + + if (g_cyServiceStarted == CPA_TRUE) { + g_cyServiceStarted = CPA_FALSE; + for (i=0; i 1) { + return 0; + } + + status = qaeMemInit(); + if (status != CPA_STATUS_SUCCESS) { + QLOG("IntelQA: Could not start qae mem for user space (status %d)\n" + "\tHas the qaeMemDrv.ko module been loaded?\n", + status); + return ASYNC_INIT_E; + } + + status = icp_sal_userStartMultiProcess(process_name, + limitDevAccess ? CPA_TRUE : CPA_FALSE); + if (status != CPA_STATUS_SUCCESS) { + QLOG("IntelQA: Could not start sal for user space! status %d\n", + status); + ret = ASYNC_INIT_E; goto error; + } + +#ifdef QAT_DEBUG + /* optionally enable debugging */ + //osalLogLevelSet(8); +#endif + + status = cpaCyGetNumInstances(&g_numInstances); + if (status != CPA_STATUS_SUCCESS || g_numInstances == 0) { + QLOG("IntelQA: Failed to get num of instances! status %d\n", status); + ret = INVALID_DEVID; goto error; + } + + /* Get handles / info */ + g_cyInstances = (CpaInstanceHandle*)XMALLOC( + sizeof(CpaInstanceHandle) * g_numInstances, NULL, DYNAMIC_TYPE_ASYNC); + if (g_cyInstances == NULL) { + QLOG("IntelQA: Failed to allocate instances\n"); + ret = INVALID_DEVID; goto error; + } + +#ifdef QAT_USE_POLLING_CHECK + g_cyPolling = (CpaBoolean*)XMALLOC(sizeof(CpaBoolean) * g_numInstances, NULL, + DYNAMIC_TYPE_ASYNC); + if (g_cyPolling == NULL) { + QLOG("IntelQA: Failed to allocate polling status\n"); + ret = INVALID_DEVID; goto error; + } + g_PollLock = (pthread_mutex_t*)XMALLOC(sizeof(pthread_mutex_t) * + g_numInstances, NULL, DYNAMIC_TYPE_ASYNC); + if (g_PollLock == NULL) { + QLOG("IntelQA: Failed to allocate polling locks\n"); + ret = INVALID_DEVID; goto error; + } + for (i=0; i> 8), + (Cpa8U)((g_cyInstanceInfo[i].physInstId.busAddress) + & 0xFF) >> 3, + (Cpa8U)((g_cyInstanceInfo[i].physInstId.busAddress) & 3), + g_cyInstanceInfo[i].isPolled); + + status = cpaCySetAddressTranslation(g_cyInstances[i], + qaeVirtToPhysNUMA); + if (status != CPA_STATUS_SUCCESS) { + QLOG("IntelQA: Error setting memory config for inst %d\n", i); + ret = INVALID_DEVID; goto error; + } + + status = cpaCyStartInstance(g_cyInstances[i]); + if (status != CPA_STATUS_SUCCESS) { + QLOG("IntelQA: Error starting crypto instance %d\n", i); + ret = INVALID_DEVID; goto error; + } + } + + QLOG("IntelQA: Instances %d\n", g_numInstances); + return ret; + +error: + IntelQaHardwareStop(); + return ret; +} + + +int IntelQaInit(void* threadId) +{ + int ret; + int devId; + (void)threadId; + + ret = pthread_mutex_lock(&g_Hwlock); + if (ret != 0) { + QLOG("IntelQaInit: mutex lock failed! %d\n", ret); + return BAD_MUTEX_E; + } + + ret = IntelQaHardwareStart(QAT_PROCESS_NAME, QAT_LIMIT_DEV_ACCESS); + if (ret != 0) { + pthread_mutex_unlock(&g_Hwlock); + return ret; + } + + if (g_numInstances <= 0) { + pthread_mutex_unlock(&g_Hwlock); + return ASYNC_INIT_E; + } + + /* assign device id */ + devId = (g_instCounter % g_numInstances); + g_instCounter++; + + pthread_mutex_unlock(&g_Hwlock); + + return devId; +} + + +int IntelQaNumInstances(void) +{ + return g_numInstances; +} + + +int IntelQaOpen(IntelQaDev* dev, int devId) +{ + if (dev == NULL) { + return BAD_FUNC_ARG; + } + + /* clear device info */ + XMEMSET(dev, 0, sizeof(IntelQaDev)); + + if (g_cyInstances == NULL) { + QLOG("IntelQA not initialized\n"); + return ASYNC_INIT_E; + } + + dev->devId = devId; + dev->handle = g_cyInstances[devId]; + +#ifdef QAT_USE_POLLING_THREAD + /* start polling thread */ + IntelQaStartPollingThread(dev); +#endif + + return 0; +} + + +#if defined(QAT_ENABLE_CRYPTO) + +static IntelQaSymCtx* IntelQaGetSymCtx(IntelQaDev* dev) +{ + return &dev->op.cipher.ctx; +} + +#endif + + +void IntelQaClose(IntelQaDev* dev) +{ + if (dev) { + QLOG("IntelQaClose %p\n", dev); + /* close any active session */ + IntelQaSymClose(dev, 1); + + #ifdef QAT_USE_POLLING_THREAD + IntelQaStopPollingThread(dev); + #endif + + dev->handle = NULL; + } +} + +void IntelQaDeInit(int devId) +{ + (void)devId; + + if (pthread_mutex_lock(&g_Hwlock) == 0) { + IntelQaHardwareStop(); + pthread_mutex_unlock(&g_Hwlock); + } +} + +int IntelQaPoll(IntelQaDev* dev) +{ + int ret = 0; + CpaStatus status; + +#ifdef QAT_USE_POLLING_CHECK + pthread_mutex_t* lock = &g_PollLock[dev->qat.devId]; + if (pthread_mutex_lock(lock) == 0) { + /* test if any other threads are polling */ + if (g_cyPolling[dev->qat.devId]) { + pthread_mutex_unlock(lock); + + /* return success even though its busy, caller will treat as WC_PENDING_E */ + return 0; + } + + g_cyPolling[dev->qat.devId] = 1; + pthread_mutex_unlock(lock); + } +#endif + + status = icp_sal_CyPollInstance(dev->handle, QAT_POLL_RESP_QUOTA); + if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_RETRY) { + QLOG("IntelQa: Poll failure %d\n", status); + ret = -1; + } + + { + if (dev->ret != WC_PENDING_E) { + /* perform cleanup */ + IntelQaFreeFunc freeFunc = dev->freeFunc; + QLOG("IntelQaOpFree: Dev %p, FreeFunc %p\n", dev, freeFunc); + if (freeFunc) { + dev->freeFunc = NULL; + freeFunc(dev); + } + } + } + +#ifdef QAT_USE_POLLING_CHECK + /* indicate we are done polling */ + if (pthread_mutex_lock(lock) == 0) { + g_cyPolling[dev->qat.devId] = 0; + pthread_mutex_unlock(lock); + } +#endif + + return ret; +} + +static int IntelQaPollBlockRet(IntelQaDev* dev, int ret_wait) +{ + int ret; + + do { + ret = IntelQaPoll(dev); + + if (dev->ret != ret_wait) { + break; + } + } while (1); + ret = dev->ret; + + return ret; +} + +int IntelQaGetCyInstanceCount(void) +{ + return g_numInstances; +} + +static WC_INLINE int IntelQaHandleCpaStatus(IntelQaDev* dev, CpaStatus status, + int* ret, byte isAsync, void* callback, int* retryCount) +{ + int retry = 0; + + if (status == CPA_STATUS_SUCCESS) { + if (isAsync && callback) { + *ret = WC_PENDING_E; + } + else { + *ret = IntelQaPollBlockRet(dev, WC_PENDING_E); + } + } + else if (status == CPA_STATUS_RETRY) { + (*retryCount)++; + if ((*retryCount % (QAT_RETRY_LIMIT + 1)) == QAT_RETRY_LIMIT) { + SyncSleep(10); + } + retry = 1; + } + else { + *ret = ASYNC_OP_E; + } + + return retry; +} + +static WC_INLINE void IntelQaOpInit(IntelQaDev* dev, IntelQaFreeFunc freeFunc) +{ + dev->ret = WC_PENDING_E; + dev->freeFunc = freeFunc; +} + + +/* -------------------------------------------------------------------------- */ +/* Symmetric Algos */ +/* -------------------------------------------------------------------------- */ + +#if defined(QAT_ENABLE_CRYPTO) + +static int IntelQaSymOpen(IntelQaDev* dev, CpaCySymSessionSetupData* setup, + CpaCySymCbFunc callback) +{ + int ret = 0; + CpaStatus status = CPA_STATUS_SUCCESS; + Cpa32U sessionCtxSize = 0; + IntelQaSymCtx* ctx; + + /* arg check */ + if (dev == NULL || setup == NULL) { + return BAD_FUNC_ARG; + } + + ctx = IntelQaGetSymCtx(dev); + + /* Determine size of session context to allocate - use max size */ + status = cpaCySymSessionCtxGetSize(dev->handle, setup, &sessionCtxSize); + + if (ctx->symCtxSize > 0 && ctx->symCtxSize > sessionCtxSize) { + QLOG("Symmetric context size error! Buf %d, Exp %d\n", + ctx->symCtxSize, sessionCtxSize); + return ASYNC_OP_E; + } + + /* make sure session context is allocated */ + if (ctx->symCtx == NULL) { + /* Allocate session context */ + ctx->symCtx = XMALLOC(sessionCtxSize, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA64); + if (ctx->symCtx == NULL) { + return MEMORY_E; + } + } + ctx->symCtxSize = sessionCtxSize; + + if (!ctx->isOpen) { + ctx->isOpen = 1; + + QLOG("IntelQaSymOpen: InitSession dev %p, symCtx %p\n", + dev, ctx->symCtx); + + /* open symmetric session */ + status = cpaCySymInitSession(dev->handle, callback, setup, ctx->symCtx); + if (status != CPA_STATUS_SUCCESS) { + QLOG("cpaCySymInitSession failed! dev %p, status %d\n", + dev, status); + XFREE(ctx->symCtx, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA64); + ctx->symCtx = NULL; + return ASYNC_INIT_E; + } + } + + if (ctx->symCtxSrc == NULL) { + ctx->symCtxSrc = ctx->symCtx; + } + + QLOG("IntelQaSymOpen: dev %p, symCtx %p (src %p), " + "symCtxSize %d, isCopy %d, isOpen %d\n", + dev, ctx->symCtx, ctx->symCtxSrc, ctx->symCtxSize, + ctx->isCopy, ctx->isOpen); + + return ret; +} + +static int IntelQaSymClose(IntelQaDev* dev, int doFree) +{ + int ret = 0; + CpaStatus status = CPA_STATUS_SUCCESS; + IntelQaSymCtx* ctx; + + if (dev == NULL) { + return BAD_FUNC_ARG; + } + + ctx = IntelQaGetSymCtx(dev); + + QLOG("IntelQaSymClose: dev %p, ctx %p, symCtx %p (src %p), " + "symCtxSize %d, isCopy %d, isOpen %d, doFree %d\n", + dev, ctx, ctx->symCtx, ctx->symCtxSrc, ctx->symCtxSize, + ctx->isCopy, ctx->isOpen, doFree); + + if (ctx->symCtx == ctx->symCtxSrc && ctx->symCtx != NULL) { + if (ctx->isOpen) { + ctx->isOpen = 0; + QLOG("IntelQaSymClose: RemoveSession dev %p, symCtx %p\n", + dev, ctx->symCtx); + status = cpaCySymRemoveSession(dev->handle, ctx->symCtx); + if (status == CPA_STATUS_RETRY) { + QLOG("cpaCySymRemoveSession retry!\n"); + /* treat this as error, since session should not be active */ + ret = ASYNC_OP_E; + } + else if (status != CPA_STATUS_SUCCESS) { + QLOG("cpaCySymRemoveSession failed! status %d\n", status); + ret = ASYNC_OP_E; + } + } + } + + if (doFree) { + XFREE(ctx->symCtx, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA64); + ctx->symCtx = NULL; + ctx->symCtxSrc = NULL; + ctx->symCtxSize = 0; + } + + return ret; +} + +#endif /* QAT_ENABLE_CRYPTO */ + + +/* -------------------------------------------------------------------------- */ +/* AES/DES Algo */ +/* -------------------------------------------------------------------------- */ + +#ifdef QAT_ENABLE_CRYPTO + +static void IntelQaSymCipherFree(IntelQaDev* dev) +{ + IntelQaSymCtx* ctx = &dev->op.cipher.ctx; + CpaCySymOpData* opData = &ctx->opData; + CpaBufferList* pDstBuffer = &dev->op.cipher.bufferList; + + if (opData) { + if (opData->pAdditionalAuthData) { + XFREE(opData->pAdditionalAuthData, dev->heap, + DYNAMIC_TYPE_ASYNC_NUMA); + opData->pAdditionalAuthData = NULL; + } + if (opData->pIv) { + XFREE(opData->pIv, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); + opData->pIv = NULL; + } + XMEMSET(opData, 0, sizeof(CpaCySymOpData)); + } + if (pDstBuffer) { + if (pDstBuffer->pBuffers) { + if (pDstBuffer->pBuffers->pData) { + XFREE(pDstBuffer->pBuffers->pData, dev->heap, + DYNAMIC_TYPE_ASYNC_NUMA); + pDstBuffer->pBuffers->pData = NULL; + } + XMEMSET(pDstBuffer->pBuffers, 0, sizeof(CpaFlatBuffer)); + } + if (pDstBuffer->pPrivateMetaData) { + XFREE(pDstBuffer->pPrivateMetaData, dev->heap, + DYNAMIC_TYPE_ASYNC_NUMA); + pDstBuffer->pPrivateMetaData = NULL; + } + XMEMSET(pDstBuffer, 0, sizeof(CpaBufferList)); + } + + /* close and free sym context */ + IntelQaSymClose(dev, 1); + + /* clear temp pointers */ + dev->out = NULL; + dev->outLen = 0; +#ifndef NO_AES + dev->op.cipher.authTag = NULL; + dev->op.cipher.authTagSz = 0; +#endif +} + +static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in, + word32 inOutSz, const byte* key, word32 keySz, const byte* iv, word32 ivSz, + CpaCySymOp symOperation, CpaCySymCipherAlgorithm cipherAlgorithm, + CpaCySymCipherDirection cipherDirection, + + /* for auth ciphers (CCM or GCM) */ + CpaCySymHashAlgorithm hashAlgorithm, + byte* authTag, word32 authTagSz, + const byte* authIn, word32 authInSz) +{ + int ret; + CpaStatus status = CPA_STATUS_SUCCESS; + CpaCySymOpData* opData = NULL; + CpaCySymSessionSetupData setup; + const Cpa32U numBuffers = 1; + CpaBufferList* bufferList = NULL; + CpaFlatBuffer* flatBuffer = NULL; + Cpa8U* ivBuf = NULL; + Cpa8U* dataBuf = NULL; + Cpa32U dataLen = inOutSz; + Cpa8U* metaBuf = NULL; + Cpa32U metaSize = 0; + Cpa8U* authInBuf = NULL; + Cpa32U authInSzAligned = authInSz; + IntelQaSymCtx* ctx; + + QLOG("IntelQaSymCipher: dev %p, out %p, in %p, inOutSz %d, op %d, " + "algo %d, dir %d, hash %d\n", + dev, out, in, inOutSz, symOperation, cipherAlgorithm, + cipherDirection, hashAlgorithm); + + /* check args */ + if (out == NULL || in == NULL || inOutSz == 0 || + key == NULL || keySz == 0 || iv == NULL || ivSz == 0) { + return BAD_FUNC_ARG; + } + if (hashAlgorithm != CPA_CY_SYM_HASH_NONE && + (authTag == NULL || authTagSz == 0)) { + return BAD_FUNC_ARG; + } + + /* get meta size */ + status = cpaCyBufferListGetMetaSize(dev->handle, numBuffers, &metaSize); + if (status != CPA_STATUS_SUCCESS && metaSize <= 0) { + ret = BUFFER_E; goto exit; + } + + /* if authtag provided then it will be appended to end of input */ + if (authTag && authTagSz > 0) { + dataLen += authTagSz; + } + + /* allocate buffers */ + ctx = &dev->op.cipher.ctx; + opData = &ctx->opData; + bufferList = &dev->op.cipher.bufferList; + flatBuffer = &dev->op.cipher.flatBuffer; + metaBuf = XMALLOC(metaSize, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); +#ifndef WOLFSSL_SNIFFER + dataBuf = XREALLOC((byte*)in, dataLen, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); + ivBuf = XREALLOC((byte*)iv, AES_BLOCK_SIZE, dev->heap, + DYNAMIC_TYPE_ASYNC_NUMA); +#else + dataBuf = XMALLOC(dataLen, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); + XMEMCPY(dataBuf, in, inOutSz); + ivBuf = XMALLOC(AES_BLOCK_SIZE, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA); + XMEMCPY(ivBuf, iv, ivSz); +#endif + + /* check allocations */ + if (ivBuf == NULL || metaBuf == NULL || dataBuf == NULL) { + ret = MEMORY_E; goto exit; + } + + /* AAD */ + if (authIn && authInSz > 0) { + /* make sure AAD is block aligned */ + if (authInSzAligned % AES_BLOCK_SIZE) { + authInSzAligned += AES_BLOCK_SIZE - + (authInSzAligned % AES_BLOCK_SIZE); + } + +#ifndef WOLFSSL_SNIFFER + authInBuf = XREALLOC((byte*)authIn, authInSzAligned, dev->heap, + DYNAMIC_TYPE_ASYNC_NUMA); +#else + authInBuf = XMALLOC(authInSzAligned, dev->heap, + DYNAMIC_TYPE_ASYNC_NUMA); + XMEMCPY(authInBuf, authIn, authInSz); +#endif + if (authInBuf == NULL) { + ret = MEMORY_E; goto exit; + } + /* clear remainder */ + XMEMSET(authInBuf + authInSz, 0, authInSzAligned - authInSz); + } + + /* init buffers */ + XMEMSET(&setup, 0, sizeof(CpaCySymSessionSetupData)); + XMEMSET(opData, 0, sizeof(CpaCySymOpData)); + XMEMSET(bufferList, 0, sizeof(CpaBufferList)); + XMEMSET(flatBuffer, 0, sizeof(CpaFlatBuffer)); + XMEMSET(metaBuf, 0, metaSize); + + bufferList->pBuffers = flatBuffer; + bufferList->numBuffers = numBuffers; + bufferList->pPrivateMetaData = metaBuf; + flatBuffer->dataLenInBytes = dataLen; + flatBuffer->pData = dataBuf; + + /* setup */ + setup.sessionPriority = CPA_CY_PRIORITY_NORMAL; + setup.symOperation = symOperation; + setup.cipherSetupData.cipherAlgorithm = cipherAlgorithm; + setup.cipherSetupData.cipherKeyLenInBytes = keySz; + setup.cipherSetupData.pCipherKey = (byte*)key; + setup.cipherSetupData.cipherDirection = cipherDirection; + + /* setup auth ciphers */ + if (hashAlgorithm != CPA_CY_SYM_HASH_NONE) { + setup.algChainOrder = + (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) ? + CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH : + CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; + + setup.hashSetupData.hashAlgorithm = hashAlgorithm; + setup.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH; + setup.hashSetupData.digestResultLenInBytes = authTagSz; + setup.hashSetupData.authModeSetupData.aadLenInBytes = authInSz; + + setup.digestIsAppended = CPA_TRUE; + } + + /* open session */ + ret = IntelQaSymOpen(dev, &setup, NULL); + if (ret != 0) { + goto exit; + } + + /* operation data */ + opData->sessionCtx = ctx->symCtx; + opData->packetType = CPA_CY_SYM_PACKET_TYPE_FULL; + opData->pIv = ivBuf; + opData->ivLenInBytes = ivSz; + opData->cryptoStartSrcOffsetInBytes = 0; + opData->messageLenToCipherInBytes = inOutSz; + if (authIn && authInSz > 0) { + opData->pAdditionalAuthData = authInBuf; + } + if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT) { + if (authTag && authTagSz > 0) { + /* append digest to end of data buffer */ + XMEMCPY(flatBuffer->pData + inOutSz, authTag, authTagSz); + } + } + + /* store info needed for output */ + dev->out = out; + dev->outLen = inOutSz; + if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) { + dev->op.cipher.authTag = authTag; + dev->op.cipher.authTagSz = authTagSz; + } + else { + dev->op.cipher.authTag = NULL; + dev->op.cipher.authTagSz = 0; + } + IntelQaOpInit(dev, IntelQaSymCipherFree); + + /* perform symmetric AES operation async */ + /* use same buffer list for in-place operation */ + status = cpaCySymPerformOp(dev->handle, dev, opData, + bufferList, bufferList, NULL); + + if (ret == WC_PENDING_E) + return ret; + +exit: + + if (ret != 0) { + QLOG("cpaCySymPerformOp Cipher failed! dev %p, status %d, ret %d\n", + dev, status, ret); + } + +#ifdef WOLFSSL_SNIFFER + /* Capture the inline decrypt into the output. */ + XMEMCPY(out, dataBuf, inOutSz); +#endif + + /* handle cleanup */ + IntelQaSymCipherFree(dev); + + return ret; +} + +#ifdef HAVE_AES_CBC +int IntelQaSymAesCbcEncrypt(IntelQaDev* dev, + byte* out, const byte* in, word32 sz, + const byte* key, word32 keySz, + const byte* iv, word32 ivSz) +{ + return IntelQaSymCipher(dev, out, in, sz, + key, keySz, iv, ivSz, + CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_AES_CBC, + CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT, + CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0); +} + +#ifdef HAVE_AES_DECRYPT +int IntelQaSymAesCbcDecrypt(IntelQaDev* dev, + byte* out, const byte* in, word32 sz, + const byte* key, word32 keySz, + const byte* iv, word32 ivSz) +{ + return IntelQaSymCipher(dev, out, in, sz, + key, keySz, iv, ivSz, + CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_AES_CBC, + CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT, + CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0); +} +#endif /* HAVE_AES_DECRYPT */ +#endif /* HAVE_AES_CBC */ + + +#ifdef HAVE_AESGCM +int IntelQaSymAesGcmEncrypt(IntelQaDev* dev, + byte* out, const byte* in, word32 sz, + const byte* key, word32 keySz, + const byte* iv, word32 ivSz, + byte* authTag, word32 authTagSz, + const byte* authIn, word32 authInSz) +{ + return IntelQaSymCipher(dev, out, in, sz, + key, keySz, iv, ivSz, + CPA_CY_SYM_OP_ALGORITHM_CHAINING, CPA_CY_SYM_CIPHER_AES_GCM, + CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT, + CPA_CY_SYM_HASH_AES_GCM, authTag, authTagSz, authIn, authInSz); +} +#ifdef HAVE_AES_DECRYPT +int IntelQaSymAesGcmDecrypt(IntelQaDev* dev, + byte* out, const byte* in, word32 sz, + const byte* key, word32 keySz, + const byte* iv, word32 ivSz, + const byte* authTag, word32 authTagSz, + const byte* authIn, word32 authInSz) +{ + return IntelQaSymCipher(dev, out, in, sz, + key, keySz, iv, ivSz, + CPA_CY_SYM_OP_ALGORITHM_CHAINING, CPA_CY_SYM_CIPHER_AES_GCM, + CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT, + CPA_CY_SYM_HASH_AES_GCM, (byte*)authTag, authTagSz, authIn, authInSz); +} +#endif /* HAVE_AES_DECRYPT */ +#endif /* HAVE_AESGCM */ + +#ifndef NO_DES3 +int IntelQaSymDes3CbcEncrypt(IntelQaDev* dev, + byte* out, const byte* in, word32 sz, + const byte* key, word32 keySz, + const byte* iv, word32 ivSz) +{ + return IntelQaSymCipher(dev, out, in, sz, + key, keySz, iv, ivSz, + CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_3DES_CBC, + CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT, + CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0); +} + +int IntelQaSymDes3CbcDecrypt(IntelQaDev* dev, + byte* out, const byte* in, word32 sz, + const byte* key, word32 keySz, + const byte* iv, word32 ivSz) +{ + return IntelQaSymCipher(dev, out, in, sz, + key, keySz, iv, ivSz, + CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_3DES_CBC, + CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT, + CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0); +} +#endif /* !NO_DES3 */ + +#endif /* QAT_ENABLE_CRYPTO */ + + +#ifdef WOLF_CRYPTO_CB + +int IntelQaSymSync_CryptoDevCb(int devId, struct wc_CryptoInfo* info, void* ctx) +{ + int rc = NOT_COMPILED_IN; /* return this to bypass HW and use SW */ + IntelQaDev* dev; + + if (info == NULL || ctx == NULL) + return BAD_FUNC_ARG; + + (void)devId; + dev = (IntelQaDev*)ctx; + + #ifdef QAT_ENABLE_CRYPTO + if (info->algo_type == WC_ALGO_TYPE_CIPHER) { + QLOG("CryptoDevCb Cipher: Type %d\n", info->cipher.type); + + #ifndef NO_AES + if (info->cipher.type == WC_CIPHER_AES_CBC) { + Aes* aes = info->cipher.aescbc.aes; + if (aes == NULL) + return BAD_FUNC_ARG; + + if (info->cipher.enc) { + rc = IntelQaSymAesCbcEncrypt(dev, + info->cipher.aescbc.out, + info->cipher.aescbc.in, + info->cipher.aescbc.sz, + (byte*)aes->devKey, aes->keylen, + (byte*)aes->reg, AES_BLOCK_SIZE); + } + else { + rc = IntelQaSymAesCbcDecrypt(dev, + info->cipher.aescbc.out, + info->cipher.aescbc.in, + info->cipher.aescbc.sz, + (byte*)aes->devKey, aes->keylen, + (byte*)aes->reg, AES_BLOCK_SIZE); + } + } + #endif /* !NO_AES */ + + #ifdef HAVE_AESGCM + if (info->cipher.type == WC_CIPHER_AES_GCM) { + if (info->cipher.enc) { + Aes* aes = info->cipher.aesgcm_enc.aes; + if (aes == NULL) + return BAD_FUNC_ARG; + + rc = IntelQaSymAesGcmEncrypt(dev, + info->cipher.aesgcm_enc.out, + info->cipher.aesgcm_enc.in, + info->cipher.aesgcm_enc.sz, + (const byte*)aes->devKey, aes->keylen, + info->cipher.aesgcm_enc.iv, + info->cipher.aesgcm_enc.ivSz, + info->cipher.aesgcm_enc.authTag, + info->cipher.aesgcm_enc.authTagSz, + info->cipher.aesgcm_enc.authIn, + info->cipher.aesgcm_enc.authInSz); + } + else { + Aes* aes = info->cipher.aesgcm_dec.aes; + if (aes == NULL) + return BAD_FUNC_ARG; + + rc = IntelQaSymAesGcmDecrypt(dev, + info->cipher.aesgcm_dec.out, + info->cipher.aesgcm_dec.in, + info->cipher.aesgcm_dec.sz, + (const byte*)aes->devKey, aes->keylen, + info->cipher.aesgcm_dec.iv, + info->cipher.aesgcm_dec.ivSz, + info->cipher.aesgcm_dec.authTag, + info->cipher.aesgcm_dec.authTagSz, + info->cipher.aesgcm_dec.authIn, + info->cipher.aesgcm_dec.authInSz); + } + } + #endif /* HAVE_AESGCM */ + + #ifndef NO_DES3 + if (info->cipher.type == WC_CIPHER_DES3) { + Des3* des = info->cipher.des3.des; + if (des == NULL) + return BAD_FUNC_ARG; + + if (info->cipher.enc) { + rc = IntelQaSymDes3CbcEncrypt(dev, + info->cipher.des3.out, + info->cipher.des3.in, + info->cipher.des3.sz, + (byte*)des->devKey, DES3_KEYLEN, + (byte*)des->reg, DES_BLOCK_SIZE); + } + else { + rc = IntelQaSymDes3CbcDecrypt(dev, + info->cipher.des3.out, + info->cipher.des3.in, + info->cipher.des3.sz, + (byte*)des->devKey, DES3_KEYLEN, + (byte*)des->reg, DES_BLOCK_SIZE); + } + } + #endif /* !NO_DES3 */ + } + #endif /* QAT_ENABLE_CRYPTO */ + + return rc; +} + +#endif /* WOLF_CRYPTO_CB */ + + +/* -------------------------------------------------------------------------- */ +/* Memory allocator and deallocator */ +/* -------------------------------------------------------------------------- */ +#include +#include +#include + +/* use thread local for QAE variables (removing mutex requirement) */ +#ifdef USE_QAE_THREAD_LS + #include /* for threadId tracking */ + #define QAE_THREAD_LS THREAD_LS_T +#else + #define QAE_THREAD_LS +#endif + +/* these are used to align memory to a byte boundary */ +#define ALIGNMENT_BASE (16ul) +#define ALIGNMENT_HW (64ul) +#define WOLF_MAGIC_NUM 0xA576F6C6641736EBUL /* (0xA)WolfAsyn(0xB) */ +#define WOLF_HEADER_ALIGN ALIGNMENT_BASE + +#define QAE_NOT_NUMA_PAGE 0xFFFF +typedef struct qaeMemHeader { +#ifdef WOLFSSL_TRACK_MEMORY + struct qaeMemHeader* next; + struct qaeMemHeader* prev; + #ifdef WOLFSSL_DEBUG_MEMORY + const char* func; + unsigned int line; + #endif +#endif + uint64_t magic; + void* heap; +#ifdef USE_QAE_THREAD_LS + pthread_t threadId; +#endif + size_t size; + word16 count; + word16 isNuma:1; + word16 reservedBits:15; /* use for future bits */ + word16 type; + word16 numa_page_offset; /* use QAE_NOT_NUMA_PAGE if not NUMA */ +} ALIGN16 qaeMemHeader; + +#ifdef WOLFSSL_TRACK_MEMORY + typedef struct qaeMemStats { + long totalAllocs; /* number of allocations */ + long totalDeallocs; /* number of deallocations */ + long totalBytes; /* total number of bytes allocated */ + long peakBytes; /* concurrent max bytes */ + long currentBytes; /* total current bytes in use */ + } qaeMemStats; + + /* track allocations and report at end */ + typedef struct qaeMemList { + qaeMemHeader* head; + qaeMemHeader* tail; + uint32_t count; + } qaeMemList; +#endif /* WOLFSSL_TRACK_MEMORY */ + + +/* local variables */ +#ifndef USE_QAE_THREAD_LS + static pthread_mutex_t g_memLock = PTHREAD_MUTEX_INITIALIZER; +#endif + + +#ifdef WOLFSSL_TRACK_MEMORY + static qaeMemStats g_memStats; + static qaeMemList g_memList; + static pthread_mutex_t g_memStatLock = PTHREAD_MUTEX_INITIALIZER; +#endif + +static WC_INLINE int qaeMemTypeIsNuma(int type) +{ + int isNuma = 0; + + switch (type) { + case DYNAMIC_TYPE_ASYNC_NUMA: + case DYNAMIC_TYPE_ASYNC_NUMA64: + case DYNAMIC_TYPE_WOLF_BIGINT: + case DYNAMIC_TYPE_PRIVATE_KEY: + case DYNAMIC_TYPE_PUBLIC_KEY: + case DYNAMIC_TYPE_AES_BUFFER: + case DYNAMIC_TYPE_RSA_BUFFER: + case DYNAMIC_TYPE_ECC_BUFFER: + case DYNAMIC_TYPE_SIGNATURE: + case DYNAMIC_TYPE_DIGEST: + case DYNAMIC_TYPE_SECRET: + case DYNAMIC_TYPE_SEED: + case DYNAMIC_TYPE_SALT: + { + isNuma = 1; + break; + } + case DYNAMIC_TYPE_OUT_BUFFER: + case DYNAMIC_TYPE_IN_BUFFER: + { + #if !defined(WC_ASYNC_NO_CRYPT) && !defined(WC_ASYNC_NO_HASH) + isNuma = 1; + #else + isNuma = 0; + #endif + break; + } + default: + isNuma = 0; + break; + } + return isNuma; +} + + +static void _qaeMemFree(void *ptr, void* heap, int type +#ifdef WOLFSSL_DEBUG_MEMORY + , const char* func, unsigned int line +#endif +) +{ + qaeMemHeader* header = NULL; + size_t size; + void* origPtr = ptr; + + if (ptr == NULL) + return; + + /* adjust for header and align */ + ptr = (byte*)(((size_t)ptr - ((size_t)ptr % WOLF_HEADER_ALIGN)) - + sizeof(qaeMemHeader)); + header = (qaeMemHeader*)ptr; + + /* check for header magic */ + if (header->magic != WOLF_MAGIC_NUM) { + printf("Free: Header magic not found! %p\n", ptr); + return; + } + + /* cache values for later */ + size = header->size; + +#ifdef WOLFSSL_DEBUG_MEMORY +#ifdef WOLFSSL_DEBUG_MEMORY_PRINT + printf("Free: %p (%u) at %s:%d, heap %p, type %d, count %d\n", + origPtr, (unsigned int)size, func, line, heap, type, header->count); +#else + (void)func; + (void)line; +#endif +#endif + (void)type; + + /* adjust free count */ + header->count--; + + /* check header count */ + if (header->count > 0) { + /* go ahead and return if still in use */ + return; + } + +#ifdef WOLFSSL_TRACK_MEMORY + if (pthread_mutex_lock(&g_memStatLock) == 0) { + g_memStats.currentBytes -= size; + g_memStats.totalDeallocs++; + + if (header == g_memList.head && header == g_memList.tail) { + g_memList.head = NULL; + g_memList.tail = NULL; + } + else if (header == g_memList.head) { + g_memList.head = header->next; + g_memList.head->prev = NULL; + } + else if (header == g_memList.tail) { + g_memList.tail = header->prev; + g_memList.tail->next = NULL; + } + else { + qaeMemHeader* next = header->next; + qaeMemHeader* prev = header->prev; + if (next) + next->prev = prev; + if (prev) + prev->next = next; + } + g_memList.count--; + + pthread_mutex_unlock(&g_memStatLock); + } +#endif + + (void)heap; + (void)size; + (void)origPtr; + +#ifdef WOLFSSL_DEBUG_MEMORY + /* make sure magic is gone */ + header->magic = 0; +#endif + + /* free type */ + if (header->isNuma && header->numa_page_offset != QAE_NOT_NUMA_PAGE) { + qaeMemFreeNUMA(&ptr); + } + else { + free(ptr); + } +} + + +static void* _qaeMemAlloc(size_t size, void* heap, int type +#ifdef WOLFSSL_DEBUG_MEMORY + , const char* func, unsigned int line +#endif +) +{ + void* ptr = NULL; + qaeMemHeader* header = NULL; + int isNuma; + int alignment = ALIGNMENT_BASE; + word16 page_offset = QAE_NOT_NUMA_PAGE; + + /* make sure all allocations are aligned */ + if ((size % WOLF_HEADER_ALIGN) != 0) { + size += (WOLF_HEADER_ALIGN - (size % WOLF_HEADER_ALIGN)); + } + + isNuma = qaeMemTypeIsNuma(type); + if (type == DYNAMIC_TYPE_ASYNC_NUMA64) + alignment = ALIGNMENT_HW; + + /* allocate type */ + if (isNuma) { + /* Node is typically 0 */ + page_offset = 0; + ptr = qaeMemAllocNUMA((Cpa32U)(size + sizeof(qaeMemHeader)), 0, + alignment); + } + else if (ptr == NULL) { + isNuma = 0; + ptr = malloc(size + sizeof(qaeMemHeader)); + } + + /* add header */ + if (ptr) { + header = (qaeMemHeader*)ptr; + ptr = (byte*)ptr + sizeof(qaeMemHeader); + header->magic = WOLF_MAGIC_NUM; + header->heap = heap; + header->size = size; + header->type = type; + header->count = 1; + header->isNuma = isNuma; + header->numa_page_offset = page_offset; + #ifdef USE_QAE_THREAD_LS + header->threadId = pthread_self(); + #endif + + #ifdef WOLFSSL_TRACK_MEMORY + if (pthread_mutex_lock(&g_memStatLock) == 0) { + g_memStats.totalAllocs++; + g_memStats.totalBytes += size; + g_memStats.currentBytes += size; + if (g_memStats.currentBytes > g_memStats.peakBytes) + g_memStats.peakBytes = g_memStats.currentBytes; + + #ifdef WOLFSSL_DEBUG_MEMORY + header->func = func; + header->line = line; + #endif + + /* Setup event */ + header->next = NULL; + if (g_memList.tail == NULL) { + g_memList.head = header; + } + else { + g_memList.tail->next = header; + header->prev = g_memList.tail; + } + g_memList.tail = header; /* add to the end either way */ + g_memList.count++; + + pthread_mutex_unlock(&g_memStatLock); + } + #endif + } + +#ifdef WOLFSSL_DEBUG_MEMORY +#ifdef WOLFSSL_DEBUG_MEMORY_PRINT + printf("Alloc: %p (%u) at %s:%d, heap %p, type %d\n", + ptr, (unsigned int)size, func, line, heap, type); +#else + (void)func; + (void)line; +#endif +#endif + + (void)heap; + + return ptr; +} + +/* Public Functions */ +void* IntelQaMalloc(size_t size, void* heap, int type +#ifdef WOLFSSL_DEBUG_MEMORY + , const char* func, unsigned int line +#endif +) +{ + void* ptr; + +#ifndef USE_QAE_THREAD_LS + int ret = pthread_mutex_lock(&g_memLock); + if (ret != 0) { + printf("Alloc: Error(%d) on mutex lock\n", ret); + return NULL; + } +#endif + + ptr = _qaeMemAlloc(size, heap, type + #ifdef WOLFSSL_DEBUG_MEMORY + , func, line + #endif + ); + +#ifndef USE_QAE_THREAD_LS + pthread_mutex_unlock(&g_memLock); +#endif + + return ptr; +} + +void IntelQaFree(void *ptr, void* heap, int type +#ifdef WOLFSSL_DEBUG_MEMORY + , const char* func, unsigned int line +#endif +) +{ +#ifndef USE_QAE_THREAD_LS + int ret = pthread_mutex_lock(&g_memLock); + if (ret != 0) { + printf("Free: Error(%d) on mutex lock\n", ret); + return; + } +#endif + + _qaeMemFree(ptr, heap, type + #ifdef WOLFSSL_DEBUG_MEMORY + , func, line + #endif + ); + +#ifndef USE_QAE_THREAD_LS + pthread_mutex_unlock(&g_memLock); +#endif +} + +void* IntelQaRealloc(void *ptr, size_t size, void* heap, int type +#ifdef WOLFSSL_DEBUG_MEMORY + , const char* func, unsigned int line +#endif +) +{ + void* newPtr = NULL; + void* origPtr = ptr; + qaeMemHeader* header = NULL; + byte allocNew = 1; + int newIsNuma = -1, ptrIsNuma = -1; + size_t copySize = 0; + +#ifndef USE_QAE_THREAD_LS + int ret = pthread_mutex_lock(&g_memLock); + if (ret != 0) { + printf("Realloc: Error(%d) on mutex lock\n", ret); + return NULL; + } +#endif + + (void)heap; + + if (ptr) { + /* get header pointer and align */ + header = (qaeMemHeader*)(((size_t)ptr - + ((size_t)ptr % WOLF_HEADER_ALIGN)) - sizeof(qaeMemHeader)); + if (header->magic == WOLF_MAGIC_NUM) { + newIsNuma = qaeMemTypeIsNuma(type); + ptrIsNuma = (header->numa_page_offset != QAE_NOT_NUMA_PAGE) ? 1 : 0; + + /* for non-NUMA, treat as normal REALLOC */ + if (newIsNuma == 0 && ptrIsNuma == 0) { + allocNew = 1; + } + /* if matching NUMA type and size fits, use existing */ + else if (newIsNuma == ptrIsNuma && header->size >= size) { + + #ifdef USE_QAE_THREAD_LS + if (header->threadId != pthread_self()) { + allocNew = 1; + #if 0 + printf("Realloc %p from different thread! orig %lx this %lx\n", + origPtr, header->threadId, pthread_self()); + #endif + } + else + #endif + { + /* use existing pointer and increment counter */ + header->count++; + newPtr = origPtr; + allocNew = 0; + } + } + + copySize = header->size; + } + else { + copySize = size; + } + } + + if (allocNew) { + newPtr = _qaeMemAlloc(size, heap, type + #ifdef WOLFSSL_DEBUG_MEMORY + , func, line + #endif + ); + if (newPtr && ptr) { + /* only copy min of new and old size to new pointer */ + if (copySize > size) + copySize = size; + XMEMCPY(newPtr, ptr, copySize); + + if (newIsNuma == 0 && ptrIsNuma == 0) { + /* for non-NUMA, treat as normal REALLOC and free old pointer */ + _qaeMemFree(ptr, heap, type + #ifdef WOLFSSL_DEBUG_MEMORY + , func, line + #endif + ); + } + } + } + +#ifndef USE_QAE_THREAD_LS + pthread_mutex_unlock(&g_memLock); +#endif + +#ifdef WOLFSSL_DEBUG_MEMORY +#ifdef WOLFSSL_DEBUG_MEMORY_PRINT + if (allocNew) { + printf("Realloc: New %p -> %p (%u) at %s:%d, heap %p, type %d\n", + origPtr, newPtr, (unsigned int)size, func, line, heap, type); + } + else { + printf("Realloc: Reuse %p (%u) at %s:%d, heap %p, type %d, count %d\n", + origPtr, (unsigned int)size, func, line, header->heap, header->type, header->count); + } +#else + (void)func; + (void)line; +#endif +#endif + + return newPtr; +} + + +#ifdef WOLFSSL_TRACK_MEMORY +int InitMemoryTracker(void) +{ + if (pthread_mutex_lock(&g_memStatLock) == 0) { + g_memStats.totalAllocs = 0; + g_memStats.totalDeallocs= 0; + g_memStats.totalBytes = 0; + g_memStats.peakBytes = 0; + g_memStats.currentBytes = 0; + + XMEMSET(&g_memList, 0, sizeof(g_memList)); + + pthread_mutex_unlock(&g_memStatLock); + } + + return 0; +} + +void ShowMemoryTracker(void) +{ + if (pthread_mutex_lock(&g_memStatLock) == 0) { + printf("total Allocs = %9ld\n", g_memStats.totalAllocs); + printf("total Deallocs = %9ld\n", g_memStats.totalDeallocs); + printf("total Bytes = %9ld\n", g_memStats.totalBytes); + printf("peak Bytes = %9ld\n", g_memStats.peakBytes); + printf("current Bytes = %9ld\n", g_memStats.currentBytes); + + if (g_memList.count > 0) { + + /* print list of allocations */ + qaeMemHeader* header; + for (header = g_memList.head; header != NULL; header = header->next) { + printf("Leak: Ptr %p, Size %u, Type %d, Heap %p" + #ifdef WOLFSSL_DEBUG_MEMORY + ", Func %s, Line %d" + #endif + "\n", + (byte*)header + sizeof(qaeMemHeader), (unsigned int)header->size, + header->type, header->heap + #ifdef WOLFSSL_DEBUG_MEMORY + , header->func, header->line + #endif + ); + } + } + + pthread_mutex_unlock(&g_memStatLock); + + /* cleanup lock */ + pthread_mutex_destroy(&g_memStatLock); + } +} +#endif /* WOLFSSL_TRACK_MEMORY */ + +#ifdef QAT_DEMO_MAIN + +/* AES GCM */ +static const byte aesgcm_k[] = { + 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, + 0x99, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, + 0x77, 0x88, 0x99, 0x00, 0x11, 0x22, 0x33, 0x44, + 0x55, 0x66, 0x77, 0x88, 0x99, 0x00, 0x11, 0x22 +}; + +static const byte aesgcm_iv[] = { + 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, + 0xca, 0xfe, 0xca, 0xfe +}; + +static const byte aesgcm_a[] = { + 0xde, 0xad, 0xde, 0xad, 0xde, 0xad, 0xde, 0xad, + 0xde, 0xad, 0xde, 0xad, 0xde, 0xad, 0xde, 0xad, + 0xde, 0xad, 0xde, 0xad +}; + +static const byte aesgcm_p[] = { + 0x79, 0x84, 0x86, 0x44, 0x68, 0x45, 0x15, 0x61, + 0x86, 0x54, 0x66, 0x56, 0x54, 0x54, 0x31, 0x54, + 0x64, 0x64, 0x68, 0x45, 0x15, 0x15, 0x61, 0x61, + 0x51, 0x51, 0x51, 0x51, 0x51, 0x56, 0x14, 0x11, + 0x72, 0x13, 0x51, 0x82, 0x84, 0x56, 0x74, 0x53, + 0x45, 0x34, 0x65, 0x15, 0x46, 0x14, 0x67, 0x55, + 0x16, 0x14, 0x67, 0x54, 0x65, 0x47, 0x14, 0x67, + 0x46, 0x74, 0x65, 0x46 +}; + +static const byte aesgcm_c[] = { + 0x59, 0x85, 0x02, 0x97, 0xE0, 0x4D, 0xFC, 0x5C, + 0x03, 0xCC, 0x83, 0x64, 0xCE, 0x28, 0x0B, 0x95, + 0x78, 0xEC, 0x93, 0x40, 0xA1, 0x8D, 0x21, 0xC5, + 0x48, 0x6A, 0x39, 0xBA, 0x4F, 0x4B, 0x8C, 0x95, + 0x6F, 0x8C, 0xF6, 0x9C, 0xD0, 0xA5, 0x8D, 0x67, + 0xA1, 0x32, 0x11, 0xE7, 0x2E, 0xF6, 0x63, 0xAF, + 0xDE, 0xD4, 0x7D, 0xEC, 0x15, 0x01, 0x58, 0xCB, + 0xE3, 0x7B, 0xC6, 0x94, +}; + +static byte aesgcm_t[] = { + 0x5D, 0x10, 0x3F, 0xC7, 0x22, 0xC7, 0x21, 0x29 +}; + + +/* simple example of using AES-GCM encrypt with Intel QA */ +int main(int argc, char** argv) +{ + int ret; + IntelQaDev dev; + byte out[256]; + word32 outLen = sizeof(out); + byte tmp[256]; + word32 tmpLen = sizeof(tmp); + +#ifdef QAT_DEBUG + wolfSSL_Debugging_ON(); +#endif + + IntelQaInit(NULL); + +#ifndef NO_AES +#ifdef HAVE_AESGCM + /* AES Test */ + IntelQaOpen(&dev, 0); + dev.event.ret = WC_PENDING_E; + tmpLen = sizeof(aesgcm_t); + XMEMSET(out, 0, sizeof(out)); + XMEMSET(tmp, 0, sizeof(tmp)); + + ret = IntelQaSymAesGcmEncrypt(&dev, out, aesgcm_p, sizeof(aesgcm_p), + aesgcm_k, sizeof(aesgcm_k), aesgcm_iv, sizeof(aesgcm_iv), + tmp, tmpLen, aesgcm_a, sizeof(aesgcm_a)); + printf("AES GCM Encrypt: Ret=%d, Tag Len=%d\n", ret, tmpLen); + IntelQaClose(&dev); +#endif /* HAVE_AESGCM */ +#endif /* NO_AES */ + + (void)tmp; + (void)tmpLen; + + IntelQaDeInit(0); + + return 0; +} + +#endif + +#endif /* HAVE_INTEL_QA_SYNC */ diff --git a/wolfcrypt/src/wc_port.c b/wolfcrypt/src/wc_port.c index 9d5307d09..ac83c68e9 100644 --- a/wolfcrypt/src/wc_port.c +++ b/wolfcrypt/src/wc_port.c @@ -74,6 +74,10 @@ #include #endif +#ifdef HAVE_INTEL_QA_SYNC + #include +#endif + #ifdef _MSC_VER /* 4996 warning to use MS extensions e.g., strcpy_s instead of strncpy */ #pragma warning(disable: 4996) @@ -124,7 +128,11 @@ int wolfCrypt_Init(void) return ret; } #endif - + + #ifdef HAVE_INTEL_QA_SYNC + ret = IntelQaHardwareStart(QAT_PROCESS_NAME, QAT_LIMIT_DEV_ACCESS); + #endif + #if defined(WOLFSSL_TRACK_MEMORY) && !defined(WOLFSSL_STATIC_MEMORY) ret = InitMemoryTracker(); if (ret != 0) { diff --git a/wolfssl/openssl/aes.h b/wolfssl/openssl/aes.h index db0d8ddf5..37d9d9b1c 100644 --- a/wolfssl/openssl/aes.h +++ b/wolfssl/openssl/aes.h @@ -59,7 +59,7 @@ typedef struct WOLFSSL_AES_KEY { #ifdef HAVE_PKCS11 void* pkcs11_holder[(AES_MAX_ID_LEN + sizeof(int)) / sizeof(void*)]; #endif - #ifdef WOLFSSL_ASYNC_CRYPT + #if defined(WOLFSSL_ASYNC_CRYPT) || defined(WOLF_CRYPTO_CB) void* async_holder[128 / sizeof(void*)]; #endif } WOLFSSL_AES_KEY; diff --git a/wolfssl/wolfcrypt/des3.h b/wolfssl/wolfcrypt/des3.h index 4aa0400b8..d231a0f35 100644 --- a/wolfssl/wolfcrypt/des3.h +++ b/wolfssl/wolfcrypt/des3.h @@ -105,6 +105,7 @@ struct Des3 { WC_ASYNC_DEV asyncDev; #endif #ifdef WOLF_CRYPTO_CB + word32 devKey[DES3_KEYLEN/sizeof(word32)]; /* raw key */ int devId; void* devCtx; #endif diff --git a/wolfssl/wolfcrypt/include.am b/wolfssl/wolfcrypt/include.am index cc43a08f9..9b9d76588 100644 --- a/wolfssl/wolfcrypt/include.am +++ b/wolfssl/wolfcrypt/include.am @@ -111,6 +111,10 @@ nobase_include_HEADERS+= wolfssl/wolfcrypt/port/intel/quickassist.h nobase_include_HEADERS+= wolfssl/wolfcrypt/port/intel/quickassist_mem.h endif +if BUILD_INTEL_QA_SYNC +nobase_include_HEADERS+= wolfssl/wolfcrypt/port/intel/quickassist_sync.h +endif + if BUILD_SP nobase_include_HEADERS+= wolfssl/wolfcrypt/sp.h nobase_include_HEADERS+= wolfssl/wolfcrypt/sp_int.h diff --git a/wolfssl/wolfcrypt/port/intel/quickassist_sync.h b/wolfssl/wolfcrypt/port/intel/quickassist_sync.h new file mode 100644 index 000000000..5b21ad4d9 --- /dev/null +++ b/wolfssl/wolfcrypt/port/intel/quickassist_sync.h @@ -0,0 +1,229 @@ +/* quickassist_sync.h + * + * Copyright (C) 2006-2019 wolfSSL Inc. + * + * This file is part of wolfSSL. (formerly known as CyaSSL) + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + */ + +#ifndef _INTEL_QUICKASSIST_SYNC_H_ +#define _INTEL_QUICKASSIST_SYNC_H_ + +#ifdef HAVE_INTEL_QA_SYNC + +#include "cpa.h" +#include "cpa_cy_im.h" +#include "cpa_cy_sym.h" +#include "cpa_cy_rsa.h" +#include "cpa_cy_ln.h" +#include "cpa_cy_ecdh.h" +#include "cpa_cy_ecdsa.h" +#include "cpa_cy_dh.h" +#include "cpa_cy_drbg.h" +#include "cpa_cy_nrbg.h" +#include "cpa_cy_prime.h" + +/* User space utils */ +#include +#include +#include +#include + + +#if 0 + /* Optional feature for partial QAT hashing support */ + /* This will process updates through hardware instead of caching them */ + #define QAT_HASH_ENABLE_PARTIAL +#endif +#ifdef QAT_HASH_ENABLE_PARTIAL + #define MAX_QAT_HASH_BUFFERS 2 +#endif + +/* Detect QAT driver version */ +#if defined(CPA_CY_API_VERSION_NUM_MAJOR) && CPA_CY_API_VERSION_NUM_MAJOR > 1 + #define QAT_V2 +#endif + +#ifdef QAT_V2 + /* quickassist/utilities/libusdm_drv/qae_mem.h */ + /* Provides user-space API's for accessing NUMA allocated memory through usdm_drv */ + #include "qae_mem.h" +#include "linux/include/qae_mem_utils.h" +#endif + +#ifdef QAT_USE_POLLING_THREAD + #include +#endif +#ifdef QA_DEMO_MAIN + #include +#endif + + +/* Tunable parameters */ +#ifndef QAT_PROCESS_NAME + #define QAT_PROCESS_NAME "SSL" +#endif +#ifndef QAT_LIMIT_DEV_ACCESS + #define QAT_LIMIT_DEV_ACCESS CPA_FALSE +#endif +#ifndef QAT_MAX_DEVICES + #define QAT_MAX_DEVICES (1) /* maximum number of QAT cards */ +#endif + +#ifndef QAT_RETRY_LIMIT + #define QAT_RETRY_LIMIT (100) +#endif +#ifndef QAT_POLL_RESP_QUOTA + #define QAT_POLL_RESP_QUOTA (0) /* all pending */ +#endif + +#if !defined(NO_AES) || !defined(NO_DES3) + #define QAT_ENABLE_CRYPTO +#endif + + +/* Pre-declarations */ +struct IntelQaDev; +struct wc_CryptoInfo; +struct WC_BIGINT; +struct WC_RNG; + + +#if defined(QAT_ENABLE_HASH) || defined(QAT_ENABLE_CRYPTO) +/* symmetric context */ +typedef struct IntelQaSymCtx { + CpaCySymOpData opData; + CpaCySymSessionCtx symCtxSrc; + CpaCySymSessionCtx symCtx; + word32 symCtxSize; + + /* flags */ + word32 isOpen:1; + word32 isCopy:1; +} IntelQaSymCtx; +#endif + +typedef void (*IntelQaFreeFunc)(struct IntelQaDev*); + + +/* QuickAssist device */ +typedef struct IntelQaDev { + CpaInstanceHandle handle; + int devId; + void* heap; + + /* callback return info */ + int ret; + byte* out; + union { + word32* outLenPtr; + word32 outLen; + }; + + /* operations */ + IntelQaFreeFunc freeFunc; + union { + #ifdef QAT_ENABLE_CRYPTO + struct { + IntelQaSymCtx ctx; + CpaBufferList bufferList; + CpaFlatBuffer flatBuffer; + byte* authTag; + word32 authTagSz; + } cipher; + #endif + } op; + +#ifdef QAT_USE_POLLING_THREAD + pthread_t pollingThread; + byte pollingCy; +#endif +} IntelQaDev; + + +/* Interface */ +WOLFSSL_LOCAL int IntelQaHardwareStart(const char*, int); +WOLFSSL_LOCAL void IntelQaHardwareStop(void); +WOLFSSL_LOCAL int IntelQaInit(void*); +WOLFSSL_LOCAL void IntelQaDeInit(int); +WOLFSSL_LOCAL int IntelQaNumInstances(void); +WOLFSSL_LOCAL int IntelQaOpen(IntelQaDev*, int); +WOLFSSL_LOCAL void IntelQaClose(IntelQaDev*); +WOLFSSL_LOCAL int IntelQaDevCopy(IntelQaDev*, IntelQaDev*); +WOLFSSL_LOCAL int IntelQaPoll(IntelQaDev*); +WOLFSSL_LOCAL int IntelQaGetCyInstanceCount(void); + +#ifndef NO_AES + #ifdef HAVE_AES_CBC + WOLFSSL_LOCAL int IntelQaSymAesCbcEncrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32); + #ifdef HAVE_AES_DECRYPT + WOLFSSL_LOCAL int IntelQaSymAesCbcDecrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32); + #endif /* HAVE_AES_DECRYPT */ + #endif /* HAVE_AES_CBC */ + + #ifdef HAVE_AESGCM + WOLFSSL_LOCAL int IntelQaSymAesGcmEncrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32, + byte*, word32, const byte*, word32); + #ifdef HAVE_AES_DECRYPT + WOLFSSL_LOCAL int IntelQaSymAesGcmDecrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte*, word32, + const byte*, word32, const byte*, word32); + #endif /* HAVE_AES_DECRYPT */ + #endif /* HAVE_AESGCM */ +#endif /* !NO_AES */ + +#ifndef NO_DES3 + WOLFSSL_LOCAL int IntelQaSymDes3CbcEncrypt(IntelQaDev*, byte*, + const byte*, word32, const byte*, word32, const byte* iv, word32); + WOLFSSL_LOCAL int IntelQaSymDes3CbcDecrypt(IntelQaDev* dev, byte*, + const byte*, word32, const byte*, word32, const byte* iv, word32); +#endif /*! NO_DES3 */ + +#ifdef WOLF_CRYPTO_CB + WOLFSSL_LOCAL int IntelQaSymSync_CryptoDevCb(int, struct wc_CryptoInfo*, + void*); +#endif /* WOLF_CRYPTO_CB */ + + +#ifdef WOLFSSL_TRACK_MEMORY + WOLFSSL_API int InitMemoryTracker(void); + WOLFSSL_API void ShowMemoryTracker(void); +#endif + + +WOLFSSL_API void* IntelQaMalloc(size_t size, void* heap, int type +#ifdef WOLFSSL_DEBUG_MEMORY + , const char* func, unsigned int line +#endif +); + +WOLFSSL_API void IntelQaFree(void *ptr, void* heap, int type +#ifdef WOLFSSL_DEBUG_MEMORY + , const char* func, unsigned int line +#endif +); + +WOLFSSL_API void* IntelQaRealloc(void *ptr, size_t size, void* heap, int type +#ifdef WOLFSSL_DEBUG_MEMORY + , const char* func, unsigned int line +#endif +); +#endif /* HAVE_INTEL_QA_SYNC */ + +#endif /* _INTEL_QUICKASSIST_SYNC_H_ */ diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index f3326f539..635c3ca73 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -233,8 +233,13 @@ WOLFSSL_API void* XMALLOC(size_t n, void* heap, int type); WOLFSSL_API void* XREALLOC(void *p, size_t n, void* heap, int type); WOLFSSL_API void XFREE(void *p, void* heap, int type); - #elif defined(WOLFSSL_ASYNC_CRYPT) && defined(HAVE_INTEL_QA) - #include + #elif (defined(WOLFSSL_ASYNC_CRYPT) && defined(HAVE_INTEL_QA)) || \ + defined(HAVE_INTEL_QA_SYNC) + #ifndef HAVE_INTEL_QA_SYNC + #include + #else + #include + #endif #undef USE_WOLFSSL_MEMORY #ifdef WOLFSSL_DEBUG_MEMORY #define XMALLOC(s, h, t) IntelQaMalloc((s), (h), (t), __func__, __LINE__)