/* server-tls-perf.c * * Copyright (C) 2006-2020 wolfSSL Inc. * * This file is part of wolfSSL. (formerly known as CyaSSL) * * wolfSSL is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * wolfSSL is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA *============================================================================= * * This is an example of a TCP Server that uses non-blocking input and output to * handle a large number of connections in multiple threads. */ #include #include #include #include /* Default port to listen on. */ #define DEFAULT_PORT 11111 /* The number of EPOLL events to accept and process at one time. */ #define EPOLL_NUM_EVENTS 10 /* The number of threads to create. */ #define NUM_THREADS 15 /* The number of concurrent connections to support. */ #define SSL_NUM_CONN 15 /* The number of bytes to read from client. */ #define NUM_READ_BYTES 16384 /* The number of bytes to write to client. */ #define NUM_WRITE_BYTES 16384 /* The maximum number of bytes to send in a run. */ #define MAX_BYTES -1 /* The maximum number of connections to perform in this run. */ #define MAX_CONNECTIONS 100 /* The maximum length of the queue of pending connections. */ #define NUM_CLIENTS 100 /* The number of wolfSSL events to accept and process at one time. */ #define MAX_WOLF_EVENTS 10 /* The command line options. */ #define OPTIONS "?p:v:al:c:k:A:t:n:N:R:W:B:" /* The default server certificate. */ #define SVR_CERT "../certs/server-cert.pem" /* The default server private key. */ #define SVR_KEY "../certs/server-key.pem" /* The default certificate/CA file for the client. */ #define CLI_CERT "../certs/client-cert.pem" /* The states of the SSL connection. */ typedef enum SSLState { ACCEPT, READ, WRITE, CLOSED } SSLState; /* Type for the SSL connection data. */ typedef struct SSLConn SSLConn; /* Data for each active connection. */ struct SSLConn { /* The socket listening on, reading from and writing to. */ int sockfd; /* The wolfSSL object to perform TLS communications. */ WOLFSSL* ssl; /* The current state of the SSL/TLS connection. */ SSLState state; /* Previous SSL connection data object. */ SSLConn* prev; /* Next SSL connection data object. */ SSLConn* next; }; /* SSL connection data for a thread. */ typedef struct ThreadData { /* The SSL/TLS context for all connections in thread. */ WOLFSSL_CTX* ctx; /* The device id for this thread. */ int devId; /* Linked list of SSL connection data. */ SSLConn *sslConn; /* Free list. */ SSLConn *freeSSLConn; /* The number of active SSL connections. */ int cnt; /* Accepting new connections. */ int accepting; /* The thread id for the handler. */ pthread_t thread_id; } ThreadData; /* The information about SSL/TLS connections. */ typedef struct SSLConn_CTX { /* An array of active connections. */ ThreadData* threadData; /* Number of threads. */ int numThreads; /* Maximum number of active connections per thread. */ int numConns; /* Size of the client data buffer. */ int bufferLen; /* Number of bytes to write. */ int replyLen; /* Number of connections handled. */ int numConnections; /* Number of resumed connections handled. */ int numResumed; /* Maximum number of connections to perform. */ int maxConnections; /* Total number of bytes read. */ int totalReadBytes; /* Total number of bytes written. */ int totalWriteBytes; /* Maximum number of bytes to read/write. */ int maxBytes; /* Total time handling accepts. */ double acceptTime; /* Total time handling accepts - resumed connections. */ double resumeTime; #ifdef WOLFSSL_ASYNC_CRYPT /* Total time handling asynchronous operations. */ double asyncTime; #endif /* Total time handling reading. */ double readTime; /* Total time handling writing. */ double writeTime; /* Total time handling connections. */ double totalTime; } SSLConn_CTX; static void SSLConn_Free(SSLConn_CTX* ctx); static void SSLConn_Close(SSLConn_CTX* ctx, ThreadData* threadData, SSLConn* sslConn); static void SSLConn_FreeSSLConn(ThreadData* threadData); static void WolfSSLCtx_Final(ThreadData* threadData); /* The index of the command line option. */ int myoptind = 0; /* The current command line option. */ char* myoptarg = NULL; /* The data to reply with. */ static char reply[NUM_WRITE_BYTES]; /* Global SSL/TLS connection data context. */ static SSLConn_CTX* sslConnCtx = NULL; /* Mutex for using connection count. */ static pthread_mutex_t sslConnMutex = PTHREAD_MUTEX_INITIALIZER; /* The port to listen on. */ static word16 port = DEFAULT_PORT; /* The size of the listen backlog. */ static int numClients = NUM_CLIENTS; /* User specified ciphersuite list. */ static char* cipherList = NULL; /* The server's certificate for authentication. */ static char* ourCert = SVR_CERT; /* The server's private key for authentication. */ static char* ourKey = SVR_KEY; /* The client's CA/certificate for client authentication. */ static char* verifyCert = CLI_CERT; /* The version of SSL/TLS to use. */ static int version = SERVER_DEFAULT_VERSION; /* The flag to indicate downgrade is allowed */ static int allowDowngrade = 0; /* The number of threads to start. */ static int numThreads = NUM_THREADS; /* The number of connections per threads to allow. */ static int numConns = SSL_NUM_CONN; /* The number of bytes to read from the client. */ static int numBytesRead = NUM_READ_BYTES; /* The number of bytes to write to the client. */ static int numBytesWrite = NUM_WRITE_BYTES; /* The maximum number of bytes to read/write in a run. */ static int maxBytes = MAX_BYTES; /* The maximum number of connections accept in a run. */ static int maxConns = MAX_CONNECTIONS; /* Get the wolfSSL server method function for the specified version. * * version Protocol version to use. * returns The server method function or NULL when version not supported. */ static wolfSSL_method_func SSL_GetMethod(int version, int allowDowngrade) { wolfSSL_method_func method = NULL; switch (version) { #ifndef NO_OLD_TLS #ifdef WOLFSSL_ALLOW_SSLV3 case 0: method = wolfSSLv3_server_method_ex; break; #endif #ifndef NO_TLS #ifdef WOLFSSL_ALLOW_TLSV10 case 1: method = wolfTLSv1_server_method_ex; break; #endif #ifndef NO_OLD_TLS case 2: method = wolfTLSv1_1_server_method_ex; break; #endif #endif #endif #ifndef NO_TLS case 3: method = allowDowngrade ? wolfSSLv23_server_method_ex : wolfTLSv1_2_server_method_ex; break; #endif } return method; } /* Write data to a client. * * ssl The wolfSSL object. * reply The data to send to the client. * replyLen The length of the data to send to the client. * totalBytes The total number of bytes sent to clients. * writeTime The amount of time spent writing data to client. * returns 0 on failure, 1 on success, 2 on want read and 3 on want write. */ static int SSL_Write(WOLFSSL* ssl, char* reply, int replyLen, int* totalBytes, double* writeTime) { int rwret = 0; int error; double start, diff; start = current_time(1); rwret = wolfSSL_write(ssl, reply, replyLen); diff = current_time(0) - start; pthread_mutex_lock(&sslConnMutex); *writeTime += diff; pthread_mutex_unlock(&sslConnMutex); if (rwret == 0) { fprintf(stderr, "The client has closed the connection - write!\n"); return 0; } if (rwret > 0) { pthread_mutex_lock(&sslConnMutex); *totalBytes += rwret; pthread_mutex_unlock(&sslConnMutex); } if (rwret == replyLen) return 1; error = wolfSSL_get_error(ssl, 0); if (error == SSL_ERROR_WANT_READ) return 2; if (error == SSL_ERROR_WANT_WRITE) return 3; if (error == WC_PENDING_E) return 4; if (error == 0) return 1; /* Cannot do anything about other errors. */ fprintf(stderr, "wolfSSL_write error = %d\n", error); return 0; } /* Reads data from a client. * * ssl The wolfSSL object. * buffer The buffer to place client data into. * len The length of the buffer. * totalBytes The total number of bytes read from clients. * readTime The amount of time spent reading data from client. * returns 0 on failure, 1 on success, 2 on want read and 3 on want write. */ static int SSL_Read(WOLFSSL* ssl, char* buffer, int len, int* totalBytes, double* readTime) { int rwret = 0; int error; double start, diff; start = current_time(1); rwret = wolfSSL_read(ssl, buffer, len); diff = current_time(0) - start; pthread_mutex_lock(&sslConnMutex); *readTime += diff; if (rwret > 0) *totalBytes += rwret; pthread_mutex_unlock(&sslConnMutex); if (rwret == 0) { return 0; } error = wolfSSL_get_error(ssl, 0); if (error == SSL_ERROR_WANT_READ) return 2; if (error == SSL_ERROR_WANT_WRITE) return 3; if (error == WC_PENDING_E) return 4; if (error == 0) return 1; /* Cannot do anything about other errors. */ fprintf(stderr, "wolfSSL_read error = %d\n", error); return 0; } /* Accept/negotiate a secure connection. * * ssl The wolfSSL object. * acceptTime The amount of time spent accepting a client. * resumeTime The amount of time spent resuming a connection with a client. * returns 0 on failure, 1 on success, 2 on want read and 3 on want write. */ static int SSL_Accept(WOLFSSL* ssl, double* acceptTime, double* resumeTime) { int ret; int error; double start, diff; /* Accept the connection. */ start = current_time(1); ret = wolfSSL_accept(ssl); diff = current_time(0) - start; pthread_mutex_lock(&sslConnMutex); if (!wolfSSL_session_reused(ssl)) *acceptTime += diff; else *resumeTime += diff; pthread_mutex_unlock(&sslConnMutex); if (ret == 0) { fprintf(stderr, "The client has closed the connection - accept!\n"); return 0; } if (ret == SSL_SUCCESS) return 1; error = wolfSSL_get_error(ssl, 0); if (error == SSL_ERROR_WANT_READ) return 2; if (error == SSL_ERROR_WANT_WRITE) return 3; if (error == WC_PENDING_E) return 4; /* Cannot do anything about other errors. */ fprintf(stderr, "wolfSSL_accept error = %d\n", error); return 0; } /* Create a new SSL/TLS connection data object. * * numThreads The number of threads to create. * numConns The maximum number of concurrent connections. * bufferLen The number of data bytes to read from client. * replyLen The number of data bytes to write to client. * maxConns The number of connections to process this run. * -1 indicates no maximum. * maxBytes The number of bytes to send this run. * -1 indicates no maximum. * returns an allocated and initialized connection data object or NULL on error. */ static SSLConn_CTX* SSLConn_New(int numThreads, int numConns, int bufferLen, int replyLen, int maxConns, int maxBytes) { SSLConn_CTX* ctx; ThreadData* threadData; int i; ctx = (SSLConn_CTX*)malloc(sizeof(*ctx)); if (ctx == NULL) return NULL; memset(ctx, 0, sizeof(*ctx)); ctx->numThreads = numThreads; ctx->numConns = numConns; ctx->bufferLen = bufferLen; ctx->replyLen = replyLen; ctx->maxConnections = maxConns; ctx->maxBytes = maxBytes; /* Pre-allocate the SSL connection data. */ ctx->threadData = (ThreadData*)malloc(ctx->numThreads * sizeof(*ctx->threadData)); if (ctx->threadData == NULL) { SSLConn_Free(ctx); return NULL; } for (i = 0; i < ctx->numThreads; i++) { threadData = &ctx->threadData[i]; threadData->ctx = NULL; threadData->devId = INVALID_DEVID; threadData->sslConn = NULL; threadData->freeSSLConn = NULL; threadData->cnt = 0; threadData->thread_id = 0; } return ctx; } /* Free the SSL/TLS connection data. * * ctx The connection data. */ static void SSLConn_Free(SSLConn_CTX* ctx) { int i; ThreadData* threadData; if (ctx == NULL) return; for (i = 0; i < ctx->numThreads; i++) { threadData = &ctx->threadData[i]; while (threadData->sslConn != NULL) SSLConn_Close(ctx, threadData, threadData->sslConn); SSLConn_FreeSSLConn(threadData); WolfSSLCtx_Final(threadData); } free(ctx->threadData); ctx->threadData = NULL; free(ctx); } /* Close an active connection. * * ctx The SSL/TLS connection data. * threadData The SSL/TLS connection data for the thread. * sslConn The SSL connection data object. */ static void SSLConn_Close(SSLConn_CTX* ctx, ThreadData* threadData, SSLConn* sslConn) { int ret; if (sslConn->state == CLOSED) return; pthread_mutex_lock(&sslConnMutex); ret = (ctx->numConnections == 0); ctx->numConnections++; if (wolfSSL_session_reused(sslConn->ssl)) ctx->numResumed++; pthread_mutex_unlock(&sslConnMutex); if (ret) { WOLFSSL_CIPHER* cipher; cipher = wolfSSL_get_current_cipher(sslConn->ssl); printf("SSL cipher suite is %s\n", wolfSSL_CIPHER_get_name(cipher)); } sslConn->state = CLOSED; /* Take it out of the double-linked list. */ if (threadData->sslConn == sslConn) threadData->sslConn = sslConn->next; if (sslConn->next != NULL) sslConn->next->prev = sslConn->prev; if (sslConn->prev != NULL) sslConn->prev->next = sslConn->next; /* Put object at head of free list */ sslConn->next = threadData->freeSSLConn; sslConn->prev = NULL; threadData->freeSSLConn = sslConn; threadData->cnt--; } /* Free the SSL/TLS connections that are closed. * * threadData The SSL/TLS connection data for the thread. */ static void SSLConn_FreeSSLConn(ThreadData* threadData) { SSLConn* sslConn = threadData->freeSSLConn; threadData->freeSSLConn = NULL; while (sslConn != NULL) { SSLConn* next = sslConn->next; #ifdef WOLFSSL_ASYNC_CRYPT /* Clear out any events. */ while (wolfSSL_AsyncPoll(sslConn->ssl, WOLF_POLL_FLAG_CHECK_HW) == 1) ; #endif wolfSSL_free(sslConn->ssl); sslConn->ssl = NULL; close(sslConn->sockfd); free(sslConn); sslConn = next; } } /* Checks whether this run is done i.e. maximum number of connections or bytes * have been server. * * ctx The SSL/TLS connection data. * returns 1 if the run is done or 0 otherwise. */ static int SSLConn_Done(SSLConn_CTX* ctx) { int ret; pthread_mutex_lock(&sslConnMutex); if (ctx->maxConnections > 0) ret = (ctx->numConnections >= ctx->maxConnections); else { ret = (ctx->totalWriteBytes >= ctx->maxBytes) && (ctx->totalReadBytes >= ctx->maxBytes); } pthread_mutex_unlock(&sslConnMutex); return ret; } /* Accepts a new connection. * * threadData The SSL/TLS connection data for a thread. * sslCtx The SSL/TLS context. * sockfd The socket file descriptor to accept on. * sslConn The newly create SSL connection data object. * returns EXIT_SUCCESS if a new connection was accepted or EXIT_FAILURE * otherwise. */ static int SSLConn_Accept(ThreadData* threadData, WOLFSSL_CTX* sslCtx, socklen_t sockfd, SSLConn** sslConn) { struct sockaddr_in clientAddr = {0}; socklen_t size = sizeof(clientAddr); SSLConn* conn; conn = malloc(sizeof(*conn)); if (conn == NULL) return EXIT_FAILURE; /* Accept the client connection. */ conn->sockfd = accept(sockfd, (struct sockaddr *)&clientAddr, &size); if (conn->sockfd == -1) { free(conn); fprintf(stderr, "ERROR: failed to accept\n"); return EXIT_FAILURE; } /* Set the new socket to be non-blocking. */ fcntl(conn->sockfd, F_SETFL, O_NONBLOCK); /* Setup SSL/TLS connection. */ if ((conn->ssl = wolfSSL_new(sslCtx)) == NULL) { free(conn); fprintf(stderr, "wolfSSL_new error.\n"); return EXIT_FAILURE; } /* Set the socket to communicate over into the wolfSSL object. */ wolfSSL_set_fd(conn->ssl, conn->sockfd); conn->state = ACCEPT; conn->next = threadData->sslConn; conn->prev = NULL; if (threadData->sslConn != NULL) threadData->sslConn->prev = conn; threadData->sslConn = conn; threadData->cnt++; *sslConn = conn; return EXIT_SUCCESS; } /* Read/write from/to client at the specified socket. * * ctx The SSL/TLS connection data. * threadData The SSL connection data for a thread. * sslConn The SSL connection data object. * returns EXIT_FAILURE on failure and EXIT_SUCCESS otherwise. */ static int SSLConn_ReadWrite(SSLConn_CTX* ctx, ThreadData* threadData, SSLConn* sslConn) { int ret; int len; /* Perform TLS handshake if in accept state. */ switch (sslConn->state) { case ACCEPT: ret = SSL_Accept(sslConn->ssl, &ctx->acceptTime, &ctx->resumeTime); if (ret == 0) { printf("ERROR: Accept failed\n"); SSLConn_Close(ctx, threadData, sslConn); return EXIT_FAILURE; } if (ret == 1) sslConn->state = READ; break; case READ: { char buffer[NUM_READ_BYTES]; len = ctx->bufferLen; if (ctx->maxBytes > 0) { len = min(len, ctx->maxBytes - ctx->totalReadBytes); } if (len == 0) break; /* Read application data. */ ret = SSL_Read(sslConn->ssl, buffer, len, &ctx->totalReadBytes, &ctx->readTime); if (ret == 0) { SSLConn_Close(ctx, threadData, sslConn); return EXIT_FAILURE; } } if (ret != 1) break; sslConn->state = WRITE; case WRITE: len = ctx->replyLen; if (ctx->maxBytes > 0) { len = min(len, ctx->maxBytes - ctx->totalWriteBytes); } if (len == 0) break; /* Write application data. */ ret = SSL_Write(sslConn->ssl, reply, len, &ctx->totalWriteBytes, &ctx->writeTime); if (ret == 0) { printf("ERROR: Write failed\n"); SSLConn_Close(ctx, threadData, sslConn); return EXIT_FAILURE; } if (ret == 1) sslConn->state = READ; break; case CLOSED: break; } return EXIT_SUCCESS; } /* Print the connection statistics. * * ctx The SSL/TLS connection data. */ static void SSLConn_PrintStats(SSLConn_CTX* ctx) { fprintf(stderr, "wolfSSL Server Benchmark %d bytes\n" "\tNum Conns : %9d\n" "\tTotal : %9.3f ms\n" "\tTotal Avg : %9.3f ms\n" "\tt/s : %9.3f\n" "\tAccept : %9.3f ms\n" "\tAccept Avg : %9.3f ms\n", ctx->replyLen, ctx->numConnections - ctx->numResumed, ctx->totalTime * 1000, ctx->totalTime * 1000 / ctx->numConnections, ctx->numConnections / ctx->totalTime, ctx->acceptTime * 1000, ctx->acceptTime * 1000 / (ctx->numConnections - ctx->numResumed)); if (ctx->numResumed > 0) { fprintf(stderr, "\tResumed Conns : %9d\n" "\tResume : %9.3f ms\n" "\tResume Avg : %9.3f ms\n", ctx->numResumed, ctx->resumeTime * 1000, ctx->resumeTime * 1000 / ctx->numResumed); } #ifdef WOLFSSL_ASYNC_CRYPT fprintf(stderr, "\tAsync : %9.3f ms\n" "\tAsync Avg : %9.3f ms\n", ctx->asyncTime * 1000, ctx->asyncTime * 1000 / ctx->numConnections); #endif fprintf(stderr, "\tTotal Read bytes : %9d bytes\n" "\tTotal Write bytes : %9d bytes\n" "\tRead : %9.3f ms (%9.3f MBps)\n" "\tWrite : %9.3f ms (%9.3f MBps)\n", ctx->totalReadBytes, ctx->totalWriteBytes, ctx->readTime * 1000, ctx->totalReadBytes / ctx->readTime / 1024 / 1024, ctx->writeTime * 1000, ctx->totalWriteBytes / ctx->writeTime / 1024 / 1024 ); } /* Initialize the wolfSSL library and create a wolfSSL context. * * version The protocol version. * cert The server's certificate. * key The server's private key matching the certificate. * verifyCert The certificate for client authentication. * cipherList The list of negotiable ciphers. * wolfsslCtx The new wolfSSL context object. * returns EXIT_SUCCESS when a wolfSSL context object is created and * EXIT_FAILURE otherwise. */ static int WolfSSLCtx_Init(ThreadData* threadData, int version, int allowDowngrade, char* cert, char* key, char* verifyCert, char* cipherList) { wolfSSL_method_func method = NULL; method = SSL_GetMethod(version, allowDowngrade); if (method == NULL) return(EXIT_FAILURE); /* Create and initialize WOLFSSL_CTX structure */ if ((threadData->ctx = wolfSSL_CTX_new(method(NULL))) == NULL) { fprintf(stderr, "wolfSSL_CTX_new error.\n"); return(EXIT_FAILURE); } #ifdef WOLFSSL_ASYNC_CRYPT #ifndef WC_NO_ASYNC_THREADING if (wolfAsync_DevOpenThread(&threadData->devId, &threadData->thread_id) < 0) #else if (wolfAsync_DevOpen(&threadData->devId) < 0) #endif { fprintf(stderr, "Async device open failed\nRunning without async\n"); } wolfSSL_CTX_UseAsync(threadData->ctx, threadData->devId); #endif /* Load server certificate into WOLFSSL_CTX */ if (wolfSSL_CTX_use_certificate_file(threadData->ctx, cert, SSL_FILETYPE_PEM) != SSL_SUCCESS) { fprintf(stderr, "Error loading %s, please check the file.\n", cert); WolfSSLCtx_Final(threadData); return(EXIT_FAILURE); } /* Load server key into WOLFSSL_CTX */ if (wolfSSL_CTX_use_PrivateKey_file(threadData->ctx, key, SSL_FILETYPE_PEM) != SSL_SUCCESS) { fprintf(stderr, "Error loading %s, please check the file.\n", key); WolfSSLCtx_Final(threadData); return(EXIT_FAILURE); } /* Setup client authentication. */ wolfSSL_CTX_set_verify(threadData->ctx, SSL_VERIFY_PEER, 0); if (wolfSSL_CTX_load_verify_locations(threadData->ctx, verifyCert, 0) != SSL_SUCCESS) { fprintf(stderr, "Error loading %s, please check the file.\n", verifyCert); WolfSSLCtx_Final(threadData); return(EXIT_FAILURE); } if (cipherList != NULL) { if (wolfSSL_CTX_set_cipher_list(threadData->ctx, cipherList) != SSL_SUCCESS) { fprintf(stderr, "Server can't set cipher list.\n"); WolfSSLCtx_Final(threadData); return(EXIT_FAILURE); } } #ifndef NO_DH SetDHCtx(threadData->ctx); #endif return EXIT_SUCCESS; } /* Cleanup the wolfSSL context and wolfSSL library. * * ctx The wolfSSL context object. */ static void WolfSSLCtx_Final(ThreadData* threadData) { wolfSSL_CTX_free(threadData->ctx); threadData->ctx = NULL; #ifdef WOLFSSL_ASYNC_CRYPT wolfAsync_DevClose(&threadData->devId); #endif } /* Create a socket to listen on and wait for first client. * * port The port to listen on. * numClients The number of clients for listen to support. * socketfd The socket file descriptor to accept on. * returns EXIT_SUCCESS on success and EXIT_FAILURE otherwise. */ static int CreateSocketListen(int port, int numClients, socklen_t* socketfd) { int ret; socklen_t sockfd; struct sockaddr_in serverAddr = {0}; int on = 1; socklen_t len = sizeof(on); /* Set the server's address. */ memset((char *)&serverAddr, 0, sizeof(serverAddr)); serverAddr.sin_family = AF_INET; serverAddr.sin_addr.s_addr = INADDR_ANY; serverAddr.sin_port = htons(port); /* Create a non-blocking socket to listen on for new connections. */ sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd == (socklen_t)-1) { fprintf(stderr, "ERROR: failed to create the socket\n"); return(EXIT_FAILURE); } if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEPORT, &on, len) < 0) fprintf(stderr, "setsockopt SO_REUSEADDR failed\n"); if (setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &on, len) < 0) fprintf(stderr, "setsockopt TCP_NODELAY failed\n"); if (bind(sockfd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { fprintf(stderr, "ERROR: failed to bind\n"); return(EXIT_FAILURE); } printf("Waiting for a connection...\n"); /* Listen for a client to connect. */ ret = listen(sockfd, numClients); if (ret == -1) { fprintf(stderr, "ERROR: failed to listen\n"); return(EXIT_FAILURE); } *socketfd = sockfd; return EXIT_SUCCESS; } /* Handles a number of connections for a thread. * * data The thread data. */ static void *ThreadHandler(void *data) { int ret; socklen_t socketfd = -1; int efd; struct epoll_event event; struct epoll_event event_conn; struct epoll_event* events = NULL; ThreadData* threadData = (ThreadData*)data; #ifdef WOLFSSL_ASYNC_CRYPT WOLF_EVENT* wolfEvents[MAX_WOLF_EVENTS]; #endif /* Initialize wolfSSL and create a context object. */ if (WolfSSLCtx_Init(threadData, version, allowDowngrade, ourCert, ourKey, verifyCert, cipherList) == -1) { exit(EXIT_FAILURE); } /* Allocate space for EPOLL events to be stored. */ events = (struct epoll_event*)malloc(EPOLL_NUM_EVENTS * sizeof(*events)); if (events == NULL) exit(EXIT_FAILURE); /* Create a socket and listen for a client. */ if (CreateSocketListen(port, numClients, &socketfd) == EXIT_FAILURE) exit(EXIT_FAILURE); /* Create an EPOLL file descriptor. */ efd = epoll_create1(0); if (efd == -1) { fprintf(stderr, "ERROR: failed to create epoll\n"); exit(EXIT_FAILURE); } /* Add the event for communications on listening socket. */ memset(&event, 0, sizeof(event)); event.events = EPOLLIN; event.data.ptr = NULL; ret = epoll_ctl(efd, EPOLL_CTL_ADD, socketfd, &event); if (ret == -1) { fprintf(stderr, "ERROR: failed to add event to epoll\n"); exit(EXIT_FAILURE); } threadData->accepting = 1; /* Keep handling clients until done. */ while (!SSLConn_Done(sslConnCtx)) { int n; int i; #ifdef WOLFSSL_ASYNC_CRYPT do { double diff, start = current_time(1); ret = wolfSSL_CTX_AsyncPoll(threadData->ctx, wolfEvents, MAX_WOLF_EVENTS, WOLF_POLL_FLAG_CHECK_HW, &n); diff = current_time(0) - start; pthread_mutex_lock(&sslConnMutex); sslConnCtx->asyncTime += diff; pthread_mutex_unlock(&sslConnMutex); for (i = 0; i < n; i++) { SSLConn* sslConn = threadData->sslConn; while (sslConn != NULL) { if (sslConn->ssl != wolfEvents[i]->context) { sslConn = sslConn->next; continue; } SSLConn_ReadWrite(sslConnCtx, threadData, sslConn); break; } } } while (n > 0); #endif SSLConn_FreeSSLConn(threadData); #ifdef WOLFSSL_ASYNC_CRYPT /* Look for events. */ n = epoll_wait(efd, events, EPOLL_NUM_EVENTS, 0); #else /* Wait a second for events. */ n = epoll_wait(efd, events, EPOLL_NUM_EVENTS, 1); #endif /* Process all returned events. */ for (i = 0; i < n; i++) { /* Error event on socket. */ if (!(events[i].events & EPOLLIN)) { if (events[i].data.ptr == NULL) { /* Not a client, therefore the listening connection. */ close(socketfd); socketfd = -1; } else { /* Client connection. */ SSLConn_Close(sslConnCtx, threadData, events[i].data.ptr); ret = epoll_ctl(efd, EPOLL_CTL_ADD, socketfd, &event); } } else if (events[i].data.ptr == NULL) { SSLConn* sslConn; /* Accept a new client on the listener. */ ret = SSLConn_Accept(threadData, threadData->ctx, socketfd, &sslConn); if (ret == EXIT_SUCCESS) { /* Set EPOLL to check for events on the new socket. */ memset(&event_conn, 0, sizeof(event_conn)); event_conn.events = EPOLLIN | EPOLLET; event_conn.data.ptr = sslConn; ret = epoll_ctl(efd, EPOLL_CTL_ADD, sslConn->sockfd, &event_conn); if (ret == -1) { fprintf(stderr, "ERROR: failed add event to epoll\n"); exit(EXIT_FAILURE); } } if (threadData->cnt == sslConnCtx->numConns) { /* Don't accept any more TCP connections. */ ret = epoll_ctl(efd, EPOLL_CTL_DEL, socketfd, &event); if (ret == -1) { fprintf(stderr, "ERROR: failed delete epoll event\n"); exit(EXIT_FAILURE); } threadData->accepting = 0; } } else { if (sslConnCtx->totalTime == 0) { pthread_mutex_lock(&sslConnMutex); if (sslConnCtx->totalTime == 0) sslConnCtx->totalTime = current_time(1); pthread_mutex_unlock(&sslConnMutex); } ret = SSLConn_ReadWrite(sslConnCtx, threadData, events[i].data.ptr); } } /* Accept more connections again up to the maximum concurrent. */ if (!threadData->accepting && threadData->cnt < sslConnCtx->numConns) { ret = epoll_ctl(efd, EPOLL_CTL_ADD, socketfd, &event); if (ret == -1) { fprintf(stderr, "ERROR: failed add event to epoll\n"); exit(EXIT_FAILURE); } threadData->accepting = 1; } } if (socketfd != -1) close(socketfd); free(events); return NULL; } /* Create a random reply. * * reply The buffer to put the random data into. * replyLen The amount of data to generate. */ static void RandomReply(char* reply, int replyLen) { int ret; WC_RNG rng; ret = wc_InitRng(&rng); if (ret != 0) { fprintf(stderr, "Error: initialize random\n"); exit(EXIT_FAILURE); } ret = wc_RNG_GenerateBlock(&rng, (byte*)reply, replyLen); wc_FreeRng(&rng); if (ret != 0) { fprintf(stderr, "Error: initialize random\n"); exit(EXIT_FAILURE); } } /* Display the usage for the program. */ static void Usage(void) { printf("perf " LIBWOLFSSL_VERSION_STRING " (NOTE: All files relative to wolfSSL home dir)\n"); printf("-? Help, print this usage\n"); printf("-p Port to listen on, not 0, default %d\n", DEFAULT_PORT); printf("-v SSL version [0-3], SSLv3(0) - TLS1.2(3)), default %d\n", SERVER_DEFAULT_VERSION); printf("-a Allow TLS version downgrade\n"); printf("-l Cipher suite list (: delimited)\n"); printf("-c Certificate file, default %s\n", SVR_CERT); printf("-k Key file, default %s\n", SVR_KEY); printf("-A Certificate Authority file, default %s\n", CLI_CERT); printf("-t threads started\n"); printf("-n Benchmark connections\n"); printf("-N concurrent connections\n"); printf("-R bytes read from client\n"); printf("-W bytes written to client\n"); printf("-B Benchmark written bytes\n"); } /* Main entry point for the program. * * argc The count of command line arguments. * argv The command line arguments. * returns 0 on success and 1 otherwise. */ int main(int argc, char* argv[]) { int i; int ch; /* Parse the command line arguments. */ while ((ch = mygetopt(argc, argv, OPTIONS)) != -1) { switch (ch) { /* Help with command line options. */ case '?': Usage(); exit(EXIT_SUCCESS); /* Port number to listen on. */ case 'p': port = (word16)atoi(myoptarg); break; /* Version of SSL/TLS to use. */ case 'v': version = atoi(myoptarg); if (version < 0 || version > 3) { Usage(); exit(MY_EX_USAGE); } break; case 'a': allowDowngrade = 1; break; /* List of cipher suites to use. */ case 'l': cipherList = myoptarg; break; /* File name of server certificate for authentication. */ case 'c': ourCert = myoptarg; break; /* File name of server private key for authentication. */ case 'k': ourKey = myoptarg; break; /* File name of client certificate/CA for peer verification. */ case 'A': verifyCert = myoptarg; break; /* Number of connections to make. */ case 't': numThreads = atoi(myoptarg); if (numThreads < 0 || numThreads > 100) { Usage(); exit(MY_EX_USAGE); } break; /* Number of connections to make. */ case 'n': maxConns = atoi(myoptarg); if (maxConns < 0 || maxConns > 1000000) { Usage(); exit(MY_EX_USAGE); } maxBytes = 0; break; /* Number of conncurrent connections to use. */ case 'N': numConns = atoi(myoptarg); if (numConns < 0 || numConns > 100000) { Usage(); exit(MY_EX_USAGE); } break; /* Number of bytes to read each call. */ case 'R': numBytesRead = atoi(myoptarg); if (numBytesRead <= 0) { Usage(); exit(MY_EX_USAGE); } break; /* Number of bytes to write each call. */ case 'W': numBytesWrite = atoi(myoptarg); if (numBytesWrite <= 0) { Usage(); exit(MY_EX_USAGE); } break; /* Maximum number of read and write bytes (separate counts). */ case 'B': maxBytes = atoi(myoptarg); if (maxBytes <= 0) { Usage(); exit(MY_EX_USAGE); } maxConns = 0; break; /* Unrecognized command line argument. */ default: Usage(); exit(MY_EX_USAGE); } } #ifdef DEBUG_WOLFSSL wolfSSL_Debugging_ON(); #endif #ifdef WOLFSSL_ASYNC_CRYPT wolfAsync_HardwareStart(); #endif /* Initialize wolfSSL */ wolfSSL_Init(); RandomReply(reply, sizeof(reply)); /* Create SSL/TLS connection data object. */ sslConnCtx = SSLConn_New(numThreads, numConns, numBytesRead, numBytesWrite, maxConns, maxBytes); if (sslConnCtx == NULL) exit(EXIT_FAILURE); for (i = 0; i < numThreads; i++) { if (pthread_create(&sslConnCtx->threadData[i].thread_id, NULL, ThreadHandler, &sslConnCtx->threadData[i]) < 0) { perror("ERROR: could not create thread"); } } /* Start all the threads. */ for (i = 0; i < numThreads; i++) pthread_join(sslConnCtx->threadData[i].thread_id, NULL) ; sslConnCtx->totalTime = current_time(0) - sslConnCtx->totalTime; SSLConn_PrintStats(sslConnCtx); SSLConn_Free(sslConnCtx); wolfSSL_Cleanup(); #ifdef WOLFSSL_ASYNC_CRYPT wolfAsync_HardwareStop(); #endif exit(EXIT_SUCCESS); }