Fixes for building with C++17. Adds new `USE_CPLUSPLUS_BYTE` build option to use `std::byte` as `byte` to avoid redefinition issues when customer application sets default namespace to "std" or wants to use "byte".

Tested using `./configure CC="g++ -std=c++17" --enable-all && make check`.

src/ssl.c

@@ -8868,8 +8868,8 @@ static int LoadSystemCaCertsMac(WOLFSSL_CTX* ctx, byte* loaded)
 
     for (i = 0; ret == WOLFSSL_SUCCESS &&
          i < sizeof(trustDomains)/sizeof(*trustDomains); ++i) {
-        stat = SecTrustSettingsCopyCertificates(trustDomains[i], &certs);
-
+        stat = SecTrustSettingsCopyCertificates(
+            (SecTrustSettingsDomain)trustDomains[i], &certs);
         if (stat == errSecSuccess) {
             numCerts = CFArrayGetCount(certs);
             for (j = 0; j < numCerts; ++j) {

wolfssl/test.h

@@ -2033,7 +2033,8 @@ static WC_INLINE void tcp_listen(SOCKET_T* sockfd, word16* port, int useAnyAddr,
 
     /* don't use INADDR_ANY by default, firewall may block, make user switch
        on */
-    build_addr(&addr, (useAnyAddr ? INADDR_ANY : wolfSSLIP), *port, udp, sctp);
+    build_addr(&addr, (useAnyAddr ? (const char*)INADDR_ANY : wolfSSLIP),
+        *port, udp, sctp);
     tcp_socket(sockfd, udp, sctp);
 
 #if !defined(USE_WINDOWS_API) && !defined(WOLFSSL_MDK_ARM)\
@@ -2111,7 +2112,8 @@ static WC_INLINE void udp_accept(SOCKET_T* sockfd, SOCKET_T* clientfd,
     SOCKADDR_IN_T addr;
 
     (void)args;
-    build_addr(&addr, (useAnyAddr ? INADDR_ANY : wolfSSLIP), port, 1, 0);
+    build_addr(&addr, (useAnyAddr ? (const char*)INADDR_ANY : wolfSSLIP),
+        port, 1, 0);
     tcp_socket(sockfd, 1, 0);
 
 

wolfssl/wolfcrypt/types.h

@@ -74,10 +74,18 @@ decouple library dependencies with standard string, memory and so on.
 
     #ifndef WOLFSSL_TYPES
         #ifndef byte
+            /* If using C++ C17 or later and getting:
+             *   "error: reference to 'byte' is ambiguous", this is caused by
+             * cstddef conflict with "std::byte" in
+             *   "enum class byte : unsigned char {};".
+             * This can occur if the user application is using "std" as the
+             * default namespace before including wolfSSL headers.
+             * Workarounds: https://github.com/wolfSSL/wolfssl/issues/5400
+             */
             typedef unsigned char  byte;
+        #endif
             typedef   signed char  sword8;
             typedef unsigned char  word8;
-        #endif
         #ifdef WC_16BIT_CPU
             typedef          int   sword16;
             typedef unsigned int   word16;