Refactor build for better platform support

* Fix some path and compiler things for Windows
* Move source code to align better with Python standards
* Have setup.py build wolfSSL and FFI
* Update Python versions in tox
* Add CMake building for Windows build
* Add missing __init__.py for tox
* Make wolfSSL a git submodule

.gitignore

@@ -16,7 +16,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lib/wolfssl"]
+	path = lib/wolfssl
+	url = https://github.com/wolfssl/wolfssl

Makefile

@@ -33,8 +33,10 @@ clean-build: ## remove build artifacts
 	rm -fr build/
 	rm -fr dist/
 	rm -fr .eggs/
+	rm -fr wolfssl/_ffi*
 	find . -name '*.egg-info' -exec rm -fr {} +
 	find . -name '*.egg' -exec rm -f {} +
+	-cd lib/wolfssl && make clean
 
 clean-pyc: ## remove Python file artifacts
 	find . -name '*.pyc' -exec rm -f {} +

README.rst

@@ -14,6 +14,41 @@ speed, and feature set. It works seamlessly in desktop, enterprise, and cloud
 environments as well. It is the crypto engine behind `wolfSSl's embedded ssl
 library <https://wolfssl.com/wolfSSL/Products-wolfssl.html>`_.
 
+Compiling
+---------
+
+Windows
+^^^^^^^
+
+Install the following on Windows:
+
+* `CMake <https://cmake.org/download/>`_
+* `Git <https://git-scm.com/download/win>`_
+* `Python 3.9 <https://www.python.org/downloads/windows/>`_
+* `Build Tools for Visual Studio <https://visualstudio.microsoft.com/downloads/>`_. This is in the "Tools for Visual Studio" section at the bottom of the page. The "Desktop development with C++" pack is needed from the installer.
+
+Then from the command line install tox and CFFI using:
+
+.. code-block:: sh
+    pip install tox cffi
+
+You can then build the source distribution packages using:
+
+.. code-block:: sh
+    python setup.py sdist
+
+
+Linux
+^^^^^
+
+The `setup.py` file covers most things you will need to do to build and install from source. As pre-requisites you will need to install either from your OS repository or pip. You'll also need the Python development package for your Python version:
+
+* `cffi`
+* `tox`
+* `pytest`
+
+To build a source package run `python setup.py sdist`, to build a wheel package run `python setup.py bdist_wheel`. To test the build run `tox`. The `tox` tests rely on Python 3.9 being installed, if you do not have this version we recommend using `pyenv` to install it.
+
 Installation
 ------------
 

lib/wolfssl

@@ -0,0 +1 @@
+Subproject commit c3513bf2573c30f6d2df815de216120e92142020

setup.py

@@ -28,7 +28,7 @@ from setuptools.command.build_ext import build_ext
 
 
 # Adding src folder to the include path in order to import from wolfcrypt
-package_dir = os.path.join(os.path.dirname(__file__), "src")
+package_dir = os.path.dirname(__file__)
 sys.path.insert(0, package_dir)
 
 import wolfcrypt
@@ -72,7 +72,7 @@ setup(
     package_dir={"":package_dir},
 
     zip_safe=False,
-    cffi_modules=["./src/wolfcrypt/_build_ffi.py:ffibuilder"],
+    cffi_modules=["./wolfcrypt/_build_ffi.py:ffibuilder"],
 
     keywords="wolfssl, wolfcrypt, security, cryptography",
     classifiers=[

src/wolfcrypt/_build_wolfssl.py

@@ -1,215 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2006-2022 wolfSSL Inc.
-#
-# This file is part of wolfSSL. (formerly known as CyaSSL)
-#
-# wolfSSL is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# wolfSSL is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
-
-import os
-import subprocess
-from contextlib import contextmanager
-from distutils.util import get_platform
-from wolfcrypt.__init__ import __wolfssl_version__ as version
-
-
-def local_path(path):
-    """ Return path relative to the root of this project
-    """
-    current = os.path.dirname(__file__)
-    gparent = os.path.dirname(os.path.dirname(current))
-    return os.path.abspath(os.path.join(gparent, path))
-
-
-WOLFSSL_GIT_ADDR = "https://github.com/wolfssl/wolfssl.git"
-WOLFSSL_SRC_PATH = local_path("lib/wolfssl/src")
-
-
-def wolfssl_inc_path():
-    wolfssl_path = os.environ.get("USE_LOCAL_WOLFSSL")
-    if wolfssl_path is None:
-        return local_path("lib/wolfssl/src")
-    else:
-        if os.path.isdir(wolfssl_path) and os.path.exists(wolfssl_path):
-            return wolfssl_path + "/include"
-        else:
-            return "/usr/local/include"
-
-
-def wolfssl_lib_path():
-    wolfssl_path = os.environ.get("USE_LOCAL_WOLFSSL")
-    if wolfssl_path is None:
-        return local_path("lib/wolfssl/{}/{}/lib".format(
-                          get_platform(), version))
-    else:
-        if os.path.isdir(wolfssl_path) and os.path.exists(wolfssl_path):
-            return wolfssl_path + "/lib"
-        else:
-            return "/usr/local/lib"
-
-
-def call(cmd):
-    print("Calling: '{}' from working directory {}".format(cmd, os.getcwd()))
-
-    old_env = os.environ["PATH"]
-    os.environ["PATH"] = "{}:{}".format(WOLFSSL_SRC_PATH, old_env)
-    subprocess.check_call(cmd, shell=True, env=os.environ)
-    os.environ["PATH"] = old_env
-
-
-@contextmanager
-def chdir(new_path, mkdir=False):
-    old_path = os.getcwd()
-
-    if mkdir:
-        try:
-            os.mkdir(new_path)
-        except OSError:
-            pass
-
-    try:
-        yield os.chdir(new_path)
-    finally:
-        os.chdir(old_path)
-
-
-def clone_wolfssl(version):
-    """ Clone wolfSSL C library repository
-    """
-    call("git clone --depth=1 --branch={} {} {}".format(
-        version, WOLFSSL_GIT_ADDR, WOLFSSL_SRC_PATH))
-
-
-def checkout_version(version):
-    """ Ensure that we have the right version
-    """
-    with chdir(WOLFSSL_SRC_PATH):
-        current = subprocess.check_output(
-            ["git", "describe", "--all", "--exact-match"]
-        ).strip().decode().split('/')[-1]
-
-        if current != version:
-            tags = subprocess.check_output(
-                ["git", "tag"]
-            ).strip().decode().split("\n")
-
-            if version != "master" and version not in tags:
-                call("git fetch --depth=1 origin tag {}".format(version))
-
-            call("git checkout --force {}".format(version))
-
-            return True  # rebuild needed
-
-    return False
-
-
-def ensure_wolfssl_src(version):
-    """ Ensure that wolfssl sources are presents and up-to-date
-    """
-    if not os.path.isdir(WOLFSSL_SRC_PATH):
-        clone_wolfssl(version)
-        return True
-
-    return checkout_version(version)
-
-
-def make_flags(prefix):
-    """ Returns compilation flags
-    """
-    flags = []
-
-    if get_platform() in ["linux-x86_64", "linux-i686"]:
-        flags.append("CFLAGS=-fPIC")
-
-    # install location
-    flags.append("--prefix={}".format(prefix))
-
-    # crypt only, lib only
-    flags.append("--enable-cryptonly")
-    flags.append("--disable-crypttests")
-    flags.append("--disable-shared")
-
-    # symmetric ciphers
-    flags.append("--enable-aes")
-    flags.append("--enable-des3")
-    flags.append("--enable-chacha")
-
-    flags.append("--disable-aesgcm")
-
-    # hashes and MACs
-    flags.append("--enable-sha")
-    flags.append("--enable-sha384")
-    flags.append("--enable-sha512")
-    flags.append("--enable-sha3")
-    flags.append("--enable-hkdf")
-
-    flags.append("--disable-md5")
-    flags.append("--disable-sha224")
-    flags.append("--disable-poly1305")
-
-    # asymmetric ciphers
-    flags.append("--enable-rsa")
-    flags.append("--enable-ecc")
-    flags.append("--enable-ed25519")
-    flags.append("--enable-ed448")
-    flags.append("--enable-curve25519")
-    flags.append("--enable-keygen")
-
-    flags.append("--disable-dh")
-
-    # pwdbased
-    flags.append("--enable-pwdbased")
-    flags.append("--enable-pkcs7")
-
-    # disabling other configs enabled by default
-    flags.append("--disable-oldtls")
-    flags.append("--disable-oldnames")
-    flags.append("--disable-extended-master")
-    flags.append("--disable-errorstrings")
-
-    return " ".join(flags)
-
-
-def make(configure_flags):
-    """ Create a release of wolfSSL C library
-    """
-    with chdir(WOLFSSL_SRC_PATH):
-        call("git clean -fdX")
-
-        try:
-            call("./autogen.sh")
-        except subprocess.CalledProcessError:
-            call("libtoolize")
-            call("./autogen.sh")
-
-        call("./configure {}".format(configure_flags))
-        call("make")
-        call("make install-exec")
-
-
-def build_wolfssl(version="master"):
-    prefix = local_path("lib/wolfssl/{}/{}".format(
-        get_platform(), version))
-    libfile = os.path.join(prefix, 'lib/libwolfssl.la')
-
-    rebuild = ensure_wolfssl_src(version)
-
-    if rebuild or not os.path.isfile(libfile):
-        make(make_flags(prefix))
-
-
-if __name__ == "__main__":
-    build_wolfssl()

tox.ini

@@ -1,6 +1,6 @@
 [tox]
-envlist = py27, py34, py35, py36
+envlist = py39, pep8
-skip_missing_interpreters = true
+skipsdist = True
 
 [testenv]
 setenv =

wolfcrypt/_build_ffi.py

@@ -24,7 +24,38 @@ import re
 from distutils.util import get_platform
 from cffi import FFI
 from wolfcrypt import __wolfssl_version__ as version
-from wolfcrypt._build_wolfssl import wolfssl_inc_path, wolfssl_lib_path
+from wolfcrypt._build_wolfssl import wolfssl_inc_path, wolfssl_lib_path, ensure_wolfssl_src, make, make_flags, local_path
+
+libwolfssl_path = ""
+
+def get_libwolfssl():
+    global libwolfssl_path
+    if sys.platform == "win32":
+        libwolfssl_path = os.path.join(wolfssl_lib_path(), "wolfssl.lib")
+        if not os.path.exists(libwolfssl_path):
+            return 0
+        else:
+            return 1
+    else:
+        libwolfssl_path = os.path.join(wolfssl_lib_path(), "libwolfssl.a")
+        if not os.path.exists(libwolfssl_path):
+            libwolfssl_path = os.path.join(wolfssl_lib_path(), "libwolfssl.so")
+            if not os.path.exists(libwolfssl_path):
+                return 0
+            else:
+                return 1
+        else:
+            return 1
+
+def generate_libwolfssl():
+    ensure_wolfssl_src(version)
+    prefix = local_path("lib/wolfssl/{}/{}".format(
+        get_platform(), version))
+    make(make_flags(prefix))
+
+if get_libwolfssl() == 0:
+    generate_libwolfssl()
+    get_libwolfssl()
 
 # detect features if user has built against local wolfSSL library
 # if they are not, we are controlling build options in _build_wolfssl.py
@@ -110,9 +141,18 @@ if RSA_BLINDING_ENABLED and FIPS_ENABLED:
 # build cffi module, wrapping native wolfSSL
 ffibuilder = FFI()
 
+cffi_libraries = ["wolfssl"]
+
+# Needed for WIN32 functions in random.c
+if sys.platform == "win32":
+    cffi_libraries.append("Advapi32")
+
 ffibuilder.set_source(
     "wolfcrypt._ffi",
     """
+#ifdef __cplusplus
+extern "C" {
+#endif
     #include <wolfssl/options.h>
     #include <wolfssl/wolfcrypt/settings.h>
 
@@ -136,6 +176,9 @@ ffibuilder.set_source(
     #include <wolfssl/wolfcrypt/ed25519.h>
     #include <wolfssl/wolfcrypt/ed448.h>
     #include <wolfssl/wolfcrypt/curve25519.h>
+#ifdef __cplusplus
+}
+#endif
 
     int MPAPI_ENABLED = """ + str(MPAPI_ENABLED) + """;
     int SHA_ENABLED = """ + str(SHA_ENABLED) + """;
@@ -162,7 +205,7 @@ ffibuilder.set_source(
     """,
     include_dirs=[wolfssl_inc_path()],
     library_dirs=[wolfssl_lib_path()],
-    libraries=["wolfssl"],
+    libraries=cffi_libraries,
 )
 
 _cdef = """
@@ -511,5 +554,4 @@ if FIPS_ENABLED and (FIPS_VERSION > 5 or (FIPS_VERSION == 5 and FIPS_VERSION >=
 
 ffibuilder.cdef(_cdef)
 
-if __name__ == "__main__":
+ffibuilder.compile(verbose=True)
-    ffibuilder.compile(verbose=True)

wolfcrypt/_build_wolfssl.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2006-2022 wolfSSL Inc.
+#
+# This file is part of wolfSSL. (formerly known as CyaSSL)
+#
+# wolfSSL is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# wolfSSL is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+import os
+import sys
+import subprocess
+from contextlib import contextmanager
+from distutils.util import get_platform
+from wolfcrypt.__init__ import __wolfssl_version__ as version
+
+
+def local_path(path):
+    """ Return path relative to the root of this project
+    """
+    current = os.path.abspath(os.getcwd())
+    return os.path.abspath(os.path.join(current, path))
+
+
+WOLFSSL_SRC_PATH = local_path("lib/wolfssl")
+
+
+def wolfssl_inc_path():
+    if sys.platform == "win32":
+        return os.path.join(WOLFSSL_SRC_PATH)
+    else:
+        wolfssl_path = os.environ.get("USE_LOCAL_WOLFSSL")
+        if wolfssl_path is None:
+            return local_path("lib/wolfssl")
+        else:
+            if os.path.isdir(wolfssl_path) and os.path.exists(wolfssl_path):
+                return wolfssl_path + "/include"
+            else:
+                return "/usr/local/include"
+
+
+def wolfssl_lib_path():
+    if sys.platform == "win32":
+        return os.path.join(WOLFSSL_SRC_PATH, "build", "Release")
+    else:
+        wolfssl_path = os.environ.get("USE_LOCAL_WOLFSSL")
+        if wolfssl_path is None:
+            return local_path("lib/wolfssl/{}/{}/lib".format(
+                              get_platform(), version))
+        else:
+            if os.path.isdir(wolfssl_path) and os.path.exists(wolfssl_path):
+                return wolfssl_path + "/lib"
+            else:
+                return "/usr/local/lib"
+
+
+def call(cmd):
+    print("Calling: '{}' from working directory {}".format(cmd, os.getcwd()))
+
+    old_env = os.environ["PATH"]
+    os.environ["PATH"] = "{}:{}".format(WOLFSSL_SRC_PATH, old_env)
+    subprocess.check_call(cmd, shell=True, env=os.environ)
+    os.environ["PATH"] = old_env
+
+
+@contextmanager
+def chdir(new_path, mkdir=False):
+    old_path = os.getcwd()
+
+    if mkdir:
+        try:
+            os.mkdir(new_path)
+        except OSError:
+            pass
+
+    try:
+        yield os.chdir(new_path)
+    finally:
+        os.chdir(old_path)
+
+
+def checkout_version(version):
+    """ Ensure that we have the right version
+    """
+    with chdir(WOLFSSL_SRC_PATH):
+        current = ""
+        try:
+            current = subprocess.check_output(
+                ["git", "describe", "--all", "--exact-match"]
+            ).strip().decode().split('/')[-1]
+        except:
+            pass
+
+        if current != version:
+            tags = subprocess.check_output(
+                ["git", "tag"]
+            ).strip().decode().split("\n")
+
+            if version != "master" and version not in tags:
+                call("git fetch --depth=1 origin tag {}".format(version))
+
+            call("git checkout --force {}".format(version))
+
+            return True  # rebuild needed
+
+    return False
+
+
+def ensure_wolfssl_src(ref):
+    """ Ensure that wolfssl sources are presents and up-to-date
+    """
+    if not os.path.isdir(WOLFSSL_SRC_PATH):
+        os.mkdir(WOLFSSL_SRC_PATH)
+
+    if not os.path.isdir(os.path.join(WOLFSSL_SRC_PATH, "wolfssl")):
+        subprocess.run(["git", "submodule", "update", "--init", "--depth=1"])
+
+    return checkout_version(version)
+
+
+def make_flags(prefix):
+    """ Returns compilation flags
+    """
+    if sys.platform == "win32":
+        flags = []
+        flags.append("-DWOLFSSL_CRYPT_TESTS=no")
+        flags.append("-DWOLFSSL_EXAMPLES=no")
+        flags.append("-DBUILD_SHARED_LIBS=off")
+        flags.append("-DWOLFSSL_CRYPT_ONLY=yes")
+        flags.append("-DWOLFSSL_AES=yes")
+        flags.append("-DWOLFSSL_DES3=yes")
+        flags.append("-DWOLFSSL_CHACHA=yes")
+        flags.append("-DWOLFSSL_AESGCM=no")
+        flags.append("-DWOLFSSL_SHA=yes")
+        flags.append("-DWOLFSSL_SHA384=yes")
+        flags.append("-DWOLFSSL_SHA512=yes")
+        flags.append("-DWOLFSSL_SHA3=yes")
+        flags.append("-DWOLFSSL_HKDF=yes")
+        flags.append("-DWOLFSSL_MD5=no")
+        flags.append("-DWOLFSSL_SHA224=no")
+        flags.append("-DWOLFSSL_POLY1305=no")
+        flags.append("-DWOLFSSL_RSA=yes")
+        flags.append("-DWOLFSSL_ECC=yes")
+        flags.append("-DWOLFSSL_ED25519=yes")
+        flags.append("-DWOLFSSL_ED448=yes")
+        flags.append("-DWOLFSSL_CURVE25519=yes")
+        flags.append("-DWOLFSSL_DH=no")
+        flags.append("-DWOLFSSL_PWDBASED=yes")
+        flags.append("-DWOLFSSL_PKCS7=yes")
+        flags.append("-DWOLFSSL_OLD_TLS=no")
+        flags.append("-DWOLFSSL_OLD_NAMES=no")
+        flags.append("-DWOLFSSL_EXTENDED_MASTER=no")
+        flags.append("-DWOLFSSL_ERROR_STRINGS=no")
+        # Part of hack for missing CMake option
+        flags.append("-DCMAKE_C_FLAGS=\"/DWOLFSSL_KEY_GEN=1 /DWOLFCRYPT_ONLY=1\"")
+
+        return " ".join(flags)
+    else:
+        flags = []
+
+        if get_platform() in ["linux-x86_64", "linux-i686"]:
+            flags.append("CFLAGS=-fPIC")
+
+        # install location
+        flags.append("--prefix={}".format(prefix))
+
+        # crypt only, lib only
+        flags.append("--enable-cryptonly")
+        flags.append("--disable-crypttests")
+        flags.append("--disable-shared")
+
+        # symmetric ciphers
+        flags.append("--enable-aes")
+        flags.append("--enable-des3")
+        flags.append("--enable-chacha")
+
+        flags.append("--disable-aesgcm")
+
+        # hashes and MACs
+        flags.append("--enable-sha")
+        flags.append("--enable-sha384")
+        flags.append("--enable-sha512")
+        flags.append("--enable-sha3")
+        flags.append("--enable-hkdf")
+
+        flags.append("--disable-md5")
+        flags.append("--disable-sha224")
+        flags.append("--disable-poly1305")
+
+        # asymmetric ciphers
+        flags.append("--enable-rsa")
+        flags.append("--enable-ecc")
+        flags.append("--enable-ed25519")
+        flags.append("--enable-ed448")
+        flags.append("--enable-curve25519")
+        flags.append("--enable-keygen")
+
+        flags.append("--disable-dh")
+
+        # pwdbased
+        flags.append("--enable-pwdbased")
+        flags.append("--enable-pkcs7")
+
+        # disabling other configs enabled by default
+        flags.append("--disable-oldtls")
+        flags.append("--disable-oldnames")
+        flags.append("--disable-extended-master")
+        flags.append("--disable-errorstrings")
+
+        return " ".join(flags)
+
+
+# Horrid hack because we have no CMake option in 5.1.1 for this
+def cmake_hack():
+    options_file = os.path.join(WOLFSSL_SRC_PATH, "wolfssl", "options.h")
+    with open(options_file, "r") as f:
+        contents = f.readlines()
+
+    contents.insert(26, "#undef WOLFSSL_KEY_GEN\n")
+    contents.insert(27, "#define WOLFSSL_KEY_GEN\n")
+    contents.insert(28, "#undef WOLFCRYPT_ONLY\n")
+    contents.insert(29, "#define WOLFCRYPT_ONLY\n")
+
+    with open(options_file, "w") as f:
+        contents = "".join(contents)
+        f.write(contents)
+
+
+def make(configure_flags):
+    """ Create a release of wolfSSL C library
+    """
+    if sys.platform == 'win32':
+        build_path = os.path.join(WOLFSSL_SRC_PATH, "build")
+        if not os.path.isdir(build_path):
+            os.mkdir(build_path)
+        with chdir(build_path):
+            call("cmake .. {}".format(configure_flags))
+            cmake_hack()
+            call("cmake --build . --config Release")
+    else:
+        with chdir(WOLFSSL_SRC_PATH):
+            call("git clean -fdX")
+
+            try:
+                call("./autogen.sh")
+            except subprocess.CalledProcessError:
+                call("libtoolize")
+                call("./autogen.sh")
+
+            call("./configure {}".format(configure_flags))
+            call("make")
+            call("make install-exec")
+
+
+def build_wolfssl(version="master"):
+    prefix = local_path("lib/wolfssl/{}/{}".format(
+        get_platform(), version))
+    if sys.platform == 'win32':
+        libfile = os.path.join(WOLFSSL_SRC_PATH, "build", "Release", "wolfssl.lib")
+    else:
+        libfile = os.path.join(prefix, 'lib/libwolfssl.la')
+
+    rebuild = ensure_wolfssl_src(version)
+
+    if rebuild or not os.path.isfile(libfile):
+        make(make_flags(prefix))
+
+if __name__ == "__main__":
+    build_wolfssl()