From d57b7e9dcd74d15c13d260bcb381bc516887eb13 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 8 Feb 2022 15:04:22 +0000 Subject: [PATCH 1/2] Refactor build to be more Python-like This ports over some of the fixes made to wolfCrypt. Some of this will be pre-requisites for Windows support. Build recursion fixes: * Don't import module being built in setup.py * Don't build the C code until we are doing binary dist or install * Tox tests bdist_wheel instead of source Other fixes: * Separate out version to separate file for modifying * Update copyright dates * Unification of _build_ffi and _build_wolfssl --- .gitignore | 4 + setup.py | 51 +++++---- tox.ini | 1 + wolfssl/__about__.py | 18 +--- wolfssl/__init__.py | 2 +- wolfssl/_build_ffi.py | 187 ++++++++++++++++++++++++++++++++- wolfssl/_build_wolfssl.py | 212 -------------------------------------- wolfssl/_methods.py | 2 +- wolfssl/_openssl.py | 2 +- wolfssl/_version.py | 11 ++ wolfssl/exceptions.py | 2 +- wolfssl/utils.py | 2 +- 12 files changed, 231 insertions(+), 263 deletions(-) delete mode 100644 wolfssl/_build_wolfssl.py create mode 100644 wolfssl/_version.py diff --git a/.gitignore b/.gitignore index 3d307ce..bec0615 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,7 @@ venv_* # code editor preferences .vscode + +# wolfSSL specific things +wolfssl/_ffi* +tmpdist/ diff --git a/setup.py b/setup.py index 848cf07..9364502 100755 --- a/setup.py +++ b/setup.py @@ -26,11 +26,21 @@ import sys from setuptools import setup from setuptools.command.build_ext import build_ext - -import wolfssl -from wolfssl._build_wolfssl import build_wolfssl -from wolfssl._build_wolfssl import wolfssl_inc_path, wolfssl_lib_path - +import re +VERSIONFILE = "wolfssl/_version.py" +verstrline = open(VERSIONFILE, "rt").read() +VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]" +mo = re.search(VSRE, verstrline, re.M) +if mo: + verstr = mo.group(1) +else: + raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,)) +VSRE = r"^__wolfssl_version__ = ['\"]([^'\"]*)['\"]" +mo = re.search(VSRE, verstrline, re.M) +if mo: + wolfverstr = mo.group(1) +else: + raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,)) # long_description with open("README.rst") as readme_file: @@ -60,30 +70,16 @@ def verify_wolfssl_config(): raise RuntimeError("wolfSSL needs to be compiled with " "--enable-opensslextra") -class cffiBuilder(build_ext, object): - - def build_extension(self, ext): - """ Compile manually the wolfssl-py extension, bypass setuptools - """ - - # if USE_LOCAL_WOLFSSL environment variable has been defined, - # do not clone and compile wolfSSL from GitHub - if os.environ.get("USE_LOCAL_WOLFSSL") is None: - build_wolfssl(wolfssl.__wolfssl_version__) - - verify_wolfssl_config() - - super(cffiBuilder, self).build_extension(ext) setup( - name=wolfssl.__title__, - version=wolfssl.__version__, - description=wolfssl.__summary__, + name="wolfssl", + version=verstr, + description="Python module that encapsulates wolfSSL's C SSL/TLS library.", long_description=long_description, - author=wolfssl.__author__, - author_email=wolfssl.__email__, - url=wolfssl.__uri__, - license=wolfssl.__license__, + author="wolfSSL Inc.", + author_email="info@wolfssl.com", + url="https://github.com/wolfssl/wolfssl-py", + license="GPLv2 or Commercial License", packages=["wolfssl"], @@ -107,6 +103,5 @@ setup( setup_requires=["cffi"], install_requires=["cffi"], test_suite="tests", - tests_require=["tox", "pytest"], - cmdclass={"build_ext" : cffiBuilder} + tests_require=["tox", "pytest"] ) diff --git a/tox.ini b/tox.ini index bb31b55..4b698fc 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ envlist = py3 [testenv] +wheel = true setenv = PYTHONPATH = {toxinidir}:{toxinidir}/wolfssl-py diff --git a/wolfssl/__about__.py b/wolfssl/__about__.py index 275a55f..6b31645 100644 --- a/wolfssl/__about__.py +++ b/wolfssl/__about__.py @@ -2,7 +2,7 @@ # # __about__.py # -# Copyright (C) 2006-2020 wolfSSL Inc. +# Copyright (C) 2006-2022 wolfSSL Inc. # # This file is part of wolfSSL. (formerly known as CyaSSL) # @@ -20,27 +20,17 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +from wolfssl._version import __version__, __wolfssl_version__ + __title__ = "wolfssl" __summary__ = "Python module that encapsulates wolfSSL's C SSL/TLS library." __uri__ = "https://github.com/wolfssl/wolfssl-py" -# When bumping the C library version, reset the POST count to 0 - -__wolfssl_version__ = "v5.1.1-stable" - -# We're using implicit post releases [PEP 440] to bump package version -# while maintaining the C library version intact for better reference. -# https://www.python.org/dev/peps/pep-0440/#implicit-post-releases -# -# MAJOR.MINOR.BUILD-POST - -__version__ = __wolfssl_version__[1:].replace("stable", "0") - __author__ = "wolfSSL Inc." __email__ = "info@wolfssl.com" __license__ = "GPLv2 or Commercial License" -__copyright__ = "Copyright (C) 2006-2020 wolfSSL Inc" +__copyright__ = "Copyright (C) 2006-2022 wolfSSL Inc" __all__ = [ "__title__", "__summary__", "__uri__", "__version__", diff --git a/wolfssl/__init__.py b/wolfssl/__init__.py index 6efa0ee..17c010f 100644 --- a/wolfssl/__init__.py +++ b/wolfssl/__init__.py @@ -2,7 +2,7 @@ # # __init__.py # -# Copyright (C) 2006-2020 wolfSSL Inc. +# Copyright (C) 2006-2022 wolfSSL Inc. # # This file is part of wolfSSL. (formerly known as CyaSSL) # diff --git a/wolfssl/_build_ffi.py b/wolfssl/_build_ffi.py index 335f4d3..b19d519 100644 --- a/wolfssl/_build_ffi.py +++ b/wolfssl/_build_ffi.py @@ -2,7 +2,7 @@ # # build_ffi.py # -# Copyright (C) 2006-2020 wolfSSL Inc. +# Copyright (C) 2006-2022 wolfSSL Inc. # # This file is part of wolfSSL. (formerly known as CyaSSL) # @@ -22,10 +22,11 @@ # pylint: disable=missing-docstring, invalid-name +import argparse +from contextlib import contextmanager from distutils.util import get_platform from cffi import FFI -from wolfssl._build_wolfssl import wolfssl_inc_path, wolfssl_lib_path, ensure_wolfssl_src, make, make_flags, local_path -from wolfssl.__about__ import __wolfssl_version__ as version +from wolfssl._version import __wolfssl_version__ as version import wolfssl._openssl as openssl import subprocess import shlex @@ -35,6 +36,180 @@ from collections import namedtuple libwolfssl_path = "" + +def local_path(path): + """ Return path relative to the root of this project + """ + current = os.path.abspath(os.getcwd()) + return os.path.abspath(os.path.join(current, path)) + + +WOLFSSL_SRC_PATH = local_path("lib/wolfssl") + + +def wolfssl_inc_path(): + wolfssl_path = os.environ.get("USE_LOCAL_WOLFSSL") + if wolfssl_path is None: + return local_path("lib/wolfssl") + else: + if os.path.isdir(wolfssl_path) and os.path.exists(wolfssl_path): + return wolfssl_path + "/include" + else: + return "/usr/local/include" + + +def wolfssl_lib_path(): + wolfssl_path = os.environ.get("USE_LOCAL_WOLFSSL") + if wolfssl_path is None: + return local_path("lib/wolfssl/{}/{}/lib".format( + get_platform(), version)) + else: + if os.path.isdir(wolfssl_path) and os.path.exists(wolfssl_path): + return wolfssl_path + "/lib" + else: + return "/usr/local/lib" + + +def call(cmd): + print("Calling: '{}' from working directory {}".format(cmd, os.getcwd())) + + old_env = os.environ["PATH"] + os.environ["PATH"] = "{}:{}".format(WOLFSSL_SRC_PATH, old_env) + subprocess.check_call(cmd, shell=True, env=os.environ) + os.environ["PATH"] = old_env + + +@contextmanager +def chdir(new_path, mkdir=False): + old_path = os.getcwd() + + if mkdir: + try: + os.mkdir(new_path) + except OSError: + pass + + try: + yield os.chdir(new_path) + finally: + os.chdir(old_path) + + +def checkout_ref(ref): + """ Ensure that we have the right version + """ + with chdir(WOLFSSL_SRC_PATH): + current = "" + try: + current = subprocess.check_output( + ["git", "describe", "--all", "--exact-match"] + ).strip().decode().split('/')[-1] + except: + pass + + if current != ref: + tags = subprocess.check_output( + ["git", "tag"] + ).strip().decode().split("\n") + + if ref != "master" and ref not in tags: + call("git fetch --depth=1 origin tag {}".format(ref)) + + call("git checkout --force {}".format(ref)) + + return True # rebuild needed + + return False + + +def ensure_wolfssl_src(ref): + """ Ensure that wolfssl sources are presents and up-to-date + """ + if not os.path.isdir("lib"): + os.mkdir("lib") + with chdir("lib"): + subprocess.run(["git", "clone", "--depth=1", "https://github.com/wolfssl/wolfssl"]) + + if not os.path.isdir(os.path.join(WOLFSSL_SRC_PATH, "wolfssl")): + subprocess.run(["git", "submodule", "update", "--init", "--depth=1"]) + + return checkout_ref(ref) + + +def make_flags(prefix, debug): + """ Returns compilation flags + """ + flags = [] + cflags = [] + + if get_platform() in ["linux-x86_64", "linux-i686"]: + cflags.append("-fpic") + + # install location + flags.append("--prefix={}".format(prefix)) + + # lib only + flags.append("--disable-shared") + flags.append("--disable-examples") + + # tls 1.3 + flags.append("--enable-tls13") + flags.append("--enable-sslv3") + + # for urllib3 - requires SNI (tlsx), options (openssl compat), peer cert + flags.append("--enable-tlsx") + flags.append("--enable-opensslextra") + cflags.append("-DKEEP_PEER_CERT") + + # for pyOpenSSL + flags.append("--enable-secure-renegotiation") + flags.append("--enable-opensslall") + cflags.append("-DFP_MAX_BITS=8192") + cflags.append("-DHAVE_EX_DATA") + cflags.append("-DOPENSSL_COMPATIBLE_DEFAULTS") + + if debug: + flags.append("--enable-debug") + + # Note: websocket-client test server (echo.websocket.org) only supports + # TLS 1.2 with TLS_RSA_WITH_AES_128_CBC_SHA + # If compiling for use with websocket-client, must enable static RSA suites. + # cflags.append("-DWOLFSSL_STATIC_RSA") + + joined_flags = " ".join(flags) + joined_cflags = " ".join(cflags) + + return joined_flags + " CFLAGS=\"" + joined_cflags + "\"" + + +def make(configure_flags): + """ Create a release of wolfSSL C library + """ + with chdir(WOLFSSL_SRC_PATH): + call("git clean -fdX") + + try: + call("./autogen.sh") + except subprocess.CalledProcessError: + call("libtoolize") + call("./autogen.sh") + + call("./configure {}".format(configure_flags)) + call("make") + call("make install") + + +def build_wolfssl(ref, debug=False): + prefix = local_path("lib/wolfssl/{}/{}".format( + get_platform(), ref)) + libfile = os.path.join(prefix, 'lib/libwolfssl.la') + + rebuild = ensure_wolfssl_src(ref) + + if rebuild or not os.path.isfile(libfile): + make(make_flags(prefix, debug)) + + def make_optional_func_list(libwolfssl_path, funcs): if libwolfssl_path.endswith(".so"): libwolfssl = cdll.LoadLibrary(libwolfssl_path) @@ -55,6 +230,7 @@ def make_optional_func_list(libwolfssl_path, funcs): return defined + def get_libwolfssl(): libwolfssl_path = os.path.join(wolfssl_lib_path(), "libwolfssl.a") if not os.path.exists(libwolfssl_path): @@ -66,12 +242,14 @@ def get_libwolfssl(): else: return 1 + def generate_libwolfssl(): ensure_wolfssl_src(version) prefix = local_path("lib/wolfssl/{}/{}".format( get_platform(), version)) make(make_flags(prefix, False)) + if get_libwolfssl() == 0: generate_libwolfssl() get_libwolfssl() @@ -294,4 +472,5 @@ for func in optional_funcs: ffi_cdef = cdef + openssl.construct_cdef(optional_funcs) ffi.cdef(ffi_cdef) -ffi.compile(verbose=True) +if __name__ == "__main__": + ffi.compile(verbose=True) diff --git a/wolfssl/_build_wolfssl.py b/wolfssl/_build_wolfssl.py deleted file mode 100644 index 7af7960..0000000 --- a/wolfssl/_build_wolfssl.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2006-2020 wolfSSL Inc. -# -# This file is part of wolfSSL. (formerly known as CyaSSL) -# -# wolfSSL is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# wolfSSL is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -import os -import subprocess -import argparse -from contextlib import contextmanager -from distutils.util import get_platform -from wolfssl.__about__ import __wolfssl_version__ as version - - -def local_path(path): - """ Return path relative to the root of this project - """ - current = os.path.abspath(os.getcwd()) - return os.path.abspath(os.path.join(current, path)) - - -WOLFSSL_SRC_PATH = local_path("lib/wolfssl") - - -def wolfssl_inc_path(): - wolfssl_path = os.environ.get("USE_LOCAL_WOLFSSL") - if wolfssl_path is None: - return local_path("lib/wolfssl") - else: - if os.path.isdir(wolfssl_path) and os.path.exists(wolfssl_path): - return wolfssl_path + "/include" - else: - return "/usr/local/include" - - -def wolfssl_lib_path(): - wolfssl_path = os.environ.get("USE_LOCAL_WOLFSSL") - if wolfssl_path is None: - return local_path("lib/wolfssl/{}/{}/lib".format( - get_platform(), version)) - else: - if os.path.isdir(wolfssl_path) and os.path.exists(wolfssl_path): - return wolfssl_path + "/lib" - else: - return "/usr/local/lib" - - -def call(cmd): - print("Calling: '{}' from working directory {}".format(cmd, os.getcwd())) - - old_env = os.environ["PATH"] - os.environ["PATH"] = "{}:{}".format(WOLFSSL_SRC_PATH, old_env) - subprocess.check_call(cmd, shell=True, env=os.environ) - os.environ["PATH"] = old_env - - -@contextmanager -def chdir(new_path, mkdir=False): - old_path = os.getcwd() - - if mkdir: - try: - os.mkdir(new_path) - except OSError: - pass - - try: - yield os.chdir(new_path) - finally: - os.chdir(old_path) - - -def checkout_ref(ref): - """ Ensure that we have the right version - """ - with chdir(WOLFSSL_SRC_PATH): - current = "" - try: - current = subprocess.check_output( - ["git", "describe", "--all", "--exact-match"] - ).strip().decode().split('/')[-1] - except: - pass - - if current != ref: - tags = subprocess.check_output( - ["git", "tag"] - ).strip().decode().split("\n") - - if ref != "master" and ref not in tags: - call("git fetch --depth=1 origin tag {}".format(ref)) - - call("git checkout --force {}".format(ref)) - - return True # rebuild needed - - return False - - -def ensure_wolfssl_src(ref): - """ Ensure that wolfssl sources are presents and up-to-date - """ - if not os.path.isdir("lib"): - os.mkdir("lib") - with chdir("lib"): - subprocess.run(["git", "clone", "--depth=1", "https://github.com/wolfssl/wolfssl"]) - - if not os.path.isdir(os.path.join(WOLFSSL_SRC_PATH, "wolfssl")): - subprocess.run(["git", "submodule", "update", "--init", "--depth=1"]) - - return checkout_ref(ref) - - -def make_flags(prefix, debug): - """ Returns compilation flags - """ - flags = [] - cflags = [] - - if get_platform() in ["linux-x86_64", "linux-i686"]: - cflags.append("-fpic") - - # install location - flags.append("--prefix={}".format(prefix)) - - # lib only - flags.append("--disable-shared") - flags.append("--disable-examples") - - # tls 1.3 - flags.append("--enable-tls13") - flags.append("--enable-sslv3") - - # for urllib3 - requires SNI (tlsx), options (openssl compat), peer cert - flags.append("--enable-tlsx") - flags.append("--enable-opensslextra") - cflags.append("-DKEEP_PEER_CERT") - - # for pyOpenSSL - flags.append("--enable-secure-renegotiation") - flags.append("--enable-opensslall") - cflags.append("-DFP_MAX_BITS=8192") - cflags.append("-DHAVE_EX_DATA") - cflags.append("-DOPENSSL_COMPATIBLE_DEFAULTS") - - if debug: - flags.append("--enable-debug") - - # Note: websocket-client test server (echo.websocket.org) only supports - # TLS 1.2 with TLS_RSA_WITH_AES_128_CBC_SHA - # If compiling for use with websocket-client, must enable static RSA suites. - # cflags.append("-DWOLFSSL_STATIC_RSA") - - joined_flags = " ".join(flags) - joined_cflags = " ".join(cflags) - - return joined_flags + " CFLAGS=\"" + joined_cflags + "\"" - - -def make(configure_flags): - """ Create a release of wolfSSL C library - """ - with chdir(WOLFSSL_SRC_PATH): - call("git clean -fdX") - - try: - call("./autogen.sh") - except subprocess.CalledProcessError: - call("libtoolize") - call("./autogen.sh") - - call("./configure {}".format(configure_flags)) - call("make") - call("make install") - - -def build_wolfssl(ref, debug=False): - prefix = local_path("lib/wolfssl/{}/{}".format( - get_platform(), ref)) - libfile = os.path.join(prefix, 'lib/libwolfssl.la') - - rebuild = ensure_wolfssl_src(ref) - - if rebuild or not os.path.isfile(libfile): - make(make_flags(prefix, debug)) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Build underlying wolfSSL library (libwolfssl).") - parser.add_argument("-d", "--debug", action="store_true", - help="Build libwolfssl with debug enabled.") - parser.add_argument("-r", '--ref', default="master", - help="Git ref to check out when cloning wolfSSL.") - args = parser.parse_args() - build_wolfssl(args.ref, args.debug) - diff --git a/wolfssl/_methods.py b/wolfssl/_methods.py index 0a578f3..c0026cf 100644 --- a/wolfssl/_methods.py +++ b/wolfssl/_methods.py @@ -2,7 +2,7 @@ # # _methods.py # -# Copyright (C) 2006-2020 wolfSSL Inc. +# Copyright (C) 2006-2022 wolfSSL Inc. # # This file is part of wolfSSL. (formerly known as CyaSSL) # diff --git a/wolfssl/_openssl.py b/wolfssl/_openssl.py index 2e09519..1e3e71b 100644 --- a/wolfssl/_openssl.py +++ b/wolfssl/_openssl.py @@ -2,7 +2,7 @@ # # _openssl.py # -# Copyright (C) 2006-2020 wolfSSL Inc. +# Copyright (C) 2006-2022 wolfSSL Inc. # # This file is part of wolfSSL. (formerly known as CyaSSL) # diff --git a/wolfssl/_version.py b/wolfssl/_version.py new file mode 100644 index 0000000..378f7c7 --- /dev/null +++ b/wolfssl/_version.py @@ -0,0 +1,11 @@ +# When bumping the C library version, reset the POST count to 0 + +__wolfssl_version__ = "v5.1.1-stable" + +# We're using implicit post releases [PEP 440] to bump package version +# while maintaining the C library version intact for better reference. +# https://www.python.org/dev/peps/pep-0440/#implicit-post-releases +# +# MAJOR.MINOR.BUILD-POST + +__version__ = "5.1.1-0" diff --git a/wolfssl/exceptions.py b/wolfssl/exceptions.py index 998559a..92533ee 100644 --- a/wolfssl/exceptions.py +++ b/wolfssl/exceptions.py @@ -2,7 +2,7 @@ # # exceptions.py # -# Copyright (C) 2006-2020 wolfSSL Inc. +# Copyright (C) 2006-2022 wolfSSL Inc. # # This file is part of wolfSSL. (formerly known as CyaSSL) # diff --git a/wolfssl/utils.py b/wolfssl/utils.py index 919037a..19757ed 100644 --- a/wolfssl/utils.py +++ b/wolfssl/utils.py @@ -2,7 +2,7 @@ # # utils.py # -# Copyright (C) 2006-2020 wolfSSL Inc. +# Copyright (C) 2006-2022 wolfSSL Inc. # # This file is part of wolfSSL. (formerly known as CyaSSL) # From 61d2f69d12400f65344036fae67a47bb4d2917bd Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 10 Feb 2022 14:40:36 +0000 Subject: [PATCH 2/2] Don't do local build if env var set --- wolfssl/_build_ffi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wolfssl/_build_ffi.py b/wolfssl/_build_ffi.py index b19d519..b5f067f 100644 --- a/wolfssl/_build_ffi.py +++ b/wolfssl/_build_ffi.py @@ -250,7 +250,8 @@ def generate_libwolfssl(): make(make_flags(prefix, False)) -if get_libwolfssl() == 0: +local_wolfssl = os.environ.get("USE_LOCAL_WOLFSSL") +if local_wolfssl and get_libwolfssl() == 0: generate_libwolfssl() get_libwolfssl()