From 28714e6b5cf47888bdb6cbfd0217f13e05824eef Mon Sep 17 00:00:00 2001 From: memento Date: Sat, 9 Mar 2024 14:01:22 -0600 Subject: [PATCH 01/48] (ref) use logging module. add CMake from _build_clib --- setup.py | 194 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 191 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9e5908884..6aed8e0bf 100644 --- a/setup.py +++ b/setup.py @@ -1,23 +1,24 @@ import errno +import logging import os import os.path import platform import shutil import subprocess +import sys import tarfile from io import BytesIO -import sys from setuptools import Distribution as _Distribution, setup, find_packages, __version__ as setuptools_version from setuptools._distutils import log from setuptools._distutils.errors import DistutilsError from setuptools.command.build_clib import build_clib as _build_clib from setuptools.command.build_ext import build_ext as _build_ext -from setuptools.extension import Extension from setuptools.command.develop import develop as _develop from setuptools.command.dist_info import dist_info as _dist_info from setuptools.command.egg_info import egg_info as _egg_info from setuptools.command.sdist import sdist as _sdist +from setuptools.extension import Extension try: from wheel.bdist_wheel import bdist_wheel as _bdist_wheel @@ -25,7 +26,7 @@ _bdist_wheel = None sys.path.append(os.path.abspath(os.path.dirname(__file__))) -from setup_support import absolute, build_flags, detect_dll, has_system_lib +from setup_support import absolute, build_flags, detect_dll, has_system_lib, subprocess_run BUILDING_FOR_WINDOWS = detect_dll() @@ -45,6 +46,47 @@ f'to a newer version (>= 3.3).' ) +LIB_NAME = 'libsecp256k1' +PKG_NAME = 'coincurve' + +# Helpers for compilation instructions +# Cross-compile for Windows/ARM64, Linux/ARM64, Darwin/ARM64, Windows/x86 (GitHub) +X_HOST = os.getenv('COINCURVE_CROSS_HOST') + +SYSTEM = platform.system() # supported: Windows, Linux, Darwin +MACHINE = platform.machine() # supported: AMD64, x86_64 + +logging.info(f'Building for {SYSTEM}:{MACHINE} with {X_HOST = }') +SECP256K1_BUILD = os.getenv('COINCURVE_SECP256K1_BUILD') or 'STATIC' +SECP256K1_IGNORE_EXT_LIB = os.getenv('COINCURVE_IGNORE_SYSTEM_LIB') + + +def define_secp256k1_local_lib_info(): + """ + Define the library name and the installation directory + The purpose is to automatically include the shared library in the package and + prevent inclusion the static library. This is probably hacky, but it works. + """ + if SECP256K1_BUILD == 'SHARED': + logging.info('Building shared library') + # This will place the shared library inside the coincurve package data + return PKG_NAME, 'lib' + + logging.info('Building static library') + # This will place the static library in a separate x_lib and in a lib_name directory + return LIB_NAME, 'x_lib' + + +def call_pkg_config(options, library, *, debug=False): + """Calls pkg-config with the given options and returns the output.""" + if SYSTEM == 'Windows': + options.append('--dont-define-prefix') + + pkg_config = shutil.which('pkg-config') + cmd = [pkg_config, *options, library] + + return subprocess_run(cmd, debug=debug) + def download_library(command): if command.dry_run: @@ -113,6 +155,151 @@ def run(self): bdist_wheel = None +class BuildClibWithCMake(_build_clib): + title = 'SECP256K1 C library build' + + def __init__(self, dist): + super().__init__(dist) + self.pkgconfig_dir = None + + self._cwd = None + self._lib_src = None + self._install_dir = None + self._install_lib_dir = None + + def get_source_files(self): + # Ensure library has been downloaded (sdist might have been skipped) + if not self.distribution.uses_system_lib(): + download_library(self) + + # This seems to create issues in MANIFEST.in + return [f for _, _, fs in os.walk(absolute(LIB_NAME)) for f in fs] + + def run(self): + if self.distribution.uses_system_lib(): + logging.info('Using system library') + return + + logging.info(self.title) + self.bc_set_dirs_download() + self.bc_prepare_build(self._install_lib_dir, self.build_temp, self._lib_src) + + try: + os.chdir(self.build_temp) + subprocess.check_call(self.bc_build_command()) # noqa S603 + finally: + os.chdir(self._cwd) + + # Register lib installation path + self.bc_update_pkg_config_path() + + def bc_set_dirs_download(self): + self._cwd = os.getcwd() + os.makedirs(self.build_temp, exist_ok=True) + self._install_dir = str(self.build_temp).replace('temp', 'lib') + + # Install path + # SHARED: lib/coincurve -> path/lib.xxx/coincurve/path # included in coincurve wheel + # STATIC: x_lib/libsecp256k1 -> path/x_lib.xxx/libsecp256k1/path # NOT included in coincurve wheel + lib, inst_dir = define_secp256k1_local_lib_info() + self._install_lib_dir = os.path.join(self._install_dir.replace('lib', inst_dir), lib) + + self._lib_src = os.path.join(self._cwd, LIB_NAME) + if not os.path.exists(self._lib_src): + self.get_source_files() + + def bc_update_pkg_config_path(self): + self.pkgconfig_dir = [ + os.path.join(self._install_lib_dir, 'lib', 'pkgconfig'), + os.path.join(self._install_lib_dir, 'lib64', 'pkgconfig'), + ] + os.environ['PKG_CONFIG_PATH'] = ( + f'{str(os.pathsep).join(self.pkgconfig_dir)}' + f'{os.pathsep}' + f'{os.getenv("PKG_CONFIG_PATH", "")}' + ).replace('\\', '/') + + # Verify installation + # subprocess.check_call(['pkg-config', '--exists', LIB_NAME]) # S603 + call_pkg_config(['--exists'], LIB_NAME) + + @staticmethod + def _generator(msvc): + if '2017' in str(msvc): + return 'Visual Studio 15 2017' + if '2019' in str(msvc): + return 'Visual Studio 16 2019' + if '2022' in str(msvc): + return 'Visual Studio 17 2022' + + @staticmethod + def bc_prepare_build(install_lib_dir, build_temp, lib_src): + cmake_args = [ + '-DCMAKE_BUILD_TYPE=Release', + f'-DCMAKE_INSTALL_PREFIX={install_lib_dir}', + f'-DCMAKE_C_FLAGS={"-fPIC" if SECP256K1_BUILD != "SHARED" and SYSTEM != "Windows" else ""}', + f'-DSECP256K1_DISABLE_SHARED={"OFF" if SECP256K1_BUILD == "SHARED" else "ON"}', + '-DSECP256K1_BUILD_BENCHMARK=OFF', + '-DSECP256K1_BUILD_TESTS=OFF', + '-DSECP256K1_ENABLE_MODULE_ECDH=ON', + '-DSECP256K1_ENABLE_MODULE_RECOVERY=ON', + '-DSECP256K1_ENABLE_MODULE_SCHNORRSIG=ON', + '-DSECP256K1_ENABLE_MODULE_EXTRAKEYS=ON', + ] + + # Windows (more complex) + if SYSTEM == 'Windows': + vswhere = shutil.which('vswhere') + cmd = [vswhere, '-latest', '-find', 'MSBuild\\**\\Bin\\MSBuild.exe'] + msvc = subprocess.check_output(cmd).strip().decode('utf-8') # noqa S603 + + # For windows x86/x64, select the correct architecture + arch = 'x64' if MACHINE == 'AMD64' else 'Win32' # Native + + if X_HOST is not None: + logging.info(f'Cross-compiling on {SYSTEM}:{MACHINE} for {X_HOST}') + if X_HOST in ['arm64', 'ARM64', 'x86']: + arch = 'Win32' if X_HOST in ['x86'] else 'arm64' + else: + raise NotImplementedError(f'Unsupported architecture: {X_HOST}') + + # Place the DLL directly in the package directory + cmake_args.append('-DCMAKE_INSTALL_BINDIR=.') + cmake_args.extend(['-G', BuildClibWithCMake._generator(msvc), f'-A{arch}']) + + elif SYSTEM == 'Darwin': + if X_HOST is None: + cmake_args.append( + f'-DCMAKE_OSX_ARCHITECTURES={MACHINE}' # Native + ) + else: + logging.info(f'Cross-compiling on {SYSTEM}:{MACHINE} for {X_HOST}') + if X_HOST in ['armv7', 'armv7s', 'arm64', 'arm64e']: + cmake_args.append( + f'-DCMAKE_OSX_ARCHITECTURES={X_HOST}' + ) + else: + if X_HOST is not None: + if X_HOST not in [ + 'arm-linux-gnueabihf', + 'x86_64-w64-mingw32', + ]: + raise NotImplementedError(f'Unsupported architecture: {X_HOST}') + + logging.info(f'Cross-compiling on {SYSTEM}:{MACHINE} for {X_HOST}') + cmake_args.append( + f'-DCMAKE_TOOLCHAIN_FILE=../cmake/{X_HOST}.toolchain.cmake' + ) + + logging.info(' Configure CMake') + subprocess.check_call(['cmake', '-S', lib_src, '-B', build_temp, *cmake_args]) # noqa S603 + + @staticmethod + def bc_build_command(): + logging.info(' Install with CMake') + return ['cmake', '--build', '.', '--target', 'install', '--config', 'Release', '--clean-first'] + + class build_clib(_build_clib): def initialize_options(self): _build_clib.initialize_options(self) @@ -260,6 +447,7 @@ class Distribution(_Distribution): def has_c_libraries(self): return not has_system_lib() + # --- SECP256K1 package definitions --- secp256k1_package = 'libsecp256k1' From 841a3af6c317f0e41f8ee69fe5d5de0ea1297bf0 Mon Sep 17 00:00:00 2001 From: memento Date: Sat, 9 Mar 2024 14:51:20 -0600 Subject: [PATCH 02/48] (feat) add shared and static linker --- setup.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/setup.py b/setup.py index 6aed8e0bf..17dd7e0bf 100644 --- a/setup.py +++ b/setup.py @@ -300,6 +300,61 @@ def bc_build_command(): return ['cmake', '--build', '.', '--target', 'install', '--config', 'Release', '--clean-first'] +class SharedLinker(object): + @staticmethod + def update_link_args(compiler, libraries, libraries_dirs, extra_link_args): + if compiler.__class__.__name__ == 'UnixCCompiler': + extra_link_args.extend([f'-l{lib}' for lib in libraries]) + if sys.platform == 'darwin': + extra_link_args.extend([ + '-Wl,-rpath,@loader_path/lib', + ]) + else: + extra_link_args.extend([ + '-Wl,-rpath,$ORIGIN/lib', + '-Wl,-rpath,$ORIGIN/lib64', + ]) + elif compiler.__class__.__name__ == 'MSVCCompiler': + for ld in libraries_dirs: + ld = ld.replace('/', '\\') + for lib in libraries: + lib_file = os.path.join(ld, f'lib{lib}.lib') + lib_path = [f'/LIBPATH:{ld}', f'lib{lib}.lib'] + if os.path.exists(lib_file): + extra_link_args.extend(lib_path) + else: + raise NotImplementedError(f'Unsupported compiler: {compiler.__class__.__name__}') + + +class StaticLinker(object): + @staticmethod + def update_link_args(compiler, libraries, libraries_dirs, extra_link_args): + if compiler.__class__.__name__ == 'UnixCCompiler': + # It is possible that the library was compiled without fPIC option + for lib in libraries: + # On MacOS the mix static/dynamic option is different + # It requires a -force_load option for each library + if sys.platform == 'darwin': + for lib_dir in libraries_dirs: + if os.path.exists(os.path.join(lib_dir, f'lib{lib}.a')): + extra_link_args.extend( + ['-Wl,-force_load', os.path.join(lib_dir, f'lib{lib}.a')] + ) + break + else: + extra_link_args.extend(['-Wl,-Bstatic', f'-l{lib}', '-Wl,-Bdynamic']) + + elif compiler.__class__.__name__ == 'MSVCCompiler': + for ld in libraries_dirs: + ld = ld.replace('/', '\\') + for lib in libraries: + lib_file = os.path.join(ld, f'lib{lib}.lib') + if os.path.exists(lib_file): + extra_link_args.append(lib_file) + else: + raise NotImplementedError(f'Unsupported compiler: {compiler.__class__.__name__}') + + class build_clib(_build_clib): def initialize_options(self): _build_clib.initialize_options(self) From 014c57445c768123f398ae72f7f12787499510b5 Mon Sep 17 00:00:00 2001 From: memento Date: Sat, 9 Mar 2024 16:47:06 -0600 Subject: [PATCH 03/48] (feat) add helper fct for pkg-config --- setup.py | 30 ++---------------------------- setup_support.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/setup.py b/setup.py index 17dd7e0bf..c90a926b7 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,8 @@ _bdist_wheel = None sys.path.append(os.path.abspath(os.path.dirname(__file__))) -from setup_support import absolute, build_flags, detect_dll, has_system_lib, subprocess_run +from setup_support import absolute, build_flags, detect_dll, has_system_lib, define_secp256k1_local_lib_info, \ + call_pkg_config BUILDING_FOR_WINDOWS = detect_dll() @@ -61,33 +62,6 @@ SECP256K1_IGNORE_EXT_LIB = os.getenv('COINCURVE_IGNORE_SYSTEM_LIB') -def define_secp256k1_local_lib_info(): - """ - Define the library name and the installation directory - The purpose is to automatically include the shared library in the package and - prevent inclusion the static library. This is probably hacky, but it works. - """ - if SECP256K1_BUILD == 'SHARED': - logging.info('Building shared library') - # This will place the shared library inside the coincurve package data - return PKG_NAME, 'lib' - - logging.info('Building static library') - # This will place the static library in a separate x_lib and in a lib_name directory - return LIB_NAME, 'x_lib' - - -def call_pkg_config(options, library, *, debug=False): - """Calls pkg-config with the given options and returns the output.""" - if SYSTEM == 'Windows': - options.append('--dont-define-prefix') - - pkg_config = shutil.which('pkg-config') - cmd = [pkg_config, *options, library] - - return subprocess_run(cmd, debug=debug) - - def download_library(command): if command.dry_run: return diff --git a/setup_support.py b/setup_support.py index 0aad51b86..358f19f21 100644 --- a/setup_support.py +++ b/setup_support.py @@ -84,3 +84,46 @@ def subprocess_run(cmd, *, debug=False): logging.error(f'An error occurred during the command execution: {e}') logging.error(f'Command log:\n{e.stderr}') raise e + + +def call_pkg_config(options, library, *, debug=False): + """Calls pkg-config with the given options and returns the output.""" + import shutil + + from setup import SYSTEM + + if SYSTEM == 'Windows': + options.append('--dont-define-prefix') + + pkg_config = shutil.which('pkg-config') + cmd = [pkg_config, *options, library] + + return subprocess_run(cmd, debug=debug) + + +def define_secp256k1_local_lib_info(): + """ + Define the library name and the installation directory + The purpose is to automatically include the shared library in the package and + prevent inclusion the static library. This is probably hacky, but it works. + """ + from setup import LIB_NAME, PKG_NAME, SECP256K1_BUILD + + if SECP256K1_BUILD == 'SHARED': + return PKG_NAME, 'lib' + return LIB_NAME, 'x_lib' + + +def update_pkg_config_path(path='.'): + """Updates the PKG_CONFIG_PATH environment variable to include the given path.""" + pkg_config_paths = [path, os.getenv('PKG_CONFIG_PATH', '').strip('"')] + + if cpf := os.getenv('CONDA_PREFIX'): + conda_paths = [os.path.join(cpf, sbd, 'pkgconfig') for sbd in ('lib', 'lib64', os.path.join('Library', 'lib'))] + pkg_config_paths.extend([p for p in conda_paths if os.path.isdir(p)]) + + if lbd := os.getenv('LIB_DIR'): + pkg_config_paths.append(os.path.join(lbd, 'pkgconfig')) + + # Update environment + os.environ['PKG_CONFIG_PATH'] = os.pathsep.join(pkg_config_paths) From cf316c0a3e4b7674cb669fd697ddf50e277d472a Mon Sep 17 00:00:00 2001 From: memento Date: Sat, 9 Mar 2024 19:33:14 -0600 Subject: [PATCH 04/48] (feat) add Extension build. use for non-Window --- setup.py | 97 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 77 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index c90a926b7..a5c48e50e 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ sys.path.append(os.path.abspath(os.path.dirname(__file__))) from setup_support import absolute, build_flags, detect_dll, has_system_lib, define_secp256k1_local_lib_info, \ - call_pkg_config + call_pkg_config, subprocess_run BUILDING_FOR_WINDOWS = detect_dll() @@ -143,14 +143,14 @@ def __init__(self, dist): def get_source_files(self): # Ensure library has been downloaded (sdist might have been skipped) - if not self.distribution.uses_system_lib(): + if not has_system_lib(): download_library(self) # This seems to create issues in MANIFEST.in return [f for _, _, fs in os.walk(absolute(LIB_NAME)) for f in fs] def run(self): - if self.distribution.uses_system_lib(): + if has_system_lib(): logging.info('Using system library') return @@ -183,18 +183,9 @@ def bc_set_dirs_download(self): self.get_source_files() def bc_update_pkg_config_path(self): - self.pkgconfig_dir = [ - os.path.join(self._install_lib_dir, 'lib', 'pkgconfig'), - os.path.join(self._install_lib_dir, 'lib64', 'pkgconfig'), - ] - os.environ['PKG_CONFIG_PATH'] = ( - f'{str(os.pathsep).join(self.pkgconfig_dir)}' - f'{os.pathsep}' - f'{os.getenv("PKG_CONFIG_PATH", "")}' - ).replace('\\', '/') - - # Verify installation - # subprocess.check_call(['pkg-config', '--exists', LIB_NAME]) # S603 + self.pkgconfig_dir = [os.path.join(self._install_lib_dir, n, 'pkgconfig') for n in ['lib', 'lib64']] + self.pkgconfig_dir.append(os.getenv('PKG_CONFIG_PATH', '')) + os.environ['PKG_CONFIG_PATH'] = os.pathsep.join(self.pkgconfig_dir) call_pkg_config(['--exists'], LIB_NAME) @staticmethod @@ -329,6 +320,66 @@ def update_link_args(compiler, libraries, libraries_dirs, extra_link_args): raise NotImplementedError(f'Unsupported compiler: {compiler.__class__.__name__}') +class BuildExtensionFromCFFI(_build_ext): + static_lib = True if SECP256K1_BUILD == 'STATIC' else False + + def update_link_args(self, libraries, libraries_dirs, extra_link_args): + if self.static_lib: + StaticLinker.update_link_args(self.compiler, libraries, libraries_dirs, extra_link_args) + else: + SharedLinker.update_link_args(self.compiler, libraries, libraries_dirs, extra_link_args) + + def create_c_files(self, ext): + # Construct C-file from CFFI + build_script = os.path.join('_cffi_build', 'build_shared.py') + for i, c_file in enumerate(ext.sources): + os.makedirs(self.build_temp, exist_ok=True) + c_file = os.path.join(self.build_temp, os.path.basename(c_file)) + # This puts c-file a temp location (and not in the coincurve src directory) + ext.sources[i] = c_file + cmd = [sys.executable, build_script, c_file, '1' if self.static_lib else '0'] + subprocess_run(cmd) + + def build_extension(self, ext, build_flags): + # Construct C-file from CFFI + self.create_c_files(ext) + + # Enforce API interface + ext.py_limited_api = False + + # Location of locally built library + lib, inst_dir = define_secp256k1_local_lib_info() + prefix = os.path.join(self.build_lib.replace('lib', inst_dir), lib) + postfix = os.path.join('pkgconfig', f'{LIB_NAME}.pc') + + c_lib_pkg = None + if not any([ + os.path.isfile(c_lib_pkg := os.path.join(prefix, 'lib', postfix)), + os.path.isfile(c_lib_pkg := os.path.join(prefix, 'lib64', postfix)), + has_system_lib() + ]): + raise RuntimeError( + f'Library not found: {os.path.join(prefix, "lib/lib64", postfix)}' + f'\nSystem lib is {has_system_lib() = }. ' + 'Please check that the library was properly built.' + ) + + # PKG_CONFIG_PATH is updated by build_clib if built locally, + # however, it would not work for a step-by-step build, thus we specify the lib path + build_flags = self.distribution.build_flags + ext.extra_compile_args.extend([f'-I{build_flags(LIB_NAME, "I", c_lib_pkg)[0]}']) + ext.library_dirs.extend(build_flags(LIB_NAME, 'L', c_lib_pkg)) + + libraries = build_flags(LIB_NAME, 'l', c_lib_pkg) + logging.info(f' Libraries:{libraries}') + + # We do not set ext.libraries, this would add the default link instruction + # Instead, we use extra_link_args to customize the link command + self.update_link_args(libraries, ext.library_dirs, ext.extra_link_args) + + super().build_extension(ext) + + class build_clib(_build_clib): def initialize_options(self): _build_clib.initialize_options(self) @@ -506,7 +557,7 @@ def has_c_libraries(self): setup_kwargs = dict( ext_modules=[extension], cmdclass={ - 'build_clib': build_clib, + 'build_clib': None if has_system_lib() else BuildClibWithCMake, 'build_ext': BuildCFFIForSharedLib, 'develop': develop, 'egg_info': egg_info, @@ -533,12 +584,18 @@ def has_c_libraries(self): return not has_system_lib() + extension = Extension( + name='coincurve._libsecp256k1', + sources=['_c_file_for_extension.c'], + py_limited_api=False, + extra_compile_args=['/d2FH4-'] if SYSTEM == 'Windows' else [], + ) + setup_kwargs = dict( - ext_package='coincurve', - cffi_modules=['_cffi_build/build.py:ffi'], + ext_modules=[extension], cmdclass={ - 'build_clib': build_clib, - 'build_ext': build_ext, + 'build_clib': None if has_system_lib() else BuildClibWithCMake, + 'build_ext': BuildCFFIForSharedLib, 'develop': develop, 'egg_info': egg_info, 'sdist': sdist, From 59c64ace0f81c19e136d8224721dba75901e19af Mon Sep 17 00:00:00 2001 From: memento Date: Sat, 9 Mar 2024 19:44:34 -0600 Subject: [PATCH 05/48] (fix) gcc failure --- .github/scripts/install-test-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/install-test-deps.sh b/.github/scripts/install-test-deps.sh index cdf183638..142e760ba 100755 --- a/.github/scripts/install-test-deps.sh +++ b/.github/scripts/install-test-deps.sh @@ -5,5 +5,5 @@ if [ "$RUNNER_OS" == "macOS" ]; then ./.github/scripts/install-macos-build-deps.sh fi -python -m pip install --upgrade cffi +python -m pip install --upgrade cffi python3-dev python -m pip install --upgrade tox codecov From 978f738290083644ed9319f0e0fa511c652df3b7 Mon Sep 17 00:00:00 2001 From: memento Date: Sun, 10 Mar 2024 19:09:01 -0500 Subject: [PATCH 06/48] (fix) correct build_shared.py for static/shared (need to rename file later) --- _cffi_build/build_shared.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/_cffi_build/build_shared.py b/_cffi_build/build_shared.py index 4ad89d38e..3d95a9ced 100644 --- a/_cffi_build/build_shared.py +++ b/_cffi_build/build_shared.py @@ -29,34 +29,27 @@ def gather_sources_from_directory(directory: str) -> List[Source]: define_static_lib = """ -#if defined(_WIN32) +#if defined(_WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__) # define SECP256K1_STATIC 1 -# define SECP256K1_API extern __declspec(dllexport) -#endif -""" - -define_shared_lib = """ -#if defined(_WIN32) -# define SECP256K1_API extern __declspec(dllimport) #endif """ def mk_ffi(sources: List[Source], - static_lib: str = '0', + static_lib: bool = False, name: str = '_libsecp256k1') -> FFI: """ Create an FFI object. :param sources: A list of Source namedtuples. - :param libraries: A list of libraries to link against. :param static_lib: Whether to generate a static lib in Windows. :param name: The name of the FFI object. :return: An FFI object. """ _ffi = FFI() - code = [define_static_lib] if static_lib == '1' else [define_shared_lib] + code = [define_static_lib] if static_lib else [] + logging.info(f' Static {static_lib}...') for source in sources: with open(os.path.join(here, source.h)) as h: logging.info(f' Including {source.h}...') @@ -75,10 +68,10 @@ def mk_ffi(sources: List[Source], logging.info('Starting CFFI build process...') parser = argparse.ArgumentParser(description='Generate C code using CFFI.') parser.add_argument('c_file', help='Generated C code filename.') - parser.add_argument('static_lib', help='Generate static lib in Windows.', default=False) + parser.add_argument('static_lib', help='Generate static lib in Windows.', default='0', type=str) args = parser.parse_args() modules = gather_sources_from_directory(here) - ffi = mk_ffi(modules, args.static_lib) + ffi = mk_ffi(modules, args.static_lib == '1') ffi.emit_c_code(args.c_file) logging.info(f' Generated C code: {args.c_file}') From 7236098054c3a41ba62573f6183da0518a5b0a50 Mon Sep 17 00:00:00 2001 From: memento Date: Sun, 10 Mar 2024 19:37:55 -0500 Subject: [PATCH 07/48] (fea) add pkg-config helpers --- setup_support.py | 75 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/setup_support.py b/setup_support.py index 358f19f21..20827a736 100644 --- a/setup_support.py +++ b/setup_support.py @@ -1,8 +1,6 @@ -import glob import logging import os import subprocess -from contextlib import suppress def absolute(*paths): @@ -25,7 +23,10 @@ def build_flags(library, type_, path): os.environ['PKG_CONFIG_PATH'] = new_path + os.pathsep + os.environ.get('PKG_CONFIG_PATH', '') options = {'I': '--cflags-only-I', 'L': '--libs-only-L', 'l': '--libs-only-l'} - cmd = ['pkg-config', options[type_], library] + if os.name == 'nt': + cmd = ['pkg-config', options[type_], '--dont-define-prefix', library] + else: + cmd = ['pkg-config', options[type_], library] flags = subprocess_run(cmd) flags = list(flags.split()) @@ -33,25 +34,24 @@ def build_flags(library, type_, path): def _find_lib(): - if 'COINCURVE_IGNORE_SYSTEM_LIB' in os.environ: + if os.getenv('COINCURVE_IGNORE_SYSTEM_LIB', '1') == '1': return False - from cffi import FFI + update_pkg_config_path() try: - cmd = ['pkg-config', '--cflags-only-I', 'libsecp256k1'] - includes = subprocess_run(cmd) + if os.name == 'nt': + cmd = ['pkg-config', '--libs-only-L', '--dont-define-prefix', 'libsecp256k1'] + else: + cmd = ['pkg-config', '--libs-only-L', 'libsecp256k1'] + lib_dir = subprocess_run(cmd) - return os.path.exists(os.path.join(includes[2:], 'secp256k1_ecdh.h')) + return verify_system_lib(lib_dir[2:].strip()) except (OSError, subprocess.CalledProcessError): - if 'LIB_DIR' in os.environ: - for path in glob.glob(os.path.join(os.environ['LIB_DIR'], '*secp256k1*')): - with suppress(OSError): - FFI().dlopen(path) - return True - # We couldn't locate libsecp256k1, so we'll use the bundled one - return False + from ctypes.util import find_library + + return bool(find_library('secp256k1')) _has_system_lib = None @@ -127,3 +127,48 @@ def update_pkg_config_path(path='.'): # Update environment os.environ['PKG_CONFIG_PATH'] = os.pathsep.join(pkg_config_paths) + + +def verify_system_lib(lib_dir): + """Verifies that the system library is installed and of the expected type.""" + import ctypes + import platform + from ctypes.util import find_library + from pathlib import Path + + LIB_NAME = 'libsecp256k1' # noqa N806 + PKG_NAME = 'coincurve' # noqa N806 + SECP256K1_BUILD = os.getenv('COINCURVE_SECP256K1_BUILD') or 'STATIC' # noqa N806 + SYSTEM = platform.system() # noqa N806 + + def load_library(lib): + try: + return ctypes.CDLL(lib) + except OSError: + return None + + logging.warning(f'find_library: {find_library(LIB_NAME[3:])}') + lib_dir = Path(lib_dir).with_name('bin') if SYSTEM == 'Windows' else Path(lib_dir) + lib_ext = '.dll' if SYSTEM == 'Windows' else '.[sd][oy]*' + logging.warning(f'dir: {lib_dir}') + logging.warning(f'patt: *{LIB_NAME[3:]}{lib_ext}') + l_dyn = list(lib_dir.glob(f'*{LIB_NAME[3:]}*{lib_ext}')) + + # Evaluates the dynamic libraries found, + logging.warning(f'Found libraries: {l_dyn}') + dyn_lib = next((lib for lib in l_dyn if load_library(lib) is not None), False) + + found = any((dyn_lib and SECP256K1_BUILD == 'SHARED', not dyn_lib and SECP256K1_BUILD != 'SHARED')) + if not found: + logging.warning( + f'WARNING: {LIB_NAME} is installed, but it is not the expected type. ' + f'Please ensure that the {SECP256K1_BUILD} library is installed.' + ) + + if dyn_lib: + lib_base = dyn_lib.stem + # Update coincurve._secp256k1_library_info + info_file = Path(PKG_NAME, '_secp256k1_library_info.py') + info_file.write_text(f"SECP256K1_LIBRARY_NAME = '{lib_base}'\nSECP256K1_LIBRARY_TYPE = 'EXTERNAL'\n") + + return found From ef73b343bf9901f6de6caed18c877483f52ac1d9 Mon Sep 17 00:00:00 2001 From: memento Date: Sun, 10 Mar 2024 19:41:13 -0500 Subject: [PATCH 08/48] (fea) wrap within main() to avoid double-run of build steps --- setup.py | 232 +++++++++++++++++++++++++++---------------------------- 1 file changed, 116 insertions(+), 116 deletions(-) diff --git a/setup.py b/setup.py index a5c48e50e..96b27b62c 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ # IMPORTANT: keep in sync with .github/workflows/build.yml # # Version of libsecp256k1 to download if none exists in the `libsecp256k1` directory -UPSTREAM_REF = os.getenv('COINCURVE_UPSTREAM_TAG') or '1ad5185cd42c0636104129fcc9f6a4bf9c67cc40' +UPSTREAM_REF = os.getenv('COINCURVE_UPSTREAM_REF') or '1ad5185cd42c0636104129fcc9f6a4bf9c67cc40' LIB_TARBALL_URL = f'https://github.com/bitcoin-core/secp256k1/archive/{UPSTREAM_REF}.tar.gz' @@ -340,7 +340,7 @@ def create_c_files(self, ext): cmd = [sys.executable, build_script, c_file, '1' if self.static_lib else '0'] subprocess_run(cmd) - def build_extension(self, ext, build_flags): + def build_extension(self, ext): # Construct C-file from CFFI self.create_c_files(ext) @@ -366,7 +366,6 @@ def build_extension(self, ext, build_flags): # PKG_CONFIG_PATH is updated by build_clib if built locally, # however, it would not work for a step-by-step build, thus we specify the lib path - build_flags = self.distribution.build_flags ext.extra_compile_args.extend([f'-I{build_flags(LIB_NAME, "I", c_lib_pkg)[0]}']) ext.library_dirs.extend(build_flags(LIB_NAME, 'L', c_lib_pkg)) @@ -521,76 +520,36 @@ def build_extensions(self): super().build_extensions() -if has_system_lib(): - - class Distribution(_Distribution): - def has_c_libraries(self): - return not has_system_lib() - - - # --- SECP256K1 package definitions --- - secp256k1_package = 'libsecp256k1' - - extension = Extension( - name='coincurve._libsecp256k1', - sources=[os.path.join('coincurve', '_libsecp256k1.c')], - # ABI?: py_limited_api=True, - ) - - extension.extra_compile_args = [ - subprocess.check_output(['pkg-config', '--cflags-only-I', 'libsecp256k1']).strip().decode('utf-8') # noqa S603 - ] - extension.extra_link_args = [ - subprocess.check_output(['pkg-config', '--libs-only-L', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 - subprocess.check_output(['pkg-config', '--libs-only-l', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 - ] - - if os.name == 'nt' or sys.platform == 'win32': - # Apparently, the linker on Windows interprets -lxxx as xxx.lib, not libxxx.lib - for i, v in enumerate(extension.__dict__.get('extra_link_args')): - extension.__dict__['extra_link_args'][i] = v.replace('-L', '/LIBPATH:') - - if v.startswith('-l'): - v = v.replace('-l', 'lib') - extension.__dict__['extra_link_args'][i] = f'{v}.lib' - - setup_kwargs = dict( - ext_modules=[extension], - cmdclass={ - 'build_clib': None if has_system_lib() else BuildClibWithCMake, - 'build_ext': BuildCFFIForSharedLib, - 'develop': develop, - 'egg_info': egg_info, - 'sdist': sdist, - 'bdist_wheel': bdist_wheel, - }, - ) - -else: - if BUILDING_FOR_WINDOWS: - - class Distribution(_Distribution): - def is_pure(self): - return False - - - package_data['coincurve'].append('libsecp256k1.dll') - setup_kwargs = {} - - else: +def main(): + if has_system_lib(): class Distribution(_Distribution): def has_c_libraries(self): return not has_system_lib() - extension = Extension( name='coincurve._libsecp256k1', - sources=['_c_file_for_extension.c'], - py_limited_api=False, - extra_compile_args=['/d2FH4-'] if SYSTEM == 'Windows' else [], + sources=[os.path.join('coincurve', '_libsecp256k1.c')], + # ABI?: py_limited_api=True, ) + extension.extra_compile_args = [ + subprocess.check_output(['pkg-config', '--cflags-only-I', 'libsecp256k1']).strip().decode('utf-8') # noqa S603 + ] + extension.extra_link_args = [ + subprocess.check_output(['pkg-config', '--libs-only-L', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 + subprocess.check_output(['pkg-config', '--libs-only-l', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 + ] + + if os.name == 'nt' or sys.platform == 'win32': + # Apparently, the linker on Windows interprets -lxxx as xxx.lib, not libxxx.lib + for i, v in enumerate(extension.__dict__.get('extra_link_args')): + extension.__dict__['extra_link_args'][i] = v.replace('-L', '/LIBPATH:') + + if v.startswith('-l'): + v = v.replace('-l', 'lib') + extension.__dict__['extra_link_args'][i] = f'{v}.lib' + setup_kwargs = dict( ext_modules=[extension], cmdclass={ @@ -603,55 +562,96 @@ def has_c_libraries(self): }, ) -setup( - name='coincurve', - version='19.0.1', - - description='Cross-platform Python CFFI bindings for libsecp256k1', - long_description=open('README.md', 'r').read(), - long_description_content_type='text/markdown', - author_email='Ofek Lev ', - license='MIT OR Apache-2.0', - - python_requires='>=3.8', - install_requires=['asn1crypto', 'cffi>=1.3.0'], - - packages=find_packages(exclude=('_cffi_build', '_cffi_build.*', 'libsecp256k1', 'tests')), - package_data=package_data, - - distclass=Distribution, - zip_safe=False, - - project_urls={ - 'Documentation': 'https://ofek.dev/coincurve/', - 'Issues': 'https://github.com/ofek/coincurve/issues', - 'Source': 'https://github.com/ofek/coincurve', - }, - keywords=[ - 'secp256k1', - 'crypto', - 'elliptic curves', - 'bitcoin', - 'ethereum', - 'cryptocurrency', - ], - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'License :: OSI Approved :: Apache Software License', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: Software Development :: Libraries', - 'Topic :: Security :: Cryptography', - ], - **setup_kwargs -) + else: + if BUILDING_FOR_WINDOWS: + + class Distribution(_Distribution): + def is_pure(self): + return False + + + package_data['coincurve'].append('libsecp256k1.dll') + setup_kwargs = {} + + else: + + class Distribution(_Distribution): + def has_c_libraries(self): + return not has_system_lib() + + + extension = Extension( + name='coincurve._libsecp256k1', + sources=['_c_file_for_extension.c'], + py_limited_api=False, + extra_compile_args=['/d2FH4-'] if SYSTEM == 'Windows' else [], + ) + + setup_kwargs = dict( + ext_modules=[extension], + cmdclass={ + 'build_clib': None if has_system_lib() else BuildClibWithCMake, + 'build_ext': BuildExtensionFromCFFI, + 'develop': develop, + 'egg_info': egg_info, + 'sdist': sdist, + 'bdist_wheel': bdist_wheel, + }, + ) + + setup( + name='coincurve', + version='19.0.1', + + description='Cross-platform Python CFFI bindings for libsecp256k1', + long_description=open('README.md', 'r').read(), + long_description_content_type='text/markdown', + author_email='Ofek Lev ', + license='MIT OR Apache-2.0', + + python_requires='>=3.8', + install_requires=['asn1crypto', 'cffi>=1.3.0'], + + packages=find_packages(exclude=('_cffi_build', '_cffi_build.*', 'libsecp256k1', 'tests')), + package_data=package_data, + + distclass=Distribution, + zip_safe=False, + + project_urls={ + 'Documentation': 'https://ofek.dev/coincurve/', + 'Issues': 'https://github.com/ofek/coincurve/issues', + 'Source': 'https://github.com/ofek/coincurve', + }, + keywords=[ + 'secp256k1', + 'crypto', + 'elliptic curves', + 'bitcoin', + 'ethereum', + 'cryptocurrency', + ], + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Topic :: Software Development :: Libraries', + 'Topic :: Security :: Cryptography', + ], + **setup_kwargs + ) + + +if __name__ == '__main__': + main() From 6153647dffeff5d0a222a1c8b0691a2bf02b6b63 Mon Sep 17 00:00:00 2001 From: memento Date: Sun, 10 Mar 2024 19:42:36 -0500 Subject: [PATCH 09/48] (ref) fine-tune workflow --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 095909e6a..fa8400134 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,10 +17,14 @@ concurrency: env: COINCURVE_UPSTREAM_REF: 1ad5185cd42c0636104129fcc9f6a4bf9c67cc40 COINCURVE_IGNORE_SYSTEM_LIB: '1' - CIBW_BEFORE_ALL_MACOS: ./.github/scripts/install-macos-build-deps.sh + # Only 'SHARED' is recognized, any other string means 'not SHARED' + COINCURVE_SECP256K1_BUILD: 'STATIC' CIBW_ENVIRONMENT_PASS_LINUX: > COINCURVE_UPSTREAM_REF COINCURVE_IGNORE_SYSTEM_LIB + COINCURVE_SECP256K1_BUILD + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.8" + CIBW_BEFORE_ALL_MACOS: ./.github/scripts/install-macos-build-deps.sh CIBW_TEST_REQUIRES: pytest pytest-benchmark CIBW_TEST_COMMAND: > python -c From ca593a7dd95f62364afc2d8a1571ffbecc29758c Mon Sep 17 00:00:00 2001 From: memento Date: Sun, 10 Mar 2024 19:43:23 -0500 Subject: [PATCH 10/48] (fix) remove non-existing python-dev --- .github/scripts/install-test-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/install-test-deps.sh b/.github/scripts/install-test-deps.sh index 142e760ba..cdf183638 100755 --- a/.github/scripts/install-test-deps.sh +++ b/.github/scripts/install-test-deps.sh @@ -5,5 +5,5 @@ if [ "$RUNNER_OS" == "macOS" ]; then ./.github/scripts/install-macos-build-deps.sh fi -python -m pip install --upgrade cffi python3-dev +python -m pip install --upgrade cffi python -m pip install --upgrade tox codecov From f7e97e6d28bc2b5e760abf74b401d6af22512c44 Mon Sep 17 00:00:00 2001 From: memento Date: Tue, 12 Mar 2024 13:53:09 -0500 Subject: [PATCH 11/48] (fix) no relative path to load external shared lib on unix --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 96b27b62c..a5b1f54b3 100644 --- a/setup.py +++ b/setup.py @@ -270,10 +270,10 @@ class SharedLinker(object): def update_link_args(compiler, libraries, libraries_dirs, extra_link_args): if compiler.__class__.__name__ == 'UnixCCompiler': extra_link_args.extend([f'-l{lib}' for lib in libraries]) - if sys.platform == 'darwin': - extra_link_args.extend([ - '-Wl,-rpath,@loader_path/lib', - ]) + if has_system_lib(): + extra_link_args.extend([f'-L{lib}' for lib in libraries_dirs]) + elif sys.platform == 'darwin': + extra_link_args.extend(['-Wl,-rpath,@loader_path/lib']) else: extra_link_args.extend([ '-Wl,-rpath,$ORIGIN/lib', From 0b41425cbfa232a4262987aa8da5f0af894be80c Mon Sep 17 00:00:00 2001 From: memento Date: Tue, 12 Mar 2024 16:36:26 -0500 Subject: [PATCH 12/48] (cos) add comments about specifying DYLD_LIBRARY_PATH --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index a5b1f54b3..c69825a87 100644 --- a/setup.py +++ b/setup.py @@ -271,6 +271,8 @@ def update_link_args(compiler, libraries, libraries_dirs, extra_link_args): if compiler.__class__.__name__ == 'UnixCCompiler': extra_link_args.extend([f'-l{lib}' for lib in libraries]) if has_system_lib(): + # This requires to add DYLD_LIBRARY_PATH to the environment + # When repairing the wheel on MacOS (linux not tested yet) extra_link_args.extend([f'-L{lib}' for lib in libraries_dirs]) elif sys.platform == 'darwin': extra_link_args.extend(['-Wl,-rpath,@loader_path/lib']) From 064c12b4040f9d30eccf625a65920a1a86df44b8 Mon Sep 17 00:00:00 2001 From: memento Date: Wed, 13 Mar 2024 11:43:42 -0500 Subject: [PATCH 13/48] (cos) clean-up make reference --- .github/scripts/install-macos-build-deps.sh | 2 +- setup.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/install-macos-build-deps.sh b/.github/scripts/install-macos-build-deps.sh index 5f9482305..ec3029b27 100755 --- a/.github/scripts/install-macos-build-deps.sh +++ b/.github/scripts/install-macos-build-deps.sh @@ -8,7 +8,7 @@ brew update brew outdated openssl || brew upgrade openssl # Install packages needed to build lib-secp256k1 -for pkg in automake libtool pkg-config; do +for pkg in pkg-config; do brew list $pkg > /dev/null || brew install $pkg brew outdated --quiet $pkg || brew upgrade $pkg done diff --git a/setup.py b/setup.py index c69825a87..e50de9e00 100644 --- a/setup.py +++ b/setup.py @@ -31,8 +31,6 @@ BUILDING_FOR_WINDOWS = detect_dll() -MAKE = 'gmake' if platform.system() in ['FreeBSD', 'OpenBSD'] else 'make' - # IMPORTANT: keep in sync with .github/workflows/build.yml # # Version of libsecp256k1 to download if none exists in the `libsecp256k1` directory From bcf1bf7f993bd02ba05939b5af764e3d4f74bb77 Mon Sep 17 00:00:00 2001 From: Claire's Monster Date: Wed, 13 Mar 2024 16:54:50 -0500 Subject: [PATCH 14/48] FEAT: Refactor to src-layout (#153) --- .coveragerc | 2 +- .github/scripts/build-windows-wheels.sh | 12 ++++++------ .gitignore | 4 ++-- MANIFEST.in | 2 +- setup.py | 8 ++++---- setup_support.py | 2 +- {coincurve => src/coincurve}/__init__.py | 0 .../coincurve}/_windows_libsecp256k1.py | 0 {coincurve => src/coincurve}/context.py | 0 {coincurve => src/coincurve}/ecdsa.py | 0 {coincurve => src/coincurve}/flags.py | 0 {coincurve => src/coincurve}/keys.py | 0 {coincurve => src/coincurve}/py.typed | 0 {coincurve => src/coincurve}/types.py | 0 {coincurve => src/coincurve}/utils.py | 0 tox.ini | 14 +++++--------- 16 files changed, 20 insertions(+), 24 deletions(-) rename {coincurve => src/coincurve}/__init__.py (100%) rename {coincurve => src/coincurve}/_windows_libsecp256k1.py (100%) rename {coincurve => src/coincurve}/context.py (100%) rename {coincurve => src/coincurve}/ecdsa.py (100%) rename {coincurve => src/coincurve}/flags.py (100%) rename {coincurve => src/coincurve}/keys.py (100%) rename {coincurve => src/coincurve}/py.typed (100%) rename {coincurve => src/coincurve}/types.py (100%) rename {coincurve => src/coincurve}/utils.py (100%) diff --git a/.coveragerc b/.coveragerc index 42d5ef84e..2e0a952c2 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,7 @@ branch = True source = - coincurve + src/coincurve tests omit = diff --git a/.github/scripts/build-windows-wheels.sh b/.github/scripts/build-windows-wheels.sh index 0dbf52273..c80205f02 100755 --- a/.github/scripts/build-windows-wheels.sh +++ b/.github/scripts/build-windows-wheels.sh @@ -12,7 +12,7 @@ sudo apt-get -f install mkdir .hidden cp * .hidden -R -mv .hidden/coincurve/_windows_libsecp256k1.py .hidden/coincurve/_libsecp256k1.py +mv .hidden/src/coincurve/_windows_libsecp256k1.py .hidden/src/coincurve/_libsecp256k1.py mv .hidden ../clean cd .. @@ -27,17 +27,17 @@ cp 64bit 32bit -R cd 64bit build_dll x86_64-w64-mingw32 # Not sure why it ended-up being a -2.dll instead of -0.dll: Researching -mv .libs/libsecp256k1-?.dll ../clean/coincurve/libsecp256k1.dll +mv .libs/libsecp256k1-?.dll ../clean/src/coincurve/libsecp256k1.dll cd ../clean -python setup.py bdist_wheel --plat-name=win_amd64 -rm coincurve/libsecp256k1.dll +python setup.py bdist_wheel --plat-name=win_amd64 -vvv +rm src/coincurve/libsecp256k1.dll cd ../32bit build_dll i686-w64-mingw32 # Not sure why it ended-up being a -2.dll instead of -0.dll: Researching -mv .libs/libsecp256k1-?.dll ../clean/coincurve/libsecp256k1.dll +mv .libs/libsecp256k1-?.dll ../clean/src/coincurve/libsecp256k1.dll cd ../clean python setup.py bdist_wheel --plat-name=win32 -mv dist/* ../coincurve/dist/ +mv dist/* ../coincurve/dist cd ../coincurve diff --git a/.gitignore b/.gitignore index a00485194..3dfbab6ea 100644 --- a/.gitignore +++ b/.gitignore @@ -13,5 +13,5 @@ /wheelhouse /pip-wheel-metadata -/coincurve/_libsecp256k1.py -/coincurve/libsecp256k1.dll +/src/coincurve/_libsecp256k1.py +/src/coincurve/libsecp256k1.dll diff --git a/MANIFEST.in b/MANIFEST.in index 6059c9358..c3dc80de2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include coincurve/py.typed +include src/coincurve/py.typed include setup_support.py recursive-include _cffi_build *.py *.h prune /* diff --git a/setup.py b/setup.py index e50de9e00..76e08c350 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ import tarfile from io import BytesIO -from setuptools import Distribution as _Distribution, setup, find_packages, __version__ as setuptools_version +from setuptools import Distribution as _Distribution, setup, __version__ as setuptools_version from setuptools._distutils import log from setuptools._distutils.errors import DistutilsError from setuptools.command.build_clib import build_clib as _build_clib @@ -529,7 +529,7 @@ def has_c_libraries(self): extension = Extension( name='coincurve._libsecp256k1', - sources=[os.path.join('coincurve', '_libsecp256k1.c')], + sources=[os.path.join('src/coincurve', '_libsecp256k1.c')], # ABI?: py_limited_api=True, ) @@ -612,8 +612,8 @@ def has_c_libraries(self): python_requires='>=3.8', install_requires=['asn1crypto', 'cffi>=1.3.0'], - packages=find_packages(exclude=('_cffi_build', '_cffi_build.*', 'libsecp256k1', 'tests')), - package_data=package_data, + packages=['coincurve'], + package_dir={'coincurve': 'src/coincurve'}, distclass=Distribution, zip_safe=False, diff --git a/setup_support.py b/setup_support.py index 20827a736..b1e60f6ff 100644 --- a/setup_support.py +++ b/setup_support.py @@ -66,7 +66,7 @@ def has_system_lib(): def detect_dll(): here = os.path.dirname(os.path.abspath(__file__)) - for fn in os.listdir(os.path.join(here, 'coincurve')): + for fn in os.listdir(os.path.join(here, 'src/coincurve')): if fn.endswith('.dll'): return True return False diff --git a/coincurve/__init__.py b/src/coincurve/__init__.py similarity index 100% rename from coincurve/__init__.py rename to src/coincurve/__init__.py diff --git a/coincurve/_windows_libsecp256k1.py b/src/coincurve/_windows_libsecp256k1.py similarity index 100% rename from coincurve/_windows_libsecp256k1.py rename to src/coincurve/_windows_libsecp256k1.py diff --git a/coincurve/context.py b/src/coincurve/context.py similarity index 100% rename from coincurve/context.py rename to src/coincurve/context.py diff --git a/coincurve/ecdsa.py b/src/coincurve/ecdsa.py similarity index 100% rename from coincurve/ecdsa.py rename to src/coincurve/ecdsa.py diff --git a/coincurve/flags.py b/src/coincurve/flags.py similarity index 100% rename from coincurve/flags.py rename to src/coincurve/flags.py diff --git a/coincurve/keys.py b/src/coincurve/keys.py similarity index 100% rename from coincurve/keys.py rename to src/coincurve/keys.py diff --git a/coincurve/py.typed b/src/coincurve/py.typed similarity index 100% rename from coincurve/py.typed rename to src/coincurve/py.typed diff --git a/coincurve/types.py b/src/coincurve/types.py similarity index 100% rename from coincurve/types.py rename to src/coincurve/types.py diff --git a/coincurve/utils.py b/src/coincurve/utils.py similarity index 100% rename from coincurve/utils.py rename to src/coincurve/utils.py diff --git a/tox.ini b/tox.ini index 811696ec5..d36522bdc 100644 --- a/tox.ini +++ b/tox.ini @@ -14,23 +14,21 @@ envlist = docs [testenv] +usedevelop = true passenv = * deps = -rrequirements-dev.txt commands = - python -c "import shutil; shutil.move('coincurve', '_coincurve')" coverage run --parallel-mode -m pytest -v --benchmark-skip {posargs} - python -c "import shutil; shutil.move('_coincurve', 'coincurve')" coverage combine coverage report -m [testenv:bench] -skip_install = true +setenv = + PYTHONPATH = {toxinidir} envdir = {toxworkdir}/{env:PYTHON_VERSION:bench} commands = - python -c "import shutil; shutil.move('coincurve', '_coincurve')" - - output=pytest -v --benchmark-only --benchmark-sort=name --benchmark-cprofile=tottime - python -c "import shutil; shutil.move('_coincurve', 'coincurve')" + pytest -v --benchmark-only --benchmark-sort=name --benchmark-cprofile=tottime tests [testenv:lint] envdir = {toxworkdir}/lint @@ -56,7 +54,7 @@ skip_install = true deps = mypy>=0.790 commands = - mypy coincurve + mypy --namespace-packages --explicit-package-bases src/coincurve [testenv:docs] usedevelop = true @@ -88,6 +86,4 @@ commands = setenv = {[testenv:docs]setenv} deps = {[testenv:docs]deps} commands = - python -c "import shutil; shutil.move('coincurve', '_coincurve')" {[testenv:docs]commands} - python -c "import shutil; shutil.move('_coincurve', 'coincurve')" From 0079c1927744e5f6603c577682c799e0810c9714 Mon Sep 17 00:00:00 2001 From: memento Date: Sat, 9 Mar 2024 14:01:22 -0600 Subject: [PATCH 15/48] (ref) use logging module. add CMake from _build_clib --- setup.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/setup.py b/setup.py index 76e08c350..1dba66df8 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,33 @@ SECP256K1_IGNORE_EXT_LIB = os.getenv('COINCURVE_IGNORE_SYSTEM_LIB') +def define_secp256k1_local_lib_info(): + """ + Define the library name and the installation directory + The purpose is to automatically include the shared library in the package and + prevent inclusion the static library. This is probably hacky, but it works. + """ + if SECP256K1_BUILD == 'SHARED': + logging.info('Building shared library') + # This will place the shared library inside the coincurve package data + return PKG_NAME, 'lib' + + logging.info('Building static library') + # This will place the static library in a separate x_lib and in a lib_name directory + return LIB_NAME, 'x_lib' + + +def call_pkg_config(options, library, *, debug=False): + """Calls pkg-config with the given options and returns the output.""" + if SYSTEM == 'Windows': + options.append('--dont-define-prefix') + + pkg_config = shutil.which('pkg-config') + cmd = [pkg_config, *options, library] + + return subprocess_run(cmd, debug=debug) + + def download_library(command): if command.dry_run: return From 39c8cb141c27d725c380702a97310fa2db6bb162 Mon Sep 17 00:00:00 2001 From: memento Date: Thu, 14 Mar 2024 10:17:05 -0500 Subject: [PATCH 16/48] (fix) src-layout editable dirs are in /tmp: first-pass. remove duplicates --- setup.py | 33 +++------------------------------ 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/setup.py b/setup.py index eb89a62e8..746d1f513 100644 --- a/setup.py +++ b/setup.py @@ -60,33 +60,6 @@ SECP256K1_IGNORE_EXT_LIB = os.getenv('COINCURVE_IGNORE_SYSTEM_LIB') -def define_secp256k1_local_lib_info(): - """ - Define the library name and the installation directory - The purpose is to automatically include the shared library in the package and - prevent inclusion the static library. This is probably hacky, but it works. - """ - if SECP256K1_BUILD == 'SHARED': - logging.info('Building shared library') - # This will place the shared library inside the coincurve package data - return PKG_NAME, 'lib' - - logging.info('Building static library') - # This will place the static library in a separate x_lib and in a lib_name directory - return LIB_NAME, 'x_lib' - - -def call_pkg_config(options, library, *, debug=False): - """Calls pkg-config with the given options and returns the output.""" - if SYSTEM == 'Windows': - options.append('--dont-define-prefix') - - pkg_config = shutil.which('pkg-config') - cmd = [pkg_config, *options, library] - - return subprocess_run(cmd, debug=debug) - - def download_library(command): if command.dry_run: return @@ -195,7 +168,7 @@ def run(self): def bc_set_dirs_download(self): self._cwd = os.getcwd() os.makedirs(self.build_temp, exist_ok=True) - self._install_dir = str(self.build_temp).replace('temp', 'lib') + self._install_dir = self.build_temp # .replace('temp', 'lib') # Install path # SHARED: lib/coincurve -> path/lib.xxx/coincurve/path # included in coincurve wheel @@ -376,7 +349,7 @@ def build_extension(self, ext): # Location of locally built library lib, inst_dir = define_secp256k1_local_lib_info() - prefix = os.path.join(self.build_lib.replace('lib', inst_dir), lib) + prefix = os.path.join(self.build_temp.replace('lib', inst_dir), lib) postfix = os.path.join('pkgconfig', f'{LIB_NAME}.pc') c_lib_pkg = None @@ -386,7 +359,7 @@ def build_extension(self, ext): has_system_lib() ]): raise RuntimeError( - f'Library not found: {os.path.join(prefix, "lib/lib64", postfix)}' + f'Library not found: {os.path.join(prefix, "(lib|lib64)", postfix)}' f'\nSystem lib is {has_system_lib() = }. ' 'Please check that the library was properly built.' ) From dccbcc016f7c394cafabc0809dc10659f0b2c12e Mon Sep 17 00:00:00 2001 From: memento Date: Thu, 14 Mar 2024 10:48:32 -0500 Subject: [PATCH 17/48] (ref) remove previous clib/ext. add native builds for windows --- .github/scripts/build-windows-wheels.sh | 2 +- .github/workflows/build.yml | 114 ++++++++++++++++++++-- setup.py | 121 ------------------------ 3 files changed, 109 insertions(+), 128 deletions(-) diff --git a/.github/scripts/build-windows-wheels.sh b/.github/scripts/build-windows-wheels.sh index c80205f02..2d38fc83b 100755 --- a/.github/scripts/build-windows-wheels.sh +++ b/.github/scripts/build-windows-wheels.sh @@ -26,7 +26,7 @@ cp 64bit 32bit -R cd 64bit build_dll x86_64-w64-mingw32 -# Not sure why it ended-up being a -2.dll instead of -0.dll: Researching +# As SECP256k1 updatest their API, so does the version of the DLL mv .libs/libsecp256k1-?.dll ../clean/src/coincurve/libsecp256k1.dll cd ../clean python setup.py bdist_wheel --plat-name=win_amd64 -vvv diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fa8400134..474382c0e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,8 +137,113 @@ jobs: path: wheelhouse/*.whl if-no-files-found: error - windows-wheels-and-sdist: - name: Build Windows wheels and source distribution + windows-wheels-x86_64: + name: Build Windows wheels AMD64 + needs: + - test + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 + env: + CIBW_ARCHS_WINDOWS: "AMD64" + CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite + + - uses: actions/upload-artifact@v3 + with: + name: artifacts + path: wheelhouse/*.whl + if-no-files-found: error + + - name: Build SHARED wheels + uses: pypa/cibuildwheel@v2.16.5 + env: + COINCURVE_SECP256K1_BUILD: 'SHARED' + CIBW_ARCHS_WINDOWS: "AMD64" + CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite + + + windows-wheels-x86: + name: Build Windows wheels x86 + needs: + - test + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 + env: + COINCURVE_CROSS_HOST: "x86" + CIBW_ARCHS_WINDOWS: "x86" + CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite + + - uses: actions/upload-artifact@v3 + with: + name: artifacts + path: wheelhouse/*.whl + if-no-files-found: error + + - name: Build SHARED wheels + uses: pypa/cibuildwheel@v2.16.5 + env: + COINCURVE_SECP256K1_BUILD: 'SHARED' + COINCURVE_CROSS_HOST: "x86" + CIBW_ARCHS_WINDOWS: "x86" + CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite + + + windows-wheels-arm64: + name: Build Windows wheels for ARM64 + needs: + - test + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 + env: + COINCURVE_CROSS_HOST: "arm64" + CIBW_ARCHS_WINDOWS: "ARM64" + CIBW_BUILD_VERBOSITY: 1 + CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite + + - uses: actions/upload-artifact@v3 + with: + name: artifacts + path: wheelhouse/*.whl + if-no-files-found: error + + - name: Build SHARED wheels + uses: pypa/cibuildwheel@v2.16.5 + env: + COINCURVE_SECP256K1_BUILD: 'SHARED' + COINCURVE_CROSS_HOST: "arm64" + CIBW_ARCHS_WINDOWS: "ARM64" + CIBW_BUILD_VERBOSITY: 1 + CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite + + + sdist: + name: Build source distribution needs: - test runs-on: ubuntu-latest @@ -152,9 +257,6 @@ jobs: - name: Build source distribution run: python -m build --sdist - - name: Build Windows wheels - run: ./.github/scripts/build-windows-wheels.sh - - uses: actions/upload-artifact@v3 with: name: artifacts @@ -196,7 +298,7 @@ jobs: - linux-wheels-standard - macos-wheels-x86-64 - macos-wheels-arm - - windows-wheels-and-sdist + - sdist - linux-wheels-arm runs-on: ubuntu-latest if: > diff --git a/setup.py b/setup.py index 746d1f513..642d3e3fd 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import errno import logging import os import os.path @@ -379,126 +378,6 @@ def build_extension(self, ext): super().build_extension(ext) -class build_clib(_build_clib): - def initialize_options(self): - _build_clib.initialize_options(self) - self.build_flags = None - - def finalize_options(self): - _build_clib.finalize_options(self) - if self.build_flags is None: - self.build_flags = {'include_dirs': [], 'library_dirs': [], 'define': []} - - def get_source_files(self): - # Ensure library has been downloaded (sdist might have been skipped) - if not has_system_lib(): - download_library(self) - - return [ - absolute(os.path.join(root, filename)) - for root, _, filenames in os.walk(absolute('libsecp256k1')) - for filename in filenames - ] - - def build_libraries(self, libraries): - raise Exception('build_libraries') - - def check_library_list(self, libraries): - raise Exception('check_library_list') - - def get_library_names(self): - return build_flags('libsecp256k1', 'l', os.path.abspath(self.build_temp)) - - def run(self): - if has_system_lib(): - log.info('Using system library') - return - - build_temp = os.path.abspath(self.build_temp) - - try: - os.makedirs(build_temp) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - if not os.path.exists(absolute('libsecp256k1')): - # library needs to be downloaded - self.get_source_files() - - if not os.path.exists(absolute('libsecp256k1/configure')): - # configure script hasn't been generated yet - autogen = absolute('libsecp256k1/autogen.sh') - os.chmod(absolute(autogen), 0o700) - subprocess.check_call([autogen], cwd=absolute('libsecp256k1')) # noqa S603 - - for filename in [ - 'libsecp256k1/configure', - 'libsecp256k1/build-aux/compile', - 'libsecp256k1/build-aux/config.guess', - 'libsecp256k1/build-aux/config.sub', - 'libsecp256k1/build-aux/depcomp', - 'libsecp256k1/build-aux/install-sh', - 'libsecp256k1/build-aux/missing', - 'libsecp256k1/build-aux/test-driver', - ]: - try: - os.chmod(absolute(filename), 0o700) - except OSError as e: - # some of these files might not exist depending on autoconf version - if e.errno != errno.ENOENT: - # If the error isn't 'No such file or directory' something - # else is wrong and we want to know about it - raise - - cmd = [ - absolute('libsecp256k1/configure'), - '--disable-shared', - '--enable-static', - '--disable-dependency-tracking', - '--with-pic', - '--enable-module-extrakeys', - '--enable-module-recovery', - '--enable-module-schnorrsig', - '--prefix', - os.path.abspath(self.build_clib), - '--enable-experimental', - '--enable-module-ecdh', - '--enable-benchmark=no', - '--enable-tests=no', - '--enable-exhaustive-tests=no', - ] - if 'COINCURVE_CROSS_HOST' in os.environ: - cmd.append(f"--host={os.environ['COINCURVE_CROSS_HOST']}") - - log.debug(f"Running configure: {' '.join(cmd)}") - subprocess.check_call(cmd, cwd=build_temp) # noqa S603 - - subprocess.check_call([MAKE], cwd=build_temp) # noqa S603 - subprocess.check_call([MAKE, 'check'], cwd=build_temp) # noqa S603 - subprocess.check_call([MAKE, 'install'], cwd=build_temp) # noqa S603 - - self.build_flags['include_dirs'].extend(build_flags('libsecp256k1', 'I', build_temp)) - self.build_flags['library_dirs'].extend(build_flags('libsecp256k1', 'L', build_temp)) - if not has_system_lib(): - self.build_flags['define'].append(('CFFI_ENABLE_RECOVERY', None)) - - -class build_ext(_build_ext): - def run(self): - if self.distribution.has_c_libraries(): - _build_clib = self.get_finalized_command('build_clib') - self.include_dirs.append(os.path.join(_build_clib.build_clib, 'include')) - self.include_dirs.extend(_build_clib.build_flags['include_dirs']) - - self.library_dirs.insert(0, os.path.join(_build_clib.build_clib, 'lib')) - self.library_dirs.extend(_build_clib.build_flags['library_dirs']) - - self.define = _build_clib.build_flags['define'] - - return _build_ext.run(self) - - class develop(_develop): def run(self): if not has_system_lib(): From b34284cda41d01b0141b5e239a184a707284d0b2 Mon Sep 17 00:00:00 2001 From: memento Date: Thu, 14 Mar 2024 11:34:09 -0500 Subject: [PATCH 18/48] (ref) Windows add 'Releases'. use a more robust method to find pkgconfig file from local build --- setup.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index 642d3e3fd..3ec93dd42 100644 --- a/setup.py +++ b/setup.py @@ -346,19 +346,13 @@ def build_extension(self, ext): # Enforce API interface ext.py_limited_api = False - # Location of locally built library - lib, inst_dir = define_secp256k1_local_lib_info() - prefix = os.path.join(self.build_temp.replace('lib', inst_dir), lib) - postfix = os.path.join('pkgconfig', f'{LIB_NAME}.pc') - - c_lib_pkg = None - if not any([ - os.path.isfile(c_lib_pkg := os.path.join(prefix, 'lib', postfix)), - os.path.isfile(c_lib_pkg := os.path.join(prefix, 'lib64', postfix)), - has_system_lib() - ]): + # Find pkgconfig file for locally built library + pkg_dirs = self.get_finalized_command('build_clib').pkgconfig_dir # type: ignore + c_lib_pkg = [d for d in pkg_dirs if os.path.isfile(os.path.join(d, f'{LIB_NAME}.pc'))][0] + + if not has_system_lib() and not c_lib_pkg: raise RuntimeError( - f'Library not found: {os.path.join(prefix, "(lib|lib64)", postfix)}' + f'pkgconfig file not found: {LIB_NAME}.pc in : {pkg_dirs}.' f'\nSystem lib is {has_system_lib() = }. ' 'Please check that the library was properly built.' ) @@ -369,7 +363,6 @@ def build_extension(self, ext): ext.library_dirs.extend(build_flags(LIB_NAME, 'L', c_lib_pkg)) libraries = build_flags(LIB_NAME, 'l', c_lib_pkg) - logging.info(f' Libraries:{libraries}') # We do not set ext.libraries, this would add the default link instruction # Instead, we use extra_link_args to customize the link command From 6b4868e2b8a35690a3abc6d4bd91cbc25b32e399 Mon Sep 17 00:00:00 2001 From: memento Date: Thu, 14 Mar 2024 13:50:54 -0500 Subject: [PATCH 19/48] (ref) use next --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3ec93dd42..364fc4c17 100644 --- a/setup.py +++ b/setup.py @@ -348,7 +348,7 @@ def build_extension(self, ext): # Find pkgconfig file for locally built library pkg_dirs = self.get_finalized_command('build_clib').pkgconfig_dir # type: ignore - c_lib_pkg = [d for d in pkg_dirs if os.path.isfile(os.path.join(d, f'{LIB_NAME}.pc'))][0] + c_lib_pkg = next((d for d in pkg_dirs if os.path.isfile(os.path.join(d, f'{LIB_NAME}.pc'))), None) if not has_system_lib() and not c_lib_pkg: raise RuntimeError( From 2a35b9b5e8a4a39fc7d2e4331d978748b874f527 Mon Sep 17 00:00:00 2001 From: memento Date: Thu, 14 Mar 2024 14:27:49 -0500 Subject: [PATCH 20/48] (dbg) 32/64 shared missing something (though ARM64 pass) --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 474382c0e..c51262f07 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -167,6 +167,7 @@ jobs: env: COINCURVE_SECP256K1_BUILD: 'SHARED' CIBW_ARCHS_WINDOWS: "AMD64" + CIBW_BUILD_VERBOSITY: 1 CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite @@ -202,6 +203,7 @@ jobs: COINCURVE_SECP256K1_BUILD: 'SHARED' COINCURVE_CROSS_HOST: "x86" CIBW_ARCHS_WINDOWS: "x86" + CIBW_BUILD_VERBOSITY: 1 CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite From f23676ebcd88c900858749371e545fec9898c4c0 Mon Sep 17 00:00:00 2001 From: memento Date: Thu, 14 Mar 2024 14:49:03 -0500 Subject: [PATCH 21/48] (dbg) install shared in build.lib for shared --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 364fc4c17..c57061c08 100644 --- a/setup.py +++ b/setup.py @@ -167,7 +167,7 @@ def run(self): def bc_set_dirs_download(self): self._cwd = os.getcwd() os.makedirs(self.build_temp, exist_ok=True) - self._install_dir = self.build_temp # .replace('temp', 'lib') + self._install_dir = self.build_temp.replace('temp', 'lib') # Install path # SHARED: lib/coincurve -> path/lib.xxx/coincurve/path # included in coincurve wheel From 5e2159b58d6d63d362ea1f8024bc51881a0600ef Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 11:45:07 -0500 Subject: [PATCH 22/48] (ref) remove unused x-compile script for windows --- .github/scripts/build-windows-wheels.sh | 43 ------------------------- 1 file changed, 43 deletions(-) delete mode 100755 .github/scripts/build-windows-wheels.sh diff --git a/.github/scripts/build-windows-wheels.sh b/.github/scripts/build-windows-wheels.sh deleted file mode 100755 index 2d38fc83b..000000000 --- a/.github/scripts/build-windows-wheels.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -ex - -build_dll() { - ./autogen.sh - ./configure --host=$1 --enable-module-recovery --enable-experimental --enable-module-ecdh --enable-module-extrakeys --enable-module-schnorrsig --enable-benchmark=no --enable-tests=no --enable-exhaustive-tests=no --enable-static --disable-dependency-tracking --with-pic - make -} - -sudo apt-get install -y mingw-w64 -sudo apt-get -f install - -mkdir .hidden -cp * .hidden -R -mv .hidden/src/coincurve/_windows_libsecp256k1.py .hidden/src/coincurve/_libsecp256k1.py -mv .hidden ../clean - -cd .. - -curl -sLO "https://github.com/bitcoin-core/secp256k1/archive/$COINCURVE_UPSTREAM_REF.tar.gz" -tar -xzf "$COINCURVE_UPSTREAM_REF.tar.gz" -mv "secp256k1-$COINCURVE_UPSTREAM_REF" secp256k1 - -mv secp256k1 64bit -cp 64bit 32bit -R - -cd 64bit -build_dll x86_64-w64-mingw32 -# As SECP256k1 updatest their API, so does the version of the DLL -mv .libs/libsecp256k1-?.dll ../clean/src/coincurve/libsecp256k1.dll -cd ../clean -python setup.py bdist_wheel --plat-name=win_amd64 -vvv -rm src/coincurve/libsecp256k1.dll - -cd ../32bit -build_dll i686-w64-mingw32 -# Not sure why it ended-up being a -2.dll instead of -0.dll: Researching -mv .libs/libsecp256k1-?.dll ../clean/src/coincurve/libsecp256k1.dll -cd ../clean -python setup.py bdist_wheel --plat-name=win32 - -mv dist/* ../coincurve/dist -cd ../coincurve From 21dcc29f9978256a49e7025d4e09d97d748be8f5 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 11:47:39 -0500 Subject: [PATCH 23/48] (ref) COINCURVE_CROSS_HOST not needed for macos arm --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c51262f07..b5ac21483 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -128,8 +128,8 @@ jobs: uses: pypa/cibuildwheel@v2.16.2 env: CIBW_ARCHS_MACOS: arm64 - COINCURVE_CROSS_HOST: aarch64-apple-darwin - CFLAGS: -target arm64-apple-macos11 + # COINCURVE_CROSS_HOST: aarch64-apple-darwin + # CFLAGS: -target arm64-apple-macos11 - uses: actions/upload-artifact@v3 with: From b4ee6918008cdb70adc696a2393d7aef54a029b8 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 13:36:53 -0500 Subject: [PATCH 24/48] (ref) remove unused build.py (will rename build_shared) --- .github/workflows/build.yml | 20 ++++++++++++++++++-- _cffi_build/build.py | 36 ------------------------------------ 2 files changed, 18 insertions(+), 38 deletions(-) delete mode 100644 _cffi_build/build.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b5ac21483..80b48418d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -128,8 +128,24 @@ jobs: uses: pypa/cibuildwheel@v2.16.2 env: CIBW_ARCHS_MACOS: arm64 - # COINCURVE_CROSS_HOST: aarch64-apple-darwin - # CFLAGS: -target arm64-apple-macos11 + + - uses: actions/upload-artifact@v3 + with: + name: artifacts + path: wheelhouse/*.whl + if-no-files-found: error + + macos-wheels-native-arm: + name: Build macOS wheels for ARM (Native) + needs: + - test + runs-on: macos-14 + + steps: + - uses: actions/checkout@v5 + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 - uses: actions/upload-artifact@v3 with: diff --git a/_cffi_build/build.py b/_cffi_build/build.py deleted file mode 100644 index f478fb58a..000000000 --- a/_cffi_build/build.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -from collections import namedtuple - -from cffi import FFI - -here = os.path.dirname(os.path.abspath(__file__)) - -Source = namedtuple('Source', ('h', 'include')) - - -def _mk_ffi(sources, name='_libsecp256k1', **kwargs): - _ffi = FFI() - code = [] - - for source in sources: - with open(os.path.join(here, source.h)) as h: - _ffi.cdef(h.read()) - code.append(source.include) - - code.append('#define PY_USE_BUNDLED') - _ffi.set_source(name, '\n'.join(code), **kwargs) - - return _ffi - - -modules = [ - Source('secp256k1.h', '#include '), - Source('secp256k1_ecdh.h', '#include '), - Source('secp256k1_extrakeys.h', '#include '), - Source('secp256k1_recovery.h', '#include '), - Source('secp256k1_schnorrsig.h', '#include '), - Source('secp256k1_ellswift.h', '#include '), - Source('secp256k1_preallocated.h', '#include '), -] - -ffi = _mk_ffi(modules, libraries=['secp256k1']) From 078e4c945b43b51f70d2a6dff7f5cdc5f3f03ac2 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 13:39:11 -0500 Subject: [PATCH 25/48] (ref) rename build_shared to build (it builds both static/shared) --- _cffi_build/{build_shared.py => build.py} | 0 setup.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename _cffi_build/{build_shared.py => build.py} (100%) diff --git a/_cffi_build/build_shared.py b/_cffi_build/build.py similarity index 100% rename from _cffi_build/build_shared.py rename to _cffi_build/build.py diff --git a/setup.py b/setup.py index c57061c08..f33c3f408 100644 --- a/setup.py +++ b/setup.py @@ -330,7 +330,7 @@ def update_link_args(self, libraries, libraries_dirs, extra_link_args): def create_c_files(self, ext): # Construct C-file from CFFI - build_script = os.path.join('_cffi_build', 'build_shared.py') + build_script = os.path.join('_cffi_build', 'build.py') for i, c_file in enumerate(ext.sources): os.makedirs(self.build_temp, exist_ok=True) c_file = os.path.join(self.build_temp, os.path.basename(c_file)) From cae1983cd22ae9e457610e61e212184a5cc6cbe2 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 13:40:59 -0500 Subject: [PATCH 26/48] (ref) used wrong class for build_ext --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f33c3f408..d3568850a 100644 --- a/setup.py +++ b/setup.py @@ -427,7 +427,7 @@ def has_c_libraries(self): ext_modules=[extension], cmdclass={ 'build_clib': None if has_system_lib() else BuildClibWithCMake, - 'build_ext': BuildCFFIForSharedLib, + 'build_ext': BuildExtensionFromCFFI, 'develop': develop, 'egg_info': egg_info, 'sdist': sdist, From 0ebe4d1e9ce2210610dc9f8b8ead7adddb764b6c Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 13:52:28 -0500 Subject: [PATCH 27/48] (fix) wrong version of checkout --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 80b48418d..cea3909ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,7 +142,7 @@ jobs: runs-on: macos-14 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 - name: Build wheels uses: pypa/cibuildwheel@v2.16.5 From e0617b71e6a4727f7c40690ccbad97ef83e94096 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 14:41:39 -0500 Subject: [PATCH 28/48] (ref) replace log with logging --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d3568850a..845887767 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,6 @@ from io import BytesIO from setuptools import Distribution as _Distribution, setup, __version__ as setuptools_version -from setuptools._distutils import log from setuptools._distutils.errors import DistutilsError from setuptools.command.build_clib import build_clib as _build_clib from setuptools.command.build_ext import build_ext as _build_ext @@ -67,7 +66,8 @@ def download_library(command): # Library already downloaded return if not os.path.exists(libdir): - command.announce('downloading libsecp256k1 source code', level=log.INFO) + from logging import INFO + command.announce('downloading libsecp256k1 source code', level=INFO) try: import requests try: From d1b643affd6227b10174efad410118e1cce1b097 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 14:42:43 -0500 Subject: [PATCH 29/48] (ref) remove unused build_ext class --- setup.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/setup.py b/setup.py index 845887767..fc3f3e0c4 100644 --- a/setup.py +++ b/setup.py @@ -384,14 +384,6 @@ def run(self): package_data = {'coincurve': ['py.typed']} -class BuildCFFIForSharedLib(_build_ext): - def build_extensions(self): - build_script = os.path.join('_cffi_build', 'build_shared.py') - c_file = self.extensions[0].sources[0] - subprocess.run([sys.executable, build_script, c_file, '0'], shell=False, check=True) # noqa S603 - super().build_extensions() - - def main(): if has_system_lib(): From 7430eb43a40586e2dc460e4789b9b1df23f4ade6 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 14:43:44 -0500 Subject: [PATCH 30/48] (ref) remove BUILDING_FOR_WINDOWS --- setup.py | 52 ++++++++++++++++++++-------------------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/setup.py b/setup.py index fc3f3e0c4..710655d99 100644 --- a/setup.py +++ b/setup.py @@ -428,41 +428,29 @@ def has_c_libraries(self): ) else: - if BUILDING_FOR_WINDOWS: - - class Distribution(_Distribution): - def is_pure(self): - return False - - - package_data['coincurve'].append('libsecp256k1.dll') - setup_kwargs = {} - - else: - - class Distribution(_Distribution): - def has_c_libraries(self): - return not has_system_lib() + class Distribution(_Distribution): + def has_c_libraries(self): + return not has_system_lib() - extension = Extension( - name='coincurve._libsecp256k1', - sources=['_c_file_for_extension.c'], - py_limited_api=False, - extra_compile_args=['/d2FH4-'] if SYSTEM == 'Windows' else [], - ) + extension = Extension( + name='coincurve._libsecp256k1', + sources=['_c_file_for_extension.c'], + py_limited_api=False, + extra_compile_args=['/d2FH4-'] if SYSTEM == 'Windows' else [], + ) - setup_kwargs = dict( - ext_modules=[extension], - cmdclass={ - 'build_clib': None if has_system_lib() else BuildClibWithCMake, - 'build_ext': BuildExtensionFromCFFI, - 'develop': develop, - 'egg_info': egg_info, - 'sdist': sdist, - 'bdist_wheel': bdist_wheel, - }, - ) + setup_kwargs = dict( + ext_modules=[extension], + cmdclass={ + 'build_clib': None if has_system_lib() else BuildClibWithCMake, + 'build_ext': BuildExtensionFromCFFI, + 'develop': develop, + 'egg_info': egg_info, + 'sdist': sdist, + 'bdist_wheel': bdist_wheel, + }, + ) setup( name='coincurve', From cae96d9bf4f22c02dcff60ccf64a32c0e67cb796 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 14:44:51 -0500 Subject: [PATCH 31/48] (ref) remove detect_dll --- setup.py | 2 -- setup_support.py | 8 -------- 2 files changed, 10 deletions(-) diff --git a/setup.py b/setup.py index 710655d99..2cb13b9a5 100644 --- a/setup.py +++ b/setup.py @@ -27,8 +27,6 @@ from setup_support import absolute, build_flags, detect_dll, has_system_lib, define_secp256k1_local_lib_info, \ call_pkg_config, subprocess_run -BUILDING_FOR_WINDOWS = detect_dll() - # IMPORTANT: keep in sync with .github/workflows/build.yml # # Version of libsecp256k1 to download if none exists in the `libsecp256k1` directory diff --git a/setup_support.py b/setup_support.py index b1e60f6ff..7c57ba1b7 100644 --- a/setup_support.py +++ b/setup_support.py @@ -64,14 +64,6 @@ def has_system_lib(): return _has_system_lib -def detect_dll(): - here = os.path.dirname(os.path.abspath(__file__)) - for fn in os.listdir(os.path.join(here, 'src/coincurve')): - if fn.endswith('.dll'): - return True - return False - - def subprocess_run(cmd, *, debug=False): try: result = subprocess.run(cmd, capture_output=True, text=True, check=True) # noqa S603 From 4c94d918c7e0843a159da8e4ff385090e0b19ee2 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 14:46:49 -0500 Subject: [PATCH 32/48] (cos) cleanup import --- setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2cb13b9a5..09ecce162 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,14 @@ _bdist_wheel = None sys.path.append(os.path.abspath(os.path.dirname(__file__))) -from setup_support import absolute, build_flags, detect_dll, has_system_lib, define_secp256k1_local_lib_info, \ - call_pkg_config, subprocess_run +from setup_support import ( # noqa: E402 + absolute, + build_flags, + has_system_lib, + define_secp256k1_local_lib_info, + call_pkg_config, + subprocess_run +) # IMPORTANT: keep in sync with .github/workflows/build.yml # From 078b0ec162cbd9be66d6bee8f83731ff9b63df77 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 14:51:11 -0500 Subject: [PATCH 33/48] (ref) refactor Distribution and Extension --- setup.py | 79 ++++++++++++++++++++------------------------------------ 1 file changed, 28 insertions(+), 51 deletions(-) diff --git a/setup.py b/setup.py index 09ecce162..2e171bb34 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ _bdist_wheel = None sys.path.append(os.path.abspath(os.path.dirname(__file__))) -from setup_support import ( # noqa: E402 +from setup_support import ( absolute, build_flags, has_system_lib, @@ -184,7 +184,7 @@ def bc_set_dirs_download(self): self.get_source_files() def bc_update_pkg_config_path(self): - self.pkgconfig_dir = [os.path.join(self._install_lib_dir, n, 'pkgconfig') for n in ['lib', 'lib64']] + self.pkgconfig_dir = [os.path.join(self._install_lib_dir, n, 'pkgconfig') for n in ['lib', 'lib64']] self.pkgconfig_dir.append(os.getenv('PKG_CONFIG_PATH', '')) os.environ['PKG_CONFIG_PATH'] = os.pathsep.join(self.pkgconfig_dir) call_pkg_config(['--exists'], LIB_NAME) @@ -388,26 +388,28 @@ def run(self): package_data = {'coincurve': ['py.typed']} -def main(): - if has_system_lib(): +class Distribution(_Distribution): + def has_c_libraries(self): + return not has_system_lib() - class Distribution(_Distribution): - def has_c_libraries(self): - return not has_system_lib() +extension = Extension( + name='coincurve._libsecp256k1', + sources=['_c_file_for_extension.c'], + py_limited_api=False, + extra_compile_args=['/d2FH4-'] if SYSTEM == 'Windows' else [], +) + - extension = Extension( - name='coincurve._libsecp256k1', - sources=[os.path.join('src/coincurve', '_libsecp256k1.c')], - # ABI?: py_limited_api=True, - ) +def main(): + if has_system_lib(): extension.extra_compile_args = [ - subprocess.check_output(['pkg-config', '--cflags-only-I', 'libsecp256k1']).strip().decode('utf-8') # noqa S603 + subprocess.check_output(['pkg-config', '--cflags-only-I', 'libsecp256k1']).strip().decode() # noqa S603 ] extension.extra_link_args = [ - subprocess.check_output(['pkg-config', '--libs-only-L', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 - subprocess.check_output(['pkg-config', '--libs-only-l', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 + subprocess.check_output(['pkg-config', '--libs-only-L', 'libsecp256k1']).strip().decode(), # noqa S603 + subprocess.check_output(['pkg-config', '--libs-only-l', 'libsecp256k1']).strip().decode(), # noqa S603 ] if os.name == 'nt' or sys.platform == 'win32': @@ -419,42 +421,17 @@ def has_c_libraries(self): v = v.replace('-l', 'lib') extension.__dict__['extra_link_args'][i] = f'{v}.lib' - setup_kwargs = dict( - ext_modules=[extension], - cmdclass={ - 'build_clib': None if has_system_lib() else BuildClibWithCMake, - 'build_ext': BuildExtensionFromCFFI, - 'develop': develop, - 'egg_info': egg_info, - 'sdist': sdist, - 'bdist_wheel': bdist_wheel, - }, - ) - - else: - class Distribution(_Distribution): - def has_c_libraries(self): - return not has_system_lib() - - - extension = Extension( - name='coincurve._libsecp256k1', - sources=['_c_file_for_extension.c'], - py_limited_api=False, - extra_compile_args=['/d2FH4-'] if SYSTEM == 'Windows' else [], - ) - - setup_kwargs = dict( - ext_modules=[extension], - cmdclass={ - 'build_clib': None if has_system_lib() else BuildClibWithCMake, - 'build_ext': BuildExtensionFromCFFI, - 'develop': develop, - 'egg_info': egg_info, - 'sdist': sdist, - 'bdist_wheel': bdist_wheel, - }, - ) + setup_kwargs = dict( + ext_modules=[extension], + cmdclass={ + 'build_clib': None if has_system_lib() else BuildClibWithCMake, + 'build_ext': BuildExtensionFromCFFI, + 'develop': develop, + 'egg_info': egg_info, + 'sdist': sdist, + 'bdist_wheel': bdist_wheel, + }, + ) setup( name='coincurve', From 5c9a245955291801b5f0231cc1dba36ca121a79a Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 15:00:49 -0500 Subject: [PATCH 34/48] (ref) refactor Distribution and Extension --- setup.py | 35 ++++++----------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/setup.py b/setup.py index 2e171bb34..4cd3e7df5 100644 --- a/setup.py +++ b/setup.py @@ -385,41 +385,18 @@ def run(self): _develop.run(self) -package_data = {'coincurve': ['py.typed']} - - class Distribution(_Distribution): def has_c_libraries(self): return not has_system_lib() -extension = Extension( - name='coincurve._libsecp256k1', - sources=['_c_file_for_extension.c'], - py_limited_api=False, - extra_compile_args=['/d2FH4-'] if SYSTEM == 'Windows' else [], -) - - def main(): - if has_system_lib(): - - extension.extra_compile_args = [ - subprocess.check_output(['pkg-config', '--cflags-only-I', 'libsecp256k1']).strip().decode() # noqa S603 - ] - extension.extra_link_args = [ - subprocess.check_output(['pkg-config', '--libs-only-L', 'libsecp256k1']).strip().decode(), # noqa S603 - subprocess.check_output(['pkg-config', '--libs-only-l', 'libsecp256k1']).strip().decode(), # noqa S603 - ] - - if os.name == 'nt' or sys.platform == 'win32': - # Apparently, the linker on Windows interprets -lxxx as xxx.lib, not libxxx.lib - for i, v in enumerate(extension.__dict__.get('extra_link_args')): - extension.__dict__['extra_link_args'][i] = v.replace('-L', '/LIBPATH:') - - if v.startswith('-l'): - v = v.replace('-l', 'lib') - extension.__dict__['extra_link_args'][i] = f'{v}.lib' + extension = Extension( + name='coincurve._libsecp256k1', + sources=['_c_file_for_extension.c'], + py_limited_api=False, + extra_compile_args=['/d2FH4-'] if SYSTEM == 'Windows' else [], + ) setup_kwargs = dict( ext_modules=[extension], From 9756f53f502e1f14c02003338f200298a3436110 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 15:26:38 -0500 Subject: [PATCH 35/48] (ref) streamline build workflow. complement publish --- .github/workflows/build.yml | 53 +++---------------------------------- 1 file changed, 3 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cea3909ca..2e735efb0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -116,26 +116,6 @@ jobs: if-no-files-found: error macos-wheels-arm: - name: Build macOS wheels for ARM - needs: - - test - runs-on: macos-12 - - steps: - - uses: actions/checkout@v4 - - - name: Build wheels - uses: pypa/cibuildwheel@v2.16.2 - env: - CIBW_ARCHS_MACOS: arm64 - - - uses: actions/upload-artifact@v3 - with: - name: artifacts - path: wheelhouse/*.whl - if-no-files-found: error - - macos-wheels-native-arm: name: Build macOS wheels for ARM (Native) needs: - test @@ -178,15 +158,6 @@ jobs: path: wheelhouse/*.whl if-no-files-found: error - - name: Build SHARED wheels - uses: pypa/cibuildwheel@v2.16.5 - env: - COINCURVE_SECP256K1_BUILD: 'SHARED' - CIBW_ARCHS_WINDOWS: "AMD64" - CIBW_BUILD_VERBOSITY: 1 - CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite - - windows-wheels-x86: name: Build Windows wheels x86 needs: @@ -213,16 +184,6 @@ jobs: path: wheelhouse/*.whl if-no-files-found: error - - name: Build SHARED wheels - uses: pypa/cibuildwheel@v2.16.5 - env: - COINCURVE_SECP256K1_BUILD: 'SHARED' - COINCURVE_CROSS_HOST: "x86" - CIBW_ARCHS_WINDOWS: "x86" - CIBW_BUILD_VERBOSITY: 1 - CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite - - windows-wheels-arm64: name: Build Windows wheels for ARM64 needs: @@ -241,7 +202,6 @@ jobs: env: COINCURVE_CROSS_HOST: "arm64" CIBW_ARCHS_WINDOWS: "ARM64" - CIBW_BUILD_VERBOSITY: 1 CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite - uses: actions/upload-artifact@v3 @@ -250,16 +210,6 @@ jobs: path: wheelhouse/*.whl if-no-files-found: error - - name: Build SHARED wheels - uses: pypa/cibuildwheel@v2.16.5 - env: - COINCURVE_SECP256K1_BUILD: 'SHARED' - COINCURVE_CROSS_HOST: "arm64" - CIBW_ARCHS_WINDOWS: "ARM64" - CIBW_BUILD_VERBOSITY: 1 - CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite - - sdist: name: Build source distribution needs: @@ -316,6 +266,9 @@ jobs: - linux-wheels-standard - macos-wheels-x86-64 - macos-wheels-arm + - windows-wheels-x86-64 + - windows-wheels-x86 + - windows-wheels-arm - sdist - linux-wheels-arm runs-on: ubuntu-latest From a4a7cf30a20e0ab74b2cd7bd4512005b81a68805 Mon Sep 17 00:00:00 2001 From: memento Date: Fri, 15 Mar 2024 15:31:04 -0500 Subject: [PATCH 36/48] (fix) typos in naming convention --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2e735efb0..edd6b7834 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -95,7 +95,7 @@ jobs: path: wheelhouse/*.whl if-no-files-found: error - macos-wheels-x86-64: + macos-wheels-x86_64: name: Build macOS wheels needs: - test @@ -184,7 +184,7 @@ jobs: path: wheelhouse/*.whl if-no-files-found: error - windows-wheels-arm64: + windows-wheels-arm: name: Build Windows wheels for ARM64 needs: - test @@ -264,9 +264,9 @@ jobs: name: Publish release needs: - linux-wheels-standard - - macos-wheels-x86-64 + - macos-wheels-x86_64 - macos-wheels-arm - - windows-wheels-x86-64 + - windows-wheels-x86_64 - windows-wheels-x86 - windows-wheels-arm - sdist From cf5e9583ecf9d20bf47744a3db2ee4789075b06a Mon Sep 17 00:00:00 2001 From: memento Date: Sat, 16 Mar 2024 11:28:53 -0500 Subject: [PATCH 37/48] (ref) simplify _find_lib --- setup_support.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/setup_support.py b/setup_support.py index 7c57ba1b7..04b37cc37 100644 --- a/setup_support.py +++ b/setup_support.py @@ -40,12 +40,7 @@ def _find_lib(): update_pkg_config_path() try: - if os.name == 'nt': - cmd = ['pkg-config', '--libs-only-L', '--dont-define-prefix', 'libsecp256k1'] - else: - cmd = ['pkg-config', '--libs-only-L', 'libsecp256k1'] - lib_dir = subprocess_run(cmd) - + lib_dir = call_pkg_config(['--libs-only-L'], 'libsecp256k1') return verify_system_lib(lib_dir[2:].strip()) except (OSError, subprocess.CalledProcessError): From d3ae46610c9453935b0e3022ccd0c298fd40f8df Mon Sep 17 00:00:00 2001 From: memento Date: Sun, 17 Mar 2024 09:39:37 -0500 Subject: [PATCH 38/48] (ref) remove unused windows_libsecp256k1.py --- src/coincurve/_windows_libsecp256k1.py | 420 ------------------------- 1 file changed, 420 deletions(-) delete mode 100644 src/coincurve/_windows_libsecp256k1.py diff --git a/src/coincurve/_windows_libsecp256k1.py b/src/coincurve/_windows_libsecp256k1.py deleted file mode 100644 index 49a324a26..000000000 --- a/src/coincurve/_windows_libsecp256k1.py +++ /dev/null @@ -1,420 +0,0 @@ -import os - -from cffi import FFI - -BASE_DEFINITIONS = """ -typedef struct secp256k1_context_struct secp256k1_context; -typedef struct secp256k1_scratch_space_struct secp256k1_scratch_space; - -typedef struct { - unsigned char data[64]; -} secp256k1_pubkey; - -typedef struct { - unsigned char data[64]; -} secp256k1_ecdsa_signature; - -typedef int (*secp256k1_nonce_function)( - unsigned char *nonce32, - const unsigned char *msg32, - const unsigned char *key32, - const unsigned char *algo16, - void *data, - unsigned int attempt -); - -#define SECP256K1_CONTEXT_NONE 1 -#define SECP256K1_EC_COMPRESSED 258 -#define SECP256K1_EC_UNCOMPRESSED 2 -#define SECP256K1_TAG_PUBKEY_EVEN 2 -#define SECP256K1_TAG_PUBKEY_ODD 3 -#define SECP256K1_TAG_PUBKEY_UNCOMPRESSED 4 -#define SECP256K1_TAG_PUBKEY_HYBRID_EVEN 6 -#define SECP256K1_TAG_PUBKEY_HYBRID_ODD 7 - -extern const secp256k1_context *secp256k1_context_static; -extern void secp256k1_selftest(void); -extern secp256k1_context *secp256k1_context_create( - unsigned int flags -); -extern secp256k1_context *secp256k1_context_clone( - const secp256k1_context *ctx -); -extern void secp256k1_context_destroy( - secp256k1_context *ctx -); -extern void secp256k1_context_set_illegal_callback( - secp256k1_context *ctx, - void (*fun)(const char *message, void *data), - const void *data -); -extern void secp256k1_context_set_error_callback( - secp256k1_context *ctx, - void (*fun)(const char *message, void *data), - const void *data -); -extern secp256k1_scratch_space *secp256k1_scratch_space_create( - const secp256k1_context *ctx, - size_t size -); -extern void secp256k1_scratch_space_destroy( - const secp256k1_context *ctx, - secp256k1_scratch_space *scratch -); -extern int secp256k1_ec_pubkey_parse( - const secp256k1_context *ctx, - secp256k1_pubkey *pubkey, - const unsigned char *input, - size_t inputlen -); -extern int secp256k1_ec_pubkey_serialize( - const secp256k1_context *ctx, - unsigned char *output, - size_t *outputlen, - const secp256k1_pubkey *pubkey, - unsigned int flags -); -extern int secp256k1_ec_pubkey_cmp( - const secp256k1_context *ctx, - const secp256k1_pubkey *pubkey1, - const secp256k1_pubkey *pubkey2 -); -extern int secp256k1_ecdsa_signature_parse_compact( - const secp256k1_context *ctx, - secp256k1_ecdsa_signature *sig, - const unsigned char *input64 -); -extern int secp256k1_ecdsa_signature_parse_der( - const secp256k1_context *ctx, - secp256k1_ecdsa_signature *sig, - const unsigned char *input, - size_t inputlen -); -extern int secp256k1_ecdsa_signature_serialize_der( - const secp256k1_context *ctx, - unsigned char *output, - size_t *outputlen, - const secp256k1_ecdsa_signature *sig -); -extern int secp256k1_ecdsa_signature_serialize_compact( - const secp256k1_context *ctx, - unsigned char *output64, - const secp256k1_ecdsa_signature *sig -); -extern int secp256k1_ecdsa_verify( - const secp256k1_context *ctx, - const secp256k1_ecdsa_signature *sig, - const unsigned char *msghash32, - const secp256k1_pubkey *pubkey -); -extern int secp256k1_ecdsa_signature_normalize( - const secp256k1_context *ctx, - secp256k1_ecdsa_signature *sigout, - const secp256k1_ecdsa_signature *sigin -); - -extern const secp256k1_nonce_function secp256k1_nonce_function_rfc6979; - -extern const secp256k1_nonce_function secp256k1_nonce_function_default; - -extern int secp256k1_ecdsa_sign( - const secp256k1_context *ctx, - secp256k1_ecdsa_signature *sig, - const unsigned char *msghash32, - const unsigned char *seckey, - secp256k1_nonce_function noncefp, - const void *ndata -); -extern int secp256k1_ec_seckey_verify( - const secp256k1_context *ctx, - const unsigned char *seckey -); -extern int secp256k1_ec_pubkey_create( - const secp256k1_context *ctx, - secp256k1_pubkey *pubkey, - const unsigned char *seckey -); -extern int secp256k1_ec_seckey_negate( - const secp256k1_context *ctx, - unsigned char *seckey -); -extern int secp256k1_ec_pubkey_negate( - const secp256k1_context *ctx, - secp256k1_pubkey *pubkey -); -extern int secp256k1_ec_seckey_tweak_add( - const secp256k1_context *ctx, - unsigned char *seckey, - const unsigned char *tweak32 -); -extern int secp256k1_ec_pubkey_tweak_add( - const secp256k1_context *ctx, - secp256k1_pubkey *pubkey, - const unsigned char *tweak32 -); -extern int secp256k1_ec_seckey_tweak_mul( - const secp256k1_context *ctx, - unsigned char *seckey, - const unsigned char *tweak32 -); -extern int secp256k1_ec_pubkey_tweak_mul( - const secp256k1_context *ctx, - secp256k1_pubkey *pubkey, - const unsigned char *tweak32 -); -extern int secp256k1_context_randomize( - secp256k1_context *ctx, - const unsigned char *seed32 -); -extern int secp256k1_ec_pubkey_combine( - const secp256k1_context *ctx, - secp256k1_pubkey *out, - const secp256k1_pubkey * const *ins, - size_t n -); -extern int secp256k1_tagged_sha256( - const secp256k1_context *ctx, - unsigned char *hash32, - const unsigned char *tag, - size_t taglen, - const unsigned char *msg, - size_t msglen -); -""" - -EXTRAKEYS_DEFINITIONS = """ -typedef struct { - unsigned char data[64]; -} secp256k1_xonly_pubkey; -typedef struct { - unsigned char data[96]; -} secp256k1_keypair; -extern int secp256k1_xonly_pubkey_parse( - const secp256k1_context *ctx, - secp256k1_xonly_pubkey *pubkey, - const unsigned char *input32 -); -extern int secp256k1_xonly_pubkey_serialize( - const secp256k1_context *ctx, - unsigned char *output32, - const secp256k1_xonly_pubkey *pubkey -); -extern int secp256k1_xonly_pubkey_cmp( - const secp256k1_context *ctx, - const secp256k1_xonly_pubkey *pk1, - const secp256k1_xonly_pubkey *pk2 -); -extern int secp256k1_xonly_pubkey_from_pubkey( - const secp256k1_context *ctx, - secp256k1_xonly_pubkey *xonly_pubkey, - int *pk_parity, - const secp256k1_pubkey *pubkey -); -extern int secp256k1_xonly_pubkey_tweak_add( - const secp256k1_context *ctx, - secp256k1_pubkey *output_pubkey, - const secp256k1_xonly_pubkey *internal_pubkey, - const unsigned char *tweak32 -); -extern int secp256k1_xonly_pubkey_tweak_add_check( - const secp256k1_context *ctx, - const unsigned char *tweaked_pubkey32, - int tweaked_pk_parity, - const secp256k1_xonly_pubkey *internal_pubkey, - const unsigned char *tweak32 -); -extern int secp256k1_keypair_create( - const secp256k1_context *ctx, - secp256k1_keypair *keypair, - const unsigned char *seckey -); -extern int secp256k1_keypair_sec( - const secp256k1_context *ctx, - unsigned char *seckey, - const secp256k1_keypair *keypair -); -extern int secp256k1_keypair_pub( - const secp256k1_context *ctx, - secp256k1_pubkey *pubkey, - const secp256k1_keypair *keypair -); -extern int secp256k1_keypair_xonly_pub( - const secp256k1_context *ctx, - secp256k1_xonly_pubkey *pubkey, - int *pk_parity, - const secp256k1_keypair *keypair -); -extern int secp256k1_keypair_xonly_tweak_add( - const secp256k1_context *ctx, - secp256k1_keypair *keypair, - const unsigned char *tweak32 -); -""" - -RECOVERY_DEFINITIONS = """ -typedef struct { - unsigned char data[65]; -} secp256k1_ecdsa_recoverable_signature; -extern int secp256k1_ecdsa_recoverable_signature_parse_compact( - const secp256k1_context *ctx, - secp256k1_ecdsa_recoverable_signature *sig, - const unsigned char *input64, - int recid -); -extern int secp256k1_ecdsa_recoverable_signature_convert( - const secp256k1_context *ctx, - secp256k1_ecdsa_signature *sig, - const secp256k1_ecdsa_recoverable_signature *sigin -); -extern int secp256k1_ecdsa_recoverable_signature_serialize_compact( - const secp256k1_context *ctx, - unsigned char *output64, - int *recid, - const secp256k1_ecdsa_recoverable_signature *sig -); -extern int secp256k1_ecdsa_sign_recoverable( - const secp256k1_context *ctx, - secp256k1_ecdsa_recoverable_signature *sig, - const unsigned char *msghash32, - const unsigned char *seckey, - secp256k1_nonce_function noncefp, - const void *ndata -); -extern int secp256k1_ecdsa_recover( - const secp256k1_context *ctx, - secp256k1_pubkey *pubkey, - const secp256k1_ecdsa_recoverable_signature *sig, - const unsigned char *msghash32 -); -""" - -SCHNORRSIG_DEFINITIONS = """ -typedef int (*secp256k1_nonce_function_hardened)( - unsigned char *nonce32, - const unsigned char *msg, - size_t msglen, - const unsigned char *key32, - const unsigned char *xonly_pk32, - const unsigned char *algo, - size_t algolen, - void *data -); -extern const secp256k1_nonce_function_hardened secp256k1_nonce_function_bip340; -typedef struct { - unsigned char magic[4]; - secp256k1_nonce_function_hardened noncefp; - void *ndata; -} secp256k1_schnorrsig_extraparams; -extern int secp256k1_schnorrsig_sign32( - const secp256k1_context *ctx, - unsigned char *sig64, - const unsigned char *msg32, - const secp256k1_keypair *keypair, - const unsigned char *aux_rand32 -); -extern int secp256k1_schnorrsig_sign_custom( - const secp256k1_context *ctx, - unsigned char *sig64, - const unsigned char *msg, - size_t msglen, - const secp256k1_keypair *keypair, - secp256k1_schnorrsig_extraparams *extraparams -); -extern int secp256k1_schnorrsig_verify( - const secp256k1_context *ctx, - const unsigned char *sig64, - const unsigned char *msg, - size_t msglen, - const secp256k1_xonly_pubkey *pubkey -); -""" - -ECDH_DEFINITIONS = """ -typedef int (*secp256k1_ecdh_hash_function)( - unsigned char *output, - const unsigned char *x32, - const unsigned char *y32, - void *data -); -extern const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_sha256; -extern const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_default; -extern int secp256k1_ecdh( - const secp256k1_context *ctx, - unsigned char *output, - const secp256k1_pubkey *pubkey, - const unsigned char *seckey, - secp256k1_ecdh_hash_function hashfp, - void *data -); -""" - -ELLSWIFT_DEFINITIONS = """ -typedef int (*secp256k1_ellswift_xdh_hash_function)( - unsigned char *output, - const unsigned char *x32, - const unsigned char *ell_a64, - const unsigned char *ell_b64, - void *data -); -extern const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_hash_function_prefix; -extern const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_hash_function_bip324; -extern int secp256k1_ellswift_encode( - const secp256k1_context *ctx, - unsigned char *ell64, - const secp256k1_pubkey *pubkey, - const unsigned char *rnd32 -); -extern int secp256k1_ellswift_decode( - const secp256k1_context *ctx, - secp256k1_pubkey *pubkey, - const unsigned char *ell64 -); -extern int secp256k1_ellswift_create( - const secp256k1_context *ctx, - unsigned char *ell64, - const unsigned char *seckey32, - const unsigned char *auxrnd32 -); -extern int secp256k1_ellswift_xdh( - const secp256k1_context *ctx, - unsigned char *output, - const unsigned char *ell_a64, - const unsigned char *ell_b64, - const unsigned char *seckey32, - int party, - secp256k1_ellswift_xdh_hash_function hashfp, - void *data -); -""" - -PREALLOCATED_DEFINITIONS = """ -extern size_t secp256k1_context_preallocated_size( - unsigned int flags -); -extern secp256k1_context *secp256k1_context_preallocated_create( - void *prealloc, - unsigned int flags -); -extern size_t secp256k1_context_preallocated_clone_size( - const secp256k1_context *ctx -); -extern secp256k1_context *secp256k1_context_preallocated_clone( - const secp256k1_context *ctx, - void *prealloc -); -extern void secp256k1_context_preallocated_destroy( - secp256k1_context *ctx -); -""" - -ffi = FFI() - -ffi.cdef(BASE_DEFINITIONS) -ffi.cdef(EXTRAKEYS_DEFINITIONS) -ffi.cdef(RECOVERY_DEFINITIONS) -ffi.cdef(SCHNORRSIG_DEFINITIONS) -ffi.cdef(ECDH_DEFINITIONS) - -here = os.path.dirname(os.path.abspath(__file__)) -lib = ffi.dlopen(os.path.join(here, 'libsecp256k1.dll')) - From 66f3bca56f1cf79ad2758fb609f7cc0b3516be8c Mon Sep 17 00:00:00 2001 From: memento Date: Sun, 17 Mar 2024 18:14:53 -0500 Subject: [PATCH 39/48] (ref) integrate setup_tools package --- MANIFEST.in | 2 +- setup.py | 368 ++------------------------------------ setup_tools/__init__.py | 0 setup_tools/build_clib.py | 139 ++++++++++++++ setup_tools/build_ext.py | 62 +++++++ setup_tools/commands.py | 77 ++++++++ setup_tools/linkers.py | 61 +++++++ setup_tools/support.py | 137 ++++++++++++++ 8 files changed, 494 insertions(+), 352 deletions(-) create mode 100644 setup_tools/__init__.py create mode 100644 setup_tools/build_clib.py create mode 100644 setup_tools/build_ext.py create mode 100644 setup_tools/commands.py create mode 100644 setup_tools/linkers.py create mode 100644 setup_tools/support.py diff --git a/MANIFEST.in b/MANIFEST.in index c3dc80de2..d1204bd6a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include src/coincurve/py.typed -include setup_support.py +recursive-include setup_tools *.py recursive-include _cffi_build *.py *.h prune /* graft libsecp256k1 diff --git a/setup.py b/setup.py index 4cd3e7df5..c6ca9be55 100644 --- a/setup.py +++ b/setup.py @@ -1,37 +1,22 @@ import logging -import os import os.path import platform -import shutil -import subprocess import sys -import tarfile -from io import BytesIO +from os.path import join +from sys import path as PATH from setuptools import Distribution as _Distribution, setup, __version__ as setuptools_version -from setuptools._distutils.errors import DistutilsError -from setuptools.command.build_clib import build_clib as _build_clib -from setuptools.command.build_ext import build_ext as _build_ext -from setuptools.command.develop import develop as _develop -from setuptools.command.dist_info import dist_info as _dist_info -from setuptools.command.egg_info import egg_info as _egg_info -from setuptools.command.sdist import sdist as _sdist from setuptools.extension import Extension -try: - from wheel.bdist_wheel import bdist_wheel as _bdist_wheel -except ImportError: - _bdist_wheel = None +# Define the package root directory and add it to the system path +COINCURVE_ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) +PATH.append(COINCURVE_ROOT_DIR) + +# Add setuptools local package path +PATH.insert(0, join(COINCURVE_ROOT_DIR, 'setup_tools')) sys.path.append(os.path.abspath(os.path.dirname(__file__))) -from setup_support import ( - absolute, - build_flags, - has_system_lib, - define_secp256k1_local_lib_info, - call_pkg_config, - subprocess_run -) +from setup_tools.support import has_system_lib # noqa: E402 # IMPORTANT: keep in sync with .github/workflows/build.yml # @@ -62,335 +47,16 @@ SECP256K1_IGNORE_EXT_LIB = os.getenv('COINCURVE_IGNORE_SYSTEM_LIB') -def download_library(command): - if command.dry_run: - return - libdir = absolute('libsecp256k1') - if os.path.exists(os.path.join(libdir, 'autogen.sh')): - # Library already downloaded - return - if not os.path.exists(libdir): - from logging import INFO - command.announce('downloading libsecp256k1 source code', level=INFO) - try: - import requests - try: - r = requests.get(LIB_TARBALL_URL, stream=True, timeout=10) - status_code = r.status_code - if status_code == 200: - content = BytesIO(r.raw.read()) - content.seek(0) - with tarfile.open(fileobj=content) as tf: - dirname = tf.getnames()[0].partition('/')[0] - tf.extractall() # noqa: S202 - shutil.move(dirname, libdir) - else: - raise SystemExit('Unable to download secp256k1 library: HTTP-Status: %d', status_code) - except requests.exceptions.RequestException as e: - raise SystemExit('Unable to download secp256k1 library: %s', str(e)) - except ImportError as e: - raise SystemExit('Unable to download secp256k1 library: %s', str(e)) - - -class egg_info(_egg_info): - def run(self): - # Ensure library has been downloaded (sdist might have been skipped) - if not has_system_lib(): - download_library(self) - - _egg_info.run(self) - - -class dist_info(_dist_info): - def run(self): - # Ensure library has been downloaded (sdist might have been skipped) - if not has_system_lib(): - download_library(self) - - _dist_info.run(self) - - -class sdist(_sdist): - def run(self): - if not has_system_lib(): - download_library(self) - _sdist.run(self) - - -if _bdist_wheel: - - class bdist_wheel(_bdist_wheel): - def run(self): - if not has_system_lib(): - download_library(self) - _bdist_wheel.run(self) - - -else: - bdist_wheel = None - - -class BuildClibWithCMake(_build_clib): - title = 'SECP256K1 C library build' - - def __init__(self, dist): - super().__init__(dist) - self.pkgconfig_dir = None - - self._cwd = None - self._lib_src = None - self._install_dir = None - self._install_lib_dir = None - - def get_source_files(self): - # Ensure library has been downloaded (sdist might have been skipped) - if not has_system_lib(): - download_library(self) - - # This seems to create issues in MANIFEST.in - return [f for _, _, fs in os.walk(absolute(LIB_NAME)) for f in fs] - - def run(self): - if has_system_lib(): - logging.info('Using system library') - return - - logging.info(self.title) - self.bc_set_dirs_download() - self.bc_prepare_build(self._install_lib_dir, self.build_temp, self._lib_src) - - try: - os.chdir(self.build_temp) - subprocess.check_call(self.bc_build_command()) # noqa S603 - finally: - os.chdir(self._cwd) - - # Register lib installation path - self.bc_update_pkg_config_path() - - def bc_set_dirs_download(self): - self._cwd = os.getcwd() - os.makedirs(self.build_temp, exist_ok=True) - self._install_dir = self.build_temp.replace('temp', 'lib') - - # Install path - # SHARED: lib/coincurve -> path/lib.xxx/coincurve/path # included in coincurve wheel - # STATIC: x_lib/libsecp256k1 -> path/x_lib.xxx/libsecp256k1/path # NOT included in coincurve wheel - lib, inst_dir = define_secp256k1_local_lib_info() - self._install_lib_dir = os.path.join(self._install_dir.replace('lib', inst_dir), lib) - - self._lib_src = os.path.join(self._cwd, LIB_NAME) - if not os.path.exists(self._lib_src): - self.get_source_files() - - def bc_update_pkg_config_path(self): - self.pkgconfig_dir = [os.path.join(self._install_lib_dir, n, 'pkgconfig') for n in ['lib', 'lib64']] - self.pkgconfig_dir.append(os.getenv('PKG_CONFIG_PATH', '')) - os.environ['PKG_CONFIG_PATH'] = os.pathsep.join(self.pkgconfig_dir) - call_pkg_config(['--exists'], LIB_NAME) - - @staticmethod - def _generator(msvc): - if '2017' in str(msvc): - return 'Visual Studio 15 2017' - if '2019' in str(msvc): - return 'Visual Studio 16 2019' - if '2022' in str(msvc): - return 'Visual Studio 17 2022' - - @staticmethod - def bc_prepare_build(install_lib_dir, build_temp, lib_src): - cmake_args = [ - '-DCMAKE_BUILD_TYPE=Release', - f'-DCMAKE_INSTALL_PREFIX={install_lib_dir}', - f'-DCMAKE_C_FLAGS={"-fPIC" if SECP256K1_BUILD != "SHARED" and SYSTEM != "Windows" else ""}', - f'-DSECP256K1_DISABLE_SHARED={"OFF" if SECP256K1_BUILD == "SHARED" else "ON"}', - '-DSECP256K1_BUILD_BENCHMARK=OFF', - '-DSECP256K1_BUILD_TESTS=OFF', - '-DSECP256K1_ENABLE_MODULE_ECDH=ON', - '-DSECP256K1_ENABLE_MODULE_RECOVERY=ON', - '-DSECP256K1_ENABLE_MODULE_SCHNORRSIG=ON', - '-DSECP256K1_ENABLE_MODULE_EXTRAKEYS=ON', - ] - - # Windows (more complex) - if SYSTEM == 'Windows': - vswhere = shutil.which('vswhere') - cmd = [vswhere, '-latest', '-find', 'MSBuild\\**\\Bin\\MSBuild.exe'] - msvc = subprocess.check_output(cmd).strip().decode('utf-8') # noqa S603 - - # For windows x86/x64, select the correct architecture - arch = 'x64' if MACHINE == 'AMD64' else 'Win32' # Native - - if X_HOST is not None: - logging.info(f'Cross-compiling on {SYSTEM}:{MACHINE} for {X_HOST}') - if X_HOST in ['arm64', 'ARM64', 'x86']: - arch = 'Win32' if X_HOST in ['x86'] else 'arm64' - else: - raise NotImplementedError(f'Unsupported architecture: {X_HOST}') - - # Place the DLL directly in the package directory - cmake_args.append('-DCMAKE_INSTALL_BINDIR=.') - cmake_args.extend(['-G', BuildClibWithCMake._generator(msvc), f'-A{arch}']) - - elif SYSTEM == 'Darwin': - if X_HOST is None: - cmake_args.append( - f'-DCMAKE_OSX_ARCHITECTURES={MACHINE}' # Native - ) - else: - logging.info(f'Cross-compiling on {SYSTEM}:{MACHINE} for {X_HOST}') - if X_HOST in ['armv7', 'armv7s', 'arm64', 'arm64e']: - cmake_args.append( - f'-DCMAKE_OSX_ARCHITECTURES={X_HOST}' - ) - else: - if X_HOST is not None: - if X_HOST not in [ - 'arm-linux-gnueabihf', - 'x86_64-w64-mingw32', - ]: - raise NotImplementedError(f'Unsupported architecture: {X_HOST}') - - logging.info(f'Cross-compiling on {SYSTEM}:{MACHINE} for {X_HOST}') - cmake_args.append( - f'-DCMAKE_TOOLCHAIN_FILE=../cmake/{X_HOST}.toolchain.cmake' - ) - - logging.info(' Configure CMake') - subprocess.check_call(['cmake', '-S', lib_src, '-B', build_temp, *cmake_args]) # noqa S603 - - @staticmethod - def bc_build_command(): - logging.info(' Install with CMake') - return ['cmake', '--build', '.', '--target', 'install', '--config', 'Release', '--clean-first'] - - -class SharedLinker(object): - @staticmethod - def update_link_args(compiler, libraries, libraries_dirs, extra_link_args): - if compiler.__class__.__name__ == 'UnixCCompiler': - extra_link_args.extend([f'-l{lib}' for lib in libraries]) - if has_system_lib(): - # This requires to add DYLD_LIBRARY_PATH to the environment - # When repairing the wheel on MacOS (linux not tested yet) - extra_link_args.extend([f'-L{lib}' for lib in libraries_dirs]) - elif sys.platform == 'darwin': - extra_link_args.extend(['-Wl,-rpath,@loader_path/lib']) - else: - extra_link_args.extend([ - '-Wl,-rpath,$ORIGIN/lib', - '-Wl,-rpath,$ORIGIN/lib64', - ]) - elif compiler.__class__.__name__ == 'MSVCCompiler': - for ld in libraries_dirs: - ld = ld.replace('/', '\\') - for lib in libraries: - lib_file = os.path.join(ld, f'lib{lib}.lib') - lib_path = [f'/LIBPATH:{ld}', f'lib{lib}.lib'] - if os.path.exists(lib_file): - extra_link_args.extend(lib_path) - else: - raise NotImplementedError(f'Unsupported compiler: {compiler.__class__.__name__}') - - -class StaticLinker(object): - @staticmethod - def update_link_args(compiler, libraries, libraries_dirs, extra_link_args): - if compiler.__class__.__name__ == 'UnixCCompiler': - # It is possible that the library was compiled without fPIC option - for lib in libraries: - # On MacOS the mix static/dynamic option is different - # It requires a -force_load option for each library - if sys.platform == 'darwin': - for lib_dir in libraries_dirs: - if os.path.exists(os.path.join(lib_dir, f'lib{lib}.a')): - extra_link_args.extend( - ['-Wl,-force_load', os.path.join(lib_dir, f'lib{lib}.a')] - ) - break - else: - extra_link_args.extend(['-Wl,-Bstatic', f'-l{lib}', '-Wl,-Bdynamic']) - - elif compiler.__class__.__name__ == 'MSVCCompiler': - for ld in libraries_dirs: - ld = ld.replace('/', '\\') - for lib in libraries: - lib_file = os.path.join(ld, f'lib{lib}.lib') - if os.path.exists(lib_file): - extra_link_args.append(lib_file) - else: - raise NotImplementedError(f'Unsupported compiler: {compiler.__class__.__name__}') - - -class BuildExtensionFromCFFI(_build_ext): - static_lib = True if SECP256K1_BUILD == 'STATIC' else False - - def update_link_args(self, libraries, libraries_dirs, extra_link_args): - if self.static_lib: - StaticLinker.update_link_args(self.compiler, libraries, libraries_dirs, extra_link_args) - else: - SharedLinker.update_link_args(self.compiler, libraries, libraries_dirs, extra_link_args) - - def create_c_files(self, ext): - # Construct C-file from CFFI - build_script = os.path.join('_cffi_build', 'build.py') - for i, c_file in enumerate(ext.sources): - os.makedirs(self.build_temp, exist_ok=True) - c_file = os.path.join(self.build_temp, os.path.basename(c_file)) - # This puts c-file a temp location (and not in the coincurve src directory) - ext.sources[i] = c_file - cmd = [sys.executable, build_script, c_file, '1' if self.static_lib else '0'] - subprocess_run(cmd) - - def build_extension(self, ext): - # Construct C-file from CFFI - self.create_c_files(ext) - - # Enforce API interface - ext.py_limited_api = False - - # Find pkgconfig file for locally built library - pkg_dirs = self.get_finalized_command('build_clib').pkgconfig_dir # type: ignore - c_lib_pkg = next((d for d in pkg_dirs if os.path.isfile(os.path.join(d, f'{LIB_NAME}.pc'))), None) - - if not has_system_lib() and not c_lib_pkg: - raise RuntimeError( - f'pkgconfig file not found: {LIB_NAME}.pc in : {pkg_dirs}.' - f'\nSystem lib is {has_system_lib() = }. ' - 'Please check that the library was properly built.' - ) - - # PKG_CONFIG_PATH is updated by build_clib if built locally, - # however, it would not work for a step-by-step build, thus we specify the lib path - ext.extra_compile_args.extend([f'-I{build_flags(LIB_NAME, "I", c_lib_pkg)[0]}']) - ext.library_dirs.extend(build_flags(LIB_NAME, 'L', c_lib_pkg)) - - libraries = build_flags(LIB_NAME, 'l', c_lib_pkg) - - # We do not set ext.libraries, this would add the default link instruction - # Instead, we use extra_link_args to customize the link command - self.update_link_args(libraries, ext.library_dirs, ext.extra_link_args) - - super().build_extension(ext) - - -class develop(_develop): - def run(self): - if not has_system_lib(): - raise DistutilsError( - "This library is not usable in 'develop' mode when using the " - 'bundled libsecp256k1. See README for details.' - ) - _develop.run(self) - - class Distribution(_Distribution): def has_c_libraries(self): return not has_system_lib() def main(): + from setup_tools.commands import BdistWheel, EggInfo, Sdist, Develop + from setup_tools.build_clib import BuildClibWithCMake + from setup_tools.build_ext import BuildExtensionFromCFFI + extension = Extension( name='coincurve._libsecp256k1', sources=['_c_file_for_extension.c'], @@ -403,10 +69,10 @@ def main(): cmdclass={ 'build_clib': None if has_system_lib() else BuildClibWithCMake, 'build_ext': BuildExtensionFromCFFI, - 'develop': develop, - 'egg_info': egg_info, - 'sdist': sdist, - 'bdist_wheel': bdist_wheel, + 'develop': Develop, + 'egg_info': EggInfo, + 'sdist': Sdist, + 'bdist_wheel': BdistWheel, }, ) diff --git a/setup_tools/__init__.py b/setup_tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/setup_tools/build_clib.py b/setup_tools/build_clib.py new file mode 100644 index 000000000..fa63ff810 --- /dev/null +++ b/setup_tools/build_clib.py @@ -0,0 +1,139 @@ +import logging +import os +import shutil +import subprocess + +from setuptools.command import build_clib + +from setup import LIB_NAME, MACHINE, SECP256K1_BUILD, SYSTEM, X_HOST +from setup_support import define_secp256k1_local_lib_info +from setup_tools.support import absolute, call_pkg_config, download_library, has_system_lib + + +class BuildClibWithCMake(build_clib.build_clib): + title = 'SECP256K1 C library build' + + def __init__(self, dist): + super().__init__(dist) + self.pkgconfig_dir = None + + self._cwd = None + self._lib_src = None + self._install_dir = None + self._install_lib_dir = None + + def get_source_files(self): + # Ensure library has been downloaded (sdist might have been skipped) + if not has_system_lib(): + download_library(self) + + # This seems to create issues in MANIFEST.in + return [f for _, _, fs in os.walk(absolute(LIB_NAME)) for f in fs] + + def run(self): + if has_system_lib(): + logging.info('Using system library') + return + + logging.info(self.title) + self.bc_set_dirs_download() + self.bc_prepare_build(self._install_lib_dir, self.build_temp, self._lib_src) + + try: + os.chdir(self.build_temp) + subprocess.check_call(self.bc_build_command()) # noqa S603 + finally: + os.chdir(self._cwd) + + # Register lib installation path + self.bc_update_pkg_config_path() + + def bc_set_dirs_download(self): + self._cwd = os.getcwd() + os.makedirs(self.build_temp, exist_ok=True) + self._install_dir = self.build_temp.replace('temp', 'lib') + + # Install path + # SHARED: lib/coincurve -> path/lib.xxx/coincurve/path # included in coincurve wheel + # STATIC: x_lib/libsecp256k1 -> path/x_lib.xxx/libsecp256k1/path # NOT included in coincurve wheel + lib, inst_dir = define_secp256k1_local_lib_info() + self._install_lib_dir = os.path.join(self._install_dir.replace('lib', inst_dir), lib) + + self._lib_src = os.path.join(self._cwd, LIB_NAME) + if not os.path.exists(self._lib_src): + self.get_source_files() + + def bc_update_pkg_config_path(self): + self.pkgconfig_dir = [os.path.join(self._install_lib_dir, n, 'pkgconfig') for n in ['lib', 'lib64']] + self.pkgconfig_dir.append(os.getenv('PKG_CONFIG_PATH', '')) + os.environ['PKG_CONFIG_PATH'] = os.pathsep.join(self.pkgconfig_dir) + call_pkg_config(['--exists'], LIB_NAME) + + @staticmethod + def _generator(msvc): + if '2017' in str(msvc): + return 'Visual Studio 15 2017' + if '2019' in str(msvc): + return 'Visual Studio 16 2019' + if '2022' in str(msvc): + return 'Visual Studio 17 2022' + + @staticmethod + def bc_prepare_build(install_lib_dir, build_temp, lib_src): + cmake_args = [ + '-DCMAKE_BUILD_TYPE=Release', + f'-DCMAKE_INSTALL_PREFIX={install_lib_dir}', + f'-DCMAKE_C_FLAGS={"-fPIC" if SECP256K1_BUILD != "SHARED" and SYSTEM != "Windows" else ""}', + f'-DSECP256K1_DISABLE_SHARED={"OFF" if SECP256K1_BUILD == "SHARED" else "ON"}', + '-DSECP256K1_BUILD_BENCHMARK=OFF', + '-DSECP256K1_BUILD_TESTS=OFF', + '-DSECP256K1_ENABLE_MODULE_ECDH=ON', + '-DSECP256K1_ENABLE_MODULE_RECOVERY=ON', + '-DSECP256K1_ENABLE_MODULE_SCHNORRSIG=ON', + '-DSECP256K1_ENABLE_MODULE_EXTRAKEYS=ON', + ] + + # Windows (more complex) + if SYSTEM == 'Windows': + vswhere = shutil.which('vswhere') + cmd = [vswhere, '-latest', '-find', 'MSBuild\\**\\Bin\\MSBuild.exe'] + msvc = subprocess.check_output(cmd).strip().decode('utf-8') # noqa S603 + + # For windows x86/x64, select the correct architecture + arch = 'x64' if MACHINE == 'AMD64' else 'Win32' # Native + + if X_HOST is not None: + logging.info(f'Cross-compiling on {SYSTEM}:{MACHINE} for {X_HOST}') + if X_HOST in ['arm64', 'ARM64', 'x86']: + arch = 'Win32' if X_HOST in ['x86'] else 'arm64' + else: + raise NotImplementedError(f'Unsupported architecture: {X_HOST}') + + # Place the DLL directly in the package directory + cmake_args.append('-DCMAKE_INSTALL_BINDIR=.') + cmake_args.extend(['-G', BuildClibWithCMake._generator(msvc), f'-A{arch}']) + + elif SYSTEM == 'Darwin': + if X_HOST is None: + cmake_args.append(f'-DCMAKE_OSX_ARCHITECTURES={MACHINE}') # Native + else: + logging.info(f'Cross-compiling on {SYSTEM}:{MACHINE} for {X_HOST}') + if X_HOST in ['armv7', 'armv7s', 'arm64', 'arm64e']: + cmake_args.append(f'-DCMAKE_OSX_ARCHITECTURES={X_HOST}') + elif X_HOST is not None: + if X_HOST not in [ + 'arm-linux-gnueabihf', + 'x86_64-w64-mingw32', + ]: + raise NotImplementedError(f'Unsupported architecture: {X_HOST}') + + logging.info(f'Cross-compiling on {SYSTEM}:{MACHINE} for {X_HOST}') + cmake_args.append(f'-DCMAKE_TOOLCHAIN_FILE=../cmake/{X_HOST}.toolchain.cmake') + + logging.info(' Configure CMake') + subprocess.check_call(['cmake', '-S', lib_src, '-B', build_temp, *cmake_args]) # noqa S603 + + @staticmethod + def bc_build_command(): + logging.info(' Install with CMake') + return ['cmake', '--build', '.', '--target', 'install', '--config', 'Release', '--clean-first'] diff --git a/setup_tools/build_ext.py b/setup_tools/build_ext.py new file mode 100644 index 000000000..9429aec5a --- /dev/null +++ b/setup_tools/build_ext.py @@ -0,0 +1,62 @@ +import os +import sys + +from setuptools.command import build_ext + +from setup import LIB_NAME +from setup_tools.linkers import SharedLinker, StaticLinker +from setup_tools.support import build_flags, has_system_lib, subprocess_run + + +class BuildExtensionFromCFFI(build_ext.build_ext): + from setup import SECP256K1_BUILD + + static_lib = True if SECP256K1_BUILD == 'STATIC' else False + + def update_link_args(self, libraries, libraries_dirs, extra_link_args): + if self.static_lib: + StaticLinker.update_link_args(self.compiler, libraries, libraries_dirs, extra_link_args) + else: + SharedLinker.update_link_args(self.compiler, libraries, libraries_dirs, extra_link_args) + + def create_c_files(self, ext): + # Construct C-file from CFFI + build_script = os.path.join('_cffi_build', 'build.py') + for i, c_file in enumerate(ext.sources): + os.makedirs(self.build_temp, exist_ok=True) + c_file = os.path.join(self.build_temp, os.path.basename(c_file)) + # This puts c-file a temp location (and not in the coincurve src directory) + ext.sources[i] = c_file + cmd = [sys.executable, build_script, c_file, '1' if self.static_lib else '0'] + subprocess_run(cmd) + + def build_extension(self, ext): + # Construct C-file from CFFI + self.create_c_files(ext) + + # Enforce API interface + ext.py_limited_api = False + + # Find pkgconfig file for locally built library + pkg_dirs = self.get_finalized_command('build_clib').pkgconfig_dir # type: ignore + c_lib_pkg = next((d for d in pkg_dirs if os.path.isfile(os.path.join(d, f'{LIB_NAME}.pc'))), None) + + if not has_system_lib() and not c_lib_pkg: + raise RuntimeError( + f'pkgconfig file not found: {LIB_NAME}.pc in : {pkg_dirs}.' + f'\nSystem lib is {has_system_lib() = }. ' + 'Please check that the library was properly built.' + ) + + # PKG_CONFIG_PATH is updated by build_clib if built locally, + # however, it would not work for a step-by-step build, thus we specify the lib path + ext.extra_compile_args.extend([f'-I{build_flags(LIB_NAME, "I", c_lib_pkg)[0]}']) + ext.library_dirs.extend(build_flags(LIB_NAME, 'L', c_lib_pkg)) + + libraries = build_flags(LIB_NAME, 'l', c_lib_pkg) + + # We do not set ext.libraries, this would add the default link instruction + # Instead, we use extra_link_args to customize the link command + self.update_link_args(libraries, ext.library_dirs, ext.extra_link_args) + + super().build_extension(ext) diff --git a/setup_tools/commands.py b/setup_tools/commands.py new file mode 100644 index 000000000..81150a87f --- /dev/null +++ b/setup_tools/commands.py @@ -0,0 +1,77 @@ +from setuptools.command import develop, dist_info, egg_info, sdist + +try: + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel +except ImportError: + _bdist_wheel = None + + +class EggInfo(egg_info.egg_info): + def run(self): + # Ensure library has been downloaded (sdist might have been skipped) + from setup_tools.support import has_system_lib + + if not has_system_lib(): + from support import download_library + + download_library(self) + + super().run() + + +class DistInfo(dist_info.dist_info): + def run(self): + # Ensure library has been downloaded (sdist might have been skipped) + from setup_tools.support import has_system_lib + + if not has_system_lib(): + from support import download_library + + download_library(self) + + super().run() + + +class Sdist(sdist.sdist): + def run(self): + from setup_tools.support import has_system_lib + + if not has_system_lib(): + from support import download_library + + download_library(self) + super().run() + + +class Develop(develop.develop): + def run(self): + from setup_tools.support import has_system_lib + + if not has_system_lib(): + raise RuntimeError( + "This library is not usable in 'develop' mode when using the " + 'bundled libsecp256k1. See README for details.' + ) + super().run() + + +try: + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel +except ImportError: + _bdist_wheel = None + + +if _bdist_wheel: + + class BdistWheel(_bdist_wheel): + def run(self): + from setup_tools.support import has_system_lib + + if not has_system_lib(): + from setup_tools.support import download_library + + download_library(self) + super().run() + +else: + BdistWheel = None diff --git a/setup_tools/linkers.py b/setup_tools/linkers.py new file mode 100644 index 000000000..87e375077 --- /dev/null +++ b/setup_tools/linkers.py @@ -0,0 +1,61 @@ +import os +import sys + +from setup_tools.support import has_system_lib + + +class SharedLinker: + @staticmethod + def update_link_args(compiler, libraries, libraries_dirs, extra_link_args): + if compiler.__class__.__name__ == 'UnixCCompiler': + extra_link_args.extend([f'-l{lib}' for lib in libraries]) + if has_system_lib(): + # This requires to add DYLD_LIBRARY_PATH to the environment + # When repairing the wheel on MacOS (linux not tested yet) + extra_link_args.extend([f'-L{lib}' for lib in libraries_dirs]) + elif sys.platform == 'darwin': + extra_link_args.extend(['-Wl,-rpath,@loader_path/lib']) + else: + extra_link_args.extend( + [ + '-Wl,-rpath,$ORIGIN/lib', + '-Wl,-rpath,$ORIGIN/lib64', + ] + ) + elif compiler.__class__.__name__ == 'MSVCCompiler': + for ld in libraries_dirs: + ld = ld.replace('/', '\\') + for lib in libraries: + lib_file = os.path.join(ld, f'lib{lib}.lib') + lib_path = [f'/LIBPATH:{ld}', f'lib{lib}.lib'] + if os.path.exists(lib_file): + extra_link_args.extend(lib_path) + else: + raise NotImplementedError(f'Unsupported compiler: {compiler.__class__.__name__}') + + +class StaticLinker: + @staticmethod + def update_link_args(compiler, libraries, libraries_dirs, extra_link_args): + if compiler.__class__.__name__ == 'UnixCCompiler': + # It is possible that the library was compiled without fPIC option + for lib in libraries: + # On MacOS the mix static/dynamic option is different + # It requires a -force_load option for each library + if sys.platform == 'darwin': + for lib_dir in libraries_dirs: + if os.path.exists(os.path.join(lib_dir, f'lib{lib}.a')): + extra_link_args.extend(['-Wl,-force_load', os.path.join(lib_dir, f'lib{lib}.a')]) + break + else: + extra_link_args.extend(['-Wl,-Bstatic', f'-l{lib}', '-Wl,-Bdynamic']) + + elif compiler.__class__.__name__ == 'MSVCCompiler': + for ld in libraries_dirs: + ld = ld.replace('/', '\\') + for lib in libraries: + lib_file = os.path.join(ld, f'lib{lib}.lib') + if os.path.exists(lib_file): + extra_link_args.append(lib_file) + else: + raise NotImplementedError(f'Unsupported compiler: {compiler.__class__.__name__}') diff --git a/setup_tools/support.py b/setup_tools/support.py new file mode 100644 index 000000000..2689b25a3 --- /dev/null +++ b/setup_tools/support.py @@ -0,0 +1,137 @@ +import glob +import logging +import os +import shutil +import subprocess +import tarfile +from contextlib import suppress +from io import BytesIO + + +def absolute(*paths): + from setup import COINCURVE_ROOT_DIR + + op = os.path + return op.realpath(op.abspath(op.join(COINCURVE_ROOT_DIR, *paths))) + + +def build_flags(library, type_, path): + """Return separated build flags from pkg-config output""" + + pkg_config_path = [path] + if 'PKG_CONFIG_PATH' in os.environ: + pkg_config_path.append(os.environ['PKG_CONFIG_PATH']) + if 'LIB_DIR' in os.environ: + pkg_config_path.append(os.environ['LIB_DIR']) + pkg_config_path.append(os.path.join(os.environ['LIB_DIR'], 'pkgconfig')) + + # Update PKG_CONFIG_PATH, it may be needed in later stages + new_path = str(os.pathsep).join(pkg_config_path) + os.environ['PKG_CONFIG_PATH'] = new_path + os.pathsep + os.environ.get('PKG_CONFIG_PATH', '') + + options = {'I': '--cflags-only-I', 'L': '--libs-only-L', 'l': '--libs-only-l'} + flags = call_pkg_config([options[type_]], library) + flags = list(flags.split()) + + return [flag.strip(f'-{type_}') for flag in flags] + + +def _find_lib(): + if 'COINCURVE_IGNORE_SYSTEM_LIB' in os.environ: + return False + + from cffi import FFI + + try: + options = ['--cflags-only-I'] + includes = call_pkg_config(options, 'libsecp256k1') + + return os.path.exists(os.path.join(includes[2:], 'secp256k1_ecdh.h')) + + except (OSError, subprocess.CalledProcessError): + if 'LIB_DIR' in os.environ: + for path in glob.glob(os.path.join(os.environ['LIB_DIR'], '*secp256k1*')): + with suppress(OSError): + FFI().dlopen(path) + return True + # We couldn't locate libsecp256k1, so we'll use the bundled one + return False + + +_has_system_lib = None + + +def has_system_lib(): + global _has_system_lib + if _has_system_lib is None: + _has_system_lib = _find_lib() + return _has_system_lib + + +def detect_dll(root_dir: str): + return any(fn.endswith('.dll') for fn in os.listdir(os.path.join(root_dir))) + + +def subprocess_run(cmd, *, debug=False): + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) # noqa S603 + if debug: + logging.info(f'Command log:\n{result.stderr}') + + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logging.error(f'An error occurred during the command execution: {e}') + logging.error(f'Command log:\n{e.stderr}') + raise e + + +def call_pkg_config(options, library, *, debug=False): + """Calls pkg-config with the given options and returns the output.""" + import shutil + from platform import system + + if system() == 'Windows': + options.append('--dont-define-prefix') + + pkg_config = shutil.which('pkg-config') + cmd = [pkg_config, *options, library] + + return subprocess_run(cmd, debug=debug) + + +def download_library(command): + if command.dry_run: + return + + from setup import COINCURVE_ROOT_DIR + + libdir = absolute(COINCURVE_ROOT_DIR, 'libsecp256k1') + if os.path.exists(os.path.join(libdir, 'autogen.sh')): + # Library already downloaded + return + if not os.path.exists(libdir): + import logging + + command.announce('downloading libsecp256k1 source code', level=logging.INFO) + try: + import requests + + try: + from setup import LIB_TARBALL_URL + + r = requests.get(LIB_TARBALL_URL, stream=True, timeout=10) + status_code = r.status_code + if status_code == 200: + content = BytesIO(r.raw.read()) + content.seek(0) + with tarfile.open(fileobj=content) as tf: + dirname = tf.getnames()[0].partition('/')[0] + tf.extractall() # noqa: S202 + shutil.move(dirname, libdir) + else: + raise SystemExit('Unable to download secp256k1 library: HTTP-Status: %d', status_code) + except requests.exceptions.RequestException as e: + raise SystemExit('Unable to download secp256k1 library: %s') from e + except ImportError as e: + raise SystemExit('Unable to download secp256k1 library: %s') from e From 605a79297aa13448bd05557921e74e3d90cd09fd Mon Sep 17 00:00:00 2001 From: memento Date: Mon, 18 Mar 2024 09:11:19 -0500 Subject: [PATCH 40/48] (fix) logic for pkgconfig file --- setup_tools/build_ext.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/setup_tools/build_ext.py b/setup_tools/build_ext.py index 9429aec5a..172fbf4c6 100644 --- a/setup_tools/build_ext.py +++ b/setup_tools/build_ext.py @@ -37,17 +37,26 @@ def build_extension(self, ext): # Enforce API interface ext.py_limited_api = False - # Find pkgconfig file for locally built library - pkg_dirs = self.get_finalized_command('build_clib').pkgconfig_dir # type: ignore - c_lib_pkg = next((d for d in pkg_dirs if os.path.isfile(os.path.join(d, f'{LIB_NAME}.pc'))), None) - - if not has_system_lib() and not c_lib_pkg: - raise RuntimeError( - f'pkgconfig file not found: {LIB_NAME}.pc in : {pkg_dirs}.' - f'\nSystem lib is {has_system_lib() = }. ' - 'Please check that the library was properly built.' + c_lib_pkg = None + if not has_system_lib(): + # Find pkgconfig file for locally built library + if (pkg_dirs := self.get_finalized_command('build_clib')) is None: + raise RuntimeError( + f'No C-lib command when System lib is {has_system_lib() = }. ' + 'The build system failed to trigger the build_clib.' + ) + + c_lib_pkg = next( + (d for d in pkg_dirs.pkgconfig_dir if os.path.isfile(os.path.join(d, f'{LIB_NAME}.pc'))), None ) + if c_lib_pkg is None: + raise RuntimeError( + f'pkgconfig file not found: {LIB_NAME}.pc in : {pkg_dirs}.' + f'\nSystem lib is {has_system_lib() = }. ' + 'Please check that the library was properly built.' + ) + # PKG_CONFIG_PATH is updated by build_clib if built locally, # however, it would not work for a step-by-step build, thus we specify the lib path ext.extra_compile_args.extend([f'-I{build_flags(LIB_NAME, "I", c_lib_pkg)[0]}']) From 82528eec8f71f80993c87eac58ee07cf59a94d6d Mon Sep 17 00:00:00 2001 From: memento Date: Mon, 18 Mar 2024 10:40:55 -0500 Subject: [PATCH 41/48] (fix) remove setup_support. --- setup_support.py | 161 -------------------------------------- setup_tools/build_clib.py | 4 +- setup_tools/support.py | 77 +++++++++++++++++- 3 files changed, 75 insertions(+), 167 deletions(-) delete mode 100644 setup_support.py diff --git a/setup_support.py b/setup_support.py deleted file mode 100644 index 04b37cc37..000000000 --- a/setup_support.py +++ /dev/null @@ -1,161 +0,0 @@ -import logging -import os -import subprocess - - -def absolute(*paths): - op = os.path - return op.realpath(op.abspath(op.join(op.dirname(__file__), *paths))) - - -def build_flags(library, type_, path): - """Return separated build flags from pkg-config output""" - - pkg_config_path = [path] - if 'PKG_CONFIG_PATH' in os.environ: - pkg_config_path.append(os.environ['PKG_CONFIG_PATH']) - if 'LIB_DIR' in os.environ: - pkg_config_path.append(os.environ['LIB_DIR']) - pkg_config_path.append(os.path.join(os.environ['LIB_DIR'], 'pkgconfig')) - - # Update PKG_CONFIG_PATH, it may be needed in later stages - new_path = str(os.pathsep).join(pkg_config_path) - os.environ['PKG_CONFIG_PATH'] = new_path + os.pathsep + os.environ.get('PKG_CONFIG_PATH', '') - - options = {'I': '--cflags-only-I', 'L': '--libs-only-L', 'l': '--libs-only-l'} - if os.name == 'nt': - cmd = ['pkg-config', options[type_], '--dont-define-prefix', library] - else: - cmd = ['pkg-config', options[type_], library] - flags = subprocess_run(cmd) - flags = list(flags.split()) - - return [flag.strip(f'-{type_}') for flag in flags] - - -def _find_lib(): - if os.getenv('COINCURVE_IGNORE_SYSTEM_LIB', '1') == '1': - return False - - update_pkg_config_path() - - try: - lib_dir = call_pkg_config(['--libs-only-L'], 'libsecp256k1') - return verify_system_lib(lib_dir[2:].strip()) - - except (OSError, subprocess.CalledProcessError): - from ctypes.util import find_library - - return bool(find_library('secp256k1')) - - -_has_system_lib = None - - -def has_system_lib(): - global _has_system_lib - if _has_system_lib is None: - _has_system_lib = _find_lib() - return _has_system_lib - - -def subprocess_run(cmd, *, debug=False): - try: - result = subprocess.run(cmd, capture_output=True, text=True, check=True) # noqa S603 - if debug: - logging.info(f'Command log:\n{result.stderr}') - - return result.stdout.strip() - - except subprocess.CalledProcessError as e: - logging.error(f'An error occurred during the command execution: {e}') - logging.error(f'Command log:\n{e.stderr}') - raise e - - -def call_pkg_config(options, library, *, debug=False): - """Calls pkg-config with the given options and returns the output.""" - import shutil - - from setup import SYSTEM - - if SYSTEM == 'Windows': - options.append('--dont-define-prefix') - - pkg_config = shutil.which('pkg-config') - cmd = [pkg_config, *options, library] - - return subprocess_run(cmd, debug=debug) - - -def define_secp256k1_local_lib_info(): - """ - Define the library name and the installation directory - The purpose is to automatically include the shared library in the package and - prevent inclusion the static library. This is probably hacky, but it works. - """ - from setup import LIB_NAME, PKG_NAME, SECP256K1_BUILD - - if SECP256K1_BUILD == 'SHARED': - return PKG_NAME, 'lib' - return LIB_NAME, 'x_lib' - - -def update_pkg_config_path(path='.'): - """Updates the PKG_CONFIG_PATH environment variable to include the given path.""" - pkg_config_paths = [path, os.getenv('PKG_CONFIG_PATH', '').strip('"')] - - if cpf := os.getenv('CONDA_PREFIX'): - conda_paths = [os.path.join(cpf, sbd, 'pkgconfig') for sbd in ('lib', 'lib64', os.path.join('Library', 'lib'))] - pkg_config_paths.extend([p for p in conda_paths if os.path.isdir(p)]) - - if lbd := os.getenv('LIB_DIR'): - pkg_config_paths.append(os.path.join(lbd, 'pkgconfig')) - - # Update environment - os.environ['PKG_CONFIG_PATH'] = os.pathsep.join(pkg_config_paths) - - -def verify_system_lib(lib_dir): - """Verifies that the system library is installed and of the expected type.""" - import ctypes - import platform - from ctypes.util import find_library - from pathlib import Path - - LIB_NAME = 'libsecp256k1' # noqa N806 - PKG_NAME = 'coincurve' # noqa N806 - SECP256K1_BUILD = os.getenv('COINCURVE_SECP256K1_BUILD') or 'STATIC' # noqa N806 - SYSTEM = platform.system() # noqa N806 - - def load_library(lib): - try: - return ctypes.CDLL(lib) - except OSError: - return None - - logging.warning(f'find_library: {find_library(LIB_NAME[3:])}') - lib_dir = Path(lib_dir).with_name('bin') if SYSTEM == 'Windows' else Path(lib_dir) - lib_ext = '.dll' if SYSTEM == 'Windows' else '.[sd][oy]*' - logging.warning(f'dir: {lib_dir}') - logging.warning(f'patt: *{LIB_NAME[3:]}{lib_ext}') - l_dyn = list(lib_dir.glob(f'*{LIB_NAME[3:]}*{lib_ext}')) - - # Evaluates the dynamic libraries found, - logging.warning(f'Found libraries: {l_dyn}') - dyn_lib = next((lib for lib in l_dyn if load_library(lib) is not None), False) - - found = any((dyn_lib and SECP256K1_BUILD == 'SHARED', not dyn_lib and SECP256K1_BUILD != 'SHARED')) - if not found: - logging.warning( - f'WARNING: {LIB_NAME} is installed, but it is not the expected type. ' - f'Please ensure that the {SECP256K1_BUILD} library is installed.' - ) - - if dyn_lib: - lib_base = dyn_lib.stem - # Update coincurve._secp256k1_library_info - info_file = Path(PKG_NAME, '_secp256k1_library_info.py') - info_file.write_text(f"SECP256K1_LIBRARY_NAME = '{lib_base}'\nSECP256K1_LIBRARY_TYPE = 'EXTERNAL'\n") - - return found diff --git a/setup_tools/build_clib.py b/setup_tools/build_clib.py index fa63ff810..bdebf950f 100644 --- a/setup_tools/build_clib.py +++ b/setup_tools/build_clib.py @@ -6,8 +6,8 @@ from setuptools.command import build_clib from setup import LIB_NAME, MACHINE, SECP256K1_BUILD, SYSTEM, X_HOST -from setup_support import define_secp256k1_local_lib_info -from setup_tools.support import absolute, call_pkg_config, download_library, has_system_lib +from setup_tools.support import absolute, call_pkg_config, download_library, has_system_lib, \ + define_secp256k1_local_lib_info class BuildClibWithCMake(build_clib.build_clib): diff --git a/setup_tools/support.py b/setup_tools/support.py index 2689b25a3..6f2fa856b 100644 --- a/setup_tools/support.py +++ b/setup_tools/support.py @@ -68,10 +68,6 @@ def has_system_lib(): return _has_system_lib -def detect_dll(root_dir: str): - return any(fn.endswith('.dll') for fn in os.listdir(os.path.join(root_dir))) - - def subprocess_run(cmd, *, debug=False): try: result = subprocess.run(cmd, capture_output=True, text=True, check=True) # noqa S603 @@ -135,3 +131,76 @@ def download_library(command): raise SystemExit('Unable to download secp256k1 library: %s') from e except ImportError as e: raise SystemExit('Unable to download secp256k1 library: %s') from e + + +def define_secp256k1_local_lib_info(): + """ + Define the library name and the installation directory + The purpose is to automatically include the shared library in the package and + prevent inclusion the static library. This is probably hacky, but it works. + """ + from setup import LIB_NAME, PKG_NAME, SECP256K1_BUILD + + if SECP256K1_BUILD == 'SHARED': + return PKG_NAME, 'lib' + return LIB_NAME, 'x_lib' + + +def update_pkg_config_path(path='.'): + """Updates the PKG_CONFIG_PATH environment variable to include the given path.""" + pkg_config_paths = [path, os.getenv('PKG_CONFIG_PATH', '').strip('"')] + + if cpf := os.getenv('CONDA_PREFIX'): + conda_paths = [os.path.join(cpf, sbd, 'pkgconfig') for sbd in ('lib', 'lib64', os.path.join('Library', 'lib'))] + pkg_config_paths.extend([p for p in conda_paths if os.path.isdir(p)]) + + if lbd := os.getenv('LIB_DIR'): + pkg_config_paths.append(os.path.join(lbd, 'pkgconfig')) + + # Update environment + os.environ['PKG_CONFIG_PATH'] = os.pathsep.join(pkg_config_paths) + + +def verify_system_lib(lib_dir): + """Verifies that the system library is installed and of the expected type.""" + import ctypes + import platform + from ctypes.util import find_library + from pathlib import Path + + LIB_NAME = 'libsecp256k1' # noqa N806 + PKG_NAME = 'coincurve' # noqa N806 + SECP256K1_BUILD = os.getenv('COINCURVE_SECP256K1_BUILD') or 'STATIC' # noqa N806 + SYSTEM = platform.system() # noqa N806 + + def load_library(lib): + try: + return ctypes.CDLL(lib) + except OSError: + return None + + logging.warning(f'find_library: {find_library(LIB_NAME[3:])}') + lib_dir = Path(lib_dir).with_name('bin') if SYSTEM == 'Windows' else Path(lib_dir) + lib_ext = '.dll' if SYSTEM == 'Windows' else '.[sd][oy]*' + logging.warning(f'dir: {lib_dir}') + logging.warning(f'patt: *{LIB_NAME[3:]}{lib_ext}') + l_dyn = list(lib_dir.glob(f'*{LIB_NAME[3:]}*{lib_ext}')) + + # Evaluates the dynamic libraries found, + logging.warning(f'Found libraries: {l_dyn}') + dyn_lib = next((lib for lib in l_dyn if load_library(lib) is not None), False) + + found = any((dyn_lib and SECP256K1_BUILD == 'SHARED', not dyn_lib and SECP256K1_BUILD != 'SHARED')) + if not found: + logging.warning( + f'WARNING: {LIB_NAME} is installed, but it is not the expected type. ' + f'Please ensure that the {SECP256K1_BUILD} library is installed.' + ) + + if dyn_lib: + lib_base = dyn_lib.stem + # Update coincurve._secp256k1_library_info + info_file = Path(PKG_NAME, '_secp256k1_library_info.py') + info_file.write_text(f"SECP256K1_LIBRARY_NAME = '{lib_base}'\nSECP256K1_LIBRARY_TYPE = 'EXTERNAL'\n") + + return found From 28f1fb20b4582ca4c6988fe347395488afe0fefa Mon Sep 17 00:00:00 2001 From: memento Date: Mon, 18 Mar 2024 10:55:24 -0500 Subject: [PATCH 42/48] (fix) implement update_pkg_config --- setup_tools/support.py | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/setup_tools/support.py b/setup_tools/support.py index 6f2fa856b..7181a140b 100644 --- a/setup_tools/support.py +++ b/setup_tools/support.py @@ -17,17 +17,7 @@ def absolute(*paths): def build_flags(library, type_, path): """Return separated build flags from pkg-config output""" - - pkg_config_path = [path] - if 'PKG_CONFIG_PATH' in os.environ: - pkg_config_path.append(os.environ['PKG_CONFIG_PATH']) - if 'LIB_DIR' in os.environ: - pkg_config_path.append(os.environ['LIB_DIR']) - pkg_config_path.append(os.path.join(os.environ['LIB_DIR'], 'pkgconfig')) - - # Update PKG_CONFIG_PATH, it may be needed in later stages - new_path = str(os.pathsep).join(pkg_config_path) - os.environ['PKG_CONFIG_PATH'] = new_path + os.pathsep + os.environ.get('PKG_CONFIG_PATH', '') + update_pkg_config_path(path) options = {'I': '--cflags-only-I', 'L': '--libs-only-L', 'l': '--libs-only-l'} flags = call_pkg_config([options[type_]], library) @@ -37,25 +27,19 @@ def build_flags(library, type_, path): def _find_lib(): - if 'COINCURVE_IGNORE_SYSTEM_LIB' in os.environ: + if os.getenv('COINCURVE_IGNORE_SYSTEM_LIB', '1') == '1': return False - from cffi import FFI + update_pkg_config_path() try: - options = ['--cflags-only-I'] - includes = call_pkg_config(options, 'libsecp256k1') - - return os.path.exists(os.path.join(includes[2:], 'secp256k1_ecdh.h')) + lib_dir = call_pkg_config(['--libs-only-L'], 'libsecp256k1') + return verify_system_lib(lib_dir[2:].strip()) except (OSError, subprocess.CalledProcessError): - if 'LIB_DIR' in os.environ: - for path in glob.glob(os.path.join(os.environ['LIB_DIR'], '*secp256k1*')): - with suppress(OSError): - FFI().dlopen(path) - return True - # We couldn't locate libsecp256k1, so we'll use the bundled one - return False + from ctypes.util import find_library + + return bool(find_library('secp256k1')) _has_system_lib = None From fc579dc371c12a62b3971cf9bb4a4fc31c32e87e Mon Sep 17 00:00:00 2001 From: memento Date: Mon, 18 Mar 2024 14:31:02 -0500 Subject: [PATCH 43/48] (fix) lint --- setup_tools/build_clib.py | 9 +++++++-- setup_tools/support.py | 2 -- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/setup_tools/build_clib.py b/setup_tools/build_clib.py index bdebf950f..65022e3be 100644 --- a/setup_tools/build_clib.py +++ b/setup_tools/build_clib.py @@ -6,8 +6,13 @@ from setuptools.command import build_clib from setup import LIB_NAME, MACHINE, SECP256K1_BUILD, SYSTEM, X_HOST -from setup_tools.support import absolute, call_pkg_config, download_library, has_system_lib, \ - define_secp256k1_local_lib_info +from setup_tools.support import ( + absolute, + call_pkg_config, + define_secp256k1_local_lib_info, + download_library, + has_system_lib, +) class BuildClibWithCMake(build_clib.build_clib): diff --git a/setup_tools/support.py b/setup_tools/support.py index 7181a140b..4c28eb5da 100644 --- a/setup_tools/support.py +++ b/setup_tools/support.py @@ -1,10 +1,8 @@ -import glob import logging import os import shutil import subprocess import tarfile -from contextlib import suppress from io import BytesIO From 3fe50befe3acf809e15338ba360d1f9db298cbe1 Mon Sep 17 00:00:00 2001 From: memento Date: Mon, 18 Mar 2024 15:52:24 -0500 Subject: [PATCH 44/48] (fix) wrong arch str for windows --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c1a893ea4..b6fbb3114 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ env: COINCURVE_UPSTREAM_REF COINCURVE_IGNORE_SYSTEM_LIB COINCURVE_SECP256K1_BUILD - CIBW_PROJECT_REQUIRES_PYTHON: ">=3.8" + CIBW_PROJECT_REQUIRES_PYTHON: '>=3.8' CIBW_BEFORE_ALL_MACOS: ./.github/scripts/install-macos-build-deps.sh CIBW_TEST_REQUIRES: pytest pytest-benchmark CIBW_TEST_COMMAND: > @@ -149,7 +149,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.16.5 env: - CIBW_ARCHS_WINDOWS: "AMD64" + CIBW_ARCHS_WINDOWS: 'AMD64' CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite - uses: actions/upload-artifact@v3 @@ -174,8 +174,8 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.16.5 env: - COINCURVE_CROSS_HOST: "x86" - CIBW_ARCHS_WINDOWS: "x86" + COINCURVE_CROSS_HOST: 'x86' + CIBW_ARCHS_WINDOWS: 'x86' CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite - uses: actions/upload-artifact@v3 @@ -196,8 +196,8 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.16.5 env: - COINCURVE_CROSS_HOST: "arm64" - CIBW_ARCHS_WINDOWS: arm64 + COINCURVE_CROSS_HOST: 'arm64' + CIBW_ARCHS_WINDOWS: 'ARM64' CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite - uses: actions/upload-artifact@v4 From 93fa4e2e0c7028df8580ed69a742cd75b97e0c11 Mon Sep 17 00:00:00 2001 From: memento Date: Mon, 18 Mar 2024 16:25:08 -0500 Subject: [PATCH 45/48] (fix) straggling artifact v3. include download fix --- .github/workflows/build.yml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b6fbb3114..0f7661066 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,7 +91,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: artifacts-linux-wheels-standard + name: artifact-linux-wheels-standard path: wheelhouse/*.whl if-no-files-found: error @@ -111,7 +111,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: artifacts-macos-wheels-x86_64 + name: artifact-macos-wheels-x86_64 path: wheelhouse/*.whl if-no-files-found: error @@ -127,9 +127,9 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.17 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: artifacts + name: artifact-macos-wheels-arm path: wheelhouse/*.whl if-no-files-found: error @@ -152,9 +152,9 @@ jobs: CIBW_ARCHS_WINDOWS: 'AMD64' CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: artifacts + name: artifact-windows-wheels-x86_64 path: wheelhouse/*.whl if-no-files-found: error @@ -178,9 +178,9 @@ jobs: CIBW_ARCHS_WINDOWS: 'x86' CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: artifacts + name: artifact-windows-wheels-x86 path: wheelhouse/*.whl if-no-files-found: error @@ -202,7 +202,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: artifacts-macos-wheels-arm + name: artifact-macos-wheels-arm path: wheelhouse/*.whl if-no-files-found: error @@ -223,7 +223,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: artifacts-sdist + name: artifact-sdist path: dist/* if-no-files-found: error @@ -252,7 +252,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: artifacts-linux-wheels-arm + name: artifact-linux-wheels-arm path: wheelhouse/*.whl if-no-files-found: error @@ -272,10 +272,13 @@ jobs: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: artifacts + pattern: artifact-* + merge-multiple: true path: dist + - run: ls -l dist - name: Push build artifacts to PyPI uses: pypa/gh-action-pypi-publish@v1.8.11 From 2f2ab835a215a99ddede6597c7ea1c110b88eae3 Mon Sep 17 00:00:00 2001 From: memento Date: Mon, 18 Mar 2024 16:41:10 -0500 Subject: [PATCH 46/48] (ref) straggling 2.16.5 --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f7661066..544a89a83 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -147,7 +147,7 @@ jobs: python-version: '3.12' - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.17 env: CIBW_ARCHS_WINDOWS: 'AMD64' CIBW_BEFORE_ALL: choco install -y --no-progress --no-color cmake>=3.28 pkgconfiglite @@ -172,7 +172,7 @@ jobs: python-version: '3.12' - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.17 env: COINCURVE_CROSS_HOST: 'x86' CIBW_ARCHS_WINDOWS: 'x86' @@ -194,7 +194,7 @@ jobs: - uses: actions/checkout@v4 - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.17 env: COINCURVE_CROSS_HOST: 'arm64' CIBW_ARCHS_WINDOWS: 'ARM64' From ea8ca97a887e24c495f76d9fe91145b394455c15 Mon Sep 17 00:00:00 2001 From: memento Date: Mon, 18 Mar 2024 16:44:52 -0500 Subject: [PATCH 47/48] (fix) read autocomplete before tab, AI doesn't have a clue --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 544a89a83..61b257ce9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -202,7 +202,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: artifact-macos-wheels-arm + name: artifact-windows-wheels-arm path: wheelhouse/*.whl if-no-files-found: error From c1917e1f7d75d48e2d252876a14a4a7894ed5920 Mon Sep 17 00:00:00 2001 From: memento Date: Tue, 19 Mar 2024 11:11:57 -0500 Subject: [PATCH 48/48] (fix) update_pkgconfig fails when path is None --- setup_tools/support.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup_tools/support.py b/setup_tools/support.py index 4c28eb5da..838a6a8b3 100644 --- a/setup_tools/support.py +++ b/setup_tools/support.py @@ -128,7 +128,7 @@ def define_secp256k1_local_lib_info(): return LIB_NAME, 'x_lib' -def update_pkg_config_path(path='.'): +def update_pkg_config_path(path=None): """Updates the PKG_CONFIG_PATH environment variable to include the given path.""" pkg_config_paths = [path, os.getenv('PKG_CONFIG_PATH', '').strip('"')] @@ -140,7 +140,7 @@ def update_pkg_config_path(path='.'): pkg_config_paths.append(os.path.join(lbd, 'pkgconfig')) # Update environment - os.environ['PKG_CONFIG_PATH'] = os.pathsep.join(pkg_config_paths) + os.environ['PKG_CONFIG_PATH'] = os.pathsep.join([p for p in pkg_config_paths if p is not None]) def verify_system_lib(lib_dir):