From d433bb1b4e82c146d8b548157eb833704296e750 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 23 May 2024 02:27:56 +0800 Subject: [PATCH 1/8] fix deprecated pkg_resources and support zipapp --- hpy/devel/__init__.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/hpy/devel/__init__.py b/hpy/devel/__init__.py index 3b7f6e6d..555c9ef4 100644 --- a/hpy/devel/__init__.py +++ b/hpy/devel/__init__.py @@ -203,14 +203,21 @@ def handle_hpy_ext_modules(dist, attr, hpy_ext_modules): # This file is automatically generated by hpy def __bootstrap__(): - - from sys import modules + from sys import modules, version_info from os import environ - from pkg_resources import resource_filename + from contextlib import nullcontext + from importlib import resources + from hpy.universal import _load_bootstrap - ext_filepath = resource_filename(__name__, {ext_file!r}) - m = _load_bootstrap({module_name!r}, __name__, __package__, ext_filepath, - __loader__, __spec__, environ) + + if version_info < (3, 9): + ctx = resources.path(__package__, {ext_file!r}) + else: + ctx = nullcontext(resources.files(__package__).joinpath({ext_file!r})) + + with ctx as ext_filepath: + m = _load_bootstrap({module_name!r}, __name__, __package__, str(ext_filepath), + __loader__, __spec__, environ) modules[__name__] = m __bootstrap__() From 6bf2ab38bf9aa977f96577a9db625440d3035ca1 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 23 May 2024 05:59:10 +0800 Subject: [PATCH 2/8] fix test --- hpy/devel/__init__.py | 257 ++++++++++++++++++++++++------------------ 1 file changed, 148 insertions(+), 109 deletions(-) diff --git a/hpy/devel/__init__.py b/hpy/devel/__init__.py index 555c9ef4..fdc7aae4 100644 --- a/hpy/devel/__init__.py +++ b/hpy/devel/__init__.py @@ -14,23 +14,28 @@ # don't care about setuptools version in that case. import setuptools import distutils -if (sys.version_info.major > 2 and - distutils is not getattr(setuptools, '_distutils', None)): + +if sys.version_info.major > 2 and distutils is not getattr( + setuptools, "_distutils", None +): raise Exception( "setuptools' monkey-patching of distutils did not work. " "Most likely this is caused by:\n" " - a too old setuptools. Try installing setuptools>=60.2\n" " - the env variable SETUPTOOLS_USE_DISTUTILS=stdlib. Try to unset it." - ) + ) from distutils import log from distutils.errors import DistutilsError import setuptools.command as cmd + try: import setuptools.command.build except ImportError: print( "warning: setuptools.command.build does not exist in setuptools", - setuptools.__version__, "on", sys.version + setuptools.__version__, + "on", + sys.version, ) setuptools.command.build = None import setuptools.command.build_ext @@ -38,70 +43,81 @@ from .abitag import get_hpy_ext_suffix -DEFAULT_HPY_ABI = 'universal' -if hasattr(sys, 'implementation') and sys.implementation.name == 'cpython': - DEFAULT_HPY_ABI = 'cpython' +DEFAULT_HPY_ABI = "universal" +if hasattr(sys, "implementation") and sys.implementation.name == "cpython": + DEFAULT_HPY_ABI = "cpython" class HPyDevel: - """ Extra sources for building HPy extensions with hpy.devel. """ + """Extra sources for building HPy extensions with hpy.devel.""" _DEFAULT_BASE_DIR = Path(__file__).parent def __init__(self, base_dir=_DEFAULT_BASE_DIR): self.base_dir = Path(base_dir) - self.include_dir = self.base_dir.joinpath('include') - self.src_dir = self.base_dir.joinpath('src', 'runtime') + self.include_dir = self.base_dir.joinpath("include") + self.src_dir = self.base_dir.joinpath("src", "runtime") self._available_static_libs = None def get_extra_include_dirs(self): - """ Extra include directories needed by extensions in both CPython and - Universal modes. + """Extra include directories needed by extensions in both CPython and + Universal modes. """ - return list(map(str, [ - self.include_dir, - ])) + return list( + map( + str, + [ + self.include_dir, + ], + ) + ) def get_include_dir_forbid_python_h(self): - return self.include_dir.joinpath('hpy', 'forbid_python_h') + return self.include_dir.joinpath("hpy", "forbid_python_h") def get_extra_sources(self): - """ Extra sources needed by extensions in both CPython and Universal - modes. + """Extra sources needed by extensions in both CPython and Universal + modes. """ - return list(map(str, [ - self.src_dir.joinpath('argparse.c'), - self.src_dir.joinpath('buildvalue.c'), - self.src_dir.joinpath('format.c'), - self.src_dir.joinpath('helpers.c'), - self.src_dir.joinpath('structseq.c'), - ])) + return list( + map( + str, + [ + self.src_dir.joinpath("argparse.c"), + self.src_dir.joinpath("buildvalue.c"), + self.src_dir.joinpath("format.c"), + self.src_dir.joinpath("helpers.c"), + self.src_dir.joinpath("structseq.c"), + ], + ) + ) def _scan_static_lib_dir(self): - """ Scan the static library directory and build a dict for all - available static libraries. The library directory contains - subdirectories for each ABI and the ABI folders then contain - the static libraries. + """Scan the static library directory and build a dict for all + available static libraries. The library directory contains + subdirectories for each ABI and the ABI folders then contain + the static libraries. """ available_libs = {} - lib_dir = self.base_dir.joinpath('lib') + lib_dir = self.base_dir.joinpath("lib") if lib_dir.exists(): for abi_dir in lib_dir.iterdir(): if abi_dir.is_dir(): abi = abi_dir.name # All files in '.../lib//' are considered to be static # libraries. - available_libs[abi] = \ - [str(x) for x in abi_dir.iterdir() if x.is_file()] + available_libs[abi] = [ + str(x) for x in abi_dir.iterdir() if x.is_file() + ] return available_libs def get_static_libs(self, hpy_abi): - """ The list of necessary static libraries an HPy extension needs to - link to or 'None' (if not available). The HPy ext needs to link to - all static libraries in the list otherwise some function may stay - unresolved. For example, there is library 'hpyextra' which contains - compiled HPy helper functions like 'HPyArg_Parse' and such. - Libraries are always specific to an ABI. + """The list of necessary static libraries an HPy extension needs to + link to or 'None' (if not available). The HPy ext needs to link to + all static libraries in the list otherwise some function may stay + unresolved. For example, there is library 'hpyextra' which contains + compiled HPy helper functions like 'HPyArg_Parse' and such. + Libraries are always specific to an ABI. """ if not self._available_static_libs: # lazily initialize the dict of available (=shipped) static libs @@ -109,14 +125,13 @@ def get_static_libs(self, hpy_abi): return self._available_static_libs.get(hpy_abi, None) def get_ctx_sources(self): - """ Extra sources needed only in the CPython ABI mode. - """ - return list(map(str, self.src_dir.glob('ctx_*.c'))) + """Extra sources needed only in the CPython ABI mode.""" + return list(map(str, self.src_dir.glob("ctx_*.c"))) def fix_distribution(self, dist): - """ Override build_ext to support hpy modules. + """Override build_ext to support hpy modules. - Used from both setup.py and hpy/test. + Used from both setup.py and hpy/test. """ # ============= Distribution ========== dist.hpydevel = self @@ -131,13 +146,13 @@ def has_ext_modules(self): if cmd.build is not None: build = dist.cmdclass.get("build", cmd.build.build) build_hpy = make_mixin(build, build_hpy_mixin) - dist.cmdclass['build'] = build_hpy + dist.cmdclass["build"] = build_hpy # ============= build_ext ========== build_ext = dist.cmdclass.get("build_ext", cmd.build_ext.build_ext) self.build_ext_sanity_check(build_ext) build_ext_hpy = make_mixin(build_ext, build_ext_hpy_mixin) - dist.cmdclass['build_ext'] = build_ext_hpy + dist.cmdclass["build_ext"] = build_ext_hpy # ============= bdist_egg ========== @monkeypatch(setuptools.command.bdist_egg) @@ -148,7 +163,7 @@ def write_stub(resource, pyfile): build_ext_hpy_mixin.write_stub. """ ext_suffix = None - if dist.hpy_abi != 'cpython': + if dist.hpy_abi != "cpython": ext_suffix = get_hpy_ext_suffix(dist.hpy_abi) # if ext_suffix and resource.endswith(ext_suffix): @@ -159,7 +174,7 @@ def write_stub(resource, pyfile): def build_ext_sanity_check(self, build_ext): # check that the supplied build_ext inherits from setuptools if isinstance(build_ext, type): - assert ('setuptools.command.build_ext', 'build_ext') in [ + assert ("setuptools.command.build_ext", "build_ext") in [ (c.__module__, c.__name__) for c in build_ext.__mro__ ], ( "dist.cmdclass['build_ext'] does not inherit from" @@ -171,28 +186,31 @@ def build_ext_sanity_check(self, build_ext): def handle_hpy_ext_modules(dist, attr, hpy_ext_modules): - """ Distuils hpy_ext_module setup(...) argument and --hpy-abi option. + """Distuils hpy_ext_module setup(...) argument and --hpy-abi option. - See hpy's setup.py where this function is registered as an entry - point. + See hpy's setup.py where this function is registered as an entry + point. """ - assert attr == 'hpy_ext_modules' + assert attr == "hpy_ext_modules" # It can happen that this hook will be called multiple times depending on # which command was used. So, skip patching if we already patched the # distribution. - if getattr(dist, 'hpydevel', None): + if getattr(dist, "hpydevel", None): return # add a global option --hpy-abi to setup.py dist.__class__.hpy_abi = DEFAULT_HPY_ABI dist.__class__.hpy_use_static_libs = False dist.__class__.global_options += [ - ('hpy-abi=', None, 'Specify the HPy ABI mode (default: %s)' - % DEFAULT_HPY_ABI), - ('hpy-use-static-libs', None, 'Use static library containing context ' - 'and helper functions for building ' - 'extensions (default: False)') + ("hpy-abi=", None, "Specify the HPy ABI mode (default: %s)" % DEFAULT_HPY_ABI), + ( + "hpy-use-static-libs", + None, + "Use static library containing context " + "and helper functions for building " + "extensions (default: False)", + ), ] hpydevel = HPyDevel() hpydevel.fix_distribution(dist) @@ -204,16 +222,21 @@ def handle_hpy_ext_modules(dist, attr, hpy_ext_modules): def __bootstrap__(): from sys import modules, version_info - from os import environ + from os import environ, path from contextlib import nullcontext from importlib import resources from hpy.universal import _load_bootstrap - if version_info < (3, 9): - ctx = resources.path(__package__, {ext_file!r}) + ext_name = {ext_file!r} + + if not __package__: + # directly called with python -m + ctx = nullcontext(path.join(path.dirname(__file__), ext_name)) + elif version_info < (3, 9): + ctx = resources.path(__package__, ext_name) else: - ctx = nullcontext(resources.files(__package__).joinpath({ext_file!r})) + ctx = nullcontext(resources.files(__package__).joinpath(ext_name)) with ctx as ext_filepath: m = _load_bootstrap({module_name!r}, __name__, __package__, str(ext_filepath), @@ -225,15 +248,15 @@ def __bootstrap__(): class HPyExtensionName(str): - """ Wrapper around str to allow HPy extension modules to be identified. + """Wrapper around str to allow HPy extension modules to be identified. - The following build_ext command methods are passed only the *name* - of the extension and not the full extension object. The - build_ext_hpy_mixin class needs to detect when HPy are extensions - passed to these methods and override the default behaviour. + The following build_ext command methods are passed only the *name* + of the extension and not the full extension object. The + build_ext_hpy_mixin class needs to detect when HPy are extensions + passed to these methods and override the default behaviour. - This str sub-class allows HPy extensions to be detected, while - still allowing the extension name to be used as an ordinary string. + This str sub-class allows HPy extensions to be detected, while + still allowing the extension name to be used as an ordinary string. """ def split(self, *args, **kw): @@ -246,14 +269,15 @@ def translate(self, *args, **kw): def is_hpy_extension(ext_name): - """ Return True if the extension name is for an HPy extension. """ + """Return True if the extension name is for an HPy extension.""" return isinstance(ext_name, HPyExtensionName) def remember_hpy_extension(f): - """ Decorator for remembering whether an extension name belongs to an - HPy extension. + """Decorator for remembering whether an extension name belongs to an + HPy extension. """ + @functools.wraps(f) def wrapper(self, ext_name): if self._only_hpy_extensions: @@ -268,6 +292,7 @@ def wrapper(self, ext_name): if is_hpy_extension(ext_name): result = HPyExtensionName(result) return result + return wrapper @@ -275,43 +300,49 @@ def wrapper(self, ext_name): # Augmented setuptools commands and monkeypatching # ================================================== + def make_mixin(base, mixin): """ Create a new class which inherits from both mixin and base, so that the methods of mixin effectively override the ones of base """ + class NewClass(mixin, base, object): _mixin_super = base - NewClass.__name__ = base.__name__ + '_hpy' + + NewClass.__name__ = base.__name__ + "_hpy" return NewClass + def monkeypatch(target): """ Decorator to monkey patch a function in a module. The original function will be available as new_function.super() """ + def decorator(fn): name = fn.__name__ fn.super = getattr(target, name) setattr(target, name, fn) return fn + return decorator class build_hpy_mixin: - """ A mixin class to override setuptools.commands.build """ + """A mixin class to override setuptools.commands.build""" def finalize_options(self): self._mixin_super.finalize_options(self) - if self.distribution.hpy_abi != 'cpython': - suffix = '-hpy-%s' % self.distribution.hpy_abi + if self.distribution.hpy_abi != "cpython": + suffix = "-hpy-%s" % self.distribution.hpy_abi self.build_platlib += suffix self.build_lib += suffix self.build_temp += suffix class build_ext_hpy_mixin: - """ A mixin class to override setuptools.commands.build_ext """ + """A mixin class to override setuptools.commands.build_ext""" # Ideally we would have simply added the HPy extensions to .extensions # at the end of .initialize_options() but the setuptools build_ext @@ -347,9 +378,10 @@ def _finalize_hpy_ext(self, ext): if static_libs: static_libs = self.hpydevel.get_static_libs(ext.hpy_abi) if static_libs is None or len(static_libs) != 1: - raise DistutilsError('Expected exactly one static library for ' - 'ABI "%s" but got: %r' % - (ext.hpy_abi, static_libs)) + raise DistutilsError( + "Expected exactly one static library for " + 'ABI "%s" but got: %r' % (ext.hpy_abi, static_libs) + ) if static_libs: ext.extra_objects += static_libs @@ -358,25 +390,27 @@ def _finalize_hpy_ext(self, ext): # not available, we just add the sources of the helpers to the # extension. They are then compiler with the extension. ext.sources += self.hpydevel.get_extra_sources() - ext.define_macros.append(('HPY', None)) - if ext.hpy_abi == 'cpython': + ext.define_macros.append(("HPY", None)) + if ext.hpy_abi == "cpython": # If the user disabled using static libs, we need to add the # context sources in this case. if not static_libs: ext.sources += self.hpydevel.get_ctx_sources() - ext.define_macros.append(('HPY_ABI_CPYTHON', None)) + ext.define_macros.append(("HPY_ABI_CPYTHON", None)) ext._hpy_needs_stub = False - elif ext.hpy_abi == 'hybrid': - ext.define_macros.append(('HPY_ABI_HYBRID', None)) + elif ext.hpy_abi == "hybrid": + ext.define_macros.append(("HPY_ABI_HYBRID", None)) ext._hpy_needs_stub = True - elif ext.hpy_abi == 'universal': - ext.define_macros.append(('HPY_ABI_UNIVERSAL', None)) + elif ext.hpy_abi == "universal": + ext.define_macros.append(("HPY_ABI_UNIVERSAL", None)) ext._hpy_needs_stub = True forbid_python_h = self.hpydevel.get_include_dir_forbid_python_h() ext.include_dirs.insert(0, forbid_python_h) else: - raise DistutilsError('Unknown HPy ABI: %s. Valid values are: ' - 'cpython, hybrid, universal' % ext.hpy_abi) + raise DistutilsError( + "Unknown HPy ABI: %s. Valid values are: " + "cpython, hybrid, universal" % ext.hpy_abi + ) def finalize_options(self): self._extensions = self.distribution.ext_modules or [] @@ -403,22 +437,23 @@ def get_ext_fullpath(self, ext_name): @remember_hpy_extension def get_ext_filename(self, ext_name): hpy_abi = self.distribution.hpy_abi - if not is_hpy_extension(ext_name) or hpy_abi == 'cpython': + if not is_hpy_extension(ext_name) or hpy_abi == "cpython": return self._mixin_super.get_ext_filename(self, ext_name) else: assert is_hpy_extension(ext_name) - assert hpy_abi in ('universal', 'hybrid') - ext_path = ext_name.split('.') + assert hpy_abi in ("universal", "hybrid") + ext_path = ext_name.split(".") ext_suffix = get_hpy_ext_suffix(hpy_abi) ext_filename = os.path.join(*ext_path) + ext_suffix return ext_filename def write_stub(self, output_dir, ext, compile=False): - if (not hasattr(ext, "hpy_abi") or - self.distribution.hpy_abi not in ('universal', 'hybrid')): - return self._mixin_super.write_stub( - self, output_dir, ext, compile=compile) - pkgs = ext._full_name.split('.') + if not hasattr(ext, "hpy_abi") or self.distribution.hpy_abi not in ( + "universal", + "hybrid", + ): + return self._mixin_super.write_stub(self, output_dir, ext, compile=compile) + pkgs = ext._full_name.split(".") if compile: # compile is true when .write_stub is called while copying # extensions to the source folder as part of build_ext --inplace. @@ -427,22 +462,24 @@ def write_stub(self, output_dir, ext, compile=False): # output_dir does not include those folders (and is just the # build_lib folder). pkgs = [pkgs[-1]] - stub_file = os.path.join(output_dir, *pkgs) + '.py' + stub_file = os.path.join(output_dir, *pkgs) + ".py" log.info( - "writing hpy universal stub loader for %s to %s", - ext._full_name, stub_file) + "writing hpy universal stub loader for %s to %s", ext._full_name, stub_file + ) ext_file = os.path.basename(ext._file_name) module_name = ext_file.split(".")[0] if not self.dry_run: - with open(stub_file, 'w', encoding='utf-8') as f: - f.write(_HPY_UNIVERSAL_MODULE_STUB_TEMPLATE.format( - ext_file=ext_file, module_name=module_name) + with open(stub_file, "w", encoding="utf-8") as f: + f.write( + _HPY_UNIVERSAL_MODULE_STUB_TEMPLATE.format( + ext_file=ext_file, module_name=module_name + ) ) def copy_extensions_to_source(self): """Override from setuptools 64.0.0 to copy our stub instead of recreating it.""" - build_py = self.get_finalized_command('build_py') + build_py = self.get_finalized_command("build_py") build_lib = build_py.build_lib for ext in self.extensions: inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext) @@ -454,18 +491,20 @@ def copy_extensions_to_source(self): self.copy_file(regular_file, inplace_file, level=self.verbose) if ext._needs_stub: - source_stub = os.path.join(build_lib, *ext._full_name.split('.')) + '.py' + source_stub = ( + os.path.join(build_lib, *ext._full_name.split(".")) + ".py" + ) inplace_stub = self._get_equivalent_stub(ext, inplace_file) self.copy_file(source_stub, inplace_stub, level=self.verbose) def get_export_symbols(self, ext): - """ Override .get_export_symbols to replace "PyInit_" - with "HPyInit_. + """Override .get_export_symbols to replace "PyInit_" + with "HPyInit_. - Only relevant on Windows, where the .pyd file (DLL) must export the - module "HPyInit_" function. + Only relevant on Windows, where the .pyd file (DLL) must export the + module "HPyInit_" function. """ exports = self._mixin_super.get_export_symbols(self, ext) - if hasattr(ext, "hpy_abi") and ext.hpy_abi in ('universal', 'hybrid'): + if hasattr(ext, "hpy_abi") and ext.hpy_abi in ("universal", "hybrid"): exports = [re.sub(r"^PyInit_", "HPyInit_", name) for name in exports] return exports From 90e7191aa1e4eedd68885dad8ef3b92b413846bd Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 23 May 2024 06:04:49 +0800 Subject: [PATCH 3/8] revert style change --- hpy/devel/__init__.py | 268 +++++++++++++++++------------------------- 1 file changed, 111 insertions(+), 157 deletions(-) diff --git a/hpy/devel/__init__.py b/hpy/devel/__init__.py index fdc7aae4..3b7f6e6d 100644 --- a/hpy/devel/__init__.py +++ b/hpy/devel/__init__.py @@ -14,28 +14,23 @@ # don't care about setuptools version in that case. import setuptools import distutils - -if sys.version_info.major > 2 and distutils is not getattr( - setuptools, "_distutils", None -): +if (sys.version_info.major > 2 and + distutils is not getattr(setuptools, '_distutils', None)): raise Exception( "setuptools' monkey-patching of distutils did not work. " "Most likely this is caused by:\n" " - a too old setuptools. Try installing setuptools>=60.2\n" " - the env variable SETUPTOOLS_USE_DISTUTILS=stdlib. Try to unset it." - ) + ) from distutils import log from distutils.errors import DistutilsError import setuptools.command as cmd - try: import setuptools.command.build except ImportError: print( "warning: setuptools.command.build does not exist in setuptools", - setuptools.__version__, - "on", - sys.version, + setuptools.__version__, "on", sys.version ) setuptools.command.build = None import setuptools.command.build_ext @@ -43,81 +38,70 @@ from .abitag import get_hpy_ext_suffix -DEFAULT_HPY_ABI = "universal" -if hasattr(sys, "implementation") and sys.implementation.name == "cpython": - DEFAULT_HPY_ABI = "cpython" +DEFAULT_HPY_ABI = 'universal' +if hasattr(sys, 'implementation') and sys.implementation.name == 'cpython': + DEFAULT_HPY_ABI = 'cpython' class HPyDevel: - """Extra sources for building HPy extensions with hpy.devel.""" + """ Extra sources for building HPy extensions with hpy.devel. """ _DEFAULT_BASE_DIR = Path(__file__).parent def __init__(self, base_dir=_DEFAULT_BASE_DIR): self.base_dir = Path(base_dir) - self.include_dir = self.base_dir.joinpath("include") - self.src_dir = self.base_dir.joinpath("src", "runtime") + self.include_dir = self.base_dir.joinpath('include') + self.src_dir = self.base_dir.joinpath('src', 'runtime') self._available_static_libs = None def get_extra_include_dirs(self): - """Extra include directories needed by extensions in both CPython and - Universal modes. + """ Extra include directories needed by extensions in both CPython and + Universal modes. """ - return list( - map( - str, - [ - self.include_dir, - ], - ) - ) + return list(map(str, [ + self.include_dir, + ])) def get_include_dir_forbid_python_h(self): - return self.include_dir.joinpath("hpy", "forbid_python_h") + return self.include_dir.joinpath('hpy', 'forbid_python_h') def get_extra_sources(self): - """Extra sources needed by extensions in both CPython and Universal - modes. + """ Extra sources needed by extensions in both CPython and Universal + modes. """ - return list( - map( - str, - [ - self.src_dir.joinpath("argparse.c"), - self.src_dir.joinpath("buildvalue.c"), - self.src_dir.joinpath("format.c"), - self.src_dir.joinpath("helpers.c"), - self.src_dir.joinpath("structseq.c"), - ], - ) - ) + return list(map(str, [ + self.src_dir.joinpath('argparse.c'), + self.src_dir.joinpath('buildvalue.c'), + self.src_dir.joinpath('format.c'), + self.src_dir.joinpath('helpers.c'), + self.src_dir.joinpath('structseq.c'), + ])) def _scan_static_lib_dir(self): - """Scan the static library directory and build a dict for all - available static libraries. The library directory contains - subdirectories for each ABI and the ABI folders then contain - the static libraries. + """ Scan the static library directory and build a dict for all + available static libraries. The library directory contains + subdirectories for each ABI and the ABI folders then contain + the static libraries. """ available_libs = {} - lib_dir = self.base_dir.joinpath("lib") + lib_dir = self.base_dir.joinpath('lib') if lib_dir.exists(): for abi_dir in lib_dir.iterdir(): if abi_dir.is_dir(): abi = abi_dir.name # All files in '.../lib//' are considered to be static # libraries. - available_libs[abi] = [ - str(x) for x in abi_dir.iterdir() if x.is_file() - ] + available_libs[abi] = \ + [str(x) for x in abi_dir.iterdir() if x.is_file()] return available_libs def get_static_libs(self, hpy_abi): - """The list of necessary static libraries an HPy extension needs to - link to or 'None' (if not available). The HPy ext needs to link to - all static libraries in the list otherwise some function may stay - unresolved. For example, there is library 'hpyextra' which contains - compiled HPy helper functions like 'HPyArg_Parse' and such. - Libraries are always specific to an ABI. + """ The list of necessary static libraries an HPy extension needs to + link to or 'None' (if not available). The HPy ext needs to link to + all static libraries in the list otherwise some function may stay + unresolved. For example, there is library 'hpyextra' which contains + compiled HPy helper functions like 'HPyArg_Parse' and such. + Libraries are always specific to an ABI. """ if not self._available_static_libs: # lazily initialize the dict of available (=shipped) static libs @@ -125,13 +109,14 @@ def get_static_libs(self, hpy_abi): return self._available_static_libs.get(hpy_abi, None) def get_ctx_sources(self): - """Extra sources needed only in the CPython ABI mode.""" - return list(map(str, self.src_dir.glob("ctx_*.c"))) + """ Extra sources needed only in the CPython ABI mode. + """ + return list(map(str, self.src_dir.glob('ctx_*.c'))) def fix_distribution(self, dist): - """Override build_ext to support hpy modules. + """ Override build_ext to support hpy modules. - Used from both setup.py and hpy/test. + Used from both setup.py and hpy/test. """ # ============= Distribution ========== dist.hpydevel = self @@ -146,13 +131,13 @@ def has_ext_modules(self): if cmd.build is not None: build = dist.cmdclass.get("build", cmd.build.build) build_hpy = make_mixin(build, build_hpy_mixin) - dist.cmdclass["build"] = build_hpy + dist.cmdclass['build'] = build_hpy # ============= build_ext ========== build_ext = dist.cmdclass.get("build_ext", cmd.build_ext.build_ext) self.build_ext_sanity_check(build_ext) build_ext_hpy = make_mixin(build_ext, build_ext_hpy_mixin) - dist.cmdclass["build_ext"] = build_ext_hpy + dist.cmdclass['build_ext'] = build_ext_hpy # ============= bdist_egg ========== @monkeypatch(setuptools.command.bdist_egg) @@ -163,7 +148,7 @@ def write_stub(resource, pyfile): build_ext_hpy_mixin.write_stub. """ ext_suffix = None - if dist.hpy_abi != "cpython": + if dist.hpy_abi != 'cpython': ext_suffix = get_hpy_ext_suffix(dist.hpy_abi) # if ext_suffix and resource.endswith(ext_suffix): @@ -174,7 +159,7 @@ def write_stub(resource, pyfile): def build_ext_sanity_check(self, build_ext): # check that the supplied build_ext inherits from setuptools if isinstance(build_ext, type): - assert ("setuptools.command.build_ext", "build_ext") in [ + assert ('setuptools.command.build_ext', 'build_ext') in [ (c.__module__, c.__name__) for c in build_ext.__mro__ ], ( "dist.cmdclass['build_ext'] does not inherit from" @@ -186,31 +171,28 @@ def build_ext_sanity_check(self, build_ext): def handle_hpy_ext_modules(dist, attr, hpy_ext_modules): - """Distuils hpy_ext_module setup(...) argument and --hpy-abi option. + """ Distuils hpy_ext_module setup(...) argument and --hpy-abi option. - See hpy's setup.py where this function is registered as an entry - point. + See hpy's setup.py where this function is registered as an entry + point. """ - assert attr == "hpy_ext_modules" + assert attr == 'hpy_ext_modules' # It can happen that this hook will be called multiple times depending on # which command was used. So, skip patching if we already patched the # distribution. - if getattr(dist, "hpydevel", None): + if getattr(dist, 'hpydevel', None): return # add a global option --hpy-abi to setup.py dist.__class__.hpy_abi = DEFAULT_HPY_ABI dist.__class__.hpy_use_static_libs = False dist.__class__.global_options += [ - ("hpy-abi=", None, "Specify the HPy ABI mode (default: %s)" % DEFAULT_HPY_ABI), - ( - "hpy-use-static-libs", - None, - "Use static library containing context " - "and helper functions for building " - "extensions (default: False)", - ), + ('hpy-abi=', None, 'Specify the HPy ABI mode (default: %s)' + % DEFAULT_HPY_ABI), + ('hpy-use-static-libs', None, 'Use static library containing context ' + 'and helper functions for building ' + 'extensions (default: False)') ] hpydevel = HPyDevel() hpydevel.fix_distribution(dist) @@ -221,26 +203,14 @@ def handle_hpy_ext_modules(dist, attr, hpy_ext_modules): # This file is automatically generated by hpy def __bootstrap__(): - from sys import modules, version_info - from os import environ, path - from contextlib import nullcontext - from importlib import resources + from sys import modules + from os import environ + from pkg_resources import resource_filename from hpy.universal import _load_bootstrap - - ext_name = {ext_file!r} - - if not __package__: - # directly called with python -m - ctx = nullcontext(path.join(path.dirname(__file__), ext_name)) - elif version_info < (3, 9): - ctx = resources.path(__package__, ext_name) - else: - ctx = nullcontext(resources.files(__package__).joinpath(ext_name)) - - with ctx as ext_filepath: - m = _load_bootstrap({module_name!r}, __name__, __package__, str(ext_filepath), - __loader__, __spec__, environ) + ext_filepath = resource_filename(__name__, {ext_file!r}) + m = _load_bootstrap({module_name!r}, __name__, __package__, ext_filepath, + __loader__, __spec__, environ) modules[__name__] = m __bootstrap__() @@ -248,15 +218,15 @@ def __bootstrap__(): class HPyExtensionName(str): - """Wrapper around str to allow HPy extension modules to be identified. + """ Wrapper around str to allow HPy extension modules to be identified. - The following build_ext command methods are passed only the *name* - of the extension and not the full extension object. The - build_ext_hpy_mixin class needs to detect when HPy are extensions - passed to these methods and override the default behaviour. + The following build_ext command methods are passed only the *name* + of the extension and not the full extension object. The + build_ext_hpy_mixin class needs to detect when HPy are extensions + passed to these methods and override the default behaviour. - This str sub-class allows HPy extensions to be detected, while - still allowing the extension name to be used as an ordinary string. + This str sub-class allows HPy extensions to be detected, while + still allowing the extension name to be used as an ordinary string. """ def split(self, *args, **kw): @@ -269,15 +239,14 @@ def translate(self, *args, **kw): def is_hpy_extension(ext_name): - """Return True if the extension name is for an HPy extension.""" + """ Return True if the extension name is for an HPy extension. """ return isinstance(ext_name, HPyExtensionName) def remember_hpy_extension(f): - """Decorator for remembering whether an extension name belongs to an - HPy extension. + """ Decorator for remembering whether an extension name belongs to an + HPy extension. """ - @functools.wraps(f) def wrapper(self, ext_name): if self._only_hpy_extensions: @@ -292,7 +261,6 @@ def wrapper(self, ext_name): if is_hpy_extension(ext_name): result = HPyExtensionName(result) return result - return wrapper @@ -300,49 +268,43 @@ def wrapper(self, ext_name): # Augmented setuptools commands and monkeypatching # ================================================== - def make_mixin(base, mixin): """ Create a new class which inherits from both mixin and base, so that the methods of mixin effectively override the ones of base """ - class NewClass(mixin, base, object): _mixin_super = base - - NewClass.__name__ = base.__name__ + "_hpy" + NewClass.__name__ = base.__name__ + '_hpy' return NewClass - def monkeypatch(target): """ Decorator to monkey patch a function in a module. The original function will be available as new_function.super() """ - def decorator(fn): name = fn.__name__ fn.super = getattr(target, name) setattr(target, name, fn) return fn - return decorator class build_hpy_mixin: - """A mixin class to override setuptools.commands.build""" + """ A mixin class to override setuptools.commands.build """ def finalize_options(self): self._mixin_super.finalize_options(self) - if self.distribution.hpy_abi != "cpython": - suffix = "-hpy-%s" % self.distribution.hpy_abi + if self.distribution.hpy_abi != 'cpython': + suffix = '-hpy-%s' % self.distribution.hpy_abi self.build_platlib += suffix self.build_lib += suffix self.build_temp += suffix class build_ext_hpy_mixin: - """A mixin class to override setuptools.commands.build_ext""" + """ A mixin class to override setuptools.commands.build_ext """ # Ideally we would have simply added the HPy extensions to .extensions # at the end of .initialize_options() but the setuptools build_ext @@ -378,10 +340,9 @@ def _finalize_hpy_ext(self, ext): if static_libs: static_libs = self.hpydevel.get_static_libs(ext.hpy_abi) if static_libs is None or len(static_libs) != 1: - raise DistutilsError( - "Expected exactly one static library for " - 'ABI "%s" but got: %r' % (ext.hpy_abi, static_libs) - ) + raise DistutilsError('Expected exactly one static library for ' + 'ABI "%s" but got: %r' % + (ext.hpy_abi, static_libs)) if static_libs: ext.extra_objects += static_libs @@ -390,27 +351,25 @@ def _finalize_hpy_ext(self, ext): # not available, we just add the sources of the helpers to the # extension. They are then compiler with the extension. ext.sources += self.hpydevel.get_extra_sources() - ext.define_macros.append(("HPY", None)) - if ext.hpy_abi == "cpython": + ext.define_macros.append(('HPY', None)) + if ext.hpy_abi == 'cpython': # If the user disabled using static libs, we need to add the # context sources in this case. if not static_libs: ext.sources += self.hpydevel.get_ctx_sources() - ext.define_macros.append(("HPY_ABI_CPYTHON", None)) + ext.define_macros.append(('HPY_ABI_CPYTHON', None)) ext._hpy_needs_stub = False - elif ext.hpy_abi == "hybrid": - ext.define_macros.append(("HPY_ABI_HYBRID", None)) + elif ext.hpy_abi == 'hybrid': + ext.define_macros.append(('HPY_ABI_HYBRID', None)) ext._hpy_needs_stub = True - elif ext.hpy_abi == "universal": - ext.define_macros.append(("HPY_ABI_UNIVERSAL", None)) + elif ext.hpy_abi == 'universal': + ext.define_macros.append(('HPY_ABI_UNIVERSAL', None)) ext._hpy_needs_stub = True forbid_python_h = self.hpydevel.get_include_dir_forbid_python_h() ext.include_dirs.insert(0, forbid_python_h) else: - raise DistutilsError( - "Unknown HPy ABI: %s. Valid values are: " - "cpython, hybrid, universal" % ext.hpy_abi - ) + raise DistutilsError('Unknown HPy ABI: %s. Valid values are: ' + 'cpython, hybrid, universal' % ext.hpy_abi) def finalize_options(self): self._extensions = self.distribution.ext_modules or [] @@ -437,23 +396,22 @@ def get_ext_fullpath(self, ext_name): @remember_hpy_extension def get_ext_filename(self, ext_name): hpy_abi = self.distribution.hpy_abi - if not is_hpy_extension(ext_name) or hpy_abi == "cpython": + if not is_hpy_extension(ext_name) or hpy_abi == 'cpython': return self._mixin_super.get_ext_filename(self, ext_name) else: assert is_hpy_extension(ext_name) - assert hpy_abi in ("universal", "hybrid") - ext_path = ext_name.split(".") + assert hpy_abi in ('universal', 'hybrid') + ext_path = ext_name.split('.') ext_suffix = get_hpy_ext_suffix(hpy_abi) ext_filename = os.path.join(*ext_path) + ext_suffix return ext_filename def write_stub(self, output_dir, ext, compile=False): - if not hasattr(ext, "hpy_abi") or self.distribution.hpy_abi not in ( - "universal", - "hybrid", - ): - return self._mixin_super.write_stub(self, output_dir, ext, compile=compile) - pkgs = ext._full_name.split(".") + if (not hasattr(ext, "hpy_abi") or + self.distribution.hpy_abi not in ('universal', 'hybrid')): + return self._mixin_super.write_stub( + self, output_dir, ext, compile=compile) + pkgs = ext._full_name.split('.') if compile: # compile is true when .write_stub is called while copying # extensions to the source folder as part of build_ext --inplace. @@ -462,24 +420,22 @@ def write_stub(self, output_dir, ext, compile=False): # output_dir does not include those folders (and is just the # build_lib folder). pkgs = [pkgs[-1]] - stub_file = os.path.join(output_dir, *pkgs) + ".py" + stub_file = os.path.join(output_dir, *pkgs) + '.py' log.info( - "writing hpy universal stub loader for %s to %s", ext._full_name, stub_file - ) + "writing hpy universal stub loader for %s to %s", + ext._full_name, stub_file) ext_file = os.path.basename(ext._file_name) module_name = ext_file.split(".")[0] if not self.dry_run: - with open(stub_file, "w", encoding="utf-8") as f: - f.write( - _HPY_UNIVERSAL_MODULE_STUB_TEMPLATE.format( - ext_file=ext_file, module_name=module_name - ) + with open(stub_file, 'w', encoding='utf-8') as f: + f.write(_HPY_UNIVERSAL_MODULE_STUB_TEMPLATE.format( + ext_file=ext_file, module_name=module_name) ) def copy_extensions_to_source(self): """Override from setuptools 64.0.0 to copy our stub instead of recreating it.""" - build_py = self.get_finalized_command("build_py") + build_py = self.get_finalized_command('build_py') build_lib = build_py.build_lib for ext in self.extensions: inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext) @@ -491,20 +447,18 @@ def copy_extensions_to_source(self): self.copy_file(regular_file, inplace_file, level=self.verbose) if ext._needs_stub: - source_stub = ( - os.path.join(build_lib, *ext._full_name.split(".")) + ".py" - ) + source_stub = os.path.join(build_lib, *ext._full_name.split('.')) + '.py' inplace_stub = self._get_equivalent_stub(ext, inplace_file) self.copy_file(source_stub, inplace_stub, level=self.verbose) def get_export_symbols(self, ext): - """Override .get_export_symbols to replace "PyInit_" - with "HPyInit_. + """ Override .get_export_symbols to replace "PyInit_" + with "HPyInit_. - Only relevant on Windows, where the .pyd file (DLL) must export the - module "HPyInit_" function. + Only relevant on Windows, where the .pyd file (DLL) must export the + module "HPyInit_" function. """ exports = self._mixin_super.get_export_symbols(self, ext) - if hasattr(ext, "hpy_abi") and ext.hpy_abi in ("universal", "hybrid"): + if hasattr(ext, "hpy_abi") and ext.hpy_abi in ('universal', 'hybrid'): exports = [re.sub(r"^PyInit_", "HPyInit_", name) for name in exports] return exports From bfb66440a678fc1051f4072a0b7df251cd26a195 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 23 May 2024 06:05:36 +0800 Subject: [PATCH 4/8] revert style change --- hpy/devel/__init__.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/hpy/devel/__init__.py b/hpy/devel/__init__.py index 3b7f6e6d..cd00c8ab 100644 --- a/hpy/devel/__init__.py +++ b/hpy/devel/__init__.py @@ -201,16 +201,23 @@ def handle_hpy_ext_modules(dist, attr, hpy_ext_modules): _HPY_UNIVERSAL_MODULE_STUB_TEMPLATE = """ # DO NOT EDIT THIS FILE! # This file is automatically generated by hpy - def __bootstrap__(): - - from sys import modules - from os import environ - from pkg_resources import resource_filename + from sys import modules, version_info + from os import environ, path + from contextlib import nullcontext + from importlib import resources from hpy.universal import _load_bootstrap - ext_filepath = resource_filename(__name__, {ext_file!r}) - m = _load_bootstrap({module_name!r}, __name__, __package__, ext_filepath, - __loader__, __spec__, environ) + ext_name = {ext_file!r} + if not __package__: + # directly called with python -m + ctx = nullcontext(path.join(path.dirname(__file__), ext_name)) + elif version_info < (3, 9): + ctx = resources.path(__package__, ext_name) + else: + ctx = nullcontext(resources.files(__package__).joinpath(ext_name)) + with ctx as ext_filepath: + m = _load_bootstrap({module_name!r}, __name__, __package__, str(ext_filepath), + __loader__, __spec__, environ) modules[__name__] = m __bootstrap__() From 3edf4ff2965f234aa678ed5cf83b1f9cfc331a51 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Sun, 2 Jun 2024 05:33:22 +0800 Subject: [PATCH 5/8] try fix bug --- hpy/devel/__init__.py | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/hpy/devel/__init__.py b/hpy/devel/__init__.py index cd00c8ab..57ae1c06 100644 --- a/hpy/devel/__init__.py +++ b/hpy/devel/__init__.py @@ -203,23 +203,41 @@ def handle_hpy_ext_modules(dist, attr, hpy_ext_modules): # This file is automatically generated by hpy def __bootstrap__(): from sys import modules, version_info - from os import environ, path - from contextlib import nullcontext - from importlib import resources + from os import environ from hpy.universal import _load_bootstrap + ext_name = {ext_file!r} - if not __package__: - # directly called with python -m - ctx = nullcontext(path.join(path.dirname(__file__), ext_name)) - elif version_info < (3, 9): - ctx = resources.path(__package__, ext_name) + module_name = {module_name!r} + + if version_info < (3, 10): + # there is a bug in importlib.resources that doesn't handle namespace package + # and also pkg_resources is available, so just use pkg_resources. + from pkg_resources import resource_filename + + ext_filepath = resource_filename(__name__, ext_name) + else: - ctx = nullcontext(resources.files(__package__).joinpath(ext_name)) - with ctx as ext_filepath: - m = _load_bootstrap({module_name!r}, __name__, __package__, str(ext_filepath), - __loader__, __spec__, environ) + from os import path + from importlib import resources + + if not __package__: + # directly called with python -m + ext_filepath = path.join(path.dirname(__file__), ext_name) + else: + ext_filepath = resources.files(__package__).joinpath(ext_name) + + m = _load_bootstrap( + module_name, + __name__, + __package__, + str(ext_filepath), + __loader__, + __spec__, + environ, + ) modules[__name__] = m + __bootstrap__() """ From 130f4f5a4ef929af3fd67457d67a666bb08af46c Mon Sep 17 00:00:00 2001 From: Trim21 Date: Mon, 3 Jun 2024 17:15:26 +0800 Subject: [PATCH 6/8] use backport --- docs/overview.rst | 4 +++- hpy/devel/__init__.py | 20 +++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/overview.rst b/docs/overview.rst index e4c9e30b..b291f636 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -177,6 +177,8 @@ different ABIs: The resulting filename is e.g. ``simple.hpy0.so``. + require backport ``importlib_resources>=5.0`` on ``CPython<3.10``. + HPy Hybrid ABI The HPy Hybrid ABI is essentially the same as the Universal ABI, with @@ -324,7 +326,7 @@ of features. As on April 2023, the following milestones have been reached: CPython. - it is possible to load HPy Universal extensions on CPython, thanks to the - ``hpy.universal`` package. + ``hpy.universal`` package (require ``importlib_resources>=5.0`` on ``CPython<3.10``). - it is possible to load HPy Universal extensions on PyPy (using the PyPy `hpy branch `_). diff --git a/hpy/devel/__init__.py b/hpy/devel/__init__.py index 57ae1c06..8472bb8e 100644 --- a/hpy/devel/__init__.py +++ b/hpy/devel/__init__.py @@ -203,28 +203,22 @@ def handle_hpy_ext_modules(dist, attr, hpy_ext_modules): # This file is automatically generated by hpy def __bootstrap__(): from sys import modules, version_info - from os import environ + from os import path, environ from hpy.universal import _load_bootstrap ext_name = {ext_file!r} module_name = {module_name!r} if version_info < (3, 10): - # there is a bug in importlib.resources that doesn't handle namespace package - # and also pkg_resources is available, so just use pkg_resources. - from pkg_resources import resource_filename - - ext_filepath = resource_filename(__name__, ext_name) - + import importlib_resources as resources else: - from os import path from importlib import resources - if not __package__: - # directly called with python -m - ext_filepath = path.join(path.dirname(__file__), ext_name) - else: - ext_filepath = resources.files(__package__).joinpath(ext_name) + if not __package__: + # directly called with python -m + ext_filepath = path.join(path.dirname(__file__), ext_name) + else: + ext_filepath = resources.files(__package__).joinpath(ext_name) m = _load_bootstrap( module_name, From c92430c73a17639c39bc202a043521f3c1a21949 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Mon, 3 Jun 2024 17:26:26 +0800 Subject: [PATCH 7/8] add importlib_resources to deps --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8dd6962d..bf03931f 100644 --- a/setup.py +++ b/setup.py @@ -264,6 +264,9 @@ def build_libraries(self, libraries): cmdclass={"build_clib": build_clib_hpy}, use_scm_version=get_scm_config, setup_requires=['setuptools_scm'], - install_requires=['setuptools>=64.0'], + install_requires=[ + 'setuptools>=64.0', + 'importlib_resources; python_version<"3.10"', + ], python_requires='>=3.8', ) From fab9589a537a4c0d563e1629e5c3e9ccfa788021 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Tue, 4 Jun 2024 17:25:59 +0800 Subject: [PATCH 8/8] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bf03931f..9a3ce045 100644 --- a/setup.py +++ b/setup.py @@ -266,7 +266,7 @@ def build_libraries(self, libraries): setup_requires=['setuptools_scm'], install_requires=[ 'setuptools>=64.0', - 'importlib_resources; python_version<"3.10"', + 'importlib_resources>=5.0; python_version<"3.10"', ], python_requires='>=3.8', )