From 352334ad1f3332b25da642d821ffc2c0d346131a Mon Sep 17 00:00:00 2001 From: KOLANICH Date: Wed, 18 Mar 2020 18:48:30 +0300 Subject: [PATCH] Improved ''setuptools.finalize_distribution_options'' hook. * Improved entry points mechanism. Entry points now can contain JSON metadata appended to a ''name'' after ''@'' character. It is a backward-incompatible change, but the breakage will unlikely happen. * Now it allows the implementers to ast setuptools to provide them various information by adding a parameter with a certain name into the function signature. Old behavior is also supported. For the list of args names see ''DistributionoptionsFinalizat ionRemap'' in ''dist.py''. * Now it tolerates errors in hooks code, so a package with a broken hook doesn't prevent from updating itself. * Implemented ''setuptools.finalize_distribution_options'' hooks prioritization. A priority can be set as an integer or float metadata. Or, if you need to set other options via providing a ''dict'', it can be added into ''order'' key. * Now it allows a hook only be called if it is mentioned in ''setup'' args or in ''pyproject.toml''. It can be done by adding ''{"only": ["toml"]}'' metadata. This should reduce breakage due to malfunctions in ''setuptools'' or hooks code. --- changelog.d/2031.change.rst | 6 + pkg_resources/__init__.py | 27 ++- pkg_resources/tests/test_resources.py | 9 +- setuptools/dist.py | 227 ++++++++++++++++++++++++-- 4 files changed, 252 insertions(+), 17 deletions(-) create mode 100644 changelog.d/2031.change.rst diff --git a/changelog.d/2031.change.rst b/changelog.d/2031.change.rst new file mode 100644 index 00000000000..05f92430459 --- /dev/null +++ b/changelog.d/2031.change.rst @@ -0,0 +1,6 @@ +Improved ''setuptools.finalize_distribution_options'' hook. +* Improved entry points mechanism. Entry points now can contain JSON metadata appended to a ''name'' after ''@'' character. It is a backward-incompatible change, but the breakage will unlikely happen. +* Now it allows the implementers to ast setuptools to provide them various information by adding a parameter with a certain n ame into the function signature. Old behavior is also supported. For the list of args names see ''DistributionoptionsFinalizationRemap'' in ''dist.py''. +* Now it tolerates errors in hooks code, so a package with a broken hook doesn't prevent from updating itself. +* Implemented ''setuptools.finalize_distribution_options'' hooks prioritization. A priority can be set as an integer or float metadata. Or, if you need to set other options via providing a ''dict'', it can be added into ''order'' key. +* Now it allows a hook only be called if it is mentioned in ''setup'' args or in ''pyproject.toml''. It can be done by adding ''{"only": ["toml"]}'' metadata. This should reduce breakage due to malfunctions in ''setuptools'' or hooks code. diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index 88d4bdcaedf..6bdb05f58c8 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -42,6 +42,7 @@ import ntpath import posixpath from pkgutil import get_importer +import json try: import _imp @@ -2410,17 +2411,27 @@ def yield_lines(strs): class EntryPoint: """Object representing an advertised importable object""" - def __init__(self, name, module_name, attrs=(), extras=(), dist=None): + __slots__ = ("name", "metadata", "module_name", "attrs", "extras", "dist") + + def __init__(self, name, module_name, attrs=(), + extras=(), dist=None, metadata=None + ): if not MODULE(module_name): raise ValueError("Invalid module name", module_name) self.name = name + self.metadata = metadata self.module_name = module_name self.attrs = tuple(attrs) self.extras = tuple(extras) self.dist = dist def __str__(self): - s = "%s = %s" % (self.name, self.module_name) + if self.metadata is not None: + encoded = " @ ".join((self.name, json.dumps(self.metadata))) + else: + encoded = self.name + + s = "%s = %s" % (encoded, self.module_name) if self.attrs: s += ':' + '.'.join(self.attrs) if self.extras: @@ -2470,7 +2481,8 @@ def require(self, env=None, installer=None): pattern = re.compile( r'\s*' - r'(?P.+?)\s*' + r'(?P[^@]+?)\s*' + r'(?:@\s*(?P.+?)\s*)?' r'=\s*' r'(?P[\w.]+)\s*' r'(:\s*(?P[\w.]+))?\s*' @@ -2493,9 +2505,16 @@ def parse(cls, src, dist=None): msg = "EntryPoint must be in 'name=module:attrs [extras]' format" raise ValueError(msg, src) res = m.groupdict() + + metadataStr = res['metadata'] + if metadataStr is not None: + metadata = json.loads(metadataStr) + else: + metadata = None + extras = cls._parse_extras(res['extras']) attrs = res['attr'].split('.') if res['attr'] else () - return cls(res['name'], res['module'], attrs, extras, dist) + return cls(res['name'], res['module'], attrs, extras, dist, metadata) @classmethod def _parse_extras(cls, extras_spec): diff --git a/pkg_resources/tests/test_resources.py b/pkg_resources/tests/test_resources.py index ed7cdfcc3f8..106d61f9c5c 100644 --- a/pkg_resources/tests/test_resources.py +++ b/pkg_resources/tests/test_resources.py @@ -430,6 +430,13 @@ def testParse(self): assert ep.attrs == ("foo",) assert ep.extras == () + ep = EntryPoint.parse('a @ {"aa:aa=b@ddd": {"bb:b@c=cc": 1}} = b:c') + assert ep.name == "a" + assert ep.module_name == "b" + assert ep.attrs == ("c",) + assert ep.metadata["aa:aa=b@ddd"]["bb:b@c=cc"] == 1 + assert ep.extras == () + # plus in the name spec = "html+mako = mako.ext.pygmentplugin:MakoHtmlLexer" ep = EntryPoint.parse(spec) @@ -447,7 +454,7 @@ def test_printable_name(self): Allow any printable character in the name. """ # Create a name with all printable characters; strip the whitespace. - name = string.printable.strip() + name = "".join(set(string.printable.strip()) - {"@"}) spec = "{name} = module:attr".format(**locals()) ep = EntryPoint.parse(spec) assert ep.name == name diff --git a/setuptools/dist.py b/setuptools/dist.py index 0f71dd98328..4d1ad43abb5 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -15,6 +15,8 @@ from distutils.debug import DEBUG from distutils.fancy_getopt import translate_longopt import itertools +import pkg_resources +import inspect from collections import defaultdict from email import message_from_file @@ -39,6 +41,12 @@ __import__('setuptools.extern.packaging.version') +try: + from collections.abc import Mapping +except ImportError: + Mapping = dict + + def _get_unpatched(cls): warnings.warn("Do not call this function", DistDeprecationWarning) return get_unpatched(cls) @@ -66,6 +74,51 @@ def get_metadata_version(self): return mv +def get_setup_py_path(): + """Returns the path of `setup.py` file used to trigger the build.""" + s = inspect.stack() + + setuptoolsDir = os.path.dirname(os.path.normpath( + os.path.abspath(__file__) + )) + s = s[1:] + + setuptoolsFrameFound = None + for i, f in enumerate(s): + if f.function == "setup": + setupFuncFileParentDir = os.path.dirname( + os.path.normpath(os.path.abspath(f.filename)) + ) + if os.path.samefile(setupFuncFileParentDir, setuptoolsDir): + setuptoolsFrameFound = i + break + + if setuptoolsFrameFound is not None: + setupPyFrame = s[setuptoolsFrameFound + 1] + if os.path.split(setupPyFrame.filename)[1] == "setup.py": + return os.path.normpath(os.path.realpath( + os.path.abspath(setupPyFrame.filename) + )) + + +def get_setup_py_dir(): + """Returns the path of parent dir of `setup.py`.""" + spp = get_setup_py_path() + if spp: + return os.path.dirname() + + +def read_toml(tomlFile): + import toml + + if os.path.isfile(tomlFile): + return toml.load(tomlFile) + + +def read_pyproject_toml(setupPyDir): + return read_toml(os.path.join(setupPyDir, 'pyproject.toml')) + + def read_pkg_file(self, file): """Reads the metadata values from a file object.""" msg = message_from_file(file) @@ -284,6 +337,7 @@ def check_requirements(dist, attr, value): def check_specifier(dist, attr, value): """Verify that value is a valid version specifier""" + try: packaging.specifiers.SpecifierSet(value) except packaging.specifiers.InvalidSpecifier as error: @@ -331,6 +385,31 @@ def check_packages(dist, attr, value): ) +class FinalizeDistributionOptionsHookArgsCache: + __slots__ = ("_setupPyDir", "_pyProjectToml") + + def __init__(self): + self._setupPyDir = None + self._pyProjectToml = None + + @property + def setupPyDir(self): + if self._setupPyDir is None: + self._setupPyDir = get_setup_py_dir() + if not self._setupPyDir: + self._setupPyDir = False + return self._setupPyDir + + @property + def pyProjectToml(self): + if self._pyProjectToml is None: + if self.setupPyDir: + self._pyProjectToml = read_pyproject_toml(self.setupPyDir) + else: + self._pyProjectToml = False + return self._pyProjectToml + + _Distribution = get_unpatched(distutils.core.Distribution) @@ -695,6 +774,31 @@ def fetch_build_eggs(self, requires): pkg_resources.working_set.add(dist, replace=True) return resolved_dists + optsFinalizationRemap = None # set later + + shouldCallFinCheckers = { + 'dist': lambda dist, fdohac, ep: hasattr(dist, ep.name), + 'toml': lambda dist, fdohac, ep: bool(fdohac.pyProjectToml) and ep.name in fdohac.pyProjectToml, + } + + def shouldCallEntryPoint(self, ep, fdohac): + if isinstance(ep.metadata, Mapping): + only = ep.metadata.get('only', None) + if only is not None: + if isinstance(only, str): + if only == '*': + only = tuple(self.shouldCallFinCheckers) + else: + only = (only,) + + for cand in only: + checker = self.shouldCallFinCheckers.get(cand, None) + if checker: + if checker(self, fdohac, ep): + return True + return False + return True + def finalize_options(self): """ Allow plugins to apply arbitrary operations to the @@ -702,30 +806,112 @@ def finalize_options(self): to influence the order of execution. Smaller numbers go first and the default is 0. """ - hook_key = 'setuptools.finalize_distribution_options' + + default_order_value = 0 def by_order(hook): - return getattr(hook, 'order', 0) + if isinstance(hook.metadata, (int, float)): + return hook.metadata + elif isinstance(hook.metadata, Mapping): + return hook.metadata.get("order", default_order_value) + return default_order_value + + fdohac = FinalizeDistributionOptionsHookArgsCache() + + def constructArgsForAFunc(f, ep, fdohac): + if sys.version_info.major >= 3: + p = inspect.getfullargspec(f).args + else: + p = inspect.getargspec(f)[0] + + if len(p) == 1: + firstParam = p[0] + if firstParam not in self.optsFinalizationRemap: + return [self] + + args = [] + for parN in p: + a = self.optsFinalizationRemap[parN](self, ep, fdohac, parN) + args.append(a) + return args + + hook_key = "setuptools.finalize_distribution_options" eps = pkg_resources.iter_entry_points(hook_key) + for ep in sorted(eps, key=by_order): - ep.load()(self) + if not self.shouldCallEntryPoint(ep, fdohac): + continue + + try: + f = ep.load() + except Exception as ex: + warnings.warn( + "Cannot load " + hook_key + " entry point " + + repr(ep) + ":\n" + str(ex) + ) + continue + + try: + args = constructArgsForAFunc(f, ep, fdohac) + except Exception as ex: + warnings.warn( + repr(ep) + " is incompatible to the current" + " version of setuptools: " + str(ex) + ) + continue + + try: + res = f(*args) + except BaseException as ex: + warnings.warn("Error when executing entry point:" + str(ex)) + continue + + if res is not None: + + def dictGetter(orig, k): + return orig[k] + + def dictSetter(orig, k, v): + orig[k] = v + + def recursive_merge(orig, patch, getter, setter): + for k, v in res.items(): + try: + ov = orig[k] + except KeyError: + ov[k] = v + continue + if isinstance(ov, Mapping) and isinstance(v, Mapping): + recursive_merge(ov, v, dictGetter, dictSetter) + else: + ov[k] = v + + if isinstance(res, dict): + recursive_merge(dist, res, getattr, setattr) + else: + warnings.warn( + f + " has returned " + repr(res) + + " which is not a `dict`" + ) - def _finalize_setup_keywords(self): + @staticmethod + def _finalize_setup_keywords(dist): for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'): - value = getattr(self, ep.name, None) + value = getattr(dist, ep.name, None) if value is not None: - ep.require(installer=self.fetch_build_egg) - ep.load()(self, ep.name, value) + ep.require(installer=dist.fetch_build_egg) + ep.load()(dist, ep.name, value) - def _finalize_2to3_doctests(self): - if getattr(self, 'convert_2to3_doctests', None): + @staticmethod + def _finalize_2to3_doctests(dist): + if getattr(dist, 'convert_2to3_doctests', None): # XXX may convert to set here when we can rely on set being builtin - self.convert_2to3_doctests = [ + dist.convert_2to3_doctests = [ os.path.abspath(p) - for p in self.convert_2to3_doctests + for p in dist.convert_2to3_doctests ] else: - self.convert_2to3_doctests = [] + dist.convert_2to3_doctests = [] def get_egg_cache_dir(self): egg_cache_dir = os.path.join(os.curdir, '.eggs') @@ -1020,6 +1206,23 @@ def handle_display_options(self, option_order): sys.stdout.detach(), encoding, errors, newline, line_buffering) +def pyProjectTomlSectionFinRemap(sself, entryPoint, fdohac, par): + return fdohac.pyProjectToml.get( + entryPoint.name, None + ) + + +Distribution.optsFinalizationRemap = { + "dist": lambda sself, entryPoint, fdohac, par: sself, + "setupPySection": lambda sself, entryPoint, fdohac, par: sself.get( + entryPoint.name, None + ), + "setupPyDir": lambda sself, entryPoint, fdohac, par: fdohac.setupPyDir, + "pyProjectTomlSection": pyProjectTomlSectionFinRemap, + "entryPoint": lambda sself, entryPoint, fdohac, par: entryPoint, +} + + class DistDeprecationWarning(SetuptoolsDeprecationWarning): """Class for warning about deprecations in dist in setuptools. Not ignored by default, unlike DeprecationWarning."""