diff --git a/newsfragments/4043.bugfix.rst b/newsfragments/4043.bugfix.rst new file mode 100644 index 0000000000..e32728fe24 --- /dev/null +++ b/newsfragments/4043.bugfix.rst @@ -0,0 +1,4 @@ +Avoid using caching attributes in ``Distribution.metadata`` for requirements. +This is done for backwards compatibility with customizations that attempt to +modify ``install_requires`` or ``extras_require`` at a late point (still not +recommended). diff --git a/setuptools/_core_metadata.py b/setuptools/_core_metadata.py index c1d41c6680..6c904c3c77 100644 --- a/setuptools/_core_metadata.py +++ b/setuptools/_core_metadata.py @@ -13,7 +13,7 @@ from distutils.util import rfc822_escape -from . import _normalization +from . import _normalization, _reqs from .extern.packaging.markers import Marker from .extern.packaging.requirements import Requirement from .extern.packaging.version import Version @@ -211,11 +211,11 @@ def write_field(key, value): def _write_requirements(self, file): - for req in self._normalized_install_requires: + for req in _reqs.parse(self.install_requires): file.write(f"Requires-Dist: {req}\n") processed_extras = {} - for augmented_extra, reqs in self._normalized_extras_require.items(): + for augmented_extra, reqs in self.extras_require.items(): # Historically, setuptools allows "augmented extras": `:` unsafe_extra, _, condition = augmented_extra.partition(":") unsafe_extra = unsafe_extra.strip() @@ -223,7 +223,7 @@ def _write_requirements(self, file): if extra: _write_provides_extra(file, processed_extras, extra, unsafe_extra) - for req in reqs: + for req in _reqs.parse_strings(reqs): r = _include_extra(req, extra, condition.strip()) file.write(f"Requires-Dist: {r}\n") @@ -231,7 +231,7 @@ def _write_requirements(self, file): def _include_extra(req: str, extra: str, condition: str) -> Requirement: - r = Requirement(req) + r = Requirement(req) # create a fresh object that can be modified parts = ( f"({r.marker})" if r.marker else None, f"({condition})" if condition else None, diff --git a/setuptools/_reqs.py b/setuptools/_reqs.py index 5d5b927fd8..7d7130d50e 100644 --- a/setuptools/_reqs.py +++ b/setuptools/_reqs.py @@ -1,3 +1,4 @@ +from functools import lru_cache from typing import Callable, Iterable, Iterator, TypeVar, Union, overload import setuptools.extern.jaraco.text as text @@ -7,6 +8,12 @@ _StrOrIter = Union[str, Iterable[str]] +parse_req: Callable[[str], Requirement] = lru_cache()(Requirement) +# Setuptools parses the same requirement many times +# (e.g. first for validation than for normalisation), +# so it might be worth to cache. + + def parse_strings(strs: _StrOrIter) -> Iterator[str]: """ Yield requirement strings for each specification in `strs`. @@ -26,7 +33,7 @@ def parse(strs: _StrOrIter, parser: Callable[[str], _T]) -> Iterator[_T]: ... -def parse(strs, parser=Requirement): +def parse(strs, parser=parse_req): """ Replacement for ``pkg_resources.parse_requirements`` that uses ``packaging``. """ diff --git a/setuptools/command/_requirestxt.py b/setuptools/command/_requirestxt.py index d223737fd4..32bae2c4b4 100644 --- a/setuptools/command/_requirestxt.py +++ b/setuptools/command/_requirestxt.py @@ -11,6 +11,7 @@ from itertools import filterfalse from typing import Dict, List, Tuple, Mapping, TypeVar +from .. import _reqs from ..extern.jaraco.text import yield_lines from ..extern.packaging.requirements import Requirement @@ -19,11 +20,11 @@ _T = TypeVar("_T") _Ordered = Dict[_T, None] _ordered = dict +_StrOrIter = _reqs._StrOrIter def _prepare( - install_requires: Dict[str, Requirement], - extras_require: Mapping[str, Dict[str, Requirement]], + install_requires: _StrOrIter, extras_require: Mapping[str, _StrOrIter] ) -> Tuple[List[str], Dict[str, List[str]]]: """Given values for ``install_requires`` and ``extras_require`` create modified versions in a way that can be written in ``requires.txt`` @@ -33,7 +34,7 @@ def _prepare( def _convert_extras_requirements( - extras_require: Dict[str, Dict[str, Requirement]], + extras_require: _StrOrIter, ) -> Mapping[str, _Ordered[Requirement]]: """ Convert requirements in `extras_require` of the form @@ -44,15 +45,14 @@ def _convert_extras_requirements( for section, v in extras_require.items(): # Do not strip empty sections. output[section] - for r in v.values(): + for r in _reqs.parse(v): output[section + _suffix_for(r)].setdefault(r) return output def _move_install_requirements_markers( - install_requires: Dict[str, Requirement], - extras_require: Mapping[str, _Ordered[Requirement]], + install_requires: _StrOrIter, extras_require: Mapping[str, _Ordered[Requirement]] ) -> Tuple[List[str], Dict[str, List[str]]]: """ The ``requires.txt`` file has an specific format: @@ -66,7 +66,7 @@ def _move_install_requirements_markers( # divide the install_requires into two sets, simple ones still # handled by install_requires and more complex ones handled by extras_require. - inst_reqs = install_requires.values() + inst_reqs = list(_reqs.parse(install_requires)) simple_reqs = filter(_no_marker, inst_reqs) complex_reqs = filterfalse(_no_marker, inst_reqs) simple_install_requires = list(map(str, simple_reqs)) @@ -90,7 +90,7 @@ def _suffix_for(req): def _clean_req(req): """Given a Requirement, remove environment markers and return it""" - r = Requirement(str(req)) # create a copy before modifying. + r = Requirement(str(req)) # create a copy before modifying r.marker = None return r @@ -111,10 +111,9 @@ def append_cr(line): def write_requirements(cmd, basename, filename): dist = cmd.distribution - meta = dist.metadata data = io.StringIO() install_requires, extras_require = _prepare( - meta._normalized_install_requires, meta._normalized_extras_require + dist.install_requires or (), dist.extras_require or {} ) _write_requirements(data, install_requires) for extra in sorted(extras_require): diff --git a/setuptools/config/_apply_pyprojecttoml.py b/setuptools/config/_apply_pyprojecttoml.py index 4b8f803c1b..b6443308bf 100644 --- a/setuptools/config/_apply_pyprojecttoml.py +++ b/setuptools/config/_apply_pyprojecttoml.py @@ -125,7 +125,7 @@ def _set_config(dist: "Distribution", field: str, value: Any): setter(value) elif hasattr(dist.metadata, field) or field in SETUPTOOLS_PATCHES: setattr(dist.metadata, field, value) - else: + if hasattr(dist, field): setattr(dist, field, value) diff --git a/setuptools/dist.py b/setuptools/dist.py index f1d361f1c3..2672f928d5 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -263,11 +263,8 @@ class Distribution(_Distribution): 'provides_extras': OrderedSet, 'license_file': lambda: None, 'license_files': lambda: None, - # Both install_requires and extras_require are needed to write PKG-INFO, - # So we take this opportunity to cache parsed requirement objects. - # These attributes are not part of the public API and intended for internal use. - '_normalized_install_requires': dict, # Dict[str, Requirement] - '_normalized_extras_require': dict, # Dict[str, Dict[str, Requirement]] + 'install_requires': list, + 'extras_require': dict, } _patched_dist = None @@ -299,14 +296,11 @@ def __init__(self, attrs=None): self.setup_requires = attrs.pop('setup_requires', []) for ep in metadata.entry_points(group='distutils.setup_keywords'): vars(self).setdefault(ep.name, None) - _Distribution.__init__( - self, - { - k: v - for k, v in attrs.items() - if k not in self._DISTUTILS_UNSUPPORTED_METADATA - }, - ) + + metadata_only = set(self._DISTUTILS_UNSUPPORTED_METADATA) + metadata_only -= {"install_requires", "extras_require"} + dist_attrs = {k: v for k, v in attrs.items() if k not in metadata_only} + _Distribution.__init__(self, dist_attrs) # Private API (setuptools-use only, not restricted to Distribution) # Stores files that are referenced by the configuration and need to be in the @@ -394,6 +388,8 @@ def _finalize_requires(self): self.metadata.python_requires = self.python_requires self._normalize_requires() + self.metadata.install_requires = self.install_requires + self.metadata.extras_require = self.extras_require if self.extras_require: for extra in self.extras_require.keys(): @@ -406,17 +402,9 @@ def _normalize_requires(self): """Make sure requirement-related attributes exist and are normalized""" install_requires = getattr(self, "install_requires", None) or [] extras_require = getattr(self, "extras_require", None) or {} - meta = self.metadata - meta._normalized_install_requires = { - str(r): r for r in _reqs.parse(install_requires) - } - meta._normalized_extras_require = { - k: {str(r): r for r in _reqs.parse(v or [])} - for k, v in extras_require.items() - } - self.install_requires = list(meta._normalized_install_requires) + self.install_requires = list(map(str, _reqs.parse(install_requires))) self.extras_require = { - k: list(v) for k, v in meta._normalized_extras_require.items() + k: list(map(str, _reqs.parse(v or []))) for k, v in extras_require.items() } def _finalize_license_files(self):