From b760e946dc794f145b507a1512d7ff7138c06ae8 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Mar 2022 13:15:21 +0100 Subject: [PATCH 01/10] Store install_requires and extras_require for future usage --- setuptools/dist.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setuptools/dist.py b/setuptools/dist.py index 865a19dd57..67c988b1a6 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -468,6 +468,10 @@ def __init__(self, attrs=None): }, ) + # Save the original dependencies before they are processed into the egg format + self._orig_extras_require = {} + self._orig_install_requires = [] + self.set_defaults = ConfigDiscovery(self) self._set_metadata_defaults(attrs) @@ -540,6 +544,8 @@ def _finalize_requires(self): self.metadata.python_requires = self.python_requires if getattr(self, 'extras_require', None): + # Save original before it is messed by _convert_extras_requirements + self._orig_extras_require = self._orig_extras_require or self.extras_require for extra in self.extras_require.keys(): # Since this gets called multiple times at points where the # keys have become 'converted' extras, ensure that we are only @@ -548,6 +554,10 @@ def _finalize_requires(self): if extra: self.metadata.provides_extras.add(extra) + if getattr(self, 'install_requires', None) and not self._orig_install_requires: + # Save original before it is messed by _move_install_requirements_markers + self._orig_install_requires = self.install_requires + self._convert_extras_requirements() self._move_install_requirements_markers() From 91f9960726a7a73f1009ec3adeace04f4dd6c66c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Mar 2022 14:57:25 +0100 Subject: [PATCH 02/10] Make sure apply function remains private --- setuptools/config/pyprojecttoml.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setuptools/config/pyprojecttoml.py b/setuptools/config/pyprojecttoml.py index d2c6c9c5d1..0ee1b8f9a1 100644 --- a/setuptools/config/pyprojecttoml.py +++ b/setuptools/config/pyprojecttoml.py @@ -9,7 +9,8 @@ from setuptools.errors import FileError, OptionError from . import expand as _expand -from ._apply_pyprojecttoml import apply, _PREVIOUSLY_DEFINED, _WouldIgnoreField +from ._apply_pyprojecttoml import apply as _apply +from ._apply_pyprojecttoml import _PREVIOUSLY_DEFINED, _WouldIgnoreField if TYPE_CHECKING: from setuptools.dist import Distribution # noqa @@ -44,13 +45,15 @@ def validate(config: dict, filepath: _Path): def apply_configuration( - dist: "Distribution", filepath: _Path, ignore_option_errors=False, + dist: "Distribution", + filepath: _Path, + ignore_option_errors=False, ) -> "Distribution": """Apply the configuration from a ``pyproject.toml`` file into an existing distribution object. """ config = read_configuration(filepath, True, ignore_option_errors, dist) - return apply(dist, config, filepath) + return _apply(dist, config, filepath) def read_configuration( From d0ee3e4944245db6b37cba2b3335dcacc2d3e6f6 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Mar 2022 15:40:57 +0100 Subject: [PATCH 03/10] Ensure pyproject.toml does not break dynamic install_requires --- .../tests/config/test_apply_pyprojecttoml.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/setuptools/tests/config/test_apply_pyprojecttoml.py b/setuptools/tests/config/test_apply_pyprojecttoml.py index c09ff3e6b8..a88bc1ec19 100644 --- a/setuptools/tests/config/test_apply_pyprojecttoml.py +++ b/setuptools/tests/config/test_apply_pyprojecttoml.py @@ -15,6 +15,7 @@ from setuptools.config import setupcfg, pyprojecttoml from setuptools.config import expand from setuptools.config._apply_pyprojecttoml import _WouldIgnoreField +from setuptools.command.egg_info import write_requirements EXAMPLES = (Path(__file__).parent / "setupcfg_examples.txt").read_text() @@ -207,12 +208,12 @@ def test_license_and_license_files(tmp_path): class TestPresetField: - def pyproject(self, tmp_path, dynamic): + def pyproject(self, tmp_path, dynamic, extra_content=""): content = f"[project]\nname = 'proj'\ndynamic = {dynamic!r}\n" if "version" not in dynamic: content += "version = '42'\n" file = tmp_path / "pyproject.toml" - file.write_text(content, encoding="utf-8") + file.write_text(content + extra_content, encoding="utf-8") return file @pytest.mark.parametrize( @@ -250,6 +251,28 @@ def test_listed_in_dynamic(self, tmp_path, attr, field, value): dist_value = getattr(dist, attr, None) or getattr(dist.metadata, attr, object()) assert dist_value == value + def test_optional_dependencies_dont_remove_env_markers(self, tmp_path): + """ + Internally setuptools converts dependencies with markers to "extras". + If ``install_requires`` is given by ``setup.py``, we have to ensure that + applying ``optional-dependencies`` does not overwrite the mandatory + dependencies with markers (see #3204). + """ + # If setuptools replace its internal mechanism that uses `requires.txt` + # this test has to be rewritten to adapt accordingly + extra = "\n[project.optional-dependencies]\nfoo = ['bar>1']\n" + pyproject = self.pyproject(tmp_path, ["dependencies"], extra) + install_req = ['importlib-resources (>=3.0.0) ; python_version < "3.7"'] + dist = makedist(tmp_path, install_requires=install_req) + dist = pyprojecttoml.apply_configuration(dist, pyproject) + assert "foo" in dist.extras_require + assert ':python_version < "3.7"' in dist.extras_require + egg_info = dist.get_command_obj("egg_info") + write_requirements(egg_info, tmp_path, tmp_path / "requires.txt") + reqs = (tmp_path / "requires.txt").read_text(encoding="utf-8") + assert "importlib-resources" in reqs + assert "bar" in reqs + # --- Auxiliary Functions --- From 1a60a4f69979a4031faede2f792bb8f0eb63c01f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Mar 2022 14:59:52 +0100 Subject: [PATCH 04/10] Merge pre-set dependencies when applying pyproject --- setuptools/config/_apply_pyprojecttoml.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/setuptools/config/_apply_pyprojecttoml.py b/setuptools/config/_apply_pyprojecttoml.py index 78a07273db..5d34cdb7d8 100644 --- a/setuptools/config/_apply_pyprojecttoml.py +++ b/setuptools/config/_apply_pyprojecttoml.py @@ -194,6 +194,16 @@ def _python_requires(dist: "Distribution", val: dict, _root_dir): _set_config(dist, "python_requires", SpecifierSet(val)) +def _dependencies(dist: "Distribution", val: list, _root_dir): + existing = getattr(dist, "install_requires", []) + _set_config(dist, "install_requires", existing + val) + + +def _optional_dependencies(dist: "Distribution", val: dict, _root_dir): + existing = getattr(dist, "extras_require", {}) + _set_config(dist, "extras_require", {**existing, **val}) + + def _unify_entry_points(project_table: dict): project = project_table entry_points = project.pop("entry-points", project.pop("entry_points", {})) @@ -303,8 +313,8 @@ def _acessor(obj): "authors": partial(_people, kind="author"), "maintainers": partial(_people, kind="maintainer"), "urls": _project_urls, - "dependencies": "install_requires", - "optional_dependencies": "extras_require", + "dependencies": _dependencies, + "optional_dependencies": _optional_dependencies, "requires_python": _python_requires, } From 988d0646e7294f4b99485a9c38740f832cea89ea Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Mar 2022 15:00:25 +0100 Subject: [PATCH 05/10] Small refactor --- setuptools/config/setupcfg.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/setuptools/config/setupcfg.py b/setuptools/config/setupcfg.py index 5ecf626968..d485a8bba8 100644 --- a/setuptools/config/setupcfg.py +++ b/setuptools/config/setupcfg.py @@ -70,7 +70,7 @@ def apply_configuration(dist: "Distribution", filepath: _Path) -> "Distribution" def _apply( dist: "Distribution", filepath: _Path, other_files: Iterable[_Path] = (), - ignore_option_errors: bool = False + ignore_option_errors: bool = False, ) -> Tuple["ConfigHandler", ...]: """Read configuration from ``filepath`` and applies to the ``dist`` object.""" from setuptools.dist import _Distribution @@ -677,9 +677,8 @@ def parse_section_extras_require(self, section_options): :param dict section_options: """ parse_list = partial(self._parse_list, separator=';') - self['extras_require'] = self._parse_section_to_dict( - section_options, parse_list - ) + parsed = self._parse_section_to_dict(section_options, parse_list) + self['extras_require'] = parsed def parse_section_data_files(self, section_options): """Parses `data_files` configuration file section. From 3f28fbc10f584da8555a6ea89155bc49ddcb18c9 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Mar 2022 15:03:11 +0100 Subject: [PATCH 06/10] Preserve _tmp_extras_require as an ordered set --- setuptools/dist.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setuptools/dist.py b/setuptools/dist.py index 67c988b1a6..2aa532d2bf 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -471,6 +471,7 @@ def __init__(self, attrs=None): # Save the original dependencies before they are processed into the egg format self._orig_extras_require = {} self._orig_install_requires = [] + self._tmp_extras_require = defaultdict(ordered_set.OrderedSet) self.set_defaults = ConfigDiscovery(self) @@ -568,7 +569,8 @@ def _convert_extras_requirements(self): `"extra:{marker}": ["barbazquux"]`. """ spec_ext_reqs = getattr(self, 'extras_require', None) or {} - self._tmp_extras_require = defaultdict(list) + tmp = defaultdict(ordered_set.OrderedSet) + self._tmp_extras_require = getattr(self, '_tmp_extras_require', tmp) for section, v in spec_ext_reqs.items(): # Do not strip empty sections. self._tmp_extras_require[section] @@ -606,7 +608,8 @@ def is_simple_req(req): for r in complex_reqs: self._tmp_extras_require[':' + str(r.marker)].append(r) self.extras_require = dict( - (k, [str(r) for r in map(self._clean_req, v)]) + # list(dict.fromkeys(...)) ensures a list of unique strings + (k, list(dict.fromkeys(str(r) for r in map(self._clean_req, v)))) for k, v in self._tmp_extras_require.items() ) From 245b8686ace004f2827bfad542a57fe226d6765f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Mar 2022 15:20:20 +0100 Subject: [PATCH 07/10] Decrease verbosity of _install_setup_requires --- setuptools/__init__.py | 8 ++++++++ setuptools/dist.py | 15 ++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 502d2a2e12..cff04323e9 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -58,6 +58,14 @@ def __init__(self, attrs): # Prevent accidentally triggering discovery with incomplete set of attrs self.set_defaults._disable() + def _get_project_config_files(self, filenames=None): + """Ignore ``pyproject.toml``, they are not related to setup_requires""" + try: + cfg, toml = super()._split_standard_project_metadata(filenames) + return cfg, () + except Exception: + return filenames, () + def finalize_options(self): """ Disable finalize_options to avoid building the working set. diff --git a/setuptools/dist.py b/setuptools/dist.py index 2aa532d2bf..215c88e3a8 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -827,10 +827,8 @@ def _set_command_options(self, command_obj, option_dict=None): # noqa: C901 except ValueError as e: raise DistutilsOptionError(e) from e - def parse_config_files(self, filenames=None, ignore_option_errors=False): - """Parses configuration files from various levels - and loads configuration. - """ + def _get_project_config_files(self, filenames): + """Add default file and split between INI and TOML""" tomlfiles = [] standard_project_metadata = Path(self.src_root or os.curdir, "pyproject.toml") if filenames is not None: @@ -839,8 +837,15 @@ def parse_config_files(self, filenames=None, ignore_option_errors=False): tomlfiles = list(parts[1]) # 2nd element => predicate is True elif standard_project_metadata.exists(): tomlfiles = [standard_project_metadata] + return filenames, tomlfiles + + def parse_config_files(self, filenames=None, ignore_option_errors=False): + """Parses configuration files from various levels + and loads configuration. + """ + inifiles, tomlfiles = self._get_project_config_files(filenames) - self._parse_config_files(filenames=filenames) + self._parse_config_files(filenames=inifiles) setupcfg.parse_configuration( self, self.command_options, ignore_option_errors=ignore_option_errors From f82f3689c93f97945a571aac30a244512eb98229 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Mar 2022 15:53:14 +0100 Subject: [PATCH 08/10] Add news fragment --- changelog.d/3222.misc.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog.d/3222.misc.rst diff --git a/changelog.d/3222.misc.rst b/changelog.d/3222.misc.rst new file mode 100644 index 0000000000..66f1489edf --- /dev/null +++ b/changelog.d/3222.misc.rst @@ -0,0 +1,2 @@ +Fixed missing requirements with environment markers when +``optional-dependencies`` is set in ``pyproject.toml``. From 2304d9992b74c3080955563cac24af0670db652b Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Mar 2022 16:07:34 +0100 Subject: [PATCH 09/10] Fix incorrect PR number --- changelog.d/{3222.misc.rst => 3223.misc.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{3222.misc.rst => 3223.misc.rst} (100%) diff --git a/changelog.d/3222.misc.rst b/changelog.d/3223.misc.rst similarity index 100% rename from changelog.d/3222.misc.rst rename to changelog.d/3223.misc.rst From 603bb9852f3a6a53c97beaccc9f58dc47771a486 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Mar 2022 16:57:26 +0100 Subject: [PATCH 10/10] Fix previous detection of empty arrays --- setuptools/config/_apply_pyprojecttoml.py | 2 +- setuptools/config/pyprojecttoml.py | 5 +++-- setuptools/tests/config/test_apply_pyprojecttoml.py | 8 +++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/setuptools/config/_apply_pyprojecttoml.py b/setuptools/config/_apply_pyprojecttoml.py index 5d34cdb7d8..fce5c40e34 100644 --- a/setuptools/config/_apply_pyprojecttoml.py +++ b/setuptools/config/_apply_pyprojecttoml.py @@ -303,7 +303,7 @@ def _some_attrgetter(*items): """ def _acessor(obj): values = (_attrgetter(i)(obj) for i in items) - return next((i for i in values if i), None) + return next((i for i in values if i is not None), None) return _acessor diff --git a/setuptools/config/pyprojecttoml.py b/setuptools/config/pyprojecttoml.py index 0ee1b8f9a1..e20d71d2ff 100644 --- a/setuptools/config/pyprojecttoml.py +++ b/setuptools/config/pyprojecttoml.py @@ -282,11 +282,12 @@ def _expand_all_dynamic(self, dist: "Distribution", package_dir: Mapping[str, st ) # `None` indicates there is nothing in `tool.setuptools.dynamic` but the value # might have already been set by setup.py/extensions, so avoid overwriting. - self.project_cfg.update({k: v for k, v in obtained_dynamic.items() if v}) + updates = {k: v for k, v in obtained_dynamic.items() if v is not None} + self.project_cfg.update(updates) def _ensure_previously_set(self, dist: "Distribution", field: str): previous = _PREVIOUSLY_DEFINED[field](dist) - if not previous and not self.ignore_option_errors: + if previous is None and not self.ignore_option_errors: msg = ( f"No configuration found for dynamic {field!r}.\n" "Some dynamic fields need to be specified via `tool.setuptools.dynamic`" diff --git a/setuptools/tests/config/test_apply_pyprojecttoml.py b/setuptools/tests/config/test_apply_pyprojecttoml.py index a88bc1ec19..b822096363 100644 --- a/setuptools/tests/config/test_apply_pyprojecttoml.py +++ b/setuptools/tests/config/test_apply_pyprojecttoml.py @@ -14,7 +14,7 @@ from setuptools.dist import Distribution from setuptools.config import setupcfg, pyprojecttoml from setuptools.config import expand -from setuptools.config._apply_pyprojecttoml import _WouldIgnoreField +from setuptools.config._apply_pyprojecttoml import _WouldIgnoreField, _some_attrgetter from setuptools.command.egg_info import write_requirements @@ -234,12 +234,14 @@ def test_not_listed_in_dynamic(self, tmp_path, attr, field, value): dist = pyprojecttoml.apply_configuration(dist, pyproject) # TODO: Once support for pyproject.toml config stabilizes attr should be None - dist_value = getattr(dist, attr, None) or getattr(dist.metadata, attr, object()) + dist_value = _some_attrgetter(f"metadata.{attr}", attr)(dist) assert dist_value == value @pytest.mark.parametrize( "attr, field, value", [ + ("install_requires", "dependencies", []), + ("extras_require", "optional-dependencies", {}), ("install_requires", "dependencies", ["six"]), ("classifiers", "classifiers", ["Private :: Classifier"]), ] @@ -248,7 +250,7 @@ def test_listed_in_dynamic(self, tmp_path, attr, field, value): pyproject = self.pyproject(tmp_path, [field]) dist = makedist(tmp_path, **{attr: value}) dist = pyprojecttoml.apply_configuration(dist, pyproject) - dist_value = getattr(dist, attr, None) or getattr(dist.metadata, attr, object()) + dist_value = _some_attrgetter(f"metadata.{attr}", attr)(dist) assert dist_value == value def test_optional_dependencies_dont_remove_env_markers(self, tmp_path):