diff --git a/conftest.py b/conftest.py index 43f33ba45a..723e5b4355 100644 --- a/conftest.py +++ b/conftest.py @@ -19,6 +19,7 @@ def pytest_addoption(parser): def pytest_configure(config): config.addinivalue_line("markers", "integration: integration tests") + config.addinivalue_line("markers", "uses_network: tests may try to download files") collect_ignore = [ diff --git a/setup.cfg b/setup.cfg index 6171f62419..7e4288508d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,6 +67,7 @@ testing = build[virtualenv] filelock>=3.4.0 pip_run>=8.8 + ini2toml[lite]>=0.9 testing-integration = pytest @@ -80,7 +81,6 @@ testing-integration = build[virtualenv] filelock>=3.4.0 - docs = # upstream sphinx diff --git a/setuptools/config/_apply_pyprojecttoml.py b/setuptools/config/_apply_pyprojecttoml.py new file mode 100644 index 0000000000..f711c8a2ec --- /dev/null +++ b/setuptools/config/_apply_pyprojecttoml.py @@ -0,0 +1,258 @@ +"""Translation layer between pyproject config and setuptools distribution and +metadata objects. + +The distribution and metadata objects are modeled after (an old version of) +core metadata, therefore configs in the format specified for ``pyproject.toml`` +need to be processed before being applied. +""" +import os +from collections.abc import Mapping +from email.headerregistry import Address +from functools import partial +from itertools import chain +from types import MappingProxyType +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, + Type, Union) + +if TYPE_CHECKING: + from setuptools._importlib import metadata # noqa + from setuptools.dist import Distribution # noqa + +EMPTY = MappingProxyType({}) # Immutable dict-like +_Path = Union[os.PathLike, str] +_DictOrStr = Union[dict, str] +_CorrespFn = Callable[["Distribution", Any, _Path], None] +_Correspondence = Union[str, _CorrespFn] + + +def apply(dist: "Distribution", config: dict, filename: _Path) -> "Distribution": + """Apply configuration dict read with :func:`read_configuration`""" + + root_dir = os.path.dirname(filename) or "." + tool_table = config.get("tool", {}).get("setuptools", {}) + project_table = config.get("project", {}).copy() + _unify_entry_points(project_table) + _dynamic_license(project_table, tool_table) + for field, value in project_table.items(): + norm_key = json_compatible_key(field) + corresp = PYPROJECT_CORRESPONDENCE.get(norm_key, norm_key) + if callable(corresp): + corresp(dist, value, root_dir) + else: + _set_config(dist, corresp, value) + + for field, value in tool_table.items(): + norm_key = json_compatible_key(field) + norm_key = TOOL_TABLE_RENAMES.get(norm_key, norm_key) + _set_config(dist, norm_key, value) + + _copy_command_options(config, dist, filename) + + current_directory = os.getcwd() + os.chdir(root_dir) + try: + dist._finalize_requires() + dist._finalize_license_files() + finally: + os.chdir(current_directory) + + return dist + + +def json_compatible_key(key: str) -> str: + """As defined in :pep:`566#json-compatible-metadata`""" + return key.lower().replace("-", "_") + + +def _set_config(dist: "Distribution", field: str, value: Any): + setter = getattr(dist.metadata, f"set_{field}", None) + if setter: + setter(value) + elif hasattr(dist.metadata, field) or field in SETUPTOOLS_PATCHES: + setattr(dist.metadata, field, value) + else: + setattr(dist, field, value) + + +_CONTENT_TYPES = { + ".md": "text/markdown", + ".rst": "text/x-rst", + ".txt": "text/plain", +} + + +def _guess_content_type(file: str) -> Optional[str]: + _, ext = os.path.splitext(file.lower()) + if not ext: + return None + + if ext in _CONTENT_TYPES: + return _CONTENT_TYPES[ext] + + valid = ", ".join(f"{k} ({v})" for k, v in _CONTENT_TYPES.items()) + msg = f"only the following file extensions are recognized: {valid}." + raise ValueError(f"Undefined content type for {file}, {msg}") + + +def _long_description(dist: "Distribution", val: _DictOrStr, root_dir: _Path): + from setuptools.config import expand + + if isinstance(val, str): + text = expand.read_files(val, root_dir) + ctype = _guess_content_type(val) + else: + text = val.get("text") or expand.read_files(val.get("file", []), root_dir) + ctype = val["content-type"] + + _set_config(dist, "long_description", text) + if ctype: + _set_config(dist, "long_description_content_type", ctype) + + +def _license(dist: "Distribution", val: Union[str, dict], _root_dir): + if isinstance(val, str): + _set_config(dist, "license", val) + elif "file" in val: + _set_config(dist, "license_files", [val["file"]]) + else: + _set_config(dist, "license", val["text"]) + + +def _people(dist: "Distribution", val: List[dict], _root_dir: _Path, kind: str): + field = [] + email_field = [] + for person in val: + if "name" not in person: + email_field.append(person["email"]) + elif "email" not in person: + field.append(person["name"]) + else: + addr = Address(display_name=person["name"], addr_spec=person["email"]) + email_field.append(str(addr)) + + if field: + _set_config(dist, kind, ", ".join(field)) + if email_field: + _set_config(dist, f"{kind}_email", ", ".join(email_field)) + + +def _project_urls(dist: "Distribution", val: dict, _root_dir): + special = {"downloadurl": "download_url", "homepage": "url"} + for key, url in val.items(): + norm_key = json_compatible_key(key).replace("_", "") + _set_config(dist, special.get(norm_key, key), url) + _set_config(dist, "project_urls", val.copy()) + + +def _python_requires(dist: "Distribution", val: dict, _root_dir): + from setuptools.extern.packaging.specifiers import SpecifierSet + + _set_config(dist, "python_requires", SpecifierSet(val)) + + +def _dynamic_license(project_table: dict, tool_table: dict): + # Dynamic license needs special handling (cannot be expanded in terms of PEP 621) + # due to the mutually exclusive `text` and `file` + dynamic_license = {"license", "license_files"} + dynamic = {json_compatible_key(k) for k in project_table.get("dynamic", [])} + dynamic_cfg = tool_table.get("dynamic", {}) + dynamic_cfg.setdefault("license_files", DEFAULT_LICENSE_FILES) + keys = set(dynamic_cfg) & dynamic_license if "license" in dynamic else set() + + for key in keys: + norm_key = json_compatible_key(key) + project_table[norm_key] = dynamic_cfg[key] + + +def _unify_entry_points(project_table: dict): + project = project_table + entry_points = project.pop("entry-points", project.pop("entry_points", {})) + renaming = {"scripts": "console_scripts", "gui_scripts": "gui_scripts"} + for key, value in list(project.items()): # eager to allow modifications + norm_key = json_compatible_key(key) + if norm_key in renaming and value: + entry_points[renaming[norm_key]] = project.pop(key) + + if entry_points: + project["entry-points"] = { + name: [f"{k} = {v}" for k, v in group.items()] + for name, group in entry_points.items() + } + + +def _copy_command_options(pyproject: dict, dist: "Distribution", filename: _Path): + from distutils import log + + tool_table = pyproject.get("tool", {}) + cmdclass = tool_table.get("setuptools", {}).get("cmdclass", {}) + valid_options = _valid_command_options(cmdclass) + + cmd_opts = dist.command_options + for cmd, config in pyproject.get("tool", {}).get("distutils", {}).items(): + cmd = json_compatible_key(cmd) + valid = valid_options.get(cmd, set()) + cmd_opts.setdefault(cmd, {}) + for key, value in config.items(): + key = json_compatible_key(key) + cmd_opts[cmd][key] = (str(filename), value) + if key not in valid: + # To avoid removing options that are specified dynamically we + # just log a warn... + log.warn(f"Command option {cmd}.{key} is not defined") + + +def _valid_command_options(cmdclass: Mapping = EMPTY) -> Dict[str, Set[str]]: + from .._importlib import metadata + from setuptools.dist import Distribution + + valid_options = {"global": _normalise_cmd_options(Distribution.global_options)} + + unloaded_entry_points = metadata.entry_points(group='distutils.commands') + loaded_entry_points = (_load_ep(ep) for ep in unloaded_entry_points) + entry_points = (ep for ep in loaded_entry_points if ep) + for cmd, cmd_class in chain(entry_points, cmdclass.items()): + opts = valid_options.get(cmd, set()) + opts = opts | _normalise_cmd_options(getattr(cmd_class, "user_options", [])) + valid_options[cmd] = opts + + return valid_options + + +def _load_ep(ep: "metadata.EntryPoint") -> Optional[Tuple[str, Type]]: + # Ignore all the errors + try: + return (ep.name, ep.load()) + except Exception as ex: + from distutils import log + msg = f"{ex.__class__.__name__} while trying to load entry-point {ep.name}" + log.warn(f"{msg}: {ex}") + return None + + +def _normalise_cmd_option_key(name: str) -> str: + return json_compatible_key(name).strip("_=") + + +def _normalise_cmd_options(desc: List[Tuple[str, Optional[str], str]]) -> Set[str]: + return {_normalise_cmd_option_key(fancy_option[0]) for fancy_option in desc} + + +PYPROJECT_CORRESPONDENCE: Dict[str, _Correspondence] = { + "readme": _long_description, + "license": _license, + "authors": partial(_people, kind="author"), + "maintainers": partial(_people, kind="maintainer"), + "urls": _project_urls, + "dependencies": "install_requires", + "optional_dependencies": "extras_require", + "requires_python": _python_requires, +} + +TOOL_TABLE_RENAMES = {"script_files": "scripts"} + +SETUPTOOLS_PATCHES = {"long_description_content_type", "project_urls", + "provides_extras", "license_file", "license_files"} + + +DEFAULT_LICENSE_FILES = ('LICEN[CS]E*', 'COPYING*', 'NOTICE*', 'AUTHORS*') +# defaults from the `wheel` package and historically used by setuptools diff --git a/setuptools/config/expand.py b/setuptools/config/expand.py index feb55be190..4778ffb6d6 100644 --- a/setuptools/config/expand.py +++ b/setuptools/config/expand.py @@ -21,6 +21,7 @@ import os import sys from glob import iglob +from configparser import ConfigParser from itertools import chain from distutils.errors import DistutilsOptionError @@ -292,3 +293,18 @@ def canonic_data_files(data_files, root_dir=None): (dest, glob_relative(patterns, root_dir)) for dest, patterns in data_files.items() ] + + +def entry_points(text, text_source="entry-points"): + """Given the contents of entry-points file, + process it into a 2-level dictionary (``dict[str, dict[str, str]]``). + The first level keys are entry-point groups, the second level keys are + entry-point names, and the second level values are references to objects + (that correspond to the entry-point value). + """ + parser = ConfigParser(default_section=None, delimiters=("=",)) + parser.optionxform = str # case sensitive + parser.read_string(text, text_source) + groups = {k: dict(v.items()) for k, v in parser.items()} + groups.pop(parser.default_section, None) + return groups diff --git a/setuptools/config/pyprojecttoml.py b/setuptools/config/pyprojecttoml.py new file mode 100644 index 0000000000..9075c79143 --- /dev/null +++ b/setuptools/config/pyprojecttoml.py @@ -0,0 +1,220 @@ +"""Load setuptools configuration from ``pyproject.toml`` files""" +import json +import os +from contextlib import contextmanager +from distutils import log +from functools import partial +from typing import TYPE_CHECKING, Union + +from setuptools.errors import FileError, OptionError + +from . import expand as _expand +from ._apply_pyprojecttoml import apply + +if TYPE_CHECKING: + from setuptools.dist import Distribution # noqa + +_Path = Union[str, os.PathLike] + + +def load_file(filepath: _Path) -> dict: + from setuptools.extern import tomli # type: ignore + + with open(filepath, "rb") as file: + return tomli.load(file) + + +def validate(config: dict, filepath: _Path): + from setuptools.extern import _validate_pyproject + from setuptools.extern._validate_pyproject import fastjsonschema_exceptions + + try: + return _validate_pyproject.validate(config) + except fastjsonschema_exceptions.JsonSchemaValueException as ex: + msg = [f"Schema: {ex}"] + if ex.value: + msg.append(f"Given value:\n{json.dumps(ex.value, indent=2)}") + if ex.rule: + msg.append(f"Offending rule: {json.dumps(ex.rule, indent=2)}") + if ex.definition: + msg.append(f"Definition:\n{json.dumps(ex.definition, indent=2)}") + + log.error("\n\n".join(msg) + "\n") + raise + + +def apply_configuration(dist: "Distribution", filepath: _Path) -> "Distribution": + """Apply the configuration from a ``pyproject.toml`` file into an existing + distribution object. + """ + config = read_configuration(filepath) + return apply(dist, config, filepath) + + +def read_configuration(filepath, expand=True, ignore_option_errors=False): + """Read given configuration file and returns options from it as a dict. + + :param str|unicode filepath: Path to configuration file in the ``pyproject.toml`` + format. + + :param bool expand: Whether to expand directives and other computed values + (i.e. post-process the given configuration) + + :param bool ignore_option_errors: Whether to silently ignore + options, values of which could not be resolved (e.g. due to exceptions + in directives such as file:, attr:, etc.). + If False exceptions are propagated as expected. + + :rtype: dict + """ + filepath = os.path.abspath(filepath) + + if not os.path.isfile(filepath): + raise FileError(f"Configuration file {filepath!r} does not exist.") + + asdict = load_file(filepath) or {} + project_table = asdict.get("project", {}) + tool_table = asdict.get("tool", {}).get("setuptools", {}) + if not asdict or not(project_table or tool_table): + return {} # User is not using pyproject to configure setuptools + + # There is an overall sense in the community that making include_package_data=True + # the default would be an improvement. + # `ini2toml` backfills include_package_data=False when nothing is explicitly given, + # therefore setting a default here is backwards compatible. + tool_table.setdefault("include-package-data", True) + + with _ignore_errors(ignore_option_errors): + # Don't complain about unrelated errors (e.g. tools not using the "tool" table) + subset = {"project": project_table, "tool": {"setuptools": tool_table}} + validate(subset, filepath) + + if expand: + root_dir = os.path.dirname(filepath) + return expand_configuration(asdict, root_dir, ignore_option_errors) + + return asdict + + +def expand_configuration(config, root_dir=None, ignore_option_errors=False): + """Given a configuration with unresolved fields (e.g. dynamic, cmdclass, ...) + find their final values. + + :param dict config: Dict containing the configuration for the distribution + :param str root_dir: Top-level directory for the distribution/project + (the same directory where ``pyproject.toml`` is place) + :param bool ignore_option_errors: see :func:`read_configuration` + + :rtype: dict + """ + root_dir = root_dir or os.getcwd() + project_cfg = config.get("project", {}) + setuptools_cfg = config.get("tool", {}).get("setuptools", {}) + package_dir = setuptools_cfg.get("package-dir") + + _expand_all_dynamic(project_cfg, setuptools_cfg, root_dir, ignore_option_errors) + _expand_packages(setuptools_cfg, root_dir, ignore_option_errors) + _canonic_package_data(setuptools_cfg) + _canonic_package_data(setuptools_cfg, "exclude-package-data") + + process = partial(_process_field, ignore_option_errors=ignore_option_errors) + cmdclass = partial(_expand.cmdclass, package_dir=package_dir, root_dir=root_dir) + data_files = partial(_expand.canonic_data_files, root_dir=root_dir) + process(setuptools_cfg, "data-files", data_files) + process(setuptools_cfg, "cmdclass", cmdclass) + + return config + + +def _expand_all_dynamic(project_cfg, setuptools_cfg, root_dir, ignore_option_errors): + silent = ignore_option_errors + dynamic_cfg = setuptools_cfg.get("dynamic", {}) + package_dir = setuptools_cfg.get("package-dir", None) + special = ("license", "readme", "version", "entry-points", "scripts", "gui-scripts") + # license-files are handled directly in the metadata, so no expansion + # readme, version and entry-points need special handling + dynamic = project_cfg.get("dynamic", []) + regular_dynamic = (x for x in dynamic if x not in special) + + for field in regular_dynamic: + value = _expand_dynamic(dynamic_cfg, field, package_dir, root_dir, silent) + project_cfg[field] = value + + if "version" in dynamic and "version" in dynamic_cfg: + version = _expand_dynamic(dynamic_cfg, "version", package_dir, root_dir, silent) + project_cfg["version"] = _expand.version(version) + + if "readme" in dynamic: + project_cfg["readme"] = _expand_readme(dynamic_cfg, root_dir, silent) + + if "entry-points" in dynamic: + field = "entry-points" + value = _expand_dynamic(dynamic_cfg, field, package_dir, root_dir, silent) + project_cfg.update(_expand_entry_points(value, dynamic)) + + +def _expand_dynamic(dynamic_cfg, field, package_dir, root_dir, ignore_option_errors): + if field in dynamic_cfg: + directive = dynamic_cfg[field] + if "file" in directive: + return _expand.read_files(directive["file"], root_dir) + if "attr" in directive: + return _expand.read_attr(directive["attr"], package_dir, root_dir) + elif not ignore_option_errors: + msg = f"Impossible to expand dynamic value of {field!r}. " + msg += f"No configuration found for `tool.setuptools.dynamic.{field}`" + raise OptionError(msg) + return None + + +def _expand_readme(dynamic_cfg, root_dir, ignore_option_errors): + silent = ignore_option_errors + return { + "text": _expand_dynamic(dynamic_cfg, "readme", None, root_dir, silent), + "content-type": dynamic_cfg["readme"].get("content-type", "text/x-rst") + } + + +def _expand_entry_points(text, dynamic): + groups = _expand.entry_points(text) + expanded = {"entry-points": groups} + if "scripts" in dynamic and "console_scripts" in groups: + expanded["scripts"] = groups.pop("console_scripts") + if "gui-scripts" in dynamic and "gui_scripts" in groups: + expanded["gui-scripts"] = groups.pop("gui_scripts") + return expanded + + +def _expand_packages(setuptools_cfg, root_dir, ignore_option_errors=False): + packages = setuptools_cfg.get("packages") + if packages is None or isinstance(packages, (list, tuple)): + return + + find = packages.get("find") + if isinstance(find, dict): + find["root_dir"] = root_dir + with _ignore_errors(ignore_option_errors): + setuptools_cfg["packages"] = _expand.find_packages(**find) + + +def _process_field(container, field, fn, ignore_option_errors=False): + if field in container: + with _ignore_errors(ignore_option_errors): + container[field] = fn(container[field]) + + +def _canonic_package_data(setuptools_cfg, field="package-data"): + package_data = setuptools_cfg.get(field, {}) + return _expand.canonic_package_data(package_data) + + +@contextmanager +def _ignore_errors(ignore_option_errors): + if not ignore_option_errors: + yield + return + + try: + yield + except Exception as ex: + log.debug(f"Ignored error: {ex.__class__.__name__} - {ex}") diff --git a/setuptools/config/setupcfg.py b/setuptools/config/setupcfg.py index 80cf454107..e4855a7680 100644 --- a/setuptools/config/setupcfg.py +++ b/setuptools/config/setupcfg.py @@ -10,7 +10,8 @@ from distutils.errors import DistutilsOptionError, DistutilsFileError from setuptools.extern.packaging.version import Version, InvalidVersion from setuptools.extern.packaging.specifiers import SpecifierSet -from setuptools.config import expand + +from . import expand def read_configuration(filepath, find_others=False, ignore_option_errors=False): @@ -29,7 +30,26 @@ def read_configuration(filepath, find_others=False, ignore_option_errors=False): :rtype: dict """ - from setuptools.dist import Distribution, _Distribution + from setuptools.dist import Distribution + + dist = Distribution() + filenames = dist.find_config_files() if find_others else [] + handlers = _apply(dist, filepath, filenames, ignore_option_errors) + return configuration_to_dict(handlers) + + +def apply_configuration(dist, filepath): + """Apply the configuration from a ``setup.cfg`` file into an existing + distribution object. + """ + _apply(dist, filepath) + dist._finalize_requires() + return dist + + +def _apply(dist, filepath, other_files=(), ignore_option_errors=False): + """Read configuration from ``filepath`` and applies to the ``dist`` object.""" + from setuptools.dist import _Distribution filepath = os.path.abspath(filepath) @@ -38,24 +58,18 @@ def read_configuration(filepath, find_others=False, ignore_option_errors=False): current_directory = os.getcwd() os.chdir(os.path.dirname(filepath)) + filenames = [*other_files, filepath] try: - dist = Distribution() - - filenames = dist.find_config_files() if find_others else [] - if filepath not in filenames: - filenames.append(filepath) - _Distribution.parse_config_files(dist, filenames=filenames) - handlers = parse_configuration( dist, dist.command_options, ignore_option_errors=ignore_option_errors ) - + dist._finalize_license_files() finally: os.chdir(current_directory) - return configuration_to_dict(handlers) + return handlers def _get_option(target_obj, key): diff --git a/setuptools/tests/config/downloads/.gitignore b/setuptools/tests/config/downloads/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/setuptools/tests/config/downloads/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/setuptools/tests/config/setupcfg_examples.txt b/setuptools/tests/config/setupcfg_examples.txt new file mode 100644 index 0000000000..5db3565464 --- /dev/null +++ b/setuptools/tests/config/setupcfg_examples.txt @@ -0,0 +1,23 @@ +# ==================================================================== +# Some popular packages that use setup.cfg (and others not so popular) +# Reference: https://hugovk.github.io/top-pypi-packages/ +# ==================================================================== +https://github.com/pypa/setuptools/raw/52c990172fec37766b3566679724aa8bf70ae06d/setup.cfg +https://github.com/pypa/wheel/raw/0acd203cd896afec7f715aa2ff5980a403459a3b/setup.cfg +https://github.com/python/importlib_metadata/raw/2f05392ca980952a6960d82b2f2d2ea10aa53239/setup.cfg +https://github.com/jaraco/skeleton/raw/d9008b5c510cd6969127a6a2ab6f832edddef296/setup.cfg +https://github.com/jaraco/zipp/raw/700d3a96390e970b6b962823bfea78b4f7e1c537/setup.cfg +https://github.com/pallets/jinja/raw/7d72eb7fefb7dce065193967f31f805180508448/setup.cfg +https://github.com/tkem/cachetools/raw/2fd87a94b8d3861d80e9e4236cd480bfdd21c90d/setup.cfg +https://github.com/aio-libs/aiohttp/raw/5e0e6b7080f2408d5f1dd544c0e1cf88378b7b10/setup.cfg +https://github.com/pallets/flask/raw/9486b6cf57bd6a8a261f67091aca8ca78eeec1e3/setup.cfg +https://github.com/pallets/click/raw/6411f425fae545f42795665af4162006b36c5e4a/setup.cfg +https://github.com/sqlalchemy/sqlalchemy/raw/533f5718904b620be8d63f2474229945d6f8ba5d/setup.cfg +https://github.com/pytest-dev/pluggy/raw/461ef63291d13589c4e21aa182cd1529257e9a0a/setup.cfg +https://github.com/pytest-dev/pytest/raw/c7be96dae487edbd2f55b561b31b68afac1dabe6/setup.cfg +https://github.com/tqdm/tqdm/raw/fc69d5dcf578f7c7986fa76841a6b793f813df35/setup.cfg +https://github.com/platformdirs/platformdirs/raw/7b7852128dd6f07511b618d6edea35046bd0c6ff/setup.cfg +https://github.com/pandas-dev/pandas/raw/bc17343f934a33dc231c8c74be95d8365537c376/setup.cfg +https://github.com/django/django/raw/4e249d11a6e56ca8feb4b055b681cec457ef3a3d/setup.cfg +https://github.com/pyscaffold/pyscaffold/raw/de7aa5dc059fbd04307419c667cc4961bc9df4b8/setup.cfg +https://github.com/pypa/virtualenv/raw/f92eda6e3da26a4d28c2663ffb85c4960bdb990c/setup.cfg diff --git a/setuptools/tests/config/test_apply_pyprojecttoml.py b/setuptools/tests/config/test_apply_pyprojecttoml.py new file mode 100644 index 0000000000..5b5a8dfa7f --- /dev/null +++ b/setuptools/tests/config/test_apply_pyprojecttoml.py @@ -0,0 +1,225 @@ +"""Make sure that applying the configuration from pyproject.toml is equivalent to +applying a similar configuration from setup.cfg +""" +import io +import re +from pathlib import Path +from urllib.request import urlopen +from unittest.mock import Mock + +import pytest +from ini2toml.api import Translator + +import setuptools # noqa ensure monkey patch to metadata +from setuptools.dist import Distribution +from setuptools.config import setupcfg, pyprojecttoml +from setuptools.config import expand + + +EXAMPLES = (Path(__file__).parent / "setupcfg_examples.txt").read_text() +EXAMPLE_URLS = [x for x in EXAMPLES.splitlines() if not x.startswith("#")] +DOWNLOAD_DIR = Path(__file__).parent / "downloads" + + +@pytest.mark.parametrize("url", EXAMPLE_URLS) +@pytest.mark.filterwarnings("ignore") +@pytest.mark.uses_network +def test_apply_pyproject_equivalent_to_setupcfg(url, monkeypatch, tmp_path): + monkeypatch.setattr(expand, "read_attr", Mock(return_value="0.0.1")) + setupcfg_example = retrieve_file(url, DOWNLOAD_DIR) + pyproject_example = Path(tmp_path, "pyproject.toml") + toml_config = Translator().translate(setupcfg_example.read_text(), "setup.cfg") + pyproject_example.write_text(toml_config) + + dist_toml = pyprojecttoml.apply_configuration(Distribution(), pyproject_example) + dist_cfg = setupcfg.apply_configuration(Distribution(), setupcfg_example) + + pkg_info_toml = core_metadata(dist_toml) + pkg_info_cfg = core_metadata(dist_cfg) + assert pkg_info_toml == pkg_info_cfg + + if any(getattr(d, "license_files", None) for d in (dist_toml, dist_cfg)): + assert set(dist_toml.license_files) == set(dist_cfg.license_files) + + if any(getattr(d, "entry_points", None) for d in (dist_toml, dist_cfg)): + print(dist_cfg.entry_points) + ep_toml = {(k, *sorted(i.replace(" ", "") for i in v)) + for k, v in dist_toml.entry_points.items()} + ep_cfg = {(k, *sorted(i.replace(" ", "") for i in v)) + for k, v in dist_cfg.entry_points.items()} + assert ep_toml == ep_cfg + + if any(getattr(d, "package_data", None) for d in (dist_toml, dist_cfg)): + pkg_data_toml = {(k, *sorted(v)) for k, v in dist_toml.package_data.items()} + pkg_data_cfg = {(k, *sorted(v)) for k, v in dist_cfg.package_data.items()} + assert pkg_data_toml == pkg_data_cfg + + if any(getattr(d, "data_files", None) for d in (dist_toml, dist_cfg)): + data_files_toml = {(k, *sorted(v)) for k, v in dist_toml.data_files} + data_files_cfg = {(k, *sorted(v)) for k, v in dist_cfg.data_files} + assert data_files_toml == data_files_cfg + + assert set(dist_toml.install_requires) == set(dist_cfg.install_requires) + if any(getattr(d, "extras_require", None) for d in (dist_toml, dist_cfg)): + if ( + "testing" in dist_toml.extras_require + and "testing" not in dist_cfg.extras_require + ): + # ini2toml can automatically convert `tests_require` to `testing` extra + dist_toml.extras_require.pop("testing") + extra_req_toml = {(k, *sorted(v)) for k, v in dist_toml.extras_require.items()} + extra_req_cfg = {(k, *sorted(v)) for k, v in dist_cfg.extras_require.items()} + assert extra_req_toml == extra_req_cfg + + +PEP621_EXAMPLE = """\ +[project] +name = "spam" +version = "2020.0.0" +description = "Lovely Spam! Wonderful Spam!" +readme = "README.rst" +requires-python = ">=3.8" +license = {file = "LICENSE.txt"} +keywords = ["egg", "bacon", "sausage", "tomatoes", "Lobster Thermidor"] +authors = [ + {email = "hi@pradyunsg.me"}, + {name = "Tzu-Ping Chung"} +] +maintainers = [ + {name = "Brett Cannon", email = "brett@python.org"} +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python" +] + +dependencies = [ + "httpx", + "gidgethub[httpx]>4.0.0", + "django>2.1; os_name != 'nt'", + "django>2.0; os_name == 'nt'" +] + +[project.optional-dependencies] +test = [ + "pytest < 5.0.0", + "pytest-cov[all]" +] + +[project.urls] +homepage = "http://example.com" +documentation = "http://readthedocs.org" +repository = "http://github.com" +changelog = "http://github.com/me/spam/blob/master/CHANGELOG.md" + +[project.scripts] +spam-cli = "spam:main_cli" + +[project.gui-scripts] +spam-gui = "spam:main_gui" + +[project.entry-points."spam.magical"] +tomatoes = "spam:main_tomatoes" +""" + +PEP621_EXAMPLE_SCRIPT = """ +def main_cli(): pass +def main_gui(): pass +def main_tomatoes(): pass +""" + + +def _pep621_example_project(tmp_path, readme="README.rst"): + pyproject = tmp_path / "pyproject.toml" + text = PEP621_EXAMPLE + replacements = {'readme = "README.rst"': f'readme = "{readme}"'} + for orig, subst in replacements.items(): + text = text.replace(orig, subst) + pyproject.write_text(text) + + (tmp_path / "README.rst").write_text("hello world") + (tmp_path / "LICENSE.txt").write_text("--- LICENSE stub ---") + (tmp_path / "spam.py").write_text(PEP621_EXAMPLE_SCRIPT) + return pyproject + + +def test_pep621_example(tmp_path): + """Make sure the example in PEP 621 works""" + pyproject = _pep621_example_project(tmp_path) + dist = pyprojecttoml.apply_configuration(Distribution(), pyproject) + assert set(dist.metadata.license_files) == {"LICENSE.txt"} + + +@pytest.mark.parametrize( + "readme, ctype", + [ + ("Readme.txt", "text/plain"), + ("readme.md", "text/markdown"), + ("text.rst", "text/x-rst"), + ] +) +def test_readme_content_type(tmp_path, readme, ctype): + pyproject = _pep621_example_project(tmp_path, readme) + dist = pyprojecttoml.apply_configuration(Distribution(), pyproject) + assert dist.metadata.long_description_content_type == ctype + + +def test_undefined_content_type(tmp_path): + pyproject = _pep621_example_project(tmp_path, "README.tex") + with pytest.raises(ValueError, match="Undefined content type for README.tex"): + pyprojecttoml.apply_configuration(Distribution(), pyproject) + + +def test_no_explicit_content_type_for_missing_extension(tmp_path): + pyproject = _pep621_example_project(tmp_path, "README") + dist = pyprojecttoml.apply_configuration(Distribution(), pyproject) + assert dist.metadata.long_description_content_type is None + + +# --- Auxiliary Functions --- + + +NAME_REMOVE = ("http://", "https://", "github.com/", "/raw/") + + +def retrieve_file(url, download_dir): + file_name = url.strip() + for part in NAME_REMOVE: + file_name = file_name.replace(part, '').strip().strip('/:').strip() + file_name = re.sub(r"[^\-_\.\w\d]+", "_", file_name) + path = Path(download_dir, file_name) + if not path.exists(): + download_dir.mkdir(exist_ok=True, parents=True) + download(url, path) + return path + + +def download(url, dest): + with urlopen(url) as f: + data = f.read() + + with open(dest, "wb") as f: + f.write(data) + + assert Path(dest).exists() + + +def core_metadata(dist) -> str: + with io.StringIO() as buffer: + dist.metadata.write_pkg_file(buffer) + value = "\n".join(buffer.getvalue().strip().splitlines()) + + # ---- DIFF NORMALISATION ---- + # PEP 621 is very particular about author/maintainer metadata conversion, so skip + value = re.sub(r"^(Author|Maintainer)(-email)?:.*$", "", value, flags=re.M) + # May be redundant with Home-page + value = re.sub(r"^Project-URL: Homepage,.*$", "", value, flags=re.M) + # May be missing in original (relying on default) but backfilled in the TOML + value = re.sub(r"^Description-Content-Type:.*$", "", value, flags=re.M) + # ini2toml can automatically convert `tests_require` to `testing` extra + value = value.replace("Provides-Extra: testing\n", "") + # Remove empty lines + value = re.sub(r"^\s*$", "", value, flags=re.M) + value = re.sub(r"^\n", "", value, flags=re.M) + + return value diff --git a/setuptools/tests/config/test_pyprojecttoml.py b/setuptools/tests/config/test_pyprojecttoml.py new file mode 100644 index 0000000000..759f04542c --- /dev/null +++ b/setuptools/tests/config/test_pyprojecttoml.py @@ -0,0 +1,167 @@ +from configparser import ConfigParser + +import pytest + +from setuptools.config.pyprojecttoml import read_configuration, expand_configuration + +EXAMPLE = """ +[project] +name = "myproj" +keywords = ["some", "key", "words"] +dynamic = ["version", "readme"] +requires-python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +dependencies = [ + 'importlib-metadata>=0.12;python_version<"3.8"', + 'importlib-resources>=1.0;python_version<"3.7"', + 'pathlib2>=2.3.3,<3;python_version < "3.4" and sys.platform != "win32"', +] + +[project.optional-dependencies] +docs = [ + "sphinx>=3", + "sphinx-argparse>=0.2.5", + "sphinx-rtd-theme>=0.4.3", +] +testing = [ + "pytest>=1", + "coverage>=3,<5", +] + +[project.scripts] +exec = "pkg.__main__:exec" + +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +package-dir = {"" = "src"} +zip-safe = true +platforms = ["any"] + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = true + +[tool.setuptools.cmdclass] +sdist = "pkg.mod.CustomSdist" + +[tool.setuptools.dynamic.version] +attr = "pkg.__version__.VERSION" + +[tool.setuptools.dynamic.readme] +file = ["README.md"] +content-type = "text/markdown" + +[tool.setuptools.package-data] +"*" = ["*.txt"] + +[tool.setuptools.data-files] +"data" = ["files/*.txt"] + +[tool.distutils.sdist] +formats = "gztar" + +[tool.distutils.bdist_wheel] +universal = true +""" + + +def test_read_configuration(tmp_path): + pyproject = tmp_path / "pyproject.toml" + + files = [ + "src/pkg/__init__.py", + "src/other/nested/__init__.py", + "files/file.txt" + ] + for file in files: + (tmp_path / file).parent.mkdir(exist_ok=True, parents=True) + (tmp_path / file).touch() + + pyproject.write_text(EXAMPLE) + (tmp_path / "README.md").write_text("hello world") + (tmp_path / "src/pkg/mod.py").write_text("class CustomSdist: pass") + (tmp_path / "src/pkg/__version__.py").write_text("VERSION = (3, 10)") + (tmp_path / "src/pkg/__main__.py").write_text("def exec(): print('hello')") + + config = read_configuration(pyproject, expand=False) + assert config["project"].get("version") is None + assert config["project"].get("readme") is None + + expanded = expand_configuration(config, tmp_path) + expanded_project = expanded["project"] + assert read_configuration(pyproject, expand=True) == expanded + assert expanded_project["version"] == "3.10" + assert expanded_project["readme"]["text"] == "hello world" + assert set(expanded["tool"]["setuptools"]["packages"]) == { + "pkg", + "other", + "other.nested", + } + assert "" in expanded["tool"]["setuptools"]["package-data"] + assert "*" not in expanded["tool"]["setuptools"]["package-data"] + assert expanded["tool"]["setuptools"]["data-files"] == [ + ("data", ["files/file.txt"]) + ] + + +ENTRY_POINTS = { + "console_scripts": {"a": "mod.a:func"}, + "gui_scripts": {"b": "mod.b:func"}, + "other": {"c": "mod.c:func [extra]"}, +} + + +def test_expand_entry_point(tmp_path): + entry_points = ConfigParser() + entry_points.read_dict(ENTRY_POINTS) + with open(tmp_path / "entry-points.txt", "w") as f: + entry_points.write(f) + + tool = {"setuptools": {"dynamic": {"entry-points": {"file": "entry-points.txt"}}}} + project = {"dynamic": ["scripts", "gui-scripts", "entry-points"]} + pyproject = {"project": project, "tool": tool} + expanded = expand_configuration(pyproject, tmp_path) + expanded_project = expanded["project"] + assert len(expanded_project["scripts"]) == 1 + assert expanded_project["scripts"]["a"] == "mod.a:func" + assert len(expanded_project["gui-scripts"]) == 1 + assert expanded_project["gui-scripts"]["b"] == "mod.b:func" + assert len(expanded_project["entry-points"]) == 1 + assert expanded_project["entry-points"]["other"]["c"] == "mod.c:func [extra]" + + project = {"dynamic": ["entry-points"]} + pyproject = {"project": project, "tool": tool} + expanded = expand_configuration(pyproject, tmp_path) + expanded_project = expanded["project"] + assert len(expanded_project["entry-points"]) == 3 + assert "scripts" not in expanded_project + assert "gui-scripts" not in expanded_project + + +EXAMPLE_INVALID_3RD_PARTY_CONFIG = """ +[project] +name = "myproj" +version = "1.2" + +[my-tool.that-disrespect.pep518] +value = 42 +""" + + +def test_ignore_unrelated_config(tmp_path): + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text(EXAMPLE_INVALID_3RD_PARTY_CONFIG) + + # Make sure no error is raised due to 3rd party configs in pyproject.toml + assert read_configuration(pyproject) is not None + + +@pytest.mark.parametrize("config", ("", "[tool.something]\nvalue = 42")) +def test_empty(tmp_path, config): + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text(config) + + # Make sure no error is raised + assert read_configuration(pyproject) == {} diff --git a/setuptools/tests/integration/test_pip_install_sdist.py b/setuptools/tests/integration/test_pip_install_sdist.py index 86cc42359d..0177c22dd4 100644 --- a/setuptools/tests/integration/test_pip_install_sdist.py +++ b/setuptools/tests/integration/test_pip_install_sdist.py @@ -112,6 +112,7 @@ def _debug_info(): @pytest.mark.parametrize('package, version', EXAMPLES) +@pytest.mark.uses_network def test_install_sdist(package, version, tmp_path, venv_python, setuptools_wheel): venv_pip = (venv_python, "-m", "pip") sdist = retrieve_sdist(package, version, tmp_path)