Skip to content

Commit

Permalink
Add config_settings support for build backend calls (#3090)
Browse files Browse the repository at this point in the history
Co-authored-by: Bernát Gábor <bgabor8@bloomberg.net>
  • Loading branch information
nschloe and gaborbernat authored Aug 29, 2023
1 parent ce3c96e commit ec7c0aa
Showing 7 changed files with 267 additions and 21 deletions.
1 change: 1 addition & 0 deletions docs/changelog/3090.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add support for setting build backend ``config_settings`` in the configuration file - by :user:`gaborbernat`.
48 changes: 48 additions & 0 deletions docs/config.rst
Original file line number Diff line number Diff line change
@@ -742,6 +742,54 @@ Python virtual environment packaging

Directory where to put project packages.

.. conf::
:keys: config_settings_get_requires_for_build_sdist
:version_added: 4.11

Config settings (``dict[str, str]``) passed to the ``get_requires_for_build_sdist`` backend API endpoint.

.. conf::
:keys: config_settings_build_sdist
:version_added: 4.11

Config settings (``dict[str, str]``) passed to the ``build_sdist`` backend API endpoint.

.. conf::
:keys: config_settings_get_requires_for_build_wheel
:version_added: 4.11

Config settings (``dict[str, str]``) passed to the ``get_requires_for_build_wheel`` backend API endpoint.

.. conf::
:keys: config_settings_prepare_metadata_for_build_wheel
:version_added: 4.11

Config settings (``dict[str, str]``) passed to the ``prepare_metadata_for_build_wheel`` backend API endpoint.

.. conf::
:keys: config_settings_build_wheel
:version_added: 4.11

Config settings (``dict[str, str]``) passed to the ``build_wheel`` backend API endpoint.

.. conf::
:keys: config_settings_get_requires_for_build_editable
:version_added: 4.11

Config settings (``dict[str, str]``) passed to the ``get_requires_for_build_editable`` backend API endpoint.

.. conf::
:keys: config_settings_prepare_metadata_for_build_editable
:version_added: 4.11

Config settings (``dict[str, str]``) passed to the ``prepare_metadata_for_build_editable`` backend API endpoint.

.. conf::
:keys: config_settings_build_editable
:version_added: 4.11

Config settings (``dict[str, str]``) passed to the ``build_editable`` backend API endpoint.

Pip installer
~~~~~~~~~~~~~

4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -50,12 +50,12 @@ dependencies = [
"cachetools>=5.3.1",
"chardet>=5.2",
"colorama>=0.4.6",
"filelock>=3.12.2",
"filelock>=3.12.3",
'importlib-metadata>=6.8; python_version < "3.8"',
"packaging>=23.1",
"platformdirs>=3.10",
"pluggy>=1.3",
"pyproject-api>=1.5.4",
"pyproject-api>=1.6.1",
'tomli>=2.0.1; python_version < "3.11"',
'typing-extensions>=4.7.1; python_version < "3.8"',
"virtualenv>=20.24.3",
69 changes: 51 additions & 18 deletions src/tox/tox_env/python/virtual_env/package/pyproject.py
Original file line number Diff line number Diff line change
@@ -8,11 +8,17 @@
from itertools import chain
from pathlib import Path
from threading import RLock
from typing import TYPE_CHECKING, Any, Dict, Generator, Iterator, NoReturn, Optional, Sequence, cast
from typing import TYPE_CHECKING, Any, Dict, Generator, Iterator, Literal, NoReturn, Optional, Sequence, cast

from cachetools import cached
from packaging.requirements import Requirement
from pyproject_api import BackendFailed, CmdStatus, Frontend
from pyproject_api import (
BackendFailed,
CmdStatus,
Frontend,
MetadataForBuildEditableResult,
MetadataForBuildWheelResult,
)

from tox.execute.pep517_backend import LocalSubProcessPep517Executor
from tox.execute.request import StdinSource
@@ -127,6 +133,23 @@ def register_config(self) -> None:
default=lambda conf, name: self.env_dir / "dist", # noqa: ARG005
desc="directory where to put project packages",
)
for key in ("sdist", "wheel", "editable"):
self._add_config_settings(key)

def _add_config_settings(self, build_type: str) -> None:
# config settings passed to PEP-517-compliant build backend https://peps.python.org/pep-0517/#config-settings
keys = {
"sdist": ["get_requires_for_build_sdist", "build_sdist"],
"wheel": ["get_requires_for_build_wheel", "prepare_metadata_for_build_wheel", "build_wheel"],
"editable": ["get_requires_for_build_editable", "prepare_metadata_for_build_editable", "build_editable"],
}
for key in keys.get(build_type, []):
self.conf.add_config(
keys=[f"config_settings_{key}"],
of_type=Dict[str, str],
default=None, # type: ignore[arg-type]
desc=f"config settings passed to the {key} backend API endpoint",
)

@property
def pkg_dir(self) -> Path:
@@ -164,7 +187,8 @@ def _setup_env(self) -> None:
self._setup_build_requires("editable")

def _setup_build_requires(self, of_type: str) -> None:
requires = getattr(self._frontend, f"get_requires_for_build_{of_type}")().requires
settings: ConfigSettings = self.conf[f"config_settings_get_requires_for_build_{of_type}"]
requires = getattr(self._frontend, f"get_requires_for_build_{of_type}")(config_settings=settings).requires
self._install(requires, PythonPackageToxEnv.__name__, f"requires_for_build_{of_type}")

def _teardown(self) -> None:
@@ -206,12 +230,15 @@ def perform_packaging(self, for_env: EnvConfigSet) -> list[Package]:
of_type: str = for_env["package"]
if of_type == "editable-legacy":
self.setup()
deps = [*self.requires(), *self._frontend.get_requires_for_build_sdist().requires, *deps]
config_settings: ConfigSettings = self.conf["config_settings_get_requires_for_build_sdist"]
sdist_requires = self._frontend.get_requires_for_build_sdist(config_settings=config_settings).requires
deps = [*self.requires(), *sdist_requires, *deps]
package: Package = EditableLegacyPackage(self.core["tox_root"], deps) # the folder itself is the package
elif of_type == "sdist":
self.setup()
with self._pkg_lock:
sdist = self._frontend.build_sdist(sdist_directory=self.pkg_dir).sdist
config_settings = self.conf["config_settings_build_sdist"]
sdist = self._frontend.build_sdist(sdist_directory=self.pkg_dir, config_settings=config_settings).sdist
sdist = create_session_view(sdist, self._package_temp_path)
self._package_paths.add(sdist)
package = SdistPackage(sdist, deps)
@@ -223,11 +250,12 @@ def perform_packaging(self, for_env: EnvConfigSet) -> list[Package]:
else:
self.setup()
method = "build_editable" if of_type == "editable" else "build_wheel"
config_settings = self.conf[f"config_settings_{method}"]
with self._pkg_lock:
wheel = getattr(self._frontend, method)(
wheel_directory=self.pkg_dir,
metadata_directory=self.meta_folder_if_populated,
config_settings=self._wheel_config_settings,
config_settings=config_settings,
).wheel
wheel = create_session_view(wheel, self._package_temp_path)
self._package_paths.add(wheel)
@@ -313,17 +341,20 @@ def _ensure_meta_present(self, for_env: EnvConfigSet) -> None:
if self._distribution_meta is not None: # pragma: no branch
return # pragma: no cover
# even if we don't build a wheel we need the requirements for it should we want to build its metadata
target = "editable" if for_env["package"] == "editable" else "wheel"
target: Literal["editable", "wheel"] = "editable" if for_env["package"] == "editable" else "wheel"
self.call_require_hooks.add(target)

self.setup()
hook = getattr(self._frontend, f"prepare_metadata_for_build_{target}")
dist_info = hook(self.meta_folder, self._wheel_config_settings).metadata
self._distribution_meta = Distribution.at(str(dist_info))

@property
def _wheel_config_settings(self) -> ConfigSettings | None:
return {"--build-option": []}
config: ConfigSettings = self.conf[f"config_settings_prepare_metadata_for_build_{target}"]
result: MetadataForBuildWheelResult | MetadataForBuildEditableResult | None = hook(self.meta_folder, config)
if result is None:
config = self.conf[f"config_settings_build_{target}"]
dist_info_path, _, __ = self._frontend.metadata_from_built(self.meta_folder, target, config)
dist_info = str(dist_info_path)
else:
dist_info = str(result.metadata)
self._distribution_meta = Distribution.at(dist_info)

def requires(self) -> tuple[Requirement, ...]:
return self._frontend.requires
@@ -353,16 +384,18 @@ def backend_cmd(self) -> Sequence[str]:

def _send(self, cmd: str, **kwargs: Any) -> tuple[Any, str, str]:
try:
if (
cmd in ("prepare_metadata_for_build_wheel", "prepare_metadata_for_build_editable")
# given we'll build a wheel we might skip the prepare step
and ("wheel" in self._tox_env.builds or "editable" in self._tox_env.builds)
):
if self._can_skip_prepare(cmd):
return None, "", "" # will need to build wheel either way, avoid prepare
return super()._send(cmd, **kwargs)
except BackendFailed as exception:
raise exception if isinstance(exception, ToxBackendFailed) else ToxBackendFailed(exception) from exception

def _can_skip_prepare(self, cmd: str) -> bool:
# given we'll build a wheel we might skip the prepare step
return cmd in ("prepare_metadata_for_build_wheel", "prepare_metadata_for_build_editable") and (
"wheel" in self._tox_env.builds or "editable" in self._tox_env.builds
)

@contextmanager
def _send_msg(
self,
2 changes: 1 addition & 1 deletion tests/demo_pkg_inline/build.py
Original file line number Diff line number Diff line change
@@ -98,7 +98,7 @@ def build_wheel(
str(Path(sub_directory) / filename),
)
else:
for arc_name, data in metadata_files.items(): # pragma: no branch
for arc_name, data in metadata_files.items():
zip_file_handler.writestr(arc_name, dedent(data).strip())
print(f"created wheel {path}") # noqa: T201
return base_name
163 changes: 163 additions & 0 deletions tests/tox_env/python/virtual_env/package/test_package_pyproject.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,19 @@
from __future__ import annotations

import json
from textwrap import dedent
from typing import TYPE_CHECKING

import pytest

from tox.execute.local_sub_process import LocalSubprocessExecuteStatus
from tox.tox_env.python.virtual_env.package.pyproject import Pep517VirtualEnvFrontend

if TYPE_CHECKING:
from pathlib import Path

from pytest_mock import MockerFixture

from tox.pytest import ToxProjectCreator


@@ -295,3 +301,160 @@ def test_pyproject_build_editable_and_wheel(tox_project: ToxProjectCreator, demo
("d", "install_package"),
(".pkg", "_exit"),
]


def test_pyproject_config_settings_sdist(
tox_project: ToxProjectCreator,
demo_pkg_setuptools: Path,
mocker: MockerFixture,
) -> None:
ini = """
[tox]
env_list = sdist
[testenv]
wheel_build_env = .pkg
package = sdist
[testenv:.pkg]
config_settings_get_requires_for_build_sdist = A = 1
config_settings_build_sdist = B = 2
config_settings_get_requires_for_build_wheel = C = 3
config_settings_prepare_metadata_for_build_wheel = D = 4
"""
proj = tox_project({"tox.ini": ini}, base=demo_pkg_setuptools)
proj.patch_execute(lambda r: 0 if "install" in r.run_id else None)

write_stdin = mocker.spy(LocalSubprocessExecuteStatus, "write_stdin")

result = proj.run("r", "--notest", from_cwd=proj.path)
result.assert_success()

found = {
message["cmd"]: message["kwargs"]["config_settings"]
for message in [json.loads(call[0][1]) for call in write_stdin.call_args_list]
if not message["cmd"].startswith("_")
}
assert found == {
"build_sdist": {"B": "2"},
"get_requires_for_build_sdist": {"A": "1"},
"get_requires_for_build_wheel": {"C": "3"},
"prepare_metadata_for_build_wheel": {"D": "4"},
}


def test_pyproject_config_settings_wheel(
tox_project: ToxProjectCreator,
demo_pkg_setuptools: Path,
mocker: MockerFixture,
) -> None:
ini = """
[tox]
env_list = wheel
[testenv]
wheel_build_env = .pkg
package = wheel
[testenv:.pkg]
config_settings_get_requires_for_build_wheel = C = 3
config_settings_prepare_metadata_for_build_wheel = D = 4
config_settings_build_wheel = E = 5
"""
proj = tox_project({"tox.ini": ini}, base=demo_pkg_setuptools)
proj.patch_execute(lambda r: 0 if "install" in r.run_id else None)

write_stdin = mocker.spy(LocalSubprocessExecuteStatus, "write_stdin")
mocker.patch.object(Pep517VirtualEnvFrontend, "_can_skip_prepare", return_value=False)

result = proj.run("r", "--notest", from_cwd=proj.path)
result.assert_success()

found = {
message["cmd"]: message["kwargs"]["config_settings"]
for message in [json.loads(call[0][1]) for call in write_stdin.call_args_list]
if not message["cmd"].startswith("_")
}
assert found == {
"get_requires_for_build_wheel": {"C": "3"},
"prepare_metadata_for_build_wheel": {"D": "4"},
"build_wheel": {"E": "5"},
}


def test_pyproject_config_settings_editable(
tox_project: ToxProjectCreator,
demo_pkg_setuptools: Path,
mocker: MockerFixture,
) -> None:
ini = """
[tox]
env_list = editable
[testenv:.pkg]
config_settings_get_requires_for_build_editable = F = 6
config_settings_prepare_metadata_for_build_editable = G = 7
config_settings_build_editable = H = 8
[testenv]
wheel_build_env = .pkg
package = editable
"""
proj = tox_project({"tox.ini": ini}, base=demo_pkg_setuptools)
proj.patch_execute(lambda r: 0 if "install" in r.run_id else None)

write_stdin = mocker.spy(LocalSubprocessExecuteStatus, "write_stdin")
mocker.patch.object(Pep517VirtualEnvFrontend, "_can_skip_prepare", return_value=False)

result = proj.run("r", "--notest", from_cwd=proj.path)
result.assert_success()

found = {
message["cmd"]: message["kwargs"]["config_settings"]
for message in [json.loads(call[0][1]) for call in write_stdin.call_args_list]
if not message["cmd"].startswith("_")
}
assert found == {
"get_requires_for_build_editable": {"F": "6"},
"prepare_metadata_for_build_editable": {"G": "7"},
"build_editable": {"H": "8"},
}


def test_pyproject_config_settings_editable_legacy(
tox_project: ToxProjectCreator,
demo_pkg_setuptools: Path,
mocker: MockerFixture,
) -> None:
ini = """
[tox]
env_list = editable
[testenv:.pkg]
config_settings_get_requires_for_build_sdist = A = 1
config_settings_get_requires_for_build_wheel = C = 3
config_settings_prepare_metadata_for_build_wheel = D = 4
[testenv]
wheel_build_env = .pkg
package = editable-legacy
"""
proj = tox_project({"tox.ini": ini}, base=demo_pkg_setuptools)
proj.patch_execute(lambda r: 0 if "install" in r.run_id else None)

write_stdin = mocker.spy(LocalSubprocessExecuteStatus, "write_stdin")
mocker.patch.object(Pep517VirtualEnvFrontend, "_can_skip_prepare", return_value=False)

result = proj.run("r", "--notest", from_cwd=proj.path)
result.assert_success()

found = {
message["cmd"]: message["kwargs"]["config_settings"]
for message in [json.loads(call[0][1]) for call in write_stdin.call_args_list]
if not message["cmd"].startswith("_")
}
assert found == {
"get_requires_for_build_sdist": {"A": "1"},
"get_requires_for_build_wheel": {"C": "3"},
"prepare_metadata_for_build_wheel": {"D": "4"},
}
Loading

0 comments on commit ec7c0aa

Please sign in to comment.