Merge pull request #1863 from j00bar/j00bar/issue-604-pyproject-toml

README.md

@@ -281,6 +281,26 @@ $ pip-compile requirements.in --pip-args "--retries 10 --timeout 30"
 
 ### Configuration
 
+You can define project-level defaults for `pip-compile` and `pip-sync` by
+writing them to a configuration file in the same directory as your requirements
+input files (or the current working directory if piping input from stdin).
+By default, both `pip-compile` and `pip-sync` will look first
+for a `.pip-tools.toml` file and then in your `pyproject.toml`. You can
+also specify an alternate TOML configuration file with the `--config` option.
+
+For example, to by default generate `pip` hashes in the resulting
+requirements file output, you can specify in a configuration file
+
+```toml
+[tool.pip-tools]
+generate-hashes = true
+
+```
+
+Options to `pip-compile` and `pip-sync` that may be used more than once
+must be defined as lists in a configuration file, even if they only have one
+value.
+
 You might be wrapping the `pip-compile` command in another script. To avoid
 confusing consumers of your custom script you can override the update command
 generated at the top of requirements files by setting the

piptools/locations.py

@@ -4,3 +4,6 @@
 
 # The user_cache_dir helper comes straight from pip itself
 CACHE_DIR = user_cache_dir("pip-tools")
+
+# The project defaults specific to pip-tools should be written to this filename
+CONFIG_FILE_NAME = ".pip-tools.toml"

piptools/scripts/compile.py

@@ -5,6 +5,7 @@
 import shlex
 import sys
 import tempfile
+from pathlib import Path
 from typing import IO, Any, BinaryIO, cast
 
 import click
@@ -19,7 +20,7 @@
 from .._compat import parse_requirements
 from ..cache import DependencyCache
 from ..exceptions import NoCandidateFound, PipToolsError
-from ..locations import CACHE_DIR
+from ..locations import CACHE_DIR, CONFIG_FILE_NAME
 from ..logging import log
 from ..repositories import LocalRequirementsRepository, PyPIRepository
 from ..repositories.base import BaseRepository
@@ -30,6 +31,7 @@
  drop_extras,
  is_pinned_requirement,
  key_from_ireq,
+ override_defaults_from_config_file,
  parse_requirements_from_wheel_metadata,
 )
 from ..writer import OutputWriter
@@ -302,6 +304,21 @@ def _determine_linesep(
  help="Specify a package to consider unsafe; may be used more than once. "
  f"Replaces default unsafe packages: {', '.join(sorted(UNSAFE_PACKAGES))}",
 )
+@click.option(
+ "--config",
+ type=click.Path(
+ exists=True,
+ file_okay=True,
+ dir_okay=False,
+ readable=True,
+ allow_dash=False,
+ path_type=str,
+ ),
+ help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or "
+ "pyproject.toml.",
+ is_eager=True,
+ callback=override_defaults_from_config_file,
+)
 def cli(
  ctx: click.Context,
  verbose: int,
@@ -340,6 +357,7 @@ def cli(
  emit_index_url: bool,
  emit_options: bool,
  unsafe_package: tuple[str, ...],
+ config: Path | None,
 ) -> None:
  """
  Compiles requirements.txt from requirements.in, pyproject.toml, setup.cfg,
@@ -391,6 +409,9 @@ def cli(
  f"input and output filenames must not be matched: {output_file.name}"
  )
 
+ if config:
+ log.info(f"Using pip-tools configuration defaults found in '{config !s}'.")
+
  if resolver_name == "legacy":
  log.warning(
  "WARNING: the legacy dependency resolver is deprecated and will be removed"

piptools/scripts/sync.py

@@ -5,6 +5,7 @@
 import shlex
 import shutil
 import sys
+from pathlib import Path
 from typing import cast
 
 import click
@@ -17,13 +18,15 @@
 from .._compat import parse_requirements
 from .._compat.pip_compat import Distribution
 from ..exceptions import PipToolsError
+from ..locations import CONFIG_FILE_NAME
 from ..logging import log
 from ..repositories import PyPIRepository
 from ..utils import (
  flat_map,
  get_pip_version_for_python_executable,
  get_required_pip_specification,
  get_sys_path_for_python_executable,
+ override_defaults_from_config_file,
 )
 
 DEFAULT_REQUIREMENTS_FILE = "requirements.txt"
@@ -86,6 +89,21 @@
 )
 @click.argument("src_files", required=False, type=click.Path(exists=True), nargs=-1)
 @click.option("--pip-args", help="Arguments to pass directly to pip install.")
+@click.option(
+ "--config",
+ type=click.Path(
+ exists=True,
+ file_okay=True,
+ dir_okay=False,
+ readable=True,
+ allow_dash=False,
+ path_type=str,
+ ),
+ help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or "
+ "pyproject.toml.",
+ is_eager=True,
+ callback=override_defaults_from_config_file,
+)
 def cli(
  ask: bool,
  dry_run: bool,
@@ -103,6 +121,7 @@ def cli(
  client_cert: str | None,
  src_files: tuple[str, ...],
  pip_args: str | None,
+ config: Path | None,
 ) -> None:
  """Synchronize virtual environment with requirements.txt."""
  log.verbosity = verbose - quiet
@@ -127,6 +146,9 @@ def cli(
  log.error("ERROR: " + msg)
  sys.exit(2)
 
+ if config:
+ log.info(f"Using pip-tools configuration defaults found in '{config !s}'.")
+
  if python_executable:
  _validate_python_executable(python_executable)
 

piptools/utils.py

@@ -2,13 +2,21 @@
 
 import collections
 import copy
+import functools
 import itertools
 import json
 import os
 import re
 import shlex
+import sys
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, TypeVar, cast
 
+if sys.version_info >= (3, 11):
+ import tomllib
+else:
+ import tomli as tomllib
+
 import click
 from click.utils import LazyFile
 from pip._internal.req import InstallRequirement
@@ -22,6 +30,7 @@
 from pip._vendor.pkg_resources import Distribution, Requirement, get_distribution
 
 from piptools._compat import PIP_VERSION
+from piptools.locations import CONFIG_FILE_NAME
 from piptools.subprocess_utils import run_python_snippet
 
 if TYPE_CHECKING:
@@ -405,9 +414,9 @@ def get_required_pip_specification() -> SpecifierSet:
  Returns pip version specifier requested by current pip-tools installation.
  """
  project_dist = get_distribution("pip-tools")
- requirement = next( # pragma: no branch
+ requirement = next(
  (r for r in project_dist.requires() if r.name == "pip"), None
- )
+ ) # pragma: no branch
  assert (
  requirement is not None
  ), "'pip' is expected to be in the list of pip-tools requirements"
@@ -522,3 +531,128 @@ def parse_requirements_from_wheel_metadata(
  markers=parts.markers,
  extras=parts.extras,
  )
+
+
+def override_defaults_from_config_file(
+ ctx: click.Context, param: click.Parameter, value: str | None
+) -> Path | None:
+ """
+ Overrides ``click.Command`` defaults based on specified or discovered config
+ file, returning the ``pathlib.Path`` of that config file if specified or
+ discovered.
+
+ ``None`` is returned if no such file is found.
+
+ ``pip-tools`` will use the first config file found, searching in this order:
+ an explicitly given config file, a d``.pip-tools.toml``, a ``pyproject.toml``
+ file. Those files are searched for in the same directory as the requirements
+ input file, or the current working directory if requirements come via stdin.
+ """
+ if value is None:
+ config_file = select_config_file(ctx.params.get("src_files", ()))
+ if config_file is None:
+ return None
+ else:
+ config_file = Path(value)
+
+ config = parse_config_file(config_file)
+ if config:
+ _assign_config_to_cli_context(ctx, config)
+
+ return config_file
+
+
+def _assign_config_to_cli_context(
+ click_context: click.Context,
+ cli_config_mapping: dict[str, Any],
+) -> None:
+ if click_context.default_map is None:
+ click_context.default_map = {}
+
+ click_context.default_map.update(cli_config_mapping)
+
+
+def select_config_file(src_files: tuple[str, ...]) -> Path | None:
+ """
+ Returns the config file to use for defaults given ``src_files`` provided.
+ """
+ # NOTE: If no src_files were specified, consider the current directory the
+ # NOTE: only config file lookup candidate. This usually happens when a
+ # NOTE: pip-tools invocation gets its incoming requirements from standard
+ # NOTE: input.
+ working_directory = Path.cwd()
+ src_files_as_paths = (
+ (working_directory / src_file).resolve() for src_file in src_files or (".",)
+ )
+ candidate_dirs = (src if src.is_dir() else src.parent for src in src_files_as_paths)
+ config_file_path = next(
+ (
+ candidate_dir / config_file
+ for candidate_dir in candidate_dirs
+ for config_file in (CONFIG_FILE_NAME, "pyproject.toml")
+ if (candidate_dir / config_file).is_file()
+ ),
+ None,
+ )
+ return config_file_path
+
+
+# Some of the defined click options have different `dest` values than the defaults
+NON_STANDARD_OPTION_DEST_MAP: dict[str, str] = {
+ "extra": "extras",
+ "upgrade_package": "upgrade_packages",
+ "resolver": "resolver_name",
+ "user": "user_only",
+}
+
+
+def get_click_dest_for_option(option_name: str) -> str:
+ """
+ Returns the click ``dest`` value for the given option name.
+ """
+ # Format the keys properly
+ option_name = option_name.lstrip("-").replace("-", "_").lower()
+ # Some options have dest values that are overrides from the click generated default
+ option_name = NON_STANDARD_OPTION_DEST_MAP.get(option_name, option_name)
+ return option_name
+
+
+# Ensure that any default overrides for these click options are lists, supporting multiple values
+MULTIPLE_VALUE_OPTIONS = [
+ "extras",
+ "upgrade_packages",
+ "unsafe_package",
+ "find_links",
+ "extra_index_url",
+ "trusted_host",
+]
+
+
+@functools.lru_cache()
+def parse_config_file(config_file: Path) -> dict[str, Any]:
+ try:
+ config = tomllib.loads(config_file.read_text(encoding="utf-8"))
+ except OSError as os_err:
+ raise click.FileError(
+ filename=str(config_file),
+ hint=f"Could not read '{config_file !s}': {os_err !s}",
+ )
+ except ValueError as value_err:
+ raise click.FileError(
+ filename=str(config_file),
+ hint=f"Could not parse '{config_file !s}': {value_err !s}",
+ )
+
+ # In a TOML file, we expect the config to be under `[tool.pip-tools]`
+ piptools_config: dict[str, Any] = config.get("tool", {}).get("pip-tools", {})
+ piptools_config = {
+ get_click_dest_for_option(k): v for k, v in piptools_config.items()
+ }
+ # Any option with multiple values needs to be a list in the pyproject.toml
+ for mv_option in MULTIPLE_VALUE_OPTIONS:
+ if not isinstance(piptools_config.get(mv_option), (list, type(None))):
+ original_option = mv_option.replace("_", "-")
+ raise click.BadOptionUsage(
+ original_option, f"Config key '{original_option}' must be a list"
+ )
+ return piptools_config

pyproject.toml

@@ -40,6 +40,7 @@ dependencies = [
  "build",
  "click >= 8",
  "pip >= 22.2",
+ "tomli; python_version < '3.11'",
  # indirect dependencies
  "setuptools", # typically needed when pip-tools invokes setup.py
  "wheel", # pip plugin needed by pip-tools
@@ -57,6 +58,7 @@ testing = [
  "pytest >= 7.2.0",
  "pytest-rerunfailures",
  "pytest-xdist",
+ "tomli-w",
  # build deps for tests
  "flit_core >=2,<4",
  "poetry_core>=1.0.0",

tests/conftest.py

@@ -8,9 +8,12 @@
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from functools import partial
+from pathlib import Path
 from textwrap import dedent
+from typing import Any
 
 import pytest
+import tomli_w
 from click.testing import CliRunner
 from pip._internal.commands.install import InstallCommand
 from pip._internal.index.package_finder import PackageFinder
@@ -26,6 +29,7 @@
 from piptools._compat.pip_compat import PIP_VERSION, uses_pkg_resources
 from piptools.cache import DependencyCache
 from piptools.exceptions import NoCandidateFound
+from piptools.locations import CONFIG_FILE_NAME
 from piptools.logging import log
 from piptools.repositories import PyPIRepository
 from piptools.repositories.base import BaseRepository
@@ -450,3 +454,25 @@ def _reset_log():
  with other tests that depend on it.
  """
  log.reset()
+
+
+@pytest.fixture
+def make_config_file(tmpdir_cwd):
+ """
+ Make a config file for pip-tools with a given parameter set to a specific
+ value, returning a ``pathlib.Path`` to the config file.
+ """
+
+ def _maker(
+ pyproject_param: str, new_default: Any, config_file_name: str = CONFIG_FILE_NAME
+ ) -> Path:
+ # Make a config file with this one config default override
+ config_path = Path(tmpdir_cwd) / pyproject_param
+ config_file = config_path / config_file_name
+ config_path.mkdir(exist_ok=True)
+
+ config_to_dump = {"tool": {"pip-tools": {pyproject_param: new_default}}}
+ config_file.write_text(tomli_w.dumps(config_to_dump))
+ return config_file
+
+ return _maker