From d1e0b521ed580276924bc7bf1a54c9c60aec7a27 Mon Sep 17 00:00:00 2001 From: Cal Jacobson Date: Fri, 3 Nov 2023 18:15:59 +0000 Subject: [PATCH 01/10] add support for multiple requirement input files add tests for multiple requirement input files rename `src` -> `srcs` to communicate that multiple input files may be provided Co-authored-by: Logan Pulley --- docs/sphinx/BUILD.bazel | 2 +- examples/build_file_generation/BUILD.bazel | 2 +- examples/bzlmod/BUILD.bazel | 4 +-- examples/bzlmod/other_module/BUILD.bazel | 2 +- .../bzlmod_build_file_generation/BUILD.bazel | 2 +- .../requirements/BUILD.bazel | 8 ++--- examples/pip_parse/BUILD.bazel | 2 +- examples/pip_parse_vendored/BUILD.bazel | 2 +- .../pip_repository_annotations/BUILD.bazel | 2 +- python/pip_install/requirements.bzl | 24 ++++++++------- .../dependency_resolver.py | 21 +++++++------ tests/compile_pip_requirements/BUILD.bazel | 6 ++-- tests/multiple_inputs/BUILD.bazel | 30 +++++++++++++++++++ tests/multiple_inputs/README.md | 3 ++ tests/multiple_inputs/a/pyproject.toml | 5 ++++ tests/multiple_inputs/b/pyproject.toml | 5 ++++ tests/multiple_inputs/multiple_inputs.txt | 18 +++++++++++ .../multiple_pyproject_toml.txt | 14 +++++++++ .../multiple_requirements_in.txt | 14 +++++++++ tests/multiple_inputs/requirements_1.in | 1 + tests/multiple_inputs/requirements_2.in | 1 + tests/pip_repository_entry_points/BUILD.bazel | 2 +- tools/publish/BUILD.bazel | 2 +- 23 files changed, 132 insertions(+), 40 deletions(-) create mode 100644 tests/multiple_inputs/BUILD.bazel create mode 100644 tests/multiple_inputs/README.md create mode 100644 tests/multiple_inputs/a/pyproject.toml create mode 100644 tests/multiple_inputs/b/pyproject.toml create mode 100644 tests/multiple_inputs/multiple_inputs.txt create mode 100644 tests/multiple_inputs/multiple_pyproject_toml.txt create mode 100644 tests/multiple_inputs/multiple_requirements_in.txt create mode 100644 tests/multiple_inputs/requirements_1.in create mode 100644 tests/multiple_inputs/requirements_2.in diff --git a/docs/sphinx/BUILD.bazel b/docs/sphinx/BUILD.bazel index 7c99f77e63..dbfed6eeaf 100644 --- a/docs/sphinx/BUILD.bazel +++ b/docs/sphinx/BUILD.bazel @@ -102,7 +102,7 @@ sphinx_build_binary( # Run bazel run //docs/sphinx:requirements.update compile_pip_requirements( name = "requirements", - src = "pyproject.toml", + srcs = ["pyproject.toml"], requirements_darwin = "requirements_darwin.txt", requirements_txt = "requirements_linux.txt", target_compatible_with = _TARGET_COMPATIBLE_WITH, diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel index 7b9766eb1a..430da8b9bc 100644 --- a/examples/build_file_generation/BUILD.bazel +++ b/examples/build_file_generation/BUILD.bazel @@ -11,7 +11,7 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") compile_pip_requirements( name = "requirements", - src = "requirements.in", + srcs = ["requirements.in"], requirements_txt = "requirements_lock.txt", requirements_windows = "requirements_windows.txt", ) diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index 5e2509af28..44d23b7876 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -16,7 +16,7 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") # with pip-compile. compile_pip_requirements_3_9( name = "requirements_3_9", - src = "requirements.in", + srcs = ["requirements.in"], requirements_txt = "requirements_lock_3_9.txt", requirements_windows = "requirements_windows_3_9.txt", ) @@ -25,7 +25,7 @@ compile_pip_requirements_3_9( # with pip-compile. compile_pip_requirements_3_10( name = "requirements_3_10", - src = "requirements.in", + srcs = ["requirements.in"], requirements_txt = "requirements_lock_3_10.txt", requirements_windows = "requirements_windows_3_10.txt", ) diff --git a/examples/bzlmod/other_module/BUILD.bazel b/examples/bzlmod/other_module/BUILD.bazel index a93b92aaed..9392c474bd 100644 --- a/examples/bzlmod/other_module/BUILD.bazel +++ b/examples/bzlmod/other_module/BUILD.bazel @@ -4,6 +4,6 @@ load("@python_versions//3.11:defs.bzl", compile_pip_requirements_311 = "compile_ # override in the MODULE.bazel. compile_pip_requirements_311( name = "requirements", - src = "requirements.in", + srcs = ["requirements.in"], requirements_txt = "requirements_lock_3_11.txt", ) diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index bca3b3681b..6b2092bbd6 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -16,7 +16,7 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") # with pip-compile. compile_pip_requirements( name = "requirements", - src = "requirements.in", + srcs = ["requirements.in"], requirements_txt = "requirements_lock.txt", requirements_windows = "requirements_windows.txt", ) diff --git a/examples/multi_python_versions/requirements/BUILD.bazel b/examples/multi_python_versions/requirements/BUILD.bazel index f67333a657..12b42ef180 100644 --- a/examples/multi_python_versions/requirements/BUILD.bazel +++ b/examples/multi_python_versions/requirements/BUILD.bazel @@ -5,24 +5,24 @@ load("@python//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requir compile_pip_requirements_3_8( name = "requirements_3_8", - src = "requirements.in", + srcs = ["requirements.in"], requirements_txt = "requirements_lock_3_8.txt", ) compile_pip_requirements_3_9( name = "requirements_3_9", - src = "requirements.in", + srcs = ["requirements.in"], requirements_txt = "requirements_lock_3_9.txt", ) compile_pip_requirements_3_10( name = "requirements_3_10", - src = "requirements.in", + srcs = ["requirements.in"], requirements_txt = "requirements_lock_3_10.txt", ) compile_pip_requirements_3_11( name = "requirements_3_11", - src = "requirements.in", + srcs = ["requirements.in"], requirements_txt = "requirements_lock_3_11.txt", ) diff --git a/examples/pip_parse/BUILD.bazel b/examples/pip_parse/BUILD.bazel index c2cc9a3276..197f341151 100644 --- a/examples/pip_parse/BUILD.bazel +++ b/examples/pip_parse/BUILD.bazel @@ -53,7 +53,7 @@ py_console_script_binary( # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", - src = "requirements.in", + srcs = ["requirements.in"], requirements_txt = "requirements_lock.txt", ) diff --git a/examples/pip_parse_vendored/BUILD.bazel b/examples/pip_parse_vendored/BUILD.bazel index 8741c5aaa7..f48e1aace5 100644 --- a/examples/pip_parse_vendored/BUILD.bazel +++ b/examples/pip_parse_vendored/BUILD.bazel @@ -6,7 +6,7 @@ load("@rules_python//python:pip.bzl", "compile_pip_requirements") # lockfile based on the requirements.in. compile_pip_requirements( name = "requirements", - src = "requirements.in", + srcs = ["requirements.in"], ) # The requirements.bzl file is generated with a reference to the interpreter for the host platform. diff --git a/examples/pip_repository_annotations/BUILD.bazel b/examples/pip_repository_annotations/BUILD.bazel index bdf9df1274..a1e1b558fd 100644 --- a/examples/pip_repository_annotations/BUILD.bazel +++ b/examples/pip_repository_annotations/BUILD.bazel @@ -9,7 +9,7 @@ exports_files( # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", - src = "requirements.in", + srcs = ["requirements.in"], ) py_test( diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 5caf7629f5..72ec6595aa 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -19,7 +19,7 @@ load("//python/pip_install:repositories.bzl", "requirement") def compile_pip_requirements( name, - src = None, + srcs = None, extra_args = [], extra_deps = [], generate_hashes = True, @@ -49,17 +49,17 @@ def compile_pip_requirements( Args: name: base name for generated targets, typically "requirements". - src: file containing inputs to dependency resolution. If not specified, - defaults to `pyproject.toml`. Supported formats are: + srcs: list of files containing inputs to dependency resolution. If not specified, + defaults to `["pyproject.toml"]`. Supported formats are: * a requirements text file, usually named `requirements.in` - * A `.toml` file, where the `project.dependencies` list is used as per + * A `pyproject.toml` file, where the `project.dependencies` list is used as per [PEP621](https://peps.python.org/pep-0621/). extra_args: passed to pip-compile. extra_deps: extra dependencies passed to pip-compile. generate_hashes: whether to put hashes in the requirements_txt file. py_binary: the py_binary rule to be used. py_test: the py_test rule to be used. - requirements_in: file expressing desired dependencies. Deprecated, use src instead. + requirements_in: file expressing desired dependencies. Deprecated, use srcs instead. requirements_txt: result of "compiling" the requirements.in file. requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes. requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes. @@ -68,10 +68,13 @@ def compile_pip_requirements( visibility: passed to both the _test and .update rules. **kwargs: other bazel attributes passed to the "_test" rule. """ - if requirements_in and src: - fail("Only one of 'src' and 'requirements_in' attributes can be used") + if requirements_in and srcs: + fail("Only one of 'srcs' and 'requirements_in' attributes can be used") + + if requirements_in: + srcs = [requirements_in] else: - src = requirements_in or src or "pyproject.toml" + srcs = srcs or ["pyproject.toml"] requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt @@ -84,7 +87,7 @@ def compile_pip_requirements( visibility = visibility, ) - data = [name, requirements_txt, src] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + data = [name, requirements_txt] + srcs + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] # Use the Label constructor so this is expanded in the context of the file # where it appears, which is to say, in @rules_python @@ -92,8 +95,7 @@ def compile_pip_requirements( loc = "$(rlocationpath {})" - args = [ - loc.format(src), + args = ["--src={}".format(loc.format(src)) for src in srcs] + [ loc.format(requirements_txt), "//%s:%s.update" % (native.package_name(), name), "--resolver=backtracking", diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 5e914bc9e9..1f3dcb9d22 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -80,7 +80,7 @@ def _locate(bazel_runfiles, file): @click.command(context_settings={"ignore_unknown_options": True}) -@click.argument("requirements_in") +@click.option("--src", "srcs", multiple=True, required=True) @click.argument("requirements_txt") @click.argument("update_target_label") @click.option("--requirements-linux") @@ -88,7 +88,7 @@ def _locate(bazel_runfiles, file): @click.option("--requirements-windows") @click.argument("extra_args", nargs=-1, type=click.UNPROCESSED) def main( - requirements_in: str, + srcs: Tuple[str, ...], requirements_txt: str, update_target_label: str, requirements_linux: Optional[str], @@ -103,7 +103,7 @@ def main( requirements_darwin=requirements_darwin, requirements_windows=requirements_windows ) - resolved_requirements_in = _locate(bazel_runfiles, requirements_in) + resolved_srcs = [_locate(bazel_runfiles, src) for src in srcs] resolved_requirements_file = _locate(bazel_runfiles, requirements_file) # Files in the runfiles directory has the following naming schema: @@ -116,12 +116,12 @@ def main( : -(len(requirements_file) - len(repository_prefix)) ] - # As requirements_in might contain references to generated files we want to + # As srcs might contain references to generated files we want to # use the runfiles file first. Thus, we need to compute the relative path # from the execution root. # Note: Windows cannot reference generated files without runfiles support enabled. - requirements_in_relative = requirements_in[len(repository_prefix):] - requirements_file_relative = requirements_file[len(repository_prefix):] + srcs_relative = [src[len(repository_prefix) :] for src in srcs] + requirements_file_relative = requirements_file[len(repository_prefix) :] # Before loading click, set the locale for its parser. # If it leaks through to the system setting, it may fail: @@ -158,10 +158,9 @@ def main( os.environ["PIP_CONFIG_FILE"] = os.getenv("PIP_CONFIG_FILE") or os.devnull argv.append(f"--output-file={requirements_file_relative if UPDATE else requirements_out}") - argv.append( - requirements_in_relative - if Path(requirements_in_relative).exists() - else resolved_requirements_in + argv.extend( + (src_relative if Path(src_relative).exists() else resolved_src) + for src_relative, resolved_src in zip(srcs_relative, resolved_srcs) ) argv.extend(extra_args) @@ -196,7 +195,7 @@ def main( print( "pip-compile exited with code 2. This means that pip-compile found " "incompatible requirements or could not find a version that matches " - f"the install requirement in {requirements_in_relative}.", + f"the install requirement in one of {srcs_relative}.", file=sys.stderr, ) sys.exit(1) diff --git a/tests/compile_pip_requirements/BUILD.bazel b/tests/compile_pip_requirements/BUILD.bazel index 6df46b8372..cf95e6c627 100644 --- a/tests/compile_pip_requirements/BUILD.bazel +++ b/tests/compile_pip_requirements/BUILD.bazel @@ -21,7 +21,7 @@ EOF compile_pip_requirements( name = "requirements", - src = "requirements.txt", + srcs = ["requirements.txt"], data = [ "requirements.in", "requirements_extra.in", @@ -31,7 +31,7 @@ compile_pip_requirements( compile_pip_requirements( name = "requirements_nohashes", - src = "requirements.txt", + srcs = ["requirements.txt"], data = [ "requirements.in", "requirements_extra.in", @@ -55,7 +55,7 @@ EOF compile_pip_requirements( name = "os_specific_requirements", - src = "requirements_os_specific.in", + srcs = ["requirements_os_specific.in"], data = [ "requirements_extra.in", "requirements_os_specific.in", diff --git a/tests/multiple_inputs/BUILD.bazel b/tests/multiple_inputs/BUILD.bazel new file mode 100644 index 0000000000..3e3cab83ca --- /dev/null +++ b/tests/multiple_inputs/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_python//python:pip.bzl", "compile_pip_requirements") + +compile_pip_requirements( + name = "multiple_requirements_in", + srcs = [ + "requirements_1.in", + "requirements_2.in", + ], + requirements_txt = "multiple_requirements_in.txt", +) + +compile_pip_requirements( + name = "multiple_pyproject_toml", + srcs = [ + "a/pyproject.toml", + "b/pyproject.toml", + ], + requirements_txt = "multiple_pyproject_toml.txt", +) + +compile_pip_requirements( + name = "multiple_inputs", + srcs = [ + "a/pyproject.toml", + "b/pyproject.toml", + "requirements_1.in", + "requirements_2.in", + ], + requirements_txt = "multiple_inputs.txt", +) diff --git a/tests/multiple_inputs/README.md b/tests/multiple_inputs/README.md new file mode 100644 index 0000000000..7b6bade122 --- /dev/null +++ b/tests/multiple_inputs/README.md @@ -0,0 +1,3 @@ +# multiple_inputs + +Test that `compile_pip_requirements` works as intended when using more than one input file. diff --git a/tests/multiple_inputs/a/pyproject.toml b/tests/multiple_inputs/a/pyproject.toml new file mode 100644 index 0000000000..91efec3821 --- /dev/null +++ b/tests/multiple_inputs/a/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "multiple_inputs_1" +version = "0.0.0" + +dependencies = ["urllib3"] diff --git a/tests/multiple_inputs/b/pyproject.toml b/tests/multiple_inputs/b/pyproject.toml new file mode 100644 index 0000000000..a461f4ed98 --- /dev/null +++ b/tests/multiple_inputs/b/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "multiple_inputs_2" +version = "0.0.0" + +dependencies = ["attrs"] diff --git a/tests/multiple_inputs/multiple_inputs.txt b/tests/multiple_inputs/multiple_inputs.txt new file mode 100644 index 0000000000..f77ec77458 --- /dev/null +++ b/tests/multiple_inputs/multiple_inputs.txt @@ -0,0 +1,18 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# bazel run //tests/multiple_inputs:multiple_inputs.update +# +attrs==23.1.0 \ + --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ + --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 + # via + # -r tests/multiple_inputs/requirements_2.in + # multiple-inputs-2 (tests/multiple_inputs/b/pyproject.toml) +urllib3==2.0.7 \ + --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ + --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e + # via + # -r tests/multiple_inputs/requirements_1.in + # multiple-inputs-1 (tests/multiple_inputs/a/pyproject.toml) diff --git a/tests/multiple_inputs/multiple_pyproject_toml.txt b/tests/multiple_inputs/multiple_pyproject_toml.txt new file mode 100644 index 0000000000..8d09185f86 --- /dev/null +++ b/tests/multiple_inputs/multiple_pyproject_toml.txt @@ -0,0 +1,14 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# bazel run //tests/multiple_inputs:multiple_pyproject_toml.update +# +attrs==23.1.0 \ + --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ + --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 + # via multiple-inputs-2 (tests/multiple_inputs/b/pyproject.toml) +urllib3==2.0.7 \ + --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ + --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e + # via multiple-inputs-1 (tests/multiple_inputs/a/pyproject.toml) diff --git a/tests/multiple_inputs/multiple_requirements_in.txt b/tests/multiple_inputs/multiple_requirements_in.txt new file mode 100644 index 0000000000..63edfe9f53 --- /dev/null +++ b/tests/multiple_inputs/multiple_requirements_in.txt @@ -0,0 +1,14 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# bazel run //tests/multiple_inputs:multiple_requirements_in.update +# +attrs==23.1.0 \ + --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ + --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 + # via -r tests/multiple_inputs/requirements_2.in +urllib3==2.0.7 \ + --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ + --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e + # via -r tests/multiple_inputs/requirements_1.in diff --git a/tests/multiple_inputs/requirements_1.in b/tests/multiple_inputs/requirements_1.in new file mode 100644 index 0000000000..a42590bebe --- /dev/null +++ b/tests/multiple_inputs/requirements_1.in @@ -0,0 +1 @@ +urllib3 diff --git a/tests/multiple_inputs/requirements_2.in b/tests/multiple_inputs/requirements_2.in new file mode 100644 index 0000000000..04cb10228e --- /dev/null +++ b/tests/multiple_inputs/requirements_2.in @@ -0,0 +1 @@ +attrs diff --git a/tests/pip_repository_entry_points/BUILD.bazel b/tests/pip_repository_entry_points/BUILD.bazel index c39b1f0a2d..6bac5ab848 100644 --- a/tests/pip_repository_entry_points/BUILD.bazel +++ b/tests/pip_repository_entry_points/BUILD.bazel @@ -5,7 +5,7 @@ load("@rules_python//python:pip.bzl", "compile_pip_requirements") # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", - src = "requirements.in", + srcs = ["requirements.in"], requirements_windows = ":requirements_windows.txt", ) diff --git a/tools/publish/BUILD.bazel b/tools/publish/BUILD.bazel index 4759a31257..296ee041b3 100644 --- a/tools/publish/BUILD.bazel +++ b/tools/publish/BUILD.bazel @@ -2,7 +2,7 @@ load("//python:pip.bzl", "compile_pip_requirements") compile_pip_requirements( name = "requirements", - src = "requirements.in", + srcs = ["requirements.in"], requirements_darwin = "requirements_darwin.txt", requirements_windows = "requirements_windows.txt", ) From c78e39a388873f70f7ef7d46c144ef8ba8727a40 Mon Sep 17 00:00:00 2001 From: Cal Jacobson Date: Mon, 6 Nov 2023 16:38:37 +0000 Subject: [PATCH 02/10] update src -> srcs in `CHANGELOG.md` --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebbde9a0af..26b268f896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,7 @@ A brief description of the categories of changes: default, which will cause `gazelle` to change third-party dependency labels from `@pip_foo//:pkg` to `@pip//foo` by default. -* The `compile_pip_requirements` now defaults to `pyproject.toml` if the `src` +* The `compile_pip_requirements` now defaults to `pyproject.toml` if the `srcs` or `requirements_in` attributes are unspecified, matching the upstream `pip-compile` behaviour more closely. From 89133652bb0247f0166e3e3393a878bdbf80f1c2 Mon Sep 17 00:00:00 2001 From: Cal Jacobson Date: Wed, 7 Aug 2024 10:29:00 -0500 Subject: [PATCH 03/10] fix failing tests --- tests/multiple_inputs/multiple_inputs.txt | 4 ++-- tests/multiple_inputs/multiple_pyproject_toml.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/multiple_inputs/multiple_inputs.txt b/tests/multiple_inputs/multiple_inputs.txt index f77ec77458..a036c3f33d 100644 --- a/tests/multiple_inputs/multiple_inputs.txt +++ b/tests/multiple_inputs/multiple_inputs.txt @@ -9,10 +9,10 @@ attrs==23.1.0 \ --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 # via # -r tests/multiple_inputs/requirements_2.in - # multiple-inputs-2 (tests/multiple_inputs/b/pyproject.toml) + # multiple_inputs_2 (tests/multiple_inputs/b/pyproject.toml) urllib3==2.0.7 \ --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e # via # -r tests/multiple_inputs/requirements_1.in - # multiple-inputs-1 (tests/multiple_inputs/a/pyproject.toml) + # multiple_inputs_1 (tests/multiple_inputs/a/pyproject.toml) diff --git a/tests/multiple_inputs/multiple_pyproject_toml.txt b/tests/multiple_inputs/multiple_pyproject_toml.txt index 8d09185f86..b8af28ac10 100644 --- a/tests/multiple_inputs/multiple_pyproject_toml.txt +++ b/tests/multiple_inputs/multiple_pyproject_toml.txt @@ -7,8 +7,8 @@ attrs==23.1.0 \ --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 - # via multiple-inputs-2 (tests/multiple_inputs/b/pyproject.toml) + # via multiple_inputs_2 (tests/multiple_inputs/b/pyproject.toml) urllib3==2.0.7 \ --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e - # via multiple-inputs-1 (tests/multiple_inputs/a/pyproject.toml) + # via multiple_inputs_1 (tests/multiple_inputs/a/pyproject.toml) From a976ed39f955370d039dc3652510e8784156ce1d Mon Sep 17 00:00:00 2001 From: Cal Jacobson Date: Thu, 8 Aug 2024 15:32:20 -0500 Subject: [PATCH 04/10] fix failing `compile_pip_requirements` on windows --- CHANGELOG.md | 3 ++- python/private/pypi/pip_compile.bzl | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a75d0cb020..066a98a3f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ A brief description of the categories of changes: Fixes [#1818](https://github.com/bazelbuild/rules_python/issues/1818). ### Added +* (rules) `compile_pip_requirements` supports multiple requirements input files as `srcs`. * (rules) `PYTHONSAFEPATH` is inherited from the calling environment to allow disabling it (Requires {obj}`--bootstrap_impl=script`) ([#2060](https://github.com/bazelbuild/rules_python/issues/2060)). @@ -624,7 +625,7 @@ A brief description of the categories of changes: default, which will cause `gazelle` to change third-party dependency labels from `@pip_foo//:pkg` to `@pip//foo` by default. -* The `compile_pip_requirements` now defaults to `pyproject.toml` if the `srcs` +* The `compile_pip_requirements` now defaults to `pyproject.toml` if the `src` or `requirements_in` attributes are unspecified, matching the upstream `pip-compile` behaviour more closely. diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index a8c498bf43..39528ae0a5 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -24,6 +24,7 @@ load("//python:defs.bzl", _py_binary = "py_binary", _py_test = "py_test") def pip_compile( name, srcs = None, + src = None, extra_args = [], extra_deps = [], generate_hashes = True, @@ -58,12 +59,17 @@ def pip_compile( * a requirements text file, usually named `requirements.in` * A `.toml` file, where the `project.dependencies` list is used as per [PEP621](https://peps.python.org/pep-0621/). + src: file containing inputs to dependency resolution. If not specified, + defaults to `pyproject.toml`. Supported formats are: + * a requirements text file, usually named `requirements.in` + * A `.toml` file, where the `project.dependencies` list is used as per + [PEP621](https://peps.python.org/pep-0621/). extra_args: passed to pip-compile. extra_deps: extra dependencies passed to pip-compile. generate_hashes: whether to put hashes in the requirements_txt file. py_binary: the py_binary rule to be used. py_test: the py_test rule to be used. - requirements_in: file expressing desired dependencies. Deprecated, use srcs instead. + requirements_in: file expressing desired dependencies. Deprecated, use src or srcs instead. requirements_txt: result of "compiling" the requirements.in file. requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes. requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes. @@ -72,11 +78,13 @@ def pip_compile( visibility: passed to both the _test and .update rules. **kwargs: other bazel attributes passed to the "_test" rule. """ - if requirements_in and srcs: - fail("Only one of 'srcs' and 'requirements_in' attributes can be used") + if len([x for x in [srcs, src, requirements_in] if x != None]) > 1: + fail("At most one of 'srcs', 'src', and 'requirements_in' attributes may be provided") if requirements_in: srcs = [requirements_in] + elif src: + srcs = [src] else: srcs = srcs or ["pyproject.toml"] @@ -149,6 +157,12 @@ def pip_compile( # cheap way to detect the bazel version _bazel_version_4_or_greater = "propeller_optimize" in dir(native) + # setuptools (the default python build tool) attempts to find user configuration in the user's home direcotory. This seems to work fine on linux and macOS, but fails on Windows. We provide a fake USERPROFILE env variable to allow setuptools to proceed without finding user-provided configuration. + kwargs["env"] = kwargs.pop("env", {}) | select({ + "@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"}, + "//conditions:default": {} + }) + # Bazel 4.0 added the "env" attribute to py_test/py_binary if _bazel_version_4_or_greater: attrs["env"] = kwargs.pop("env", {}) From 9eb19143ac03c28fb1074426d6bb52977185e804 Mon Sep 17 00:00:00 2001 From: Cal Jacobson Date: Thu, 8 Aug 2024 15:40:07 -0500 Subject: [PATCH 05/10] try `@@platforms`...? --- python/private/pypi/pip_compile.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index 39528ae0a5..5620786fe9 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -159,10 +159,10 @@ def pip_compile( # setuptools (the default python build tool) attempts to find user configuration in the user's home direcotory. This seems to work fine on linux and macOS, but fails on Windows. We provide a fake USERPROFILE env variable to allow setuptools to proceed without finding user-provided configuration. kwargs["env"] = kwargs.pop("env", {}) | select({ - "@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"}, - "//conditions:default": {} + "@@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"}, + "//conditions:default": {}, }) - + # Bazel 4.0 added the "env" attribute to py_test/py_binary if _bazel_version_4_or_greater: attrs["env"] = kwargs.pop("env", {}) From 9a5653ca7f32fbf698dfb1ba1b7c425dc521c7d5 Mon Sep 17 00:00:00 2001 From: Cal Jacobson Date: Thu, 8 Aug 2024 16:04:01 -0500 Subject: [PATCH 06/10] use `src` rather than `srcs` when only one requirements file is needed --- docs/sphinx/BUILD.bazel | 2 +- examples/build_file_generation/BUILD.bazel | 2 +- examples/bzlmod/BUILD.bazel | 4 ++-- examples/bzlmod/other_module/BUILD.bazel | 2 +- examples/bzlmod_build_file_generation/BUILD.bazel | 2 +- examples/multi_python_versions/requirements/BUILD.bazel | 8 ++++---- examples/pip_parse/BUILD.bazel | 2 +- examples/pip_parse_vendored/BUILD.bazel | 2 +- examples/pip_repository_annotations/BUILD.bazel | 2 +- examples/wheel/BUILD.bazel | 2 +- tests/integration/compile_pip_requirements/BUILD.bazel | 6 +++--- tools/publish/BUILD.bazel | 2 +- 12 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/sphinx/BUILD.bazel b/docs/sphinx/BUILD.bazel index f3ca85b552..f82d43a45a 100644 --- a/docs/sphinx/BUILD.bazel +++ b/docs/sphinx/BUILD.bazel @@ -130,7 +130,7 @@ sphinx_build_binary( # Run bazel run //docs/sphinx:requirements.update compile_pip_requirements( name = "requirements", - srcs = ["pyproject.toml"], + src = "pyproject.toml", requirements_txt = "requirements.txt", target_compatible_with = _TARGET_COMPATIBLE_WITH, ) diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel index 9ff277c5a2..4d270dd850 100644 --- a/examples/build_file_generation/BUILD.bazel +++ b/examples/build_file_generation/BUILD.bazel @@ -11,7 +11,7 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") compile_pip_requirements( name = "requirements", - srcs = ["requirements.in"], + src = "requirements.in", requirements_txt = "requirements_lock.txt", requirements_windows = "requirements_windows.txt", ) diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index 840c681fc1..bb16f98a6f 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -16,7 +16,7 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") # with pip-compile. compile_pip_requirements_3_9( name = "requirements_3_9", - srcs = ["requirements.in"], + src = "requirements.in", requirements_txt = "requirements_lock_3_9.txt", requirements_windows = "requirements_windows_3_9.txt", ) @@ -26,7 +26,7 @@ compile_pip_requirements_3_9( compile_pip_requirements_3_10( name = "requirements_3_10", timeout = "moderate", - srcs = ["requirements.in"], + src = "requirements.in", requirements_txt = "requirements_lock_3_10.txt", requirements_windows = "requirements_windows_3_10.txt", ) diff --git a/examples/bzlmod/other_module/BUILD.bazel b/examples/bzlmod/other_module/BUILD.bazel index 9392c474bd..a93b92aaed 100644 --- a/examples/bzlmod/other_module/BUILD.bazel +++ b/examples/bzlmod/other_module/BUILD.bazel @@ -4,6 +4,6 @@ load("@python_versions//3.11:defs.bzl", compile_pip_requirements_311 = "compile_ # override in the MODULE.bazel. compile_pip_requirements_311( name = "requirements", - srcs = ["requirements.in"], + src = "requirements.in", requirements_txt = "requirements_lock_3_11.txt", ) diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index b46919c257..33d01f4119 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -16,7 +16,7 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") # with pip-compile. compile_pip_requirements( name = "requirements", - srcs = ["requirements.in"], + src = "requirements.in", requirements_txt = "requirements_lock.txt", requirements_windows = "requirements_windows.txt", ) diff --git a/examples/multi_python_versions/requirements/BUILD.bazel b/examples/multi_python_versions/requirements/BUILD.bazel index 12b42ef180..f67333a657 100644 --- a/examples/multi_python_versions/requirements/BUILD.bazel +++ b/examples/multi_python_versions/requirements/BUILD.bazel @@ -5,24 +5,24 @@ load("@python//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requir compile_pip_requirements_3_8( name = "requirements_3_8", - srcs = ["requirements.in"], + src = "requirements.in", requirements_txt = "requirements_lock_3_8.txt", ) compile_pip_requirements_3_9( name = "requirements_3_9", - srcs = ["requirements.in"], + src = "requirements.in", requirements_txt = "requirements_lock_3_9.txt", ) compile_pip_requirements_3_10( name = "requirements_3_10", - srcs = ["requirements.in"], + src = "requirements.in", requirements_txt = "requirements_lock_3_10.txt", ) compile_pip_requirements_3_11( name = "requirements_3_11", - srcs = ["requirements.in"], + src = "requirements.in", requirements_txt = "requirements_lock_3_11.txt", ) diff --git a/examples/pip_parse/BUILD.bazel b/examples/pip_parse/BUILD.bazel index 5d8a28682c..fd744a2836 100644 --- a/examples/pip_parse/BUILD.bazel +++ b/examples/pip_parse/BUILD.bazel @@ -55,7 +55,7 @@ py_console_script_binary( # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", - srcs = ["requirements.in"], + src = "requirements.in", requirements_txt = "requirements_lock.txt", requirements_windows = "requirements_windows.txt", ) diff --git a/examples/pip_parse_vendored/BUILD.bazel b/examples/pip_parse_vendored/BUILD.bazel index 2926ba2f9a..e2b1f5d49b 100644 --- a/examples/pip_parse_vendored/BUILD.bazel +++ b/examples/pip_parse_vendored/BUILD.bazel @@ -9,7 +9,7 @@ load("//:requirements.bzl", "all_data_requirements", "all_requirements", "all_wh # lockfile based on the requirements.in. compile_pip_requirements( name = "requirements", - srcs = ["requirements.in"], + src = "requirements.in", ) # The requirements.bzl file is using the hub repo to access packages via the diff --git a/examples/pip_repository_annotations/BUILD.bazel b/examples/pip_repository_annotations/BUILD.bazel index a1e1b558fd..bdf9df1274 100644 --- a/examples/pip_repository_annotations/BUILD.bazel +++ b/examples/pip_repository_annotations/BUILD.bazel @@ -9,7 +9,7 @@ exports_files( # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", - srcs = ["requirements.in"], + src = "requirements.in", ) py_test( diff --git a/examples/wheel/BUILD.bazel b/examples/wheel/BUILD.bazel index 5d08a8e0a8..aa063ce542 100644 --- a/examples/wheel/BUILD.bazel +++ b/examples/wheel/BUILD.bazel @@ -360,7 +360,7 @@ py_test( compile_pip_requirements( name = "requirements_server", - srcs = ["requirements_server.in"], + src = "requirements_server.in", ) py_test( diff --git a/tests/integration/compile_pip_requirements/BUILD.bazel b/tests/integration/compile_pip_requirements/BUILD.bazel index cf95e6c627..6df46b8372 100644 --- a/tests/integration/compile_pip_requirements/BUILD.bazel +++ b/tests/integration/compile_pip_requirements/BUILD.bazel @@ -21,7 +21,7 @@ EOF compile_pip_requirements( name = "requirements", - srcs = ["requirements.txt"], + src = "requirements.txt", data = [ "requirements.in", "requirements_extra.in", @@ -31,7 +31,7 @@ compile_pip_requirements( compile_pip_requirements( name = "requirements_nohashes", - srcs = ["requirements.txt"], + src = "requirements.txt", data = [ "requirements.in", "requirements_extra.in", @@ -55,7 +55,7 @@ EOF compile_pip_requirements( name = "os_specific_requirements", - srcs = ["requirements_os_specific.in"], + src = "requirements_os_specific.in", data = [ "requirements_extra.in", "requirements_os_specific.in", diff --git a/tools/publish/BUILD.bazel b/tools/publish/BUILD.bazel index 5935111d39..a51693b9fc 100644 --- a/tools/publish/BUILD.bazel +++ b/tools/publish/BUILD.bazel @@ -4,7 +4,7 @@ load("//python/entry_points:py_console_script_binary.bzl", "py_console_script_bi compile_pip_requirements( name = "requirements", - srcs = ["requirements.in"], + src = "requirements.in", requirements_darwin = "requirements_darwin.txt", requirements_windows = "requirements_windows.txt", ) From 2f80e2b8d904aca2eb02d4b6401bc27e312b608a Mon Sep 17 00:00:00 2001 From: Cal Jacobson Date: Fri, 9 Aug 2024 10:09:12 -0500 Subject: [PATCH 07/10] review feedback --- CHANGELOG.md | 3 +++ python/private/pypi/pip_compile.bzl | 16 +++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 066a98a3f7..762104d7d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ A brief description of the categories of changes: * `3.12 -> 3.12.4` ### Fixed +* (rules) `compile_pip_requirements` now sets the `USERPROFILE` env variable on + windows to work around an issue where `setuptools` fails to locate the user's + home directory. * (rules) Signals are properly received when using {obj}`--bootstrap_impl=script` (for non-zip builds). ([#2043](https://github.com/bazelbuild/rules_python/issues/2043)) diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index 5620786fe9..102d40ee6e 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -157,11 +157,17 @@ def pip_compile( # cheap way to detect the bazel version _bazel_version_4_or_greater = "propeller_optimize" in dir(native) - # setuptools (the default python build tool) attempts to find user configuration in the user's home direcotory. This seems to work fine on linux and macOS, but fails on Windows. We provide a fake USERPROFILE env variable to allow setuptools to proceed without finding user-provided configuration. - kwargs["env"] = kwargs.pop("env", {}) | select({ - "@@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"}, - "//conditions:default": {}, - }) + # setuptools (the default python build tool) attempts to find user + # configuration in the user's home direcotory. This seems to work fine on + # linux and macOS, but fails on Windows, so we conditionally provide a fake + # USERPROFILE env variable to allow setuptools to proceed without finding + # user-provided configuration. + env = kwargs.get("env", {}) + if "USERPROFILE" not in env: + kwargs["env"] = env | select({ + "@@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"}, + "//conditions:default": {}, + }) # Bazel 4.0 added the "env" attribute to py_test/py_binary if _bazel_version_4_or_greater: From 1f12338428e102caa5ebff87d0c9500aaa994185 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:29:17 +0300 Subject: [PATCH 08/10] Update python/private/pypi/pip_compile.bzl Co-authored-by: Cal Jacobson --- python/private/pypi/pip_compile.bzl | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index 102d40ee6e..b97d94b616 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -162,12 +162,10 @@ def pip_compile( # linux and macOS, but fails on Windows, so we conditionally provide a fake # USERPROFILE env variable to allow setuptools to proceed without finding # user-provided configuration. - env = kwargs.get("env", {}) - if "USERPROFILE" not in env: - kwargs["env"] = env | select({ - "@@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"}, - "//conditions:default": {}, - }) + kwargs["env"] = select({ + "@@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"}, + "//conditions:default": {}, + }) | kwargs.get("env", {}) # Bazel 4.0 added the "env" attribute to py_test/py_binary if _bazel_version_4_or_greater: From fc1fb1228bd7ff0c31e79f6ceeb2cfafe0e8b189 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:29:59 +0300 Subject: [PATCH 09/10] cleanup the code and fix the indentation --- python/private/pypi/pip_compile.bzl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index b97d94b616..a6cabf70c1 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -154,23 +154,16 @@ def pip_compile( "visibility": visibility, } - # cheap way to detect the bazel version - _bazel_version_4_or_greater = "propeller_optimize" in dir(native) - # setuptools (the default python build tool) attempts to find user # configuration in the user's home direcotory. This seems to work fine on # linux and macOS, but fails on Windows, so we conditionally provide a fake # USERPROFILE env variable to allow setuptools to proceed without finding # user-provided configuration. - kwargs["env"] = select({ + kwargs["env"] = select({ "@@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"}, "//conditions:default": {}, }) | kwargs.get("env", {}) - # Bazel 4.0 added the "env" attribute to py_test/py_binary - if _bazel_version_4_or_greater: - attrs["env"] = kwargs.pop("env", {}) - py_binary( name = name + ".update", **attrs From a10e9e855cd65f2bb75a3ca6b16368a07a5b098a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 15 Aug 2024 12:20:29 -0700 Subject: [PATCH 10/10] typo: windows -> Windows --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d44c81e1a..9c50e1e6d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,7 @@ A brief description of the categories of changes: ### Fixed * (rules) `compile_pip_requirements` now sets the `USERPROFILE` env variable on - windows to work around an issue where `setuptools` fails to locate the user's + Windows to work around an issue where `setuptools` fails to locate the user's home directory. * (rules) correctly handle absolute URLs in parse_simpleapi_html.bzl. * (rules) Fixes build targets linking against `@rules_python//python/cc:current_py_cc_libs`