diff --git a/CHANGELOG.md b/CHANGELOG.md index c87e76e5be..9c50e1e6d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ A brief description of the categories of changes: * `3.12 -> 3.12.4` ### Fixed +* (rules) `compile_pip_requirements` now sets the `USERPROFILE` env variable on + Windows to work around an issue where `setuptools` fails to locate the user's + home directory. * (rules) correctly handle absolute URLs in parse_simpleapi_html.bzl. * (rules) Fixes build targets linking against `@rules_python//python/cc:current_py_cc_libs` in host platform builds on macOS, by editing the `LC_ID_DYLIB` field of the hermetic interpreter's @@ -69,6 +72,7 @@ A brief description of the categories of changes: Fixes [#1818](https://github.com/bazelbuild/rules_python/issues/1818). ### Added +* (rules) `compile_pip_requirements` supports multiple requirements input files as `srcs`. * (rules) `PYTHONSAFEPATH` is inherited from the calling environment to allow disabling it (Requires {obj}`--bootstrap_impl=script`) ([#2060](https://github.com/bazelbuild/rules_python/issues/2060)). diff --git a/python/private/pypi/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py index afe5076b4f..0ff9b2fb7c 100644 --- a/python/private/pypi/dependency_resolver/dependency_resolver.py +++ b/python/private/pypi/dependency_resolver/dependency_resolver.py @@ -80,7 +80,7 @@ def _locate(bazel_runfiles, file): @click.command(context_settings={"ignore_unknown_options": True}) -@click.argument("requirements_in") +@click.option("--src", "srcs", multiple=True, required=True) @click.argument("requirements_txt") @click.argument("update_target_label") @click.option("--requirements-linux") @@ -88,7 +88,7 @@ def _locate(bazel_runfiles, file): @click.option("--requirements-windows") @click.argument("extra_args", nargs=-1, type=click.UNPROCESSED) def main( - requirements_in: str, + srcs: Tuple[str, ...], requirements_txt: str, update_target_label: str, requirements_linux: Optional[str], @@ -105,7 +105,7 @@ def main( requirements_windows=requirements_windows, ) - resolved_requirements_in = _locate(bazel_runfiles, requirements_in) + resolved_srcs = [_locate(bazel_runfiles, src) for src in srcs] resolved_requirements_file = _locate(bazel_runfiles, requirements_file) # Files in the runfiles directory has the following naming schema: @@ -118,11 +118,11 @@ def main( : -(len(requirements_file) - len(repository_prefix)) ] - # As requirements_in might contain references to generated files we want to + # As srcs might contain references to generated files we want to # use the runfiles file first. Thus, we need to compute the relative path # from the execution root. # Note: Windows cannot reference generated files without runfiles support enabled. - requirements_in_relative = requirements_in[len(repository_prefix) :] + srcs_relative = [src[len(repository_prefix) :] for src in srcs] requirements_file_relative = requirements_file[len(repository_prefix) :] # Before loading click, set the locale for its parser. @@ -162,10 +162,9 @@ def main( argv.append( f"--output-file={requirements_file_relative if UPDATE else requirements_out}" ) - argv.append( - requirements_in_relative - if Path(requirements_in_relative).exists() - else resolved_requirements_in + argv.extend( + (src_relative if Path(src_relative).exists() else resolved_src) + for src_relative, resolved_src in zip(srcs_relative, resolved_srcs) ) argv.extend(extra_args) @@ -200,7 +199,7 @@ def main( print( "pip-compile exited with code 2. This means that pip-compile found " "incompatible requirements or could not find a version that matches " - f"the install requirement in {requirements_in_relative}.", + f"the install requirement in one of {srcs_relative}.", file=sys.stderr, ) sys.exit(1) diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index f284a00f68..a6cabf70c1 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -23,6 +23,7 @@ load("//python:defs.bzl", _py_binary = "py_binary", _py_test = "py_test") def pip_compile( name, + srcs = None, src = None, extra_args = [], extra_deps = [], @@ -53,6 +54,11 @@ def pip_compile( Args: name: base name for generated targets, typically "requirements". + srcs: a list of files containing inputs to dependency resolution. If not specified, + defaults to `["pyproject.toml"]`. Supported formats are: + * a requirements text file, usually named `requirements.in` + * A `.toml` file, where the `project.dependencies` list is used as per + [PEP621](https://peps.python.org/pep-0621/). src: file containing inputs to dependency resolution. If not specified, defaults to `pyproject.toml`. Supported formats are: * a requirements text file, usually named `requirements.in` @@ -63,7 +69,7 @@ def pip_compile( generate_hashes: whether to put hashes in the requirements_txt file. py_binary: the py_binary rule to be used. py_test: the py_test rule to be used. - requirements_in: file expressing desired dependencies. Deprecated, use src instead. + requirements_in: file expressing desired dependencies. Deprecated, use src or srcs instead. requirements_txt: result of "compiling" the requirements.in file. requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes. requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes. @@ -72,10 +78,15 @@ def pip_compile( visibility: passed to both the _test and .update rules. **kwargs: other bazel attributes passed to the "_test" rule. """ - if requirements_in and src: - fail("Only one of 'src' and 'requirements_in' attributes can be used") + if len([x for x in [srcs, src, requirements_in] if x != None]) > 1: + fail("At most one of 'srcs', 'src', and 'requirements_in' attributes may be provided") + + if requirements_in: + srcs = [requirements_in] + elif src: + srcs = [src] else: - src = requirements_in or src or "pyproject.toml" + srcs = srcs or ["pyproject.toml"] requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt @@ -88,7 +99,7 @@ def pip_compile( visibility = visibility, ) - data = [name, requirements_txt, src] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + data = [name, requirements_txt] + srcs + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] # Use the Label constructor so this is expanded in the context of the file # where it appears, which is to say, in @rules_python @@ -96,8 +107,7 @@ def pip_compile( loc = "$(rlocationpath {})" - args = [ - loc.format(src), + args = ["--src=%s" % loc.format(src) for src in srcs] + [ loc.format(requirements_txt), "//%s:%s.update" % (native.package_name(), name), "--resolver=backtracking", @@ -144,12 +154,15 @@ def pip_compile( "visibility": visibility, } - # cheap way to detect the bazel version - _bazel_version_4_or_greater = "propeller_optimize" in dir(native) - - # Bazel 4.0 added the "env" attribute to py_test/py_binary - if _bazel_version_4_or_greater: - attrs["env"] = kwargs.pop("env", {}) + # setuptools (the default python build tool) attempts to find user + # configuration in the user's home direcotory. This seems to work fine on + # linux and macOS, but fails on Windows, so we conditionally provide a fake + # USERPROFILE env variable to allow setuptools to proceed without finding + # user-provided configuration. + kwargs["env"] = select({ + "@@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"}, + "//conditions:default": {}, + }) | kwargs.get("env", {}) py_binary( name = name + ".update", diff --git a/tests/multiple_inputs/BUILD.bazel b/tests/multiple_inputs/BUILD.bazel new file mode 100644 index 0000000000..3e3cab83ca --- /dev/null +++ b/tests/multiple_inputs/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_python//python:pip.bzl", "compile_pip_requirements") + +compile_pip_requirements( + name = "multiple_requirements_in", + srcs = [ + "requirements_1.in", + "requirements_2.in", + ], + requirements_txt = "multiple_requirements_in.txt", +) + +compile_pip_requirements( + name = "multiple_pyproject_toml", + srcs = [ + "a/pyproject.toml", + "b/pyproject.toml", + ], + requirements_txt = "multiple_pyproject_toml.txt", +) + +compile_pip_requirements( + name = "multiple_inputs", + srcs = [ + "a/pyproject.toml", + "b/pyproject.toml", + "requirements_1.in", + "requirements_2.in", + ], + requirements_txt = "multiple_inputs.txt", +) diff --git a/tests/multiple_inputs/README.md b/tests/multiple_inputs/README.md new file mode 100644 index 0000000000..7b6bade122 --- /dev/null +++ b/tests/multiple_inputs/README.md @@ -0,0 +1,3 @@ +# multiple_inputs + +Test that `compile_pip_requirements` works as intended when using more than one input file. diff --git a/tests/multiple_inputs/a/pyproject.toml b/tests/multiple_inputs/a/pyproject.toml new file mode 100644 index 0000000000..91efec3821 --- /dev/null +++ b/tests/multiple_inputs/a/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "multiple_inputs_1" +version = "0.0.0" + +dependencies = ["urllib3"] diff --git a/tests/multiple_inputs/b/pyproject.toml b/tests/multiple_inputs/b/pyproject.toml new file mode 100644 index 0000000000..a461f4ed98 --- /dev/null +++ b/tests/multiple_inputs/b/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "multiple_inputs_2" +version = "0.0.0" + +dependencies = ["attrs"] diff --git a/tests/multiple_inputs/multiple_inputs.txt b/tests/multiple_inputs/multiple_inputs.txt new file mode 100644 index 0000000000..a036c3f33d --- /dev/null +++ b/tests/multiple_inputs/multiple_inputs.txt @@ -0,0 +1,18 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# bazel run //tests/multiple_inputs:multiple_inputs.update +# +attrs==23.1.0 \ + --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ + --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 + # via + # -r tests/multiple_inputs/requirements_2.in + # multiple_inputs_2 (tests/multiple_inputs/b/pyproject.toml) +urllib3==2.0.7 \ + --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ + --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e + # via + # -r tests/multiple_inputs/requirements_1.in + # multiple_inputs_1 (tests/multiple_inputs/a/pyproject.toml) diff --git a/tests/multiple_inputs/multiple_pyproject_toml.txt b/tests/multiple_inputs/multiple_pyproject_toml.txt new file mode 100644 index 0000000000..b8af28ac10 --- /dev/null +++ b/tests/multiple_inputs/multiple_pyproject_toml.txt @@ -0,0 +1,14 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# bazel run //tests/multiple_inputs:multiple_pyproject_toml.update +# +attrs==23.1.0 \ + --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ + --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 + # via multiple_inputs_2 (tests/multiple_inputs/b/pyproject.toml) +urllib3==2.0.7 \ + --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ + --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e + # via multiple_inputs_1 (tests/multiple_inputs/a/pyproject.toml) diff --git a/tests/multiple_inputs/multiple_requirements_in.txt b/tests/multiple_inputs/multiple_requirements_in.txt new file mode 100644 index 0000000000..63edfe9f53 --- /dev/null +++ b/tests/multiple_inputs/multiple_requirements_in.txt @@ -0,0 +1,14 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# bazel run //tests/multiple_inputs:multiple_requirements_in.update +# +attrs==23.1.0 \ + --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ + --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 + # via -r tests/multiple_inputs/requirements_2.in +urllib3==2.0.7 \ + --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ + --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e + # via -r tests/multiple_inputs/requirements_1.in diff --git a/tests/multiple_inputs/requirements_1.in b/tests/multiple_inputs/requirements_1.in new file mode 100644 index 0000000000..a42590bebe --- /dev/null +++ b/tests/multiple_inputs/requirements_1.in @@ -0,0 +1 @@ +urllib3 diff --git a/tests/multiple_inputs/requirements_2.in b/tests/multiple_inputs/requirements_2.in new file mode 100644 index 0000000000..04cb10228e --- /dev/null +++ b/tests/multiple_inputs/requirements_2.in @@ -0,0 +1 @@ +attrs