From cc640222d575859dae37b7145348747d37dc2f50 Mon Sep 17 00:00:00 2001 From: Adam Liddell Date: Mon, 8 Mar 2021 21:57:46 +0000 Subject: [PATCH 1/6] Run pip within the directory containing the requirements.txt file This allows for relative requirements to be resolved in the way that standalone pip would without patching or parsing the requirements file manually --- python/pip_install/extract_wheels/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/pip_install/extract_wheels/__init__.py b/python/pip_install/extract_wheels/__init__.py index fe8b8ef7ea..5abace6107 100644 --- a/python/pip_install/extract_wheels/__init__.py +++ b/python/pip_install/extract_wheels/__init__.py @@ -8,6 +8,7 @@ import argparse import glob import os +import pathlib import subprocess import sys import json @@ -79,12 +80,15 @@ def main() -> None: ) args = parser.parse_args() - pip_args = [sys.executable, "-m", "pip", "--isolated", "wheel", "-r", args.requirements] + # Pip is run with the working directory changed to the folder containing the requirements.txt file, to allow for + # relative requirements to be correctly resolved. The --wheel-dir is therefore required to be repointed back to the + # current calling working directory (the repo root in .../external/name), where the wheel files should be written to + pip_args = [sys.executable, "-m", "pip", "--isolated", "wheel", "-r", args.requirements, "--wheel-dir", os.getcwd()] if args.extra_pip_args: pip_args += json.loads(args.extra_pip_args)["args"] # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails - subprocess.run(pip_args, check=True) + subprocess.run(pip_args, check=True, cwd=pathlib.Path(args.requirements).parent.resolve()) extras = requirements.parse_extras(args.requirements) From dd1c8cf298a32a84819e6aa7d10e34da790a5344 Mon Sep 17 00:00:00 2001 From: Adam Liddell Date: Mon, 8 Mar 2021 22:39:26 +0000 Subject: [PATCH 2/6] Print test errors of sub-workspaces in bazel_integration_test Prior to this, errors would be written to logs in the temp directory that would promptly be deleted on parent test teardown. This means at least the test logs will be printed on failure --- tools/bazel_integration_test/bazel_integration_test.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bazel_integration_test/bazel_integration_test.bzl b/tools/bazel_integration_test/bazel_integration_test.bzl index caac0d9343..a6fb50d25a 100644 --- a/tools/bazel_integration_test/bazel_integration_test.bzl +++ b/tools/bazel_integration_test/bazel_integration_test.bzl @@ -13,8 +13,8 @@ _ATTRS = { It is assumed by the test runner that the bazel binary is found at label_workspace/bazel (wksp/bazel.exe on Windows)""", ), "bazel_commands": attr.string_list( - default = ["info", "test ..."], - doc = """The list of bazel commands to run. Defaults to `["info", "test ..."]`. + default = ["info", "test --test_output=errors ..."], + doc = """The list of bazel commands to run. Defaults to `["info", "test --test_output=errors ..."]`. Note that if a command contains a bare `--` argument, the --test_arg passed to Bazel will appear before it. """, From e3b8daababe5b90f741e429bbdafe21996037e4a Mon Sep 17 00:00:00 2001 From: Adam Liddell Date: Mon, 8 Mar 2021 22:57:21 +0000 Subject: [PATCH 3/6] Add example of relative requirement as test --- .bazelrc | 4 ++-- examples/BUILD | 5 +++++ examples/relative_requirements/BUILD | 10 ++++++++++ examples/relative_requirements/README.md | 4 ++++ examples/relative_requirements/WORKSPACE | 15 +++++++++++++++ examples/relative_requirements/main.py | 5 +++++ .../relative_package_name/__init__.py | 2 ++ .../relative_package/setup.py | 7 +++++++ examples/relative_requirements/requirements.txt | 1 + 9 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 examples/relative_requirements/BUILD create mode 100644 examples/relative_requirements/README.md create mode 100644 examples/relative_requirements/WORKSPACE create mode 100644 examples/relative_requirements/main.py create mode 100644 examples/relative_requirements/relative_package/relative_package_name/__init__.py create mode 100644 examples/relative_requirements/relative_package/setup.py create mode 100644 examples/relative_requirements/requirements.txt diff --git a/.bazelrc b/.bazelrc index 1afaedbc24..8381bea0a7 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,7 +3,7 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install -query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install +build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/relative_requirements +query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/relative_requirements test --test_output=errors diff --git a/examples/BUILD b/examples/BUILD index 092ad40902..32ddf12a17 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -26,3 +26,8 @@ bazel_integration_test( name = "pip_install_example", timeout = "long", ) + +bazel_integration_test( + name = "relative_requirements_example", + timeout = "long", +) diff --git a/examples/relative_requirements/BUILD b/examples/relative_requirements/BUILD new file mode 100644 index 0000000000..d24ee5f72b --- /dev/null +++ b/examples/relative_requirements/BUILD @@ -0,0 +1,10 @@ +load("@pip//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_test") + +py_test( + name = "main", + srcs = ["main.py"], + deps = [ + requirement("relative_package_name"), + ], +) diff --git a/examples/relative_requirements/README.md b/examples/relative_requirements/README.md new file mode 100644 index 0000000000..4b9258e370 --- /dev/null +++ b/examples/relative_requirements/README.md @@ -0,0 +1,4 @@ +# relative_requirements example + +This example shows how to use pip to fetch relative dependencies from a requirements.txt file, +then use them in BUILD files as dependencies of Bazel targets. diff --git a/examples/relative_requirements/WORKSPACE b/examples/relative_requirements/WORKSPACE new file mode 100644 index 0000000000..1b37fbdbbb --- /dev/null +++ b/examples/relative_requirements/WORKSPACE @@ -0,0 +1,15 @@ +workspace(name = "example_repo") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_python", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", + sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", +) + +load("@rules_python//python:pip.bzl", "pip_install") + +pip_install( + requirements = "//:requirements.txt", +) diff --git a/examples/relative_requirements/main.py b/examples/relative_requirements/main.py new file mode 100644 index 0000000000..b8ac021e90 --- /dev/null +++ b/examples/relative_requirements/main.py @@ -0,0 +1,5 @@ +import relative_package_name + +if __name__ == "__main__": + # Run a function from the relative package + print(relative_package_name.test()) diff --git a/examples/relative_requirements/relative_package/relative_package_name/__init__.py b/examples/relative_requirements/relative_package/relative_package_name/__init__.py new file mode 100644 index 0000000000..c031192907 --- /dev/null +++ b/examples/relative_requirements/relative_package/relative_package_name/__init__.py @@ -0,0 +1,2 @@ +def test(): + return True diff --git a/examples/relative_requirements/relative_package/setup.py b/examples/relative_requirements/relative_package/setup.py new file mode 100644 index 0000000000..3fd85c12ae --- /dev/null +++ b/examples/relative_requirements/relative_package/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='relative_package_name', + version='1.0.0', + packages=['relative_package_name'], +) diff --git a/examples/relative_requirements/requirements.txt b/examples/relative_requirements/requirements.txt new file mode 100644 index 0000000000..9a81317e1e --- /dev/null +++ b/examples/relative_requirements/requirements.txt @@ -0,0 +1 @@ +./relative_package From 10e1624fa580c4204a70eb3dfcbd4b20dbcf9667 Mon Sep 17 00:00:00 2001 From: Adam Liddell Date: Mon, 8 Mar 2021 23:03:24 +0000 Subject: [PATCH 4/6] Explicitly cast Path to str to support Python 3.5 --- python/pip_install/extract_wheels/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pip_install/extract_wheels/__init__.py b/python/pip_install/extract_wheels/__init__.py index 5abace6107..8a38733df4 100644 --- a/python/pip_install/extract_wheels/__init__.py +++ b/python/pip_install/extract_wheels/__init__.py @@ -88,7 +88,7 @@ def main() -> None: pip_args += json.loads(args.extra_pip_args)["args"] # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails - subprocess.run(pip_args, check=True, cwd=pathlib.Path(args.requirements).parent.resolve()) + subprocess.run(pip_args, check=True, cwd=str(pathlib.Path(args.requirements).parent.resolve())) extras = requirements.parse_extras(args.requirements) From 550afc99a5998bd5dd05e840c8181e2d7f7af0f7 Mon Sep 17 00:00:00 2001 From: Adam Liddell Date: Wed, 22 Sep 2021 13:00:23 +0000 Subject: [PATCH 5/6] Fix multiple declarations of pip_args --- python/pip_install/extract_wheels/__init__.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/python/pip_install/extract_wheels/__init__.py b/python/pip_install/extract_wheels/__init__.py index f5967c8c71..5fdf9eae75 100644 --- a/python/pip_install/extract_wheels/__init__.py +++ b/python/pip_install/extract_wheels/__init__.py @@ -64,20 +64,17 @@ def main() -> None: deserialized_args = dict(vars(args)) arguments.deserialize_structured_args(deserialized_args) + # Pip is run with the working directory changed to the folder containing the requirements.txt file, to allow for + # relative requirements to be correctly resolved. The --wheel-dir is therefore required to be repointed back to the + # current calling working directory (the repo root in .../external/name), where the wheel files should be written to pip_args = ( [sys.executable, "-m", "pip"] + (["--isolated"] if args.isolated else []) + ["wheel", "-r", args.requirements] + + ["--wheel-dir", os.getcwd()] + deserialized_args["extra_pip_args"] ) - # Pip is run with the working directory changed to the folder containing the requirements.txt file, to allow for - # relative requirements to be correctly resolved. The --wheel-dir is therefore required to be repointed back to the - # current calling working directory (the repo root in .../external/name), where the wheel files should be written to - pip_args = [sys.executable, "-m", "pip", "--isolated", "wheel", "-r", args.requirements, "--wheel-dir", os.getcwd()] - if args.extra_pip_args: - pip_args += json.loads(args.extra_pip_args)["args"] - env = os.environ.copy() env.update(deserialized_args["environment"]) From 96c30bef520f9b8530e1ecd4cb461bfd2655769b Mon Sep 17 00:00:00 2001 From: Adam Liddell Date: Wed, 22 Sep 2021 13:05:40 +0000 Subject: [PATCH 6/6] Apply buildifier fix --- examples/relative_requirements/WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/relative_requirements/WORKSPACE b/examples/relative_requirements/WORKSPACE index 1b37fbdbbb..505fa9ebc8 100644 --- a/examples/relative_requirements/WORKSPACE +++ b/examples/relative_requirements/WORKSPACE @@ -4,8 +4,8 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "rules_python", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", ) load("@rules_python//python:pip.bzl", "pip_install")