diff --git a/docs/pip.md b/docs/pip.md index 0d657b4a42..61d8f976ef 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -134,6 +134,17 @@ alias( ) ``` +Similar to the `entry_point` macro, `setup_py_script` allows accessing +[`setup.py` generated scripts][scripts]: +```python +load("@pip_deps//:requirements.bzl", "setup_py_script") +alias( + name = "flake8", + actual = setup_py_script("flake8"), +) +``` +[scripts]: https://docs.python.org/3/distutils/setupscript.html#installing-scripts + **PARAMETERS** @@ -218,6 +229,17 @@ alias( ) ``` +Similar to the `entry_point` macro, `setup_py_script` allows accessing +[`setup.py` generated scripts][scripts]: +```python +load("@pip_deps//:requirements.bzl", "setup_py_script") +alias( + name = "flake8", + actual = setup_py_script("flake8"), +) +``` +[scripts]: https://docs.python.org/3/distutils/setupscript.html#installing-scripts + **PARAMETERS** diff --git a/examples/pip_install/BUILD b/examples/pip_install/BUILD index ecc083fdec..932bff05e1 100644 --- a/examples/pip_install/BUILD +++ b/examples/pip_install/BUILD @@ -6,6 +6,7 @@ load( "dist_info_requirement", "entry_point", "requirement", + "setup_py_script", ) load("@rules_python//python:defs.bzl", "py_binary", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -58,6 +59,11 @@ alias( actual = entry_point("yamllint"), ) +alias( + name = "s3cmd", + actual = setup_py_script("s3cmd"), +) + # Check that our compiled requirements are up-to-date compile_pip_requirements( name = "requirements", @@ -69,11 +75,13 @@ py_test( name = "pip_install_test", srcs = ["pip_install_test.py"], data = [ + ":s3cmd", ":yamllint", data_requirement("s3cmd"), dist_info_requirement("boto3"), ], env = { + "S3CMD_SETUP_PY_SCRIPT": "$(rootpath :s3cmd)", "WHEEL_DATA_CONTENTS": "$(rootpaths {})".format(data_requirement("s3cmd")), "WHEEL_DIST_INFO_CONTENTS": "$(rootpaths {})".format(dist_info_requirement("boto3")), "YAMLLINT_ENTRY_POINT": "$(rootpath :yamllint)", diff --git a/examples/pip_install/pip_install_test.py b/examples/pip_install/pip_install_test.py index f9a62ca6e8..7503fb5db6 100644 --- a/examples/pip_install/pip_install_test.py +++ b/examples/pip_install/pip_install_test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import platform import subprocess import unittest from pathlib import Path @@ -31,6 +32,28 @@ def test_entry_point(self): ) self.assertEqual(proc.stdout.decode("utf-8").strip(), "yamllint 1.26.3") + @unittest.skipIf( + platform.system() == "Windows", + "The `setup_py_script` macro does not currently support windows", + ) + def test_setup_py_script_s3cmd(self): + env = os.environ.get("S3CMD_SETUP_PY_SCRIPT") + self.assertIsNotNone(env) + + r = runfiles.Create() + + # To find an external target, this must use `{workspace_name}/$(rootpath @external_repo//:target)` + script = Path(r.Rlocation("rules_python_pip_install_example/{}".format(env))) + self.assertTrue(script.exists()) + + proc = subprocess.run( + [str(script), "--version"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(proc.stdout.decode("utf-8").strip(), "s3cmd version 2.1.0") + def test_data(self): env = os.environ.get("WHEEL_DATA_CONTENTS") self.assertIsNotNone(env) diff --git a/examples/pip_parse/BUILD b/examples/pip_parse/BUILD index 92b59ae775..378a163d83 100644 --- a/examples/pip_parse/BUILD +++ b/examples/pip_parse/BUILD @@ -3,6 +3,7 @@ load( "data_requirement", "dist_info_requirement", "entry_point", + "setup_py_script", ) load("@rules_python//python:defs.bzl", "py_binary", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -55,6 +56,11 @@ alias( actual = entry_point("yamllint"), ) +alias( + name = "s3cmd", + actual = setup_py_script("s3cmd"), +) + # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", @@ -68,11 +74,13 @@ py_test( name = "pip_parse_test", srcs = ["pip_parse_test.py"], data = [ + ":s3cmd", ":yamllint", data_requirement("s3cmd"), dist_info_requirement("requests"), ], env = { + "S3CMD_SETUP_PY_SCRIPT": "$(rootpath :s3cmd)", "WHEEL_DATA_CONTENTS": "$(rootpaths {})".format(data_requirement("s3cmd")), "WHEEL_DIST_INFO_CONTENTS": "$(rootpaths {})".format(dist_info_requirement("requests")), "YAMLLINT_ENTRY_POINT": "$(rootpath :yamllint)", diff --git a/examples/pip_parse/pip_parse_test.py b/examples/pip_parse/pip_parse_test.py index ef684c4294..46fe8ee048 100644 --- a/examples/pip_parse/pip_parse_test.py +++ b/examples/pip_parse/pip_parse_test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import platform import subprocess import unittest from pathlib import Path @@ -29,6 +30,28 @@ def test_entry_point(self): ) self.assertEqual(proc.stdout.decode("utf-8").strip(), "yamllint 1.26.3") + @unittest.skipIf( + platform.system() == "Windows", + "The `setup_py_script` macro does not currently support windows", + ) + def test_setup_py_script_s3cmd(self): + env = os.environ.get("S3CMD_SETUP_PY_SCRIPT") + self.assertIsNotNone(env) + + r = runfiles.Create() + + # To find an external target, this must use `{workspace_name}/$(rootpath @external_repo//:target)` + script = Path(r.Rlocation("rules_python_pip_parse_example/{}".format(env))) + self.assertTrue(script.exists()) + + proc = subprocess.run( + [str(script), "--version"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(proc.stdout.decode("utf-8").strip(), "s3cmd version 2.1.0") + def test_data(self): env = os.environ.get("WHEEL_DATA_CONTENTS") self.assertIsNotNone(env) diff --git a/python/pip.bzl b/python/pip.bzl index 0530a2c9f1..acf2c7c929 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -85,6 +85,17 @@ def pip_install(requirements, name = "pip", **kwargs): ) ``` + Similar to the `entry_point` macro, `setup_py_script` allows accessing + [`setup.py` generated scripts][scripts]: + ```python + load("@pip_deps//:requirements.bzl", "setup_py_script") + alias( + name = "flake8", + actual = setup_py_script("flake8"), + ) + ``` + [scripts]: https://docs.python.org/3/distutils/setupscript.html#installing-scripts + Args: requirements (Label): A 'requirements.txt' pip requirements file. name (str, optional): A unique name for the created external repository (default 'pip'). @@ -167,6 +178,17 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): ) ``` + Similar to the `entry_point` macro, `setup_py_script` allows accessing + [`setup.py` generated scripts][scripts]: + ```python + load("@pip_deps//:requirements.bzl", "setup_py_script") + alias( + name = "flake8", + actual = setup_py_script("flake8"), + ) + ``` + [scripts]: https://docs.python.org/3/distutils/setupscript.html#installing-scripts + Args: requirements_lock (Label): A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead diff --git a/python/pip_install/extract_wheels/__init__.py b/python/pip_install/extract_wheels/__init__.py index cda6d541b0..6637e3cb88 100644 --- a/python/pip_install/extract_wheels/__init__.py +++ b/python/pip_install/extract_wheels/__init__.py @@ -117,6 +117,9 @@ def main() -> None: enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs, repo_prefix=args.repo_prefix, annotation=annotations.get(name), + # Only `pip_install` uses this path so the `repo` arg + # is what's expected by this parameter. + pip_install_repo_name=args.repo, ), ) for whl, name in reqs.items() diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index 3c8697bfa7..c87fe29d9f 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py @@ -18,6 +18,7 @@ DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" +SETUP_PY_SCRIPT_PREFIX = "rules_python_setup_py_script" def generate_entry_point_contents( @@ -82,6 +83,74 @@ def generate_entry_point_rule(script: str, pkg: str) -> str: ) +def generate_setup_py_script_contents( + script: Path, repository_name: str, shebang: str = "#!/usr/bin/env python3" +) -> str: + """Generate the contents of a `setup.py` generated script + For details on `setup.py` generated scripts, see: + https://docs.python.org/3/distutils/setupscript.html#installing-scripts + Args: + script (Path): The path to the `setup.py` script + repository_name (str): The name of the repository for the current wheel. + shebang (str, optional): The shebang to use for the generated python + file. + Returns: + str: A string of python code. + """ + script_path = Path("external") / repository_name / script + return textwrap.dedent( + """\ + {shebang} + import os + import sys + if __name__ == "__main__": + script = "{script}" + if not os.path.exists(script): + raise FileNotFoundError(script) + args = sys.argv.copy() + args[0] = script + os.execv(sys.executable, [sys.executable] + args) + """.format( + shebang=shebang, script=str(script_path) + ) + ) + + +def generate_setup_py_script_rule(src: Path, script: Path, pkg: str) -> str: + """Generate a Bazel `py_binary` rule for a `setup.py` generated script. + For details on `setup.py` generated scripts, see: + https://docs.python.org/3/distutils/setupscript.html#installing-scripts + Args: + src (Path): The path to the source file generated by + `generate_setup_py_script_contents`. + script (Path): The path to the `setup.py` script. + pkg (str): The package owning the entry point. This is expected to + match up with the `py_library` defined for each repository. + Returns: + str: A `py_binary` instantiation. + """ + name = os.path.splitext(src.name)[0] + return textwrap.dedent( + """\ + py_binary( + name = "{name}", + srcs = ["{src}"], + main = "{src}", + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["."], + deps = ["{pkg}"], + data = ["{script}"], + ) + """.format( + name=name, + src=str(src.name).replace("\\", "/"), + pkg=pkg, + script=str(script).replace("\\", "/"), + ) + ) + + def generate_copy_commands(src, dest, is_executable=False) -> str: """Generate a [@bazel_skylib//rules:copy_file.bzl%copy_file][cf] target @@ -156,6 +225,7 @@ def generate_build_file_contents( "**/*.pyc", "BUILD.bazel", "WORKSPACE", + f"{SETUP_PY_SCRIPT_PREFIX}*.py", f"{WHEEL_ENTRY_POINT_PREFIX}*.py", ] + data_exclude @@ -260,6 +330,11 @@ def entry_point(pkg, script = None): script = pkg return requirement(pkg) + ":{entry_point_prefix}_" + script + def setup_py_script(pkg, script = None): + if not script: + script = pkg + return requirement(pkg) + ":{setup_py_script_prefix}_" + script + def install_deps(): fail("install_deps() only works if you are creating an incremental repo. Did you mean to use pip_parse()?") """.format( @@ -270,6 +345,7 @@ def install_deps(): data_label=DATA_LABEL, dist_info_label=DIST_INFO_LABEL, entry_point_prefix=WHEEL_ENTRY_POINT_PREFIX, + setup_py_script_prefix=SETUP_PY_SCRIPT_PREFIX, ) ) @@ -348,6 +424,7 @@ def extract_wheel( pip_data_exclude: List[str], enable_implicit_namespace_pkgs: bool, repo_prefix: str, + pip_install_repo_name: str = "", incremental: bool = False, incremental_dir: Path = Path("."), annotation: Optional[annotation.Annotation] = None, @@ -359,6 +436,8 @@ def extract_wheel( extras: a list of extras to add as dependencies for the installed wheel pip_data_exclude: list of file patterns to exclude from the generated data section of the py_library enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is + repo_prefix: Prefix to prepend to packages. + pip_install_repo_name: The external `pip_install` repo name to install dependencies. In the format `@{REPO_NAME}` incremental: If true the extract the wheel in a format suitable for an external repository. This effects the names of libraries and their dependencies, which point to other external repositories. incremental_dir: An optional override for the working directory of incremental builds. @@ -408,6 +487,39 @@ def extract_wheel( ) directory_path = Path(directory) + + # Generate scripts and `py_binary` targets for any scripts produced by `setup.py`. + setup_py_scripts = [] + wheel_data_dir = wheel.get_dot_data_directory(directory) + if wheel_data_dir: + scripts_dir = Path(wheel_data_dir) / "scripts" + for root, _dirnames, filenames in os.walk(str(scripts_dir)): + if pip_install_repo_name: + repository_name = pip_install_repo_name + else: + repository_name = "{}{}".format(repo_prefix, whl.name.replace("-", "_")) + + for filename in filenames: + setup_py_script = Path(root) / filename + setup_py_script_wrapper = ( + directory_path + / f"{SETUP_PY_SCRIPT_PREFIX}_{setup_py_script.name}.py" + ) + setup_py_script_wrapper.write_text( + generate_setup_py_script_contents( + script=setup_py_script, + repository_name=repository_name, + ) + ) + setup_py_scripts.append( + generate_setup_py_script_rule( + src=setup_py_script_wrapper, + script=setup_py_script.relative_to(directory_path), + pkg=library_name, + ) + ) + + # Generate scripts and `py_binary` targets for wheel entry points entry_points = [] for name, entry_point in sorted(whl.entry_points().items()): entry_point_script = f"{WHEEL_ENTRY_POINT_PREFIX}_{name}.py" @@ -422,7 +534,7 @@ def extract_wheel( ) with open(os.path.join(directory, "BUILD.bazel"), "w") as build_file: - additional_content = entry_points + additional_content = entry_points + setup_py_scripts data = [] data_exclude = pip_data_exclude srcs_exclude = [] diff --git a/python/pip_install/extract_wheels/lib/whl_filegroup_test.py b/python/pip_install/extract_wheels/lib/whl_filegroup_test.py index f5577136ff..8a52c3744b 100644 --- a/python/pip_install/extract_wheels/lib/whl_filegroup_test.py +++ b/python/pip_install/extract_wheels/lib/whl_filegroup_test.py @@ -23,7 +23,7 @@ def _run( incremental: bool = False, ) -> None: generated_bazel_dir = bazel.extract_wheel( - self.wheel_path, + wheel_file=self.wheel_path, extras={}, pip_data_exclude=[], enable_implicit_namespace_pkgs=False, diff --git a/python/pip_install/parse_requirements_to_bzl/__init__.py b/python/pip_install/parse_requirements_to_bzl/__init__.py index 3fc22898cb..0300a7f01e 100644 --- a/python/pip_install/parse_requirements_to_bzl/__init__.py +++ b/python/pip_install/parse_requirements_to_bzl/__init__.py @@ -132,6 +132,11 @@ def entry_point(pkg, script = None): script = pkg return "@{repo_prefix}" + _clean_name(pkg) + "//:{entry_point_prefix}_" + script + def setup_py_script(pkg, script = None): + if not script: + script = pkg + return "@{repo_prefix}" + _clean_name(pkg) + "//:{setup_py_script_prefix}_" + script + def _get_annotation(requirement): # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` # down wo `setuptools`. @@ -158,6 +163,7 @@ def install_deps(): repo_names_and_reqs=repo_names_and_reqs, repo_prefix=repo_prefix, wheel_file_label=bazel.WHEEL_FILE_LABEL, + setup_py_script_prefix=bazel.SETUP_PY_SCRIPT_PREFIX, ) )