Added setup_py_script macro to pip_parse and pip_install.

docs/pip.md

@@ -134,6 +134,20 @@ alias(
 )
 ```
 
+Similar to the `entry_point` macro, `setup_py_script` allows accessing
+[`setup.py` generated scripts][scripts]:
+
+```python
+load("@pip_deps//:requirements.bzl", "setup_py_script")
+
+alias(
+    name = "flake8",
+    actual = setup_py_script("flake8"),
+)
+```
+
+[scripts]: https://docs.python.org/3/distutils/setupscript.html#installing-scripts
+
 
 **PARAMETERS**
 
@@ -218,6 +232,20 @@ alias(
 )
 ```
 
+Similar to the `entry_point` macro, `setup_py_script` allows accessing
+[`setup.py` generated scripts][scripts]:
+
+```python
+load("@pip_deps//:requirements.bzl", "setup_py_script")
+
+alias(
+    name = "flake8",
+    actual = setup_py_script("flake8"),
+)
+```
+
+[scripts]: https://docs.python.org/3/distutils/setupscript.html#installing-scripts
+
 
 **PARAMETERS**
 

examples/pip_install/BUILD

@@ -6,6 +6,7 @@ load(
     "dist_info_requirement",
     "entry_point",
     "requirement",
+    "setup_py_script",
 )
 load("@rules_python//python:defs.bzl", "py_binary", "py_test")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
@@ -66,6 +67,11 @@ alias(
     actual = entry_point("yamllint"),
 )
 
+alias(
+    name = "s3cmd",
+    actual = setup_py_script("s3cmd"),
+)
+
 # Check that our compiled requirements are up-to-date
 compile_pip_requirements(
     name = "requirements",
@@ -77,12 +83,14 @@ py_test(
     name = "pip_install_test",
     srcs = ["pip_install_test.py"],
     data = [
+        ":s3cmd",
         ":sphinx-build",
         ":yamllint",
         data_requirement("s3cmd"),
         dist_info_requirement("boto3"),
     ],
     env = {
+        "S3CMD_SETUP_PY_SCRIPT": "$(rootpath :s3cmd)",
         "SPHINX_BUILD_ENTRY_POINT": "$(rootpath :sphinx-build)",
         "WHEEL_DATA_CONTENTS": "$(rootpaths {})".format(data_requirement("s3cmd")),
         "WHEEL_DIST_INFO_CONTENTS": "$(rootpaths {})".format(dist_info_requirement("boto3")),

examples/pip_install/pip_install_test.py

@@ -60,6 +60,21 @@ def test_entry_point_int_return(self):
             )
         self.assertIn("returned non-zero exit status 2", str(context.exception))
 
+    def test_setup_py_script_s3cmd(self):
+        env = os.environ.get("S3CMD_SETUP_PY_SCRIPT")
+        self.assertIsNotNone(env)
+
+        script = Path(env)
+        self.assertTrue(script.exists())
+
+        proc = subprocess.run(
+            [script, "--version"],
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        self.assertEqual(proc.stdout.decode("utf-8").strip(), "s3cmd version 2.1.0")
+
     def test_data(self):
         env = os.environ.get("WHEEL_DATA_CONTENTS")
         self.assertIsNotNone(env)

examples/pip_parse/BUILD

@@ -3,6 +3,7 @@ load(
     "data_requirement",
     "dist_info_requirement",
     "entry_point",
+    "setup_py_script",
 )
 load("@rules_python//python:defs.bzl", "py_binary", "py_test")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
@@ -63,6 +64,11 @@ alias(
     actual = entry_point("yamllint"),
 )
 
+alias(
+    name = "s3cmd",
+    actual = setup_py_script("s3cmd"),
+)
+
 # This rule adds a convenient way to update the requirements file.
 compile_pip_requirements(
     name = "requirements",
@@ -76,12 +82,14 @@ py_test(
     name = "pip_parse_test",
     srcs = ["pip_parse_test.py"],
     data = [
+        ":s3cmd",
         ":sphinx-build",
         ":yamllint",
         data_requirement("s3cmd"),
         dist_info_requirement("requests"),
     ],
     env = {
+        "S3CMD_SETUP_PY_SCRIPT": "$(rootpath :s3cmd)",
         "SPHINX_BUILD_ENTRY_POINT": "$(rootpath :sphinx-build)",
         "WHEEL_DATA_CONTENTS": "$(rootpaths {})".format(data_requirement("s3cmd")),
         "WHEEL_DIST_INFO_CONTENTS": "$(rootpaths {})".format(dist_info_requirement("requests")),

examples/pip_parse/pip_parse_test.py

@@ -60,6 +60,21 @@ def test_entry_point_int_return(self):
             )
         self.assertIn("returned non-zero exit status 2", str(context.exception))
 
+    def test_setup_py_script_s3cmd(self):
+        env = os.environ.get("S3CMD_SETUP_PY_SCRIPT")
+        self.assertIsNotNone(env)
+
+        script = Path(env)
+        self.assertTrue(script.exists())
+
+        proc = subprocess.run(
+            [script, "--version"],
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        self.assertEqual(proc.stdout.decode("utf-8").strip(), "s3cmd version 2.1.0")
+
     def test_data(self):
         env = os.environ.get("WHEEL_DATA_CONTENTS")
         self.assertIsNotNone(env)

python/pip.bzl

@@ -85,6 +85,20 @@ def pip_install(requirements, name = "pip", **kwargs):
     )
     ```
 
+    Similar to the `entry_point` macro, `setup_py_script` allows accessing
+    [`setup.py` generated scripts][scripts]:
+
+    ```python
+    load("@pip_deps//:requirements.bzl", "setup_py_script")
+
+    alias(
+        name = "flake8",
+        actual = setup_py_script("flake8"),
+    )
+    ```
+
+    [scripts]: https://docs.python.org/3/distutils/setupscript.html#installing-scripts
+
     Args:
         requirements (Label): A 'requirements.txt' pip requirements file.
         name (str, optional): A unique name for the created external repository (default 'pip').
@@ -167,6 +181,20 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs):
     )
     ```
 
+    Similar to the `entry_point` macro, `setup_py_script` allows accessing
+    [`setup.py` generated scripts][scripts]:
+
+    ```python
+    load("@pip_deps//:requirements.bzl", "setup_py_script")
+
+    alias(
+        name = "flake8",
+        actual = setup_py_script("flake8"),
+    )
+    ```
+
+    [scripts]: https://docs.python.org/3/distutils/setupscript.html#installing-scripts
+
     Args:
         requirements_lock (Label): A fully resolved 'requirements.txt' pip requirement file
             containing the transitive set of your dependencies. If this file is passed instead

python/pip_install/extract_wheels/__init__.py

@@ -116,6 +116,7 @@ def main() -> None:
                 pip_data_exclude=deserialized_args["pip_data_exclude"],
                 enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs,
                 repo_prefix=args.repo_prefix,
+                pip_install_repo_name=args.repo,
                 annotation=annotations.get(name),
             ),
         )

python/pip_install/extract_wheels/lib/bazel.py

@@ -18,6 +18,7 @@
 DATA_LABEL = "data"
 DIST_INFO_LABEL = "dist_info"
 WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point"
+SETUP_PY_SCRIPT_PREFIX = "rules_python_setup_py_script"
 
 
 def generate_entry_point_contents(
@@ -82,6 +83,80 @@ def generate_entry_point_rule(script: str, pkg: str) -> str:
     )
 
 
+def generate_setup_py_script_contents(
+    script: Path, repository_name: str, shebang: str = "#!/usr/bin/env python3"
+) -> str:
+    """Generate the contents of a `setup.py` generated script
+
+    For details on `setup.py` generated scripts, see:
+    https://docs.python.org/3/distutils/setupscript.html#installing-scripts
+
+    Args:
+        script (Path): The path to the `setup.py` script
+        repository_name (str): The name of the repository for the current wheel.
+        shebang (str, optional): The shebang to use for the generated python
+            file.
+
+    Returns:
+        str: A string of python code.
+    """
+    script_path = Path("external") / repository_name / script
+    return textwrap.dedent(
+        """\
+        {shebang}
+        import os
+        import sys
+        if __name__ == "__main__":
+            script = "{script}"
+            if not os.path.exists(script):
+                raise FileNotFoundError(script)
+            args = sys.argv.copy()
+            args[0] = script
+            os.execv(sys.executable, [sys.executable] + args)
+        """.format(
+            shebang=shebang, script=str(script_path)
+        )
+    )
+
+
+def generate_setup_py_script_rule(src: Path, script: Path, pkg: str) -> str:
+    """Generate a Bazel `py_binary` rule for a `setup.py` generated script.
+
+    For details on `setup.py` generated scripts, see:
+    https://docs.python.org/3/distutils/setupscript.html#installing-scripts
+
+    Args:
+        src (Path): The path to the source file generated by
+            `generate_setup_py_script_contents`.
+        script (Path): The path to the `setup.py` script.
+        pkg (str): The package owning the entry point. This is expected to
+            match up with the `py_library` defined for each repository.
+
+    Returns:
+        str: A `py_binary` instantiation.
+    """
+    name = os.path.splitext(src.name)[0]
+    return textwrap.dedent(
+        """\
+        py_binary(
+            name = "{name}",
+            srcs = ["{src}"],
+            main = "{src}",
+            # This makes this directory a top-level in the python import
+            # search path for anything that depends on this.
+            imports = ["."],
+            deps = ["{pkg}"],
+            data = ["{script}"],
+        )
+        """.format(
+            name=name,
+            src=str(src.name).replace("\\", "/"),
+            pkg=pkg,
+            script=str(script).replace("\\", "/"),
+        )
+    )
+
+
 def generate_copy_commands(src, dest, is_executable=False) -> str:
     """Generate a [@bazel_skylib//rules:copy_file.bzl%copy_file][cf] target
 
@@ -149,6 +224,7 @@ def generate_build_file_contents(
                 "**/*.pyc",
                 "BUILD.bazel",
                 "WORKSPACE",
+                f"{SETUP_PY_SCRIPT_PREFIX}*.py",
                 f"{WHEEL_ENTRY_POINT_PREFIX}*.py",
             ]
             + data_exclude
@@ -252,6 +328,11 @@ def entry_point(pkg, script = None):
                 script = pkg
             return requirement(pkg) + ":{entry_point_prefix}_" + script
 
+        def setup_py_script(pkg, script = None):
+            if not script:
+                script = pkg
+            return requirement(pkg) + ":{setup_py_script_prefix}_" + script
+
         def install_deps():
             fail("install_deps() only works if you are creating an incremental repo. Did you mean to use pip_parse()?")
         """.format(
@@ -262,6 +343,7 @@ def install_deps():
             data_label=DATA_LABEL,
             dist_info_label=DIST_INFO_LABEL,
             entry_point_prefix=WHEEL_ENTRY_POINT_PREFIX,
+            setup_py_script_prefix=SETUP_PY_SCRIPT_PREFIX,
         )
     )
 
@@ -340,6 +422,7 @@ def extract_wheel(
     pip_data_exclude: List[str],
     enable_implicit_namespace_pkgs: bool,
     repo_prefix: str,
+    pip_install_repo_name: str = "",
     incremental: bool = False,
     incremental_dir: Path = Path("."),
     annotation: Optional[annotation.Annotation] = None,
@@ -351,6 +434,8 @@ def extract_wheel(
         extras: a list of extras to add as dependencies for the installed wheel
         pip_data_exclude: list of file patterns to exclude from the generated data section of the py_library
         enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is
+        repo_prefix: Prefix to prepend to packages.
+        pip_install_repo_name: The external `pip_install` repo name to install dependencies. In the format `@{REPO_NAME}`
         incremental: If true the extract the wheel in a format suitable for an external repository. This
             effects the names of libraries and their dependencies, which point to other external repositories.
         incremental_dir: An optional override for the working directory of incremental builds.
@@ -400,6 +485,39 @@ def extract_wheel(
     )
 
     directory_path = Path(directory)
+
+    # Generate scripts and `py_binary` targets for any scripts produced by `setup.py`.
+    setup_py_scripts = []
+    wheel_data_dir = wheel.get_dot_data_directory(directory)
+    if wheel_data_dir:
+        scripts_dir = Path(wheel_data_dir) / "scripts"
+        for root, _dirnames, filenames in os.walk(str(scripts_dir)):
+            if pip_install_repo_name:
+                repository_name = pip_install_repo_name
+            else:
+                repository_name = "{}{}".format(repo_prefix, whl.name.replace("-", "_"))
+
+            for filename in filenames:
+                setup_py_script = Path(root) / filename
+                setup_py_script_wrapper = (
+                    directory_path
+                    / f"{SETUP_PY_SCRIPT_PREFIX}_{setup_py_script.name}.py"
+                )
+                setup_py_script_wrapper.write_text(
+                    generate_setup_py_script_contents(
+                        script=setup_py_script,
+                        repository_name=repository_name,
+                    )
+                )
+                setup_py_scripts.append(
+                    generate_setup_py_script_rule(
+                        src=setup_py_script_wrapper,
+                        script=setup_py_script.relative_to(directory_path),
+                        pkg=library_name,
+                    )
+                )
+
+    # Generate scripts and `py_binary` targets for wheel entry points
     entry_points = []
     for name, entry_point in sorted(whl.entry_points().items()):
         entry_point_script = f"{WHEEL_ENTRY_POINT_PREFIX}_{name}.py"
@@ -414,7 +532,7 @@ def extract_wheel(
         )
 
     with open(os.path.join(directory, "BUILD.bazel"), "w") as build_file:
-        additional_content = entry_points
+        additional_content = entry_points + setup_py_scripts
         data = []
         data_exclude = pip_data_exclude
         srcs_exclude = []

python/pip_install/extract_wheels/lib/whl_filegroup_test.py

@@ -23,7 +23,7 @@ def _run(
         incremental: bool = False,
     ) -> None:
         generated_bazel_dir = bazel.extract_wheel(
-            self.wheel_path,
+            wheel_file=self.wheel_path,
             extras={},
             pip_data_exclude=[],
             enable_implicit_namespace_pkgs=False,