feat: (WIP) Add pypi_install() rule

.bazelignore

@@ -23,6 +23,7 @@ examples/multi_python_versions/bazel-multi_python_versions
 examples/pip_parse/bazel-pip_parse
 examples/pip_parse_vendored/bazel-pip_parse_vendored
 examples/py_proto_library/bazel-py_proto_library
+examples/pypi_install/bazel-pypi_install
 tests/integration/compile_pip_requirements/bazel-compile_pip_requirements
 tests/integration/ignore_root_user_error/bazel-ignore_root_user_error
 tests/integration/pip_repository_entry_points/bazel-pip_repository_entry_points

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/pypi_install,examples/pypi_install/third_party,examples/pypi_install/wheels,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/pypi_install,examples/pypi_install/third_party,examples/pypi_install/wheels,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

BUILD.bazel

@@ -40,6 +40,7 @@ filegroup(
         "version.bzl",
         "//python:distribution",
         "//python/pip_install:distribution",
+        "//third_party/rules_pycross/pycross/private:distribution",
         "//tools:distribution",
         "@rules_python_gazelle_plugin//:distribution",
     ],

MODULE.bazel

@@ -28,6 +28,7 @@ use_repo(
     "pypi__pep517",
     "pypi__pip",
     "pypi__pip_tools",
+    "pypi__pypiserver",
     "pypi__pyproject_hooks",
     "pypi__setuptools",
     "pypi__tomli",

examples/BUILD.bazel

@@ -12,4 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("//tests/integration:integration_test.bzl", "rules_python_integration_test")
+
 licenses(["notice"])  # Apache 2.0
+
+rules_python_integration_test(
+    name = "pypi_install",
+    bzlmod = False,
+)

examples/pypi_install/.bazelrc

@@ -0,0 +1,8 @@
+# https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file
+try-import %workspace%/user.bazelrc
+test --test_output=errors
+
+common --noenable_bzlmod
+
+build:py310 --@rules_python//python/config_settings:python_version=3.10
+build:py311 --@rules_python//python/config_settings:python_version=3.11

examples/pypi_install/.gitignore

@@ -0,0 +1,4 @@
+# git ignore patterns
+
+/bazel-*
+user.bazelrc

examples/pypi_install/BUILD.bazel

@@ -0,0 +1,122 @@
+load("@rules_python//python:defs.bzl", "py_binary", "py_test")
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("@bazel_skylib//lib:versions.bzl", "versions")
+load("@bazel_skylib//lib:selects.bzl", "selects")
+load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
+
+py_binary(
+    name = "main",
+    srcs = ["main.py"],
+    deps = [
+        "@pypi//pkg_a",
+    ],
+)
+
+py_test(
+    name = "test",
+    srcs = ["test.py"],
+    deps = [":main"],
+)
+
+# This rule adds a convenient way to update the requirements file.
+compile_pip_requirements(
+    name = "requirements",
+    intermediate_file = {
+        "@//:py310_linux_aarch64": "intermediate_file_py310_linux_aarch64.json",
+        "@//:py310_linux_x86_64": "intermediate_file_py310_linux_x86_64.json",
+        "@//:py311_linux_aarch64": "intermediate_file_py311_linux_aarch64.json",
+        "@//:py311_linux_x86_64": "intermediate_file_py311_linux_x86_64.json",
+    },
+    # That binary will let users modify the intermediate file. Then they can:
+    # - Inject dependencies.
+    # - Remove dependencies.
+    # - Drop whole packages (is there such a use case?).
+    # - Add patches.
+    # - Override/create aliases (support needed).
+    intermediate_file_patcher = "//:intermediate_file_patcher",
+    requirements_in = "requirements.in",
+    # Pass in a select-like value that lets the macro decide what to do with
+    # the file. Having these values as dictionaries allows us to do some
+    # validation before the underlying targets get instantiated.
+    requirements_txt = {
+        "@//:py310_linux_aarch64": "requirements_lock_py310_linux_aarch64.txt",
+        "@//:py310_linux_x86_64": "requirements_lock_py310_linux_x86_64.txt",
+        "@//:py311_linux_aarch64": "requirements_lock_py311_linux_aarch64.txt",
+        "@//:py311_linux_x86_64": "requirements_lock_py311_linux_x86_64.txt",
+    },
+    extra_args = [
+        # Since we may frequently tweak the wheels for development, we can't
+        # really rely on consistent hashes.
+        "--no-reuse-hashes",
+        # Point at the server run via:
+        # bazel run //wheels:pypiserver_runner
+        "--index-url",
+        "http://localhost:8989/simple/",
+    ],
+)
+
+py_binary(
+    name = "intermediate_file_patcher",
+    srcs = ["intermediate_file_patcher.py"],
+)
+
+config_setting(
+    name = "linux_aarch64",
+    constraint_values = [
+        "@platforms//cpu:aarch64",
+        "@platforms//os:linux",
+    ],
+)
+
+config_setting(
+    name = "linux_x86_64",
+    constraint_values = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:linux",
+    ],
+)
+
+[selects.config_setting_group(
+    name = "py{}_{}".format(py_version.replace(".", ""), platform),
+    match_all = [
+        "@rules_python//python/config_settings:is_python_{}".format(py_version),
+        ":{}".format(platform),
+    ],
+) for py_version in (
+    "3.10",
+    "3.11",
+) for platform in (
+    "linux_aarch64",
+    "linux_x86_64",
+)]
+
+genquery(
+    name = "pkg_a_deps",
+    expression = """filter(@pypi//, kind(".* rule", deps(@pypi//pkg_a)))""",
+    opts = ["--notool_deps"],
+    scope = ["@pypi//pkg_a"],
+)
+
+# TODO(phil): This doesn't work. Find a way to make this work.
+#IS_BAZEL_6 = versions.parse(versions.get())[0] < 7
+
+diff_test(
+    name = "pkg_a_deps_test",
+    file1 = ":pkg_a_deps",
+    #file2 = "pkg_a_deps_bazel_6.txt" if IS_BAZEL_6 else "pkg_a_deps.txt",
+    file2 = "pkg_a_deps.txt",
+)
+
+genquery(
+    name = "pkg_d_deps",
+    expression = """filter(@pypi//, kind(".* rule", deps(@pypi//pkg_d)))""",
+    opts = ["--notool_deps"],
+    scope = ["@pypi//pkg_d"],
+)
+
+diff_test(
+    name = "pkg_d_deps_test",
+    file1 = ":pkg_d_deps",
+    #file2 = "pkg_d_deps_bazel_6.txt" if IS_BAZEL_6 else "pkg_d_deps.txt",
+    file2 = "pkg_d_deps.txt",
+)

examples/pypi_install/MODULE.bazel

@@ -0,0 +1,6 @@
+###############################################################################
+# Bazel now uses Bzlmod by default to manage external dependencies.
+# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.
+#
+# For more details, please check https://github.com/bazelbuild/bazel/issues/18958
+###############################################################################

examples/pypi_install/WORKSPACE

@@ -0,0 +1,49 @@
+workspace(name = "rules_python_pypi_install_example")
+
+local_repository(
+    name = "rules_python",
+    path = "../..",
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_multi_toolchains")
+
+py_repositories()
+
+load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
+
+pip_install_dependencies()
+
+python_register_multi_toolchains(
+    name = "python",
+    default_version = "3.10",
+    python_versions = [
+        "3.10",
+        "3.11",
+    ],
+)
+
+load("@rules_python//python:pypi.bzl", "pypi_install")
+
+pypi_install(
+    name = "pypi_install",
+    # Use fully resolved labels so that they're not accidentally interpreted as
+    # repo-relative.
+    # TODO(phil): Add test to make sure that the config settings are mutually
+    # exclusive.
+    pip_installation_report = [
+        "//:intermediate_file_py310_linux_x86_64.json",
+        "//:intermediate_file_py310_linux_aarch64.json",
+        "//:intermediate_file_py311_linux_x86_64.json",
+        "//:intermediate_file_py311_linux_aarch64.json",
+    ],
+)
+
+load("@pypi_install//:packages.bzl", "load_pypi_packages")
+
+load_pypi_packages(
+    name = "pypi",
+)
+
+register_toolchains(
+    "//cc:linux_toolchain_definition",
+)

examples/pypi_install/cc/BUILD.bazel

@@ -0,0 +1,29 @@
+load(":fake_cc_toolchain_config.bzl", "fake_cc_toolchain_config")
+
+filegroup(name = "empty")
+
+fake_cc_toolchain_config(
+    name = "linux_toolchain_config",
+    target_cpu = "k8",
+    toolchain_identifier = "linux-toolchain",
+)
+
+cc_toolchain(
+    name = "linux_toolchain",
+    all_files = ":empty",
+    compiler_files = ":empty",
+    dwp_files = ":empty",
+    linker_files = ":empty",
+    objcopy_files = ":empty",
+    strip_files = ":empty",
+    supports_param_files = 0,
+    toolchain_config = ":linux_toolchain_config",
+    toolchain_identifier = "linux-toolchain",
+)
+
+toolchain(
+    name = "linux_toolchain_definition",
+    target_compatible_with = ["@platforms//os:linux"],
+    toolchain = ":linux_toolchain",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)

examples/pypi_install/cc/fake_cc_toolchain_config.bzl

@@ -0,0 +1,23 @@
+"""Fake for providing CcToolchainConfigInfo."""
+
+def _impl(ctx):
+    return cc_common.create_cc_toolchain_config_info(
+        ctx = ctx,
+        toolchain_identifier = ctx.attr.toolchain_identifier,
+        host_system_name = "local",
+        target_system_name = "local",
+        target_cpu = ctx.attr.target_cpu,
+        target_libc = "unknown",
+        compiler = "clang",
+        abi_version = "unknown",
+        abi_libc_version = "unknown",
+    )
+
+fake_cc_toolchain_config = rule(
+    implementation = _impl,
+    attrs = {
+        "target_cpu": attr.string(),
+        "toolchain_identifier": attr.string(),
+    },
+    provides = [CcToolchainConfigInfo],
+)

examples/pypi_install/intermediate_file_patcher.py

@@ -0,0 +1,35 @@
+import json
+import sys
+from pathlib import Path
+
+UPDATES = {
+    "requests": {
+        "patches": [
+            "@//third_party:requests/0001-Add-a-dummy-patch-for-requests.patch",
+        ],
+        "patch_args": [
+            "-p1",
+        ],
+    },
+}
+
+def patch_intermediate_file(content):
+    for package, info_per_config in content.items():
+        for config, info in info_per_config.items():
+            if package in UPDATES:
+                info.update(UPDATES[package])
+
+def main(argv):
+    intermediate_file = Path(argv[1])
+
+    with intermediate_file.open("r") as file:
+        content = json.load(file)
+
+    patch_intermediate_file(content)
+
+    with intermediate_file.open("w") as file:
+        json.dump(content, file, indent=4)
+        file.write("\n")
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

examples/pypi_install/intermediate_file_py310_linux_aarch64.json

@@ -0,0 +1,45 @@
+{
+    "pkg-a": {
+        "@//:py310_linux_aarch64": {
+            "url": "http://localhost:8989/packages/pkg_a-1.0-py3-none-any.whl",
+            "sha256": "a0d9f8c13927e9a2b11260c7add67c827921d9727242247f31c605704bff5ff4",
+            "deps": [
+                "pkg-b"
+            ]
+        }
+    },
+    "pkg-b": {
+        "@//:py310_linux_aarch64": {
+            "url": "http://localhost:8989/packages/pkg_b-1.1-py3-none-any.whl",
+            "sha256": "05cd20f93e030fcb81b83190bb9926590954d94a3552f0f0567232743e9e8c23",
+            "deps": [
+                "pkg-c",
+                "pkg-d",
+                "pkg-e"
+            ]
+        }
+    },
+    "pkg-c": {
+        "@//:py310_linux_aarch64": {
+            "url": "http://localhost:8989/packages/pkg_c-2.0-py3-none-any.whl",
+            "sha256": "1fd7e8c2c3a8bfb045b365f4c805d40986f1b7a164537458164d82ffc6aed636",
+            "deps": []
+        }
+    },
+    "pkg-d": {
+        "@//:py310_linux_aarch64": {
+            "url": "http://localhost:8989/packages/pkg_d-3.0-py3-none-any.whl",
+            "sha256": "de78d858ac71ffe1a5f9c7ddc58a8b26938e62f6583f1143fde0957389d3986f",
+            "deps": [
+                "pkg-b"
+            ]
+        }
+    },
+    "pkg-e": {
+        "@//:py310_linux_aarch64": {
+            "url": "http://localhost:8989/packages/pkg_e-4.0-py3-none-any.whl",
+            "sha256": "9d2fcec39f4f7e8b4e0d0d9bc8f1b06ee582af6dea8db258d031aa64089ca363",
+            "deps": []
+        }
+    }
+}