exp: multi-arch whl_library in bzlmod

MODULE.bazel

@@ -4,7 +4,7 @@ module(
     compatibility_level = 1,
 )
 
-bazel_dep(name = "bazel_features", version = "1.1.1")
+bazel_dep(name = "bazel_features", version = "1.2.0")
 bazel_dep(name = "bazel_skylib", version = "1.3.0")
 bazel_dep(name = "platforms", version = "0.0.4")
 
@@ -50,6 +50,17 @@ python.toolchain(
 )
 use_repo(python, "pythons_hub")
 
+pip = use_extension("//python/extensions:pip.bzl", "pip")
+
+# use the following repo, which contains references to things we download from
+# PyPI via the experimental rules, this makes it possible to pass regular labels
+# to the whl_library rule instantiation within the pip extension itself.
+#
+# However, I am not sure how this plays with `dev_dependency = True` and/or
+# `isolated = True`. It could be that it might fail.
+# TODO @aignas 2023-12-24: test this.
+use_repo(pip, "pypi_whl")
+
 # This call registers the Python toolchains.
 register_toolchains("@pythons_hub//:all")
 

examples/bzlmod/.gitignore

@@ -1 +1,2 @@
 bazel-*
+MODULE.bazel.lock

examples/bzlmod/MODULE.bazel

@@ -51,6 +51,15 @@ use_repo(python, "python_3_10", "python_3_9", "python_versions")
 # You are able to set a hub name, so that you can have different modifications of the same
 # wheel in different pip hubs.
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.experimental_target_platforms(
+    enabled = True,
+    # Specify PyPI indexes in the order of preference. If the package is not found in the first
+    # index, it will be looked up in the next one, which means that it makes it useful to order
+    # the index URLs by the number of packages it contains.
+    extra_index_urls = [],
+    # Specify the main PyPI index URL to use. This is value the default.
+    index_url = "https://pypi.org/simple",
+)
 
 # Call whl_mods.create for the requests package.
 pip.whl_mods(

examples/bzlmod/whl_mods/appended_build_content.BUILD

@@ -8,9 +8,5 @@ write_file(
 
 filegroup(
     name = "whl_orig",
-    srcs = glob(
-        ["*.whl"],
-        allow_empty = False,
-        exclude = ["*-patched-*.whl"],
-    ),
+    srcs = ["_whl"],
 )

python/pip_install/pip_repository.bzl

@@ -713,18 +713,37 @@ def _whl_library_impl(rctx):
     # Manually construct the PYTHONPATH since we cannot use the toolchain here
     environment = _create_repository_execution_environment(rctx, python_interpreter)
 
-    result = rctx.execute(
-        args,
-        environment = environment,
-        quiet = rctx.attr.quiet,
-        timeout = rctx.attr.timeout,
-    )
-    if result.return_code:
-        fail("whl_library %s failed: %s (%s) error code: '%s'" % (rctx.attr.name, result.stdout, result.stderr, result.return_code))
+    whl_path = None
+    whl_label = None
+    if rctx.attr.experimental_whl_label:
+        # This label may be a hub repo if that is the case, resolve it to the
+        # spoke repo label. For reasons why see `../private/bzlmod/pypi_metadata.bzl`.
+        #
+        # This assumes that whls are only downloaded with the pypi_file repo rule.
+        whl_label = rctx.attr.experimental_whl_label
+        whl_path = rctx.path(Label("@@{hub_repo_name}_{dist}//:{filename}".format(
+            hub_repo_name = whl_label.workspace_name,
+            dist = whl_label.package,
+            filename = whl_label.name,
+        )))
+
+        if whl_path.basename.endswith("tar.gz"):
+            whl_path = None
+            whl_label = None
+
+    if whl_path == None:
+        result = rctx.execute(
+            args,
+            environment = environment,
+            quiet = rctx.attr.quiet,
+            timeout = rctx.attr.timeout,
+        )
+        if result.return_code:
+            fail("whl_library %s failed: %s (%s) error code: '%s'" % (rctx.attr.name, result.stdout, result.stderr, result.return_code))
 
-    whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
-    if not rctx.delete("whl_file.json"):
-        fail("failed to delete the whl_file.json file")
+        whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
+        if not rctx.delete("whl_file.json"):
+            fail("failed to delete the whl_file.json file")
 
     if rctx.attr.whl_patches:
         patches = {}
@@ -791,7 +810,7 @@ def _whl_library_impl(rctx):
 
     build_file_contents = generate_whl_library_build_bazel(
         repo_prefix = rctx.attr.repo_prefix,
-        whl_name = whl_path.basename,
+        whl_name = whl_label or whl_path.basename,
         dependencies = metadata["deps"],
         dependencies_by_platform = metadata["deps_by_platform"],
         group_name = rctx.attr.group_name,
@@ -803,6 +822,10 @@ def _whl_library_impl(rctx):
         ],
         entry_points = entry_points,
         annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
+        impl_vis = None if not rctx.attr.experimental_whl_label else "@{}{}//:__pkg__".format(
+            rctx.attr.repo_prefix,
+            normalize_name(metadata["name"]),
+        ),
     )
     rctx.file("BUILD.bazel", build_file_contents)
 
@@ -844,6 +867,20 @@ whl_library_attrs = {
         ),
         allow_files = True,
     ),
+    "experimental_whl_label": attr.label(
+        doc = """\
+The label of the whl file to use. This allows one to pass a whl file to be used, but at the same
+time it changes the assumed whl_library layout. With this parameter set, the pip repository layout
+becomes as following:
+* pip has aliases to wheel libraries based on the version of the toolchain in use.
+* downloaded whls and sdists are in separate `pypi_file` repos.
+* per each downloaded `whl` or `sdist` there is a `whl_library` that creates a `py_library` target.
+* there is a alias `whl_library` repo that allows selecting which implementation `whl_library` repo to use.
+
+In the future the `whl_library` alias repo might be merged into the main `pip` repo but for that to
+happen, the `dependency` closures need to be generated for the selected target python versions.
+""",
+    ),
     "group_deps": attr.string_list(
         doc = "List of dependencies to skip in order to break the cycles within a dependency group.",
         default = [],

python/pip_install/private/generate_whl_library_build_bazel.bzl

@@ -137,7 +137,8 @@ def generate_whl_library_build_bazel(
         entry_points,
         annotation = None,
         group_name = None,
-        group_deps = []):
+        group_deps = [],
+        impl_vis = None):
     """Generate a BUILD file for an unzipped Wheel
 
     Args:
@@ -155,6 +156,7 @@ def generate_whl_library_build_bazel(
         group_deps: List[str]; names of fellow members of the group (if any). These will be excluded
           from generated deps lists so as to avoid direct cycles. These dependencies will be provided
           at runtime by the group rules which wrap this library and its fellows together.
+        impl_vis: str; override the visibility of the implementation labels.
 
     Returns:
         A complete BUILD file as a string
@@ -267,12 +269,12 @@ config_setting(
         group_repo = repo_prefix + "_groups"
         library_impl_label = "@%s//:%s_%s" % (group_repo, normalize_name(group_name), PY_LIBRARY_PUBLIC_LABEL)
         whl_impl_label = "@%s//:%s_%s" % (group_repo, normalize_name(group_name), WHEEL_FILE_PUBLIC_LABEL)
-        impl_vis = "@%s//:__pkg__" % (group_repo,)
+        impl_vis = impl_vis or "@%s//:__pkg__" % (group_repo,)
 
     else:
         library_impl_label = PY_LIBRARY_IMPL_LABEL
         whl_impl_label = WHEEL_FILE_IMPL_LABEL
-        impl_vis = "//visibility:private"
+        impl_vis = impl_vis or "//visibility:private"
 
     contents = "\n".join(
         [