exp: multi-arch whl_library in bzlmod

MODULE.bazel

@@ -4,7 +4,7 @@ module(
  compatibility_level = 1,
 )
 
-bazel_dep(name = "bazel_features", version = "1.1.1")
+bazel_dep(name = "bazel_features", version = "1.2.0")
 bazel_dep(name = "bazel_skylib", version = "1.3.0")
 bazel_dep(name = "platforms", version = "0.0.4")
 

examples/bzlmod/.bazelversion

@@ -1 +1 @@
-6.0.0
+7.0.0

examples/bzlmod/.gitignore

@@ -1 +1,2 @@
 bazel-*
+MODULE.bazel.lock

examples/bzlmod/MODULE.bazel

@@ -45,6 +45,15 @@ use_repo(python, "python_3_10", "python_3_9", "python_versions")
 # You are able to set a hub name, so that you can have different modifications of the same
 # wheel in different pip hubs.
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.experimental_target_platforms(
+ enabled = True,
+ # Specify PyPI indexes in the order of preference. If the package is not found in the first
+ # index, it will be looked up in the next one, which means that it makes it useful to order
+ # the index URLs by the number of packages it contains.
+ extra_index_urls = [],
+ # Specify the main PyPI index URL to use. This is value the default.
+ index_url = "https://pypi.org/simple",
+)
 
 # Call whl_mods.create for the requests package.
 pip.whl_mods(

examples/bzlmod/whl_mods/BUILD.bazel

@@ -8,9 +8,13 @@ exports_files(
 py_test(
  name = "pip_whl_mods_test",
  srcs = ["pip_whl_mods_test.py"],
+ data = [
+ "@pip//requests:dist_info",
+ "@pip//wheel:dist_info",
+ ],
  env = {
- "REQUESTS_PKG_DIR": "pip_39_requests",
- "WHEEL_PKG_DIR": "pip_39_wheel",
+ "REQUESTS_DISTINFO": "$(rlocationpaths @pip//requests:dist_info)",
+ "WHEEL_DISTINFO": "$(rlocationpaths @pip//wheel:dist_info)",
  },
  main = "pip_whl_mods_test.py",
  deps = [

examples/bzlmod/whl_mods/appended_build_content.BUILD

@@ -8,9 +8,5 @@ write_file(
 
 filegroup(
  name = "whl_orig",
- srcs = glob(
- ["*.whl"],
- allow_empty = False,
- exclude = ["*-patched-*.whl"],
- ),
+ srcs = ["_whl"],
 )

examples/bzlmod/whl_mods/pip_whl_mods_test.py

@@ -30,19 +30,21 @@ class PipWhlModsTest(unittest.TestCase):
  def package_path(self) -> str:
  return "rules_python~override~pip~"
 
- def wheel_pkg_dir(self) -> str:
- env = os.environ.get("WHEEL_PKG_DIR")
- self.assertIsNotNone(env)
- return env
+ def wheel_pkg_dir(self) -> Path:
+ distinfo = os.environ.get("WHEEL_DISTINFO")
+ self.assertIsNotNone(distinfo)
+ return Path(distinfo.split(" ")[0]).parents[2]
+
+ def rlocation(self, runfiles, dir: Path, *segments):
+ absolute_path = dir
+ for segment in segments:
+ absolute_path = absolute_path / segment
+
+ return runfiles.Rlocation(str(absolute_path.relative_to(dir.parent)))
 
  def test_build_content_and_data(self):
  r = runfiles.Create()
- rpath = r.Rlocation(
- "{}{}/generated_file.txt".format(
- self.package_path(), 
- self.wheel_pkg_dir(),
- ),
- )
+ rpath = self.rlocation(r, self.wheel_pkg_dir(), "generated_file.txt")
  generated_file = Path(rpath)
  self.assertTrue(generated_file.exists())
 
@@ -51,27 +53,22 @@ def test_build_content_and_data(self):
 
  def test_copy_files(self):
  r = runfiles.Create()
- rpath = r.Rlocation(
- "{}{}/copied_content/file.txt".format(
- self.package_path(), 
- self.wheel_pkg_dir(),
- )
- )
+ rpath = self.rlocation(r, self.wheel_pkg_dir(), "copied_content", "file.txt")
  copied_file = Path(rpath)
  self.assertTrue(copied_file.exists())
 
  content = copied_file.read_text().rstrip()
  self.assertEqual(content, "Hello world from copied file")
 
  def test_copy_executables(self):
+ executable_name = (
+ "executable.exe" if platform.system() == "windows" else "executable.py"
+ )
+
  r = runfiles.Create()
- rpath = r.Rlocation(
- "{}{}/copied_content/executable{}".format(
- self.package_path(), 
- self.wheel_pkg_dir(),
- ".exe" if platform.system() == "windows" else ".py",
- )
- )
+ rpath = self.rlocation(
+ r, self.wheel_pkg_dir(), "copied_content", executable_name
+ )
  executable = Path(rpath)
  self.assertTrue(executable.exists())
 
@@ -88,37 +85,37 @@ def test_data_exclude_glob(self):
  current_wheel_version = "0.40.0"
 
  r = runfiles.Create()
- dist_info_dir = "{}{}/site-packages/wheel-{}.dist-info".format(
- self.package_path(), 
- self.wheel_pkg_dir(),
- current_wheel_version,
- )
+ wheel_pkg_dir = self.wheel_pkg_dir()
+ dist_info_dir = f"wheel-{current_wheel_version}.dist-info"
 
  # Note: `METADATA` is important as it's consumed by https://docs.python.org/3/library/importlib.metadata.html
  # `METADATA` is expected to be there to show dist-info files are included in the runfiles.
- metadata_path = r.Rlocation("{}/METADATA".format(dist_info_dir))
+ metadata_path = Path(
+ self.rlocation(r, wheel_pkg_dir, "site-packages", dist_info_dir, "METADATA")
+ )
 
  # However, `WHEEL` was explicitly excluded, so it should be missing
- wheel_path = r.Rlocation("{}/WHEEL".format(dist_info_dir))
+ wheel_path = Path(
+ self.rlocation(r, wheel_pkg_dir, "site-packages", dist_info_dir, "WHEEL")
+ )
 
- self.assertTrue(Path(metadata_path).exists())
- self.assertFalse(Path(wheel_path).exists())
+ self.assertTrue(
+ metadata_path.exists(), f"METADATA was not found in {metadata_path}"
+ )
+ self.assertTrue(wheel_path.exists(), f"WHEEL was not found in {wheel_path}")
 
- def requests_pkg_dir(self) -> str:
- env = os.environ.get("REQUESTS_PKG_DIR")
- self.assertIsNotNone(env)
- return env
+ def requests_pkg_dir(self) -> Path:
+ distinfo = os.environ.get("REQUESTS_DISTINFO")
+ self.assertIsNotNone(distinfo)
+ pkgdir = Path(distinfo.split(" ")[0]).parents[2]
+ return pkgdir
 
  def test_extra(self):
  # This test verifies that annotations work correctly for pip packages with extras
  # specified, in this case requests[security].
  r = runfiles.Create()
- rpath = r.Rlocation(
- "{}{}/generated_file.txt".format(
- self.package_path(), 
- self.requests_pkg_dir(),
- ),
- )
+ rpath = self.rlocation(r, self.requests_pkg_dir(), "generated_file.txt")
+
  generated_file = Path(rpath)
  self.assertTrue(generated_file.exists())
 

python/BUILD.bazel

@@ -196,6 +196,7 @@ bzl_library(
  srcs = ["repositories.bzl"],
  deps = [
  ":versions_bzl",
+ "//python/private:auth_bzl",
  "//python/private:bazel_tools_bzl",
  "//python/private:bzlmod_enabled_bzl",
  "//python/private:coverage_deps_bzl",

python/pip_install/pip_repository.bzl

@@ -714,18 +714,28 @@ def _whl_library_impl(rctx):
  # Manually construct the PYTHONPATH since we cannot use the toolchain here
  environment = _create_repository_execution_environment(rctx, python_interpreter)
 
- result = rctx.execute(
- args,
- environment = environment,
- quiet = rctx.attr.quiet,
- timeout = rctx.attr.timeout,
- )
- if result.return_code:
- fail("whl_library %s failed: %s (%s) error code: '%s'" % (rctx.attr.name, result.stdout, result.stderr, result.return_code))
+ whl_path = None
+ whl_label = None
+ if rctx.attr.experimental_whl_file:
+ whl_label = rctx.attr.experimental_whl_file
+ whl_path = rctx.path(whl_label).realpath
+ if whl_path.basename.endswith("tar.gz"):
+ whl_path = None
+ whl_label = None
+
+ if whl_path == None:
+ result = rctx.execute(
+ args,
+ environment = environment,
+ quiet = rctx.attr.quiet,
+ timeout = rctx.attr.timeout,
+ )
+ if result.return_code:
+ fail("whl_library %s failed: %s (%s) error code: '%s'" % (rctx.attr.name, result.stdout, result.stderr, result.return_code))
 
- whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
- if not rctx.delete("whl_file.json"):
- fail("failed to delete the whl_file.json file")
+  whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
+  if not rctx.delete("whl_file.json"):
+  fail("failed to delete the whl_file.json file")
 
  if rctx.attr.whl_patches:
  patches = {}
@@ -781,7 +791,7 @@ def _whl_library_impl(rctx):
 
  build_file_contents = generate_whl_library_build_bazel(
  repo_prefix = rctx.attr.repo_prefix,
- whl_name = whl_path.basename,
+ whl_name = whl_label or whl_path.basename,
  dependencies = metadata["deps"],
  dependencies_by_platform = metadata["deps_by_platform"],
  group_name = rctx.attr.group_name,
@@ -793,6 +803,10 @@ def _whl_library_impl(rctx):
  ],
  entry_points = entry_points,
  annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
+ impl_vis = None if not rctx.attr.experimental_whl_file else "@{}{}//:__pkg__".format(
+ rctx.attr.repo_prefix,
+ normalize_name(metadata["name"]),
+ ),
  )
  rctx.file("BUILD.bazel", build_file_contents)
 
@@ -834,6 +848,20 @@ whl_library_attrs = {
  ),
  allow_files = True,
  ),
+ "experimental_whl_file": attr.label(
+ doc = """\
+The label of the whl file to use. This allows one to pass a whl file to be used, but at the same
+time it changes the assumed whl_library layout. With this parameter set, the pip repository layout
+becomes as following:
+* pip has aliases to wheel libraries based on the version of the toolchain in use.
+* downloaded whls and sdists are in separate `pypi_file` repos.
+* per each downloaded `whl` or `sdist` there is a `whl_library` that creates a `py_library` target.
+* there is a alias `whl_library` repo that allows selecting which implementation `whl_library` repo to use.
+
+In the future the `whl_library` alias repo might be merged into the main `pip` repo but for that to
+happen, the `dependency` closures need to be generated for the selected target python versions.
+""",
+ ),
  "group_deps": attr.string_list(
  doc = "List of dependencies to skip in order to break the cycles within a dependency group.",
  default = [],

python/pip_install/private/generate_whl_library_build_bazel.bzl

@@ -137,7 +137,8 @@ def generate_whl_library_build_bazel(
  entry_points,
  annotation = None,
  group_name = None,
- group_deps = []):
+ group_deps = [],
+ impl_vis = None):
  """Generate a BUILD file for an unzipped Wheel
 
  Args:
@@ -155,6 +156,7 @@ def generate_whl_library_build_bazel(
  group_deps: List[str]; names of fellow members of the group (if any). These will be excluded
  from generated deps lists so as to avoid direct cycles. These dependencies will be provided
  at runtime by the group rules which wrap this library and its fellows together.
+ impl_vis: str; override the visibility of the implementation labels.
 
  Returns:
  A complete BUILD file as a string
@@ -267,12 +269,12 @@ config_setting(
  group_repo = repo_prefix + "_groups"
  library_impl_label = "@%s//:%s_%s" % (group_repo, normalize_name(group_name), PY_LIBRARY_PUBLIC_LABEL)
  whl_impl_label = "@%s//:%s_%s" % (group_repo, normalize_name(group_name), WHEEL_FILE_PUBLIC_LABEL)
- impl_vis = "@%s//:__pkg__" % (group_repo,)
+ impl_vis = impl_vis or "@%s//:__pkg__" % (group_repo,)
 
  else:
  library_impl_label = PY_LIBRARY_IMPL_LABEL
  whl_impl_label = WHEEL_FILE_IMPL_LABEL
- impl_vis = "//visibility:private"
+ impl_vis = impl_vis or "//visibility:private"
 
  contents = "\n".join(
  [

python/private/BUILD.bazel

@@ -48,6 +48,14 @@ filegroup(
  visibility = ["//python:__pkg__"],
 )
 
+bzl_library(
+ name = "auth_bzl",
+ srcs = ["auth.bzl"],
+ deps = [
+ "//python:py_runtime_bzl",
+ ],
+)
+
 bzl_library(
  name = "autodetecting_toolchain_bzl",
  srcs = ["autodetecting_toolchain.bzl"],

python/private/auth.bzl

@@ -0,0 +1,42 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers copied from http_file source to be reused here.
+
+The implementation below is copied directly from Bazel's implementation of `http_archive`.
+Accordingly, the return value of this function should be used identically as the `auth` parameter of `http_archive`.
+Reference: https://github.com/bazelbuild/bazel/blob/6.3.2/tools/build_defs/repo/http.bzl#L109
+"""
+
+# TODO @aignas 2023-12-18: use the following instead when available.
+# load("@bazel_tools//tools/build_defs/repo:utils.bzl", "get_auth")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "read_netrc", "read_user_netrc", "use_netrc")
+
+def get_auth(rctx, urls):
+ """Utility for retrieving netrc-based authentication parameters for repository download rules used in python_repository.
+
+ Args:
+ rctx (repository_ctx): The repository rule's context object.
+ urls: A list of URLs from which assets will be downloaded.
+
+ Returns:
+ dict: A map of authentication parameters by URL.
+ """
+ if rctx.attr.netrc:
+ netrc = read_netrc(rctx, rctx.attr.netrc)
+ elif "NETRC" in rctx.os.environ:
+ netrc = read_netrc(rctx, rctx.os.environ["NETRC"])
+ else:
+ netrc = read_user_netrc(rctx)
+ return use_netrc(netrc, urls, rctx.attr.auth_patterns)

python/private/bzlmod/label.bzl

@@ -0,0 +1,23 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"A small utility to create a label usable internally"
+
+_this = str(Label("//:unknown"))
+
+def label(label):
+ """This function allows us to construct labels to pass to rules."""
+ prefix, _, _ = _this.partition("//")
+ prefix = prefix + "~pip~"
+ return Label(label.replace("@", prefix))