feat(toolchain, pip.parse): introduce a new '_host' toolchain repo

CHANGELOG.md

@@ -25,8 +25,21 @@ A brief description of the categories of changes:
 
 ### Fixed
 
+* (bzlmod pip.parse) Use a platform-independent reference to the interpreter
+ pip uses. This reduces (but doesn't eliminate) the amount of
+ platform-specific content in `MODULE.bazel.lock` files; Follow
+ [#1643](https://github.com/bazelbuild/rules_python/issues/1643) for removing
+ platform-specific content in `MODULE.bazel.lock` files.
+
 ### Added
 
+* (toolchains) `python_register_toolchains` now also generates a repository
+ that is suffixed with `_host`, that has a single label `:python` that is a
+ symlink to the python interpreter for the host platform. The intended use is
+ mainly in `repository_rule`, which are always run using `host` platform
+ Python. This means that `WORKSPACE` users can now copy the `requirements.bzl`
+ file for vendoring as seen in the updated `pip_parse_vendored` example.
+
 ## [0.28.0] - 2024-01-07
 
 [0.28.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.28.0

examples/build_file_generation/WORKSPACE

@@ -84,8 +84,6 @@ python_register_toolchains(
  python_version = "3.9",
 )
 
-# Load the interpreter and pip_parse rules.
-load("@python39//:defs.bzl", "interpreter")
 load("@rules_python//python:pip.bzl", "pip_parse")
 
 # This macro wraps the `pip_repository` rule that invokes `pip`, with `incremental` set.
@@ -114,7 +112,7 @@ pip_parse(
  # 3. Wrapper script, like in the autodetecting python toolchain.
  #
  # Here, we use the interpreter constant that resolves to the host interpreter from the default Python toolchain.
- python_interpreter_target = interpreter,
+ python_interpreter_target = "@python39_host//:python",
  # Set the location of the lock file.
  requirements_lock = "//:requirements_lock.txt",
  requirements_windows = "//:requirements_windows.txt",

examples/multi_python_versions/WORKSPACE

@@ -28,19 +28,15 @@ python_register_multi_toolchains(
 )
 
 load("@python//:pip.bzl", "multi_pip_parse")
-load("@python//3.10:defs.bzl", interpreter_3_10 = "interpreter")
-load("@python//3.11:defs.bzl", interpreter_3_11 = "interpreter")
-load("@python//3.8:defs.bzl", interpreter_3_8 = "interpreter")
-load("@python//3.9:defs.bzl", interpreter_3_9 = "interpreter")
 
 multi_pip_parse(
  name = "pypi",
  default_version = default_python_version,
  python_interpreter_target = {
- "3.10": interpreter_3_10,
- "3.11": interpreter_3_11,
- "3.8": interpreter_3_8,
- "3.9": interpreter_3_9,
+ "3.10": "@python_3_10_host//:python",
+ "3.11": "@python_3_11_host//:python",
+ "3.8": "@python_3_8_host//:python",
+ "3.9": "@python_3_9_host//:python",
  },
  requirements_lock = {
  "3.10": "//requirements:requirements_lock_3_10.txt",

examples/pip_parse/WORKSPACE

@@ -14,7 +14,6 @@ python_register_toolchains(
  python_version = "3.9",
 )
 
-load("@python39//:defs.bzl", "interpreter")
 load("@rules_python//python:pip.bzl", "pip_parse")
 
 pip_parse(
@@ -52,7 +51,7 @@ pip_parse(
  # 3. Wrapper script, like in the autodetecting python toolchain.
  #
  # Here, we use the interpreter constant that resolves to the host interpreter from the default Python toolchain.
- python_interpreter_target = interpreter,
+ python_interpreter_target = "@python39_host//:python",
 
  # (Optional) You can set quiet to False if you want to see pip output.
  #quiet = False,

examples/pip_parse_vendored/BUILD.bazel

@@ -9,24 +9,6 @@ compile_pip_requirements(
  src = "requirements.in",
 )
 
-# The requirements.bzl file is generated with a reference to the interpreter for the host platform.
-# In order to check in a platform-agnostic file, we have to replace that reference with the symbol
-# loaded from our python toolchain.
-genrule(
- name = "make_platform_agnostic",
- srcs = ["@pip//:requirements.bzl"],
- outs = ["requirements.clean.bzl"],
- cmd = " | ".join([
- "cat $<",
- # Insert our load statement after the existing one so we don't produce a file with buildifier warnings
- """sed -e '/^load.*.pip.bzl/i\\'$$'\\n''load("@python39//:defs.bzl", "interpreter")'""",
- # Replace the bazel 6.0.0 specific comment with something that bazel 5.4.0 would produce.
- # This enables this example to be run as a test under bazel 5.4.0.
- """sed -e 's#@//#//#'""",
- """sed 's#"@python39_.*//:bin/python3"#interpreter#' >$@""",
- ]),
-)
-
 write_file(
  name = "gen_update",
  out = "update.sh",
@@ -35,14 +17,14 @@ write_file(
  "#!/usr/bin/env bash",
  # Bazel gives us a way to access the source folder!
  "cd $BUILD_WORKSPACE_DIRECTORY",
- "cp -fv bazel-bin/requirements.clean.bzl requirements.bzl",
+ "cp -fv bazel-pip_parse_vendored/external/pip/requirements.bzl requirements.bzl",
  ],
 )
 
 sh_binary(
  name = "vendor_requirements",
  srcs = ["update.sh"],
- data = [":make_platform_agnostic"],
+ data = ["@pip//:requirements.bzl"],
 )
 
 # Similarly ensures that the requirements.bzl file is updated
@@ -51,5 +33,5 @@ diff_test(
  name = "test_vendored",
  failure_message = "Please run: bazel run //:vendor_requirements",
  file1 = "requirements.bzl",
- file2 = ":make_platform_agnostic",
+ file2 = "@pip//:requirements.bzl",
 )

examples/pip_parse_vendored/WORKSPACE

@@ -14,14 +14,13 @@ python_register_toolchains(
  python_version = "3.9",
 )
 
-load("@python39//:defs.bzl", "interpreter")
 load("@rules_python//python:pip.bzl", "pip_parse")
 
 # This repository isn't referenced, except by our test that asserts the requirements.bzl is updated.
 # It also wouldn't be needed by users of this ruleset.
 pip_parse(
  name = "pip",
- python_interpreter_target = interpreter,
+ python_interpreter_target = "@python39_host//:python",
  requirements_lock = "//:requirements.txt",
 )
 

examples/pip_parse_vendored/requirements.bzl

@@ -1,10 +1,9 @@
 """Starlark representation of locked requirements.
 
 @generated by rules_python pip_parse repository rule
-from //:requirements.txt
+from @//:requirements.txt
 """
 
-load("@python39//:defs.bzl", "interpreter")
 load("@rules_python//python:pip.bzl", "pip_utils")
 load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")
 
@@ -17,7 +16,7 @@ all_whl_requirements = all_whl_requirements_by_package.values()
 all_data_requirements = ["@pip//certifi:data", "@pip//charset_normalizer:data", "@pip//idna:data", "@pip//requests:data", "@pip//urllib3:data"]
 
 _packages = [("pip_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("pip_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")]
-_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
+_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
 _annotations = {}
 
 def requirement(name):

examples/pip_repository_annotations/WORKSPACE

@@ -14,7 +14,6 @@ python_register_toolchains(
  python_version = "3.9",
 )
 
-load("@python39//:defs.bzl", "interpreter")
 load("@rules_python//python:pip.bzl", "package_annotation", "pip_parse")
 
 # Here we can see an example of annotations being applied to an arbitrary
@@ -54,7 +53,7 @@ write_file(
 pip_parse(
  name = "pip",
  annotations = ANNOTATIONS,
- python_interpreter_target = interpreter,
+ python_interpreter_target = "@python39_host//:python",
  requirements_lock = "//:requirements.txt",
 )
 

python/pip_install/pip_repository.bzl

@@ -22,7 +22,6 @@ load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse
 load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
 load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
 load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
-load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
 load("//python/private:normalize_name.bzl", "normalize_name")
 load("//python/private:parse_whl_name.bzl", "parse_whl_name")
 load("//python/private:patch_whl.bzl", "patch_whl")
@@ -87,13 +86,12 @@ def _resolve_python_interpreter(rctx):
  if rctx.attr.python_interpreter_target != None:
  python_interpreter = rctx.path(rctx.attr.python_interpreter_target)
 
- if BZLMOD_ENABLED:
- (os, _) = get_host_os_arch(rctx)
+ (os, _) = get_host_os_arch(rctx)
 
-  # On Windows, the symlink doesn't work because Windows attempts to find
-  # Python DLLs where the symlink is, not where the symlink points.
-  if os == WINDOWS_NAME:
-  python_interpreter = python_interpreter.realpath
+ # On Windows, the symlink doesn't work because Windows attempts to find
+ # Python DLLs where the symlink is, not where the symlink points.
+ if os == WINDOWS_NAME:
+ python_interpreter = python_interpreter.realpath
  elif "/" not in python_interpreter:
  # It's a plain command, e.g. "python3", to look up in the environment.
  found_python_interpreter = rctx.which(python_interpreter)

python/private/bzlmod/pip.bzl

@@ -87,8 +87,10 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides):
  # we programmatically find it.
  hub_name = pip_attr.hub_name
  if python_interpreter_target == None and not pip_attr.python_interpreter:
- python_name = "python_" + version_label(pip_attr.python_version, sep = "_")
- if python_name not in INTERPRETER_LABELS.keys():
+ python_name = "python_{}_host".format(
+ version_label(pip_attr.python_version, sep = "_"),
+ )
+ if python_name not in INTERPRETER_LABELS:
  fail((
  "Unable to find interpreter for pip hub '{hub_name}' for " +
  "python_version={version}: Make sure a corresponding " +

python/private/bzlmod/pythons_hub.bzl

@@ -78,7 +78,7 @@ DEFAULT_PYTHON_VERSION = "{default_python_version}"
 """
 
 _line_for_hub_template = """\
- "{name}": Label("@{name}_{platform}//:{path}"),
+ "{key}": Label("@{name}_{platform}//:{path}"),
 """
 
 def _hub_repo_impl(rctx):
@@ -103,11 +103,22 @@ def _hub_repo_impl(rctx):
 
  # Create a dict that is later used to create
  # a symlink to a interpreter.
- interpreter_labels = "".join([_line_for_hub_template.format(
- name = name,
- platform = platform,
- path = path,
- ) for name in rctx.attr.toolchain_user_repository_names])
+ interpreter_labels = "".join([
+ _line_for_hub_template.format(
+ key = name + ("" if platform_str != "host" else "_host"),
+ name = name,
+ platform = platform_str,
+ path = p,
+ )
+ for name in rctx.attr.toolchain_user_repository_names
+ for platform_str, p in {
+ # NOTE @aignas 2023-12-21: maintaining the `platform` specific key
+ # here may be unneeded in the long term, but I am not sure if there
+ # are other users that depend on it.
+ platform: path,
+ "host": "python",
+ }.items()
+ ])
 
  rctx.file(
  "interpreters.bzl",

python/private/toolchains_repo.bzl

@@ -240,8 +240,50 @@ def compile_pip_requirements(name, **kwargs):
 
 toolchain_aliases = repository_rule(
  _toolchain_aliases_impl,
- doc = """Creates a repository with a shorter name meant for the host platform, which contains
- a BUILD.bazel file declaring aliases to the host platform's targets.
+ doc = """\
+Creates a repository with a shorter name only referencing the python version,
+it contains a BUILD.bazel file declaring aliases to the host platform's targets
+and is a great fit for any usage related to setting up toolchains for build
+actions.""",
+ attrs = {
+ "platforms": attr.string_list(
+ doc = "List of platforms for which aliases shall be created",
+ ),
+ "python_version": attr.string(doc = "The Python version."),
+ "user_repository_name": attr.string(
+ mandatory = True,
+ doc = "The base name for all created repositories, like 'python38'.",
+ ),
+ "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")),
+ },
+)
+
+def _host_toolchain_impl(rctx):
+ rctx.file("BUILD.bazel", """\
+# Generated by python/private/toolchains_repo.bzl
+
+exports_files(["python"], visibility = ["//visibility:public"])
+""")
+
+ (os_name, arch) = get_host_os_arch(rctx)
+ host_platform = get_host_platform(os_name, arch)
+ host_python = rctx.path(
+ Label(
+ "@@{py_repository}_{host_platform}//:python".format(
+ py_repository = rctx.attr.name[:-len("_host")],
+ host_platform = host_platform,
+ ),
+ ),
+ )
+ rctx.symlink(host_python, "python")
+
+host_toolchain = repository_rule(
+ _host_toolchain_impl,
+ doc = """\
+Creates a repository with a shorter name meant to be used in the repository_ctx,
+which needs to have `symlinks` for the interpreter. This is separate from the
+toolchain_aliases repo because referencing the `python` interpreter target from
+this repo causes an eager fetch of the toolchain for the host platform.
  """,
  attrs = {
  "platforms": attr.string_list(

python/repositories.bzl

@@ -27,6 +27,7 @@ load("//python/private:full_version.bzl", "full_version")
 load("//python/private:internal_config_repo.bzl", "internal_config_repo")
 load(
  "//python/private:toolchains_repo.bzl",
+ "host_toolchain",
  "multi_toolchain_aliases",
  "toolchain_aliases",
  "toolchains_repo",
@@ -589,6 +590,13 @@ def python_register_toolchains(
  platform = platform,
  ))
 
+ host_toolchain(
+ name = name + "_host",
+ python_version = python_version,
+ user_repository_name = name,
+ platforms = loaded_platforms,
+ )
+
  toolchain_aliases(
  name = name,
  python_version = python_version,