feat(bzlmod): support cross-platform whl setups within the main hub repo

CHANGELOG.md

@@ -51,6 +51,14 @@ A brief description of the categories of changes:
   replaced by whl_modifications.
 * (pip) Correctly select wheels when the python tag includes minor versions.
   See ([#1930](https://github.com/bazelbuild/rules_python/issues/1930))
+* (pip.parse): The lock file is now reproducible on any host platform if the
+  `experimental_index_url` is not used by any of the modules in the dependency
+  chain. To make the lock file identical on each `os` and `arch`, please use
+  the `experimental_index_url` feature which will fetch metadata from PyPI or a
+  different private index and write the contents to the lock file. Fixes
+  [#1643](https://github.com/bazelbuild/rules_python/issues/1643).
+* (pip.parse): Install `yanked` packages and print a warning instead of
+  ignoring them. This better matches the behaviour of `uv pip install`.
 
 ### Added
 * (rules) Precompiling Python source at build time is available. but is

MODULE.bazel

@@ -56,9 +56,8 @@ register_toolchains("@pythons_hub//:all")
 #####################
 # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
 
-pip = use_extension("//python/extensions:pip.bzl", "pip")
+pip = use_extension("//python/private/bzlmod:pip.bzl", "pip_internal")
 pip.parse(
-    experimental_index_url = "https://pypi.org/simple",
     hub_name = "rules_python_publish_deps",
     python_version = "3.11",
     requirements_by_platform = {
@@ -80,13 +79,11 @@ bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_nam
 bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle")
 
 dev_pip = use_extension(
-    "//python/extensions:pip.bzl",
-    "pip",
+    "//python/private/bzlmod:pip.bzl",
+    "pip_internal",
     dev_dependency = True,
 )
 dev_pip.parse(
-    envsubst = ["PIP_INDEX_URL"],
-    experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
     experimental_requirement_cycles = {
         "sphinx": [
             "sphinx",
@@ -99,11 +96,11 @@ dev_pip.parse(
     },
     hub_name = "dev_pip",
     python_version = "3.11",
-    requirements_lock = "//docs/sphinx:requirements.txt",
+    requirements_by_platform = {
+        "//docs/sphinx:requirements.txt": "linux_*,osx_*",
+    },
 )
 dev_pip.parse(
-    envsubst = ["PIP_INDEX_URL"],
-    experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
     hub_name = "pypiserver",
     python_version = "3.11",
     requirements_lock = "//examples/wheel:requirements_server.txt",

docs/sphinx/api/python/config_settings/index.md

@@ -3,11 +3,18 @@
 
 # //python/config_settings
 
+:::{bzl:flag} python_version
+Determines the default hermetic Python toolchain version. This can be set to
+one of the values that `rules_python` maintains.
+:::
+
 :::{bzl:flag} precompile
 Determines if Python source files should be compiled at build time.
 
-NOTE: The flag value is overridden by the target level `precompile` attribute,
+```{note}
+The flag value is overridden by the target level `precompile` attribute,
 except for the case of `force_enabled` and `forced_disabled`.
+```
 
 Values:
 
@@ -31,8 +38,10 @@ Values:
 Determines, when a source file is compiled, if the source file is kept
 in the resulting output or not.
 
-NOTE: This flag is overridden by the target level `precompile_source_retention`
+```{note}
+This flag is overridden by the target level `precompile_source_retention`
 attribute.
+```
 
 Values:
 
@@ -60,9 +69,61 @@ Values:
 :::{bzl:flag} pyc_collection
 Determine if `py_binary` collects transitive pyc files.
 
-NOTE: This flag is overridden by the target level `pyc_collection` attribute.
+```{note}
+This flag is overridden by the target level `pyc_collection` attribute.
+```
 
 Values:
 * `include_pyc`: Include `PyInfo.transitive_pyc_files` as part of the binary.
 * `disabled`: Don't include `PyInfo.transitive_pyc_files` as part of the binary.
 :::
+
+:::{bzl:flag} py_linux_libc
+Set what libc is used for the target platform. This will affect which whl binaries will be pulled and what toolchain will be auto-detected. Currently `rules_python` only supplies toolchains compatible with `glibc`.
+
+Values:
+* `glibc`: Use `glibc`, default.
+* `muslc`: Use `muslc`.
+:::
+
+:::{bzl:flag} pip_whl
+Set what distributions are used in the `pip` integration.
+
+Values:
+* `auto`: Prefer `whl` distributions if they are compatible with a target
+  platform, but fallback to `sdist`. This is the default.
+* `only`: Only use `whl` distributions and error out if it is not available.
+* `no`: Only use `sdist` distributions. The wheels will be built non-hermetically in the `whl_library` repository rule.
+:::
+
+:::{bzl:flag} pip_whl_osx_arch
+Set what wheel types we should prefer when building on the OSX platform.
+
+Values:
+* `arch`: Prefer architecture specific wheels.
+* `universal`: Prefer universal wheels that usually are bigger and contain binaries for both, Intel and ARM architectures in the same wheel.
+:::
+
+:::{bzl:flag} pip_whl_glibc_version
+Set the minimum `glibc` version that the `py_binary` using `whl` distributions from a PyPI index should support.
+
+Values:
+* `""`: Select the lowest available version of each wheel giving you the maximum compatibility. This is the default.
+* `X.Y`: The string representation of a `glibc` version. The allowed values depend on the `requirements.txt` lock file contents.
+:::
+
+:::{bzl:flag} pip_whl_muslc_version
+Set the minimum `muslc` version that the `py_binary` using `whl` distributions from a PyPI index should support.
+
+Values:
+* `""`: Select the lowest available version of each wheel giving you the maximum compatibility. This is the default.
+* `X.Y`: The string representation of a `muslc` version. The allowed values depend on the `requirements.txt` lock file contents.
+:::
+
+:::{bzl:flag} pip_whl_osx_version
+Set the minimum `osx` version that the `py_binary` using `whl` distributions from a PyPI index should support.
+
+Values:
+* `""`: Select the lowest available version of each wheel giving you the maximum compatibility. This is the default.
+* `X.Y`: The string representation of the MacOS version. The allowed values depend on the `requirements.txt` lock file contents.
+:::

python/config_settings/BUILD.bazel

@@ -6,6 +6,13 @@ load(
     "PrecompileSourceRetentionFlag",
     "PycCollectionFlag",
 )
+load(
+    "//python/private:pip_flags.bzl",
+    "INTERNAL_FLAGS",
+    "UniversalWhlFlag",
+    "UseWhlFlag",
+    "WhlLibcFlag",
+)
 load(":config_settings.bzl", "construct_config_settings")
 
 filegroup(
@@ -52,3 +59,63 @@ string_flag(
     # NOTE: Only public because its an implicit dependency
     visibility = ["//visibility:public"],
 )
+
+# This is used for pip and hermetic toolchain resolution.
+string_flag(
+    name = "py_linux_libc",
+    build_setting_default = WhlLibcFlag.GLIBC,
+    values = sorted(WhlLibcFlag.__members__.values()),
+    # NOTE: Only public because it is used in pip hub and toolchain repos.
+    visibility = ["//visibility:public"],
+)
+
+# pip.parse related flags
+
+string_flag(
+    name = "pip_whl",
+    build_setting_default = UseWhlFlag.AUTO,
+    values = sorted(UseWhlFlag.__members__.values()),
+    # NOTE: Only public because it is used in pip hub repos.
+    visibility = ["//visibility:public"],
+)
+
+string_flag(
+    name = "pip_whl_osx_arch",
+    build_setting_default = UniversalWhlFlag.ARCH,
+    values = sorted(UniversalWhlFlag.__members__.values()),
+    # NOTE: Only public because it is used in pip hub repos.
+    visibility = ["//visibility:public"],
+)
+
+string_flag(
+    name = "pip_whl_glibc_version",
+    build_setting_default = "",
+    # NOTE: Only public because it is used in pip hub repos.
+    visibility = ["//visibility:public"],
+)
+
+string_flag(
+    name = "pip_whl_muslc_version",
+    build_setting_default = "",
+    # NOTE: Only public because it is used in pip hub repos.
+    visibility = ["//visibility:public"],
+)
+
+string_flag(
+    name = "pip_whl_osx_version",
+    build_setting_default = "",
+    # NOTE: Only public because it is used in pip hub repos.
+    visibility = ["//visibility:public"],
+)
+
+# private pip whl related flags. Their values cannot be changed and they
+# are an implementation detail of how `pip_config_settings` work.
+[
+    string_flag(
+        name = "_internal_pip_" + flag,
+        build_setting_default = "",
+        values = [""],
+        visibility = ["//visibility:public"],
+    )
+    for flag in INTERNAL_FLAGS
+]

python/pip_install/tools/wheel_installer/wheel.py

@@ -51,6 +51,7 @@ class Arch(Enum):
     aarch64 = 3
     ppc = 4
     s390x = 5
+    arm = 6
     amd64 = x86_64
     arm64 = aarch64
     i386 = x86_32

python/pip_install/tools/wheel_installer/wheel_test.py

@@ -371,17 +371,17 @@ def test_can_get_specific_from_string(self):
 
     def test_can_get_all_for_py_version(self):
         cp39 = wheel.Platform.all(minor_version=9)
-        self.assertEqual(15, len(cp39), f"Got {cp39}")
+        self.assertEqual(18, len(cp39), f"Got {cp39}")
         self.assertEqual(cp39, wheel.Platform.from_string("cp39_*"))
 
     def test_can_get_all_for_os(self):
         linuxes = wheel.Platform.all(wheel.OS.linux, minor_version=9)
-        self.assertEqual(5, len(linuxes))
+        self.assertEqual(6, len(linuxes))
         self.assertEqual(linuxes, wheel.Platform.from_string("cp39_linux_*"))
 
     def test_can_get_all_for_os_for_host_python(self):
         linuxes = wheel.Platform.all(wheel.OS.linux)
-        self.assertEqual(5, len(linuxes))
+        self.assertEqual(6, len(linuxes))
         self.assertEqual(linuxes, wheel.Platform.from_string("linux_*"))
 
     def test_specific_version_specializations(self):
@@ -425,6 +425,7 @@ def test_linux_specializations(self):
             wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.aarch64),
             wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.ppc),
             wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.s390x),
+            wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.arm),
         ]
         self.assertEqual(want, all_specializations)
 
@@ -441,6 +442,7 @@ def test_osx_specializations(self):
             wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64),
             wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.ppc),
             wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.s390x),
+            wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.arm),
         ]
         self.assertEqual(want, all_specializations)
 

python/private/BUILD.bazel

@@ -145,6 +145,23 @@ bzl_library(
     srcs = ["parse_whl_name.bzl"],
 )
 
+bzl_library(
+    name = "pip_flags_bzl",
+    srcs = ["pip_flags.bzl"],
+    deps = [
+        ":enum_bzl",
+    ],
+)
+
+bzl_library(
+    name = "pip_repo_name_bzl",
+    srcs = ["pip_repo_name.bzl"],
+    deps = [
+        ":normalize_name_bzl",
+        ":parse_whl_name_bzl",
+    ],
+)
+
 bzl_library(
     name = "pypi_index_bzl",
     srcs = ["pypi_index.bzl"],
@@ -235,6 +252,8 @@ bzl_library(
     name = "py_toolchain_suite_bzl",
     srcs = ["py_toolchain_suite.bzl"],
     deps = [
+        ":config_settings_bzl",
+        ":text_util_bzl",
         ":toolchain_types_bzl",
         "@bazel_skylib//lib:selects",
     ],