From 9d35b35a449f575364d47a6fe6f008d066876c80 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 3 Jun 2024 00:47:46 +0900 Subject: [PATCH 01/18] fix(pip.parse): make the pip extension reproducible if PyPI is not called With this PR we can finally have fewer lock file entries in setups which do not use the `experimental_index_url` feature yet. This is fully backwards compatible change as it relies on `bazel` doing the right thing and regenerating the lock file. Fixes #1643. --- CHANGELOG.md | 6 +++ MODULE.bazel | 6 +-- python/private/bzlmod/pip.bzl | 88 +++++++++++++++++++++++++++++------ 3 files changed, 84 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fdd039a5d..56c284af87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,12 @@ A brief description of the categories of changes: replaced by whl_modifications. * (pip) Correctly select wheels when the python tag includes minor versions. See ([#1930](https://github.com/bazelbuild/rules_python/issues/1930)) +* (pip.parse): The lock file is now reproducible on any host platform if the + `experimental_index_url` is not used by any of the modules in the dependency + chain. To make the lock file identical on each `os` and `arch`, please use + the `experimental_index_url` feature which will fetch metadata from PyPI or a + different private index and write the contents to the lock file. Fixes + [#1643](https://github.com/bazelbuild/rules_python/issues/1643). ### Added * (rules) Precompiling Python source at build time is available. but is diff --git a/MODULE.bazel b/MODULE.bazel index 7064dfc84f..093bc31ef8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -56,7 +56,7 @@ register_toolchains("@pythons_hub//:all") ##################### # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it. -pip = use_extension("//python/extensions:pip.bzl", "pip") +pip = use_extension("//python/private/bzlmod:pip.bzl", "pip_internal") pip.parse( experimental_index_url = "https://pypi.org/simple", hub_name = "rules_python_publish_deps", @@ -80,8 +80,8 @@ bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_nam bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle") dev_pip = use_extension( - "//python/extensions:pip.bzl", - "pip", + "//python/private/bzlmod:pip.bzl", + "pip_internal", dev_dependency = True, ) dev_pip.parse( diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl index 9e29877b42..77152248c5 100644 --- a/python/private/bzlmod/pip.bzl +++ b/python/private/bzlmod/pip.bzl @@ -103,6 +103,7 @@ You cannot use both the additive_build_content and additive_build_content_file a def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache): logger = repo_utils.logger(module_ctx) python_interpreter_target = pip_attr.python_interpreter_target + is_hub_reproducible = True # if we do not have the python_interpreter set in the attributes # we programmatically find it. @@ -274,6 +275,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s logger.debug(lambda: "Selected: {}".format(distribution)) if distribution: + is_hub_reproducible = False whl_library_args["requirement"] = requirement.srcs.requirement whl_library_args["urls"] = [distribution.url] whl_library_args["sha256"] = distribution.sha256 @@ -303,6 +305,8 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s ), ) + return is_hub_reproducible + def _pip_impl(module_ctx): """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories. @@ -412,6 +416,7 @@ def _pip_impl(module_ctx): hub_group_map = {} simpleapi_cache = {} + is_extension_reproducible = True for mod in module_ctx.modules: for pip_attr in mod.tags.parse: @@ -448,7 +453,8 @@ def _pip_impl(module_ctx): else: pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) - _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache) + is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache) + is_extension_reproducible = is_extension_reproducible and is_hub_reproducible for hub_name, whl_map in hub_whl_map.items(): pip_repository( @@ -462,6 +468,26 @@ def _pip_impl(module_ctx): groups = hub_group_map.get(hub_name), ) + if bazel_features.external_deps.extension_metadata_has_reproducible: + # If we are not using the `experimental_index_url feature, the extension is fully + # deterministic and we don't need to create a lock entry for it. + # + # In order to be able to dogfood the `experimental_index_url` feature before it gets + # stabilized, we have created the `_pip_non_reproducible` function, that will result + # in extra entries in the lock file. + return module_ctx.extension_metadata(reproducible = is_extension_reproducible) + else: + return None + +def _pip_non_reproducible(module_ctx): + _pip_impl(module_ctx) + + if bazel_features.external_deps.extension_metadata_has_reproducible: + # We allow for calling the PyPI index and that will go into the MODULE.bazel.lock file + return module_ctx.extension_metadata(reproducible = False) + else: + return None + def _pip_parse_ext_attrs(): attrs = dict({ "experimental_extra_index_urls": attr.string_list( @@ -661,17 +687,6 @@ Apply any overrides (e.g. patches) to a given Python distribution defined by other tags in this extension.""", ) -def _extension_extra_args(): - args = {} - - if bazel_features.external_deps.module_extension_has_os_arch_dependent: - args = args | { - "arch_dependent": True, - "os_dependent": True, - } - - return args - pip = module_extension( doc = """\ This extension is used to make dependencies from pip available. @@ -714,7 +729,54 @@ extension. """, ), }, - **_extension_extra_args() +) + +pip_internal = module_extension( + doc = """\ +This extension is used to make dependencies from pypi available. + +For now this is intended to be used internally so that usage of the `pip` +extension in `rules_python` does not affect the evaluations of the extension +for the consumers. + +pip.parse: +To use, call `pip.parse()` and specify `hub_name` and your requirements file. +Dependencies will be downloaded and made available in a repo named after the +`hub_name` argument. + +Each `pip.parse()` call configures a particular Python version. Multiple calls +can be made to configure different Python versions, and will be grouped by +the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy` +to automatically resolve to different, Python version-specific, libraries. + +pip.whl_mods: +This tag class is used to help create JSON files to describe modifications to +the BUILD files for wheels. +""", + implementation = _pip_non_reproducible, + tag_classes = { + "override": _override_tag, + "parse": tag_class( + attrs = _pip_parse_ext_attrs(), + doc = """\ +This tag class is used to create a pypi hub and all of the spokes that are part of that hub. +This tag class reuses most of the pypi attributes that are found in +@rules_python//python/pip_install:pip_repository.bzl. +The exception is it does not use the arg 'repo_prefix'. We set the repository +prefix for the user and the alias arg is always True in bzlmod. +""", + ), + "whl_mods": tag_class( + attrs = _whl_mod_attrs(), + doc = """\ +This tag class is used to create JSON file that are used when calling wheel_builder.py. These +JSON files contain instructions on how to modify a wheel's project. Each of the attributes +create different modifications based on the type of attribute. Previously to bzlmod these +JSON files where referred to as annotations, and were renamed to whl_modifications in this +extension. +""", + ), + }, ) def _whl_mods_repo_impl(rctx): From 3fd6821666d595223f708d1dd4d6c51f9d02b5ea Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:50:43 +0900 Subject: [PATCH 02/18] comment: address comments and set the internal pip extension to use PyPI by default --- MODULE.bazel | 5 ----- python/private/bzlmod/pip.bzl | 24 +++++++++++++++++------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 093bc31ef8..7f9ec9739e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -58,7 +58,6 @@ register_toolchains("@pythons_hub//:all") pip = use_extension("//python/private/bzlmod:pip.bzl", "pip_internal") pip.parse( - experimental_index_url = "https://pypi.org/simple", hub_name = "rules_python_publish_deps", python_version = "3.11", requirements_by_platform = { @@ -85,8 +84,6 @@ dev_pip = use_extension( dev_dependency = True, ) dev_pip.parse( - envsubst = ["PIP_INDEX_URL"], - experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}", experimental_requirement_cycles = { "sphinx": [ "sphinx", @@ -102,8 +99,6 @@ dev_pip.parse( requirements_lock = "//docs/sphinx:requirements.txt", ) dev_pip.parse( - envsubst = ["PIP_INDEX_URL"], - experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}", hub_name = "pypiserver", python_version = "3.11", requirements_lock = "//examples/wheel:requirements_server.txt", diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl index 77152248c5..8702f1fbe7 100644 --- a/python/private/bzlmod/pip.bzl +++ b/python/private/bzlmod/pip.bzl @@ -482,13 +482,20 @@ def _pip_impl(module_ctx): def _pip_non_reproducible(module_ctx): _pip_impl(module_ctx) - if bazel_features.external_deps.extension_metadata_has_reproducible: - # We allow for calling the PyPI index and that will go into the MODULE.bazel.lock file - return module_ctx.extension_metadata(reproducible = False) - else: - return None + # We default to calling the PyPI index and that will go into the + # MODULE.bazel.lock file, hence return nothing here. + return None -def _pip_parse_ext_attrs(): +def _pip_parse_ext_attrs(**kwargs): + """Get the attributes for the pip extension. + + Args: + **kwargs: A kwarg for setting defaults for the specific attributes. The + key is expected to be the same as the attribute key. + + Returns: + A dict of attributes. + """ attrs = dict({ "experimental_extra_index_urls": attr.string_list( doc = """\ @@ -503,6 +510,7 @@ This is equivalent to `--extra-index-urls` `pip` option. default = [], ), "experimental_index_url": attr.string( + default = kwargs.get("experimental_index_url", ""), doc = """\ The index URL to use for downloading wheels using bazel downloader. This value is going to be subject to `envsubst` substitutions if necessary. @@ -757,7 +765,9 @@ the BUILD files for wheels. tag_classes = { "override": _override_tag, "parse": tag_class( - attrs = _pip_parse_ext_attrs(), + attrs = _pip_parse_ext_attrs( + experimental_index_url = "https://pypi.org/simple", + ), doc = """\ This tag class is used to create a pypi hub and all of the spokes that are part of that hub. This tag class reuses most of the pypi attributes that are found in From 4c4b46fe6bba353234a86889bd515364f5dc91ac Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:08:08 +0900 Subject: [PATCH 03/18] fix(bzlmod): do not throw away yanked PyPI packages The `uv pip install` would only warn the user of yanked packages but would not refuse to install them. Update our implementation to better match the same behaviour. --- CHANGELOG.md | 2 ++ python/private/parse_requirements.bzl | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56c284af87..c16501019b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,8 @@ A brief description of the categories of changes: the `experimental_index_url` feature which will fetch metadata from PyPI or a different private index and write the contents to the lock file. Fixes [#1643](https://github.com/bazelbuild/rules_python/issues/1643). +* (pip.parse): Install `yanked` packages and print a warning instead of + ignoring them. This better matches the behaviour of `uv pip install`. ### Added * (rules) Precompiling Python source at build time is available. but is diff --git a/python/private/parse_requirements.bzl b/python/private/parse_requirements.bzl index c6a498539f..cb5024c841 100644 --- a/python/private/parse_requirements.bzl +++ b/python/private/parse_requirements.bzl @@ -451,6 +451,18 @@ def _add_dists(requirement, index_urls, python_version, logger = None): if logger: logger.warn("Could not find a whl or an sdist with sha256={}".format(sha256)) + yanked = {} + for dist in whls + [sdist]: + if dist and dist.yanked: + yanked.setdefault(dist.yanked, []).append(dist.filename) + if yanked: + logger.warn(lambda: "\n".join([ + "the following distributions got yanked:", + ] + [ + "reason: {}\n {}".format(reason, "\n".join(sorted(dists))) + for reason, dists in yanked.items() + ])) + # Filter out the wheels that are incompatible with the target_platforms. whls = select_whls( whls = whls, From 0587c374a8b0e8687a84354484019f13e6f02afa Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:59:38 +0900 Subject: [PATCH 04/18] fix(whl_library): add `arm` to the list of known architectures A previous PR has added the support for passing it in the `requirements_by_platform` and this just add the necessary code to make sure that we can also do the dependency management when parsing the `whl` `METADATA` files. --- python/pip_install/tools/wheel_installer/wheel.py | 1 + python/pip_install/tools/wheel_installer/wheel_test.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/python/pip_install/tools/wheel_installer/wheel.py b/python/pip_install/tools/wheel_installer/wheel.py index b84c214018..3d6780de9a 100644 --- a/python/pip_install/tools/wheel_installer/wheel.py +++ b/python/pip_install/tools/wheel_installer/wheel.py @@ -51,6 +51,7 @@ class Arch(Enum): aarch64 = 3 ppc = 4 s390x = 5 + arm = 6 amd64 = x86_64 arm64 = aarch64 i386 = x86_32 diff --git a/python/pip_install/tools/wheel_installer/wheel_test.py b/python/pip_install/tools/wheel_installer/wheel_test.py index acf2315ee9..3ddfaf7f2e 100644 --- a/python/pip_install/tools/wheel_installer/wheel_test.py +++ b/python/pip_install/tools/wheel_installer/wheel_test.py @@ -371,17 +371,17 @@ def test_can_get_specific_from_string(self): def test_can_get_all_for_py_version(self): cp39 = wheel.Platform.all(minor_version=9) - self.assertEqual(15, len(cp39), f"Got {cp39}") + self.assertEqual(18, len(cp39), f"Got {cp39}") self.assertEqual(cp39, wheel.Platform.from_string("cp39_*")) def test_can_get_all_for_os(self): linuxes = wheel.Platform.all(wheel.OS.linux, minor_version=9) - self.assertEqual(5, len(linuxes)) + self.assertEqual(6, len(linuxes)) self.assertEqual(linuxes, wheel.Platform.from_string("cp39_linux_*")) def test_can_get_all_for_os_for_host_python(self): linuxes = wheel.Platform.all(wheel.OS.linux) - self.assertEqual(5, len(linuxes)) + self.assertEqual(6, len(linuxes)) self.assertEqual(linuxes, wheel.Platform.from_string("linux_*")) def test_specific_version_specializations(self): @@ -425,6 +425,7 @@ def test_linux_specializations(self): wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.aarch64), wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.ppc), wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.s390x), + wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.arm), ] self.assertEqual(want, all_specializations) @@ -441,6 +442,7 @@ def test_osx_specializations(self): wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64), wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.ppc), wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.s390x), + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.arm), ] self.assertEqual(want, all_specializations) From 52a7a5e4d72be667e3acd9b244d40474351be916 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:06:12 +0900 Subject: [PATCH 05/18] fix(dev_pip): we only have `//dev_pip:sphinx` dependencies for non-windows This fixes a small issue with our dependency configuration for the documentation building. It is missing `colorama` and probably a few other dependencies to work on windows, but I am not sure if the other `sphinx` machinery supports non UNIX. --- MODULE.bazel | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 7f9ec9739e..53d845ed88 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -96,7 +96,9 @@ dev_pip.parse( }, hub_name = "dev_pip", python_version = "3.11", - requirements_lock = "//docs/sphinx:requirements.txt", + requirements_by_platform = { + "//docs/sphinx:requirements.txt": "linux_*,osx_*", + }, ) dev_pip.parse( hub_name = "pypiserver", From cc1857bda2f0787ef7a31154a3ba06eb1ca57b8b Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:57:06 +0900 Subject: [PATCH 06/18] feat(internal): add a function to get a unique pip repo name With this we can ensure that the name is unique (we use `sha256` and prefix it with `pip_hub` repo), valid (we use `sha256` instead of the version or the build tag of the whl filename) and easy to understand (we add the python tag, abi tag and platform tags to the end of the name to ensure that users have an easy way to navigate the files). --- python/private/BUILD.bazel | 9 ++++ python/private/pip_repo_name.bzl | 52 +++++++++++++++++++ tests/private/pip_repo_name/BUILD.bazel | 3 ++ .../pip_repo_name/pip_repo_name_tests.bzl | 52 +++++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 python/private/pip_repo_name.bzl create mode 100644 tests/private/pip_repo_name/BUILD.bazel create mode 100644 tests/private/pip_repo_name/pip_repo_name_tests.bzl diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 3e56208859..3d4ba4715d 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -145,6 +145,15 @@ bzl_library( srcs = ["parse_whl_name.bzl"], ) +bzl_library( + name = "pip_repo_name_bzl", + srcs = ["pip_repo_name.bzl"], + deps = [ + ":normalize_name_bzl", + ":parse_whl_name_bzl", + ], +) + bzl_library( name = "pypi_index_bzl", srcs = ["pypi_index.bzl"], diff --git a/python/private/pip_repo_name.bzl b/python/private/pip_repo_name.bzl new file mode 100644 index 0000000000..bef4304e15 --- /dev/null +++ b/python/private/pip_repo_name.bzl @@ -0,0 +1,52 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A function to convert a dist name to a valid bazel repo name. +""" + +load(":normalize_name.bzl", "normalize_name") +load(":parse_whl_name.bzl", "parse_whl_name") + +def pip_repo_name(prefix, filename, sha256): + """Return a valid whl_library repo name given a distribution filename. + + Args: + prefix: str, the prefix of the whl_library. + filename: str, the filename of the distribution. + sha256: str, the sha256 of the distribution. + + Returns: + a string that can be used in `whl_library`. + """ + parts = [prefix] + + if not filename.endswith(".whl"): + # Then the filename is basically foo-3.2.1. + parts.append(normalize_name(filename.rpartition("-")[0])) + parts.append("sdist") + else: + parsed = parse_whl_name(filename) + name = normalize_name(parsed.distribution) + python_tag, _, _ = parsed.python_tag.partition(".") + abi_tag, _, _ = parsed.abi_tag.partition(".") + platform_tag, _, _ = parsed.platform_tag.partition(".") + + parts.append(name) + parts.append(python_tag) + parts.append(abi_tag) + parts.append(platform_tag) + + parts.append(sha256[:8]) + + return "_".join(parts) diff --git a/tests/private/pip_repo_name/BUILD.bazel b/tests/private/pip_repo_name/BUILD.bazel new file mode 100644 index 0000000000..7c6782daaf --- /dev/null +++ b/tests/private/pip_repo_name/BUILD.bazel @@ -0,0 +1,3 @@ +load(":pip_repo_name_tests.bzl", "pip_repo_name_test_suite") + +pip_repo_name_test_suite(name = "pip_repo_name_tests") diff --git a/tests/private/pip_repo_name/pip_repo_name_tests.bzl b/tests/private/pip_repo_name/pip_repo_name_tests.bzl new file mode 100644 index 0000000000..574d558277 --- /dev/null +++ b/tests/private/pip_repo_name/pip_repo_name_tests.bzl @@ -0,0 +1,52 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:pip_repo_name.bzl", "pip_repo_name") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_simple(env): + got = pip_repo_name("prefix", "foo-1.2.3-py3-none-any.whl", "deadbeef") + env.expect.that_str(got).equals("prefix_foo_py3_none_any_deadbeef") + +_tests.append(_test_simple) + +def _test_sdist(env): + got = pip_repo_name("prefix", "foo-1.2.3.tar.gz", "deadbeef000deadbeef") + env.expect.that_str(got).equals("prefix_foo_sdist_deadbeef") + +_tests.append(_test_sdist) + +def _test_platform_whl(env): + got = pip_repo_name( + "prefix", + "foo-1.2.3-cp39.cp310-abi3-manylinux1_x86_64.manylinux_2_17_x86_64.whl", + "deadbeef000deadbeef", + ) + + # We only need the first segment of each + env.expect.that_str(got).equals("prefix_foo_cp39_abi3_manylinux_2_5_x86_64_deadbeef") + +_tests.append(_test_platform_whl) + +def pip_repo_name_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 870cada6cefefa0289cf5416a3efc5982ddaf1d2 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:04:38 +0900 Subject: [PATCH 07/18] feat(internal): add a function to generate config settings for all whls This also changes `is_python_config_setting` so that it can accept extra `flag_values` so that we can reuse it for the cases where we want something that is more specific than `is_python_config_setting` with a `python_version` alone. --- .../api/python/config_settings/index.md | 67 ++- python/config_settings/BUILD.bazel | 67 +++ python/private/BUILD.bazel | 8 + python/private/config_settings.bzl | 8 +- python/private/pip_config_settings.bzl | 354 ++++++++++++ python/private/pip_flags.bzl | 69 +++ tests/private/pip_config_settings/BUILD.bazel | 5 + .../pip_config_settings_tests.bzl | 544 ++++++++++++++++++ tests/support/BUILD.bazel | 16 + 9 files changed, 1133 insertions(+), 5 deletions(-) create mode 100644 python/private/pip_config_settings.bzl create mode 100644 python/private/pip_flags.bzl create mode 100644 tests/private/pip_config_settings/BUILD.bazel create mode 100644 tests/private/pip_config_settings/pip_config_settings_tests.bzl diff --git a/docs/sphinx/api/python/config_settings/index.md b/docs/sphinx/api/python/config_settings/index.md index 82a5b2a520..a667eb417f 100644 --- a/docs/sphinx/api/python/config_settings/index.md +++ b/docs/sphinx/api/python/config_settings/index.md @@ -3,11 +3,18 @@ # //python/config_settings +:::{bzl:flag} python_version +Determines the default hermetic Python toolchain version. This can be set to +one of the values that `rules_python` maintains. +::: + :::{bzl:flag} precompile Determines if Python source files should be compiled at build time. -NOTE: The flag value is overridden by the target level `precompile` attribute, +```{note} +The flag value is overridden by the target level `precompile` attribute, except for the case of `force_enabled` and `forced_disabled`. +``` Values: @@ -31,8 +38,10 @@ Values: Determines, when a source file is compiled, if the source file is kept in the resulting output or not. -NOTE: This flag is overridden by the target level `precompile_source_retention` +```{note} +This flag is overridden by the target level `precompile_source_retention` attribute. +``` Values: @@ -60,9 +69,61 @@ Values: :::{bzl:flag} pyc_collection Determine if `py_binary` collects transitive pyc files. -NOTE: This flag is overridden by the target level `pyc_collection` attribute. +```{note} +This flag is overridden by the target level `pyc_collection` attribute. +``` Values: * `include_pyc`: Include `PyInfo.transitive_pyc_files` as part of the binary. * `disabled`: Don't include `PyInfo.transitive_pyc_files` as part of the binary. ::: + +:::{bzl:flag} py_linux_libc +Set what libc is used for the target platform. This will affect which whl binaries will be pulled and what toolchain will be auto-detected. Currently `rules_python` only supplies toolchains compatible with `glibc`. + +Values: +* `glibc`: Use `glibc`, default. +* `muslc`: Use `muslc`. +::: + +:::{bzl:flag} pip_whl +Set what distributions are used in the `pip` integration. + +Values: +* `auto`: Prefer `whl` distributions if they are compatible with a target + platform, but fallback to `sdist`. This is the default. +* `only`: Only use `whl` distributions and error out if it is not available. +* `no`: Only use `sdist` distributions. The wheels will be built non-hermetically in the `whl_library` repository rule. +::: + +:::{bzl:flag} pip_whl_osx_arch +Set what wheel types we should prefer when building on the OSX platform. + +Values: +* `arch`: Prefer architecture specific wheels. +* `universal`: Prefer universal wheels that usually are bigger and contain binaries for both, Intel and ARM architectures in the same wheel. +::: + +:::{bzl:flag} pip_whl_glibc_version +Set the minimum `glibc` version that the `py_binary` using `whl` distributions from a PyPI index should support. + +Values: +* `""`: Select the lowest available version of each wheel giving you the maximum compatibility. This is the default. +* `X.Y`: The string representation of a `glibc` version. The allowed values depend on the `requirements.txt` lock file contents. +::: + +:::{bzl:flag} pip_whl_muslc_version +Set the minimum `muslc` version that the `py_binary` using `whl` distributions from a PyPI index should support. + +Values: +* `""`: Select the lowest available version of each wheel giving you the maximum compatibility. This is the default. +* `X.Y`: The string representation of a `muslc` version. The allowed values depend on the `requirements.txt` lock file contents. +::: + +:::{bzl:flag} pip_whl_osx_version +Set the minimum `osx` version that the `py_binary` using `whl` distributions from a PyPI index should support. + +Values: +* `""`: Select the lowest available version of each wheel giving you the maximum compatibility. This is the default. +* `X.Y`: The string representation of the MacOS version. The allowed values depend on the `requirements.txt` lock file contents. +::: diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index a0e59f70c0..c5fdd645b6 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -6,6 +6,13 @@ load( "PrecompileSourceRetentionFlag", "PycCollectionFlag", ) +load( + "//python/private:pip_flags.bzl", + "INTERNAL_FLAGS", + "UniversalWhlFlag", + "UseWhlFlag", + "WhlLibcFlag", +) load(":config_settings.bzl", "construct_config_settings") filegroup( @@ -52,3 +59,63 @@ string_flag( # NOTE: Only public because its an implicit dependency visibility = ["//visibility:public"], ) + +# This is used for pip and hermetic toolchain resolution. +string_flag( + name = "py_linux_libc", + build_setting_default = WhlLibcFlag.GLIBC, + values = sorted(WhlLibcFlag.__members__.values()), + # NOTE: Only public because it is used in pip hub and toolchain repos. + visibility = ["//visibility:public"], +) + +# pip.parse related flags + +string_flag( + name = "pip_whl", + build_setting_default = UseWhlFlag.AUTO, + values = sorted(UseWhlFlag.__members__.values()), + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +string_flag( + name = "pip_whl_osx_arch", + build_setting_default = UniversalWhlFlag.ARCH, + values = sorted(UniversalWhlFlag.__members__.values()), + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +string_flag( + name = "pip_whl_glibc_version", + build_setting_default = "", + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +string_flag( + name = "pip_whl_muslc_version", + build_setting_default = "", + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +string_flag( + name = "pip_whl_osx_version", + build_setting_default = "", + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +# private pip whl related flags. Their values cannot be changed and they +# are an implementation detail of how `pip_config_settings` work. +[ + string_flag( + name = "_internal_pip_" + flag, + build_setting_default = "", + values = [""], + visibility = ["//visibility:public"], + ) + for flag in INTERNAL_FLAGS +] diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 3d4ba4715d..24b4840266 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -145,6 +145,14 @@ bzl_library( srcs = ["parse_whl_name.bzl"], ) +bzl_library( + name = "pip_flags_bzl", + srcs = ["pip_flags.bzl"], + deps = [ + ":enum_bzl", + ], +) + bzl_library( name = "pip_repo_name_bzl", srcs = ["pip_repo_name.bzl"], diff --git a/python/private/config_settings.bzl b/python/private/config_settings.bzl index 75f88de4ac..92b96b3264 100644 --- a/python/private/config_settings.bzl +++ b/python/private/config_settings.bzl @@ -103,12 +103,16 @@ def is_python_config_setting(name, *, python_version, reuse_conditions = None, * fail("The 'python_version' must be known to 'rules_python', choose from the values: {}".format(VERSION_FLAG_VALUES.keys())) python_versions = VERSION_FLAG_VALUES[python_version] + extra_flag_values = kwargs.pop("flag_values", {}) + if _PYTHON_VERSION_FLAG in extra_flag_values: + fail("Cannot set '{}' in the flag values".format(_PYTHON_VERSION_FLAG)) + if len(python_versions) == 1: native.config_setting( name = name, flag_values = { _PYTHON_VERSION_FLAG: python_version, - }, + } | extra_flag_values, **kwargs ) return @@ -138,7 +142,7 @@ def is_python_config_setting(name, *, python_version, reuse_conditions = None, * for name_, flag_values_ in create_config_settings.items(): native.config_setting( name = name_, - flag_values = flag_values_, + flag_values = flag_values_ | extra_flag_values, **kwargs ) diff --git a/python/private/pip_config_settings.bzl b/python/private/pip_config_settings.bzl new file mode 100644 index 0000000000..211faabc6a --- /dev/null +++ b/python/private/pip_config_settings.bzl @@ -0,0 +1,354 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module is used to construct the config settings for selecting which distribution is used in the pip hub repository. + +Bazel's selects work by selecting the most-specialized configuration setting +that matches the target platform. We can leverage this fact to ensure that the +most specialized wheels are used by default with the users being able to +configure string_flag values to select the less specialized ones. + +The list of specialization of the dists goes like follows: +* sdist +* py*-none-any.whl +* py*-abi3-any.whl +* py*-cpxy-any.whl +* cp*-none-any.whl +* cp*-abi3-any.whl +* cp*-cpxy-plat.whl +* py*-none-plat.whl +* py*-abi3-plat.whl +* py*-cpxy-plat.whl +* cp*-none-plat.whl +* cp*-abi3-plat.whl +* cp*-cpxy-plat.whl + +Note, that here the specialization of musl vs manylinux wheels is the same in +order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa. +""" + +load(":config_settings.bzl", "is_python_config_setting") +load( + ":pip_flags.bzl", + "INTERNAL_FLAGS", + "UniversalWhlFlag", + "UseWhlFlag", + "WhlLibcFlag", +) + +FLAGS = struct( + **{ + f: str(Label("//python/config_settings:" + f)) + for f in [ + "python_version", + "pip_whl", + "pip_whl_glibc_version", + "pip_whl_muslc_version", + "pip_whl_osx_arch", + "pip_whl_osx_version", + "py_linux_libc", + ] + } +) + +# Here we create extra string flags that are just to work with the select +# selecting the most specialized match. We don't allow the user to change +# them. +_flags = struct( + **{ + f: str(Label("//python/config_settings:_internal_pip_" + f)) + for f in INTERNAL_FLAGS + } +) + +def pip_config_settings( + *, + python_versions = [], + glibc_versions = [], + muslc_versions = [], + osx_versions = [], + target_platforms = [], + name = None, + visibility = None): + """Generate all of the pip config settings. + + Args: + name (str): Currently unused. + python_versions (list[str]): The list of python versions to configure + config settings for. + glibc_versions (list[str]): The list of glibc version of the wheels to + configure config settings for. + muslc_versions (list[str]): The list of musl version of the wheels to + configure config settings for. + osx_versions (list[str]): The list of OSX OS versions to configure + config settings for. + target_platforms (list[str]): The list of "{os}_{cpu}" for deriving + constraint values for each condition. + visibility (list[str], optional): The visibility to be passed to the + exposed labels. All other labels will be private. + """ + + glibc_versions = [""] + glibc_versions + muslc_versions = [""] + muslc_versions + osx_versions = [""] + osx_versions + target_platforms = [("", "")] + [ + t.split("_", 1) + for t in target_platforms + ] + + for version in python_versions: + is_python = "is_python_{}".format(version) + native.alias( + name = is_python, + actual = Label("//python/config_settings:" + is_python), + visibility = visibility, + ) + + for os, cpu in target_platforms: + constraint_values = [] + suffix = "" + if os: + constraint_values.append("@platforms//os:" + os) + suffix += "_" + os + if cpu: + constraint_values.append("@platforms//cpu:" + cpu) + suffix += "_" + cpu + + _sdist_config_setting( + name = "sdist" + suffix, + constraint_values = constraint_values, + visibility = visibility, + ) + for python_version in python_versions: + _sdist_config_setting( + name = "cp{}_sdist{}".format(python_version, suffix), + python_version = python_version, + constraint_values = constraint_values, + visibility = visibility, + ) + + for python_version in [""] + python_versions: + _whl_config_settings( + suffix = suffix, + plat_flag_values = _plat_flag_values( + os = os, + cpu = cpu, + osx_versions = osx_versions, + glibc_versions = glibc_versions, + muslc_versions = muslc_versions, + ), + constraint_values = constraint_values, + python_version = python_version, + visibility = visibility, + ) + +def _whl_config_settings(*, suffix, plat_flag_values, **kwargs): + # With the following three we cover different per-version wheels + python_version = kwargs.get("python_version") + py = "cp{}_py".format(python_version) if python_version else "py" + pycp = "cp{}_cp3x".format(python_version) if python_version else "cp3x" + + flag_values = {} + + for n, f in { + "{}_none_any{}".format(py, suffix): None, + "{}3_none_any{}".format(py, suffix): _flags.whl_py3, + "{}3_abi3_any{}".format(py, suffix): _flags.whl_py3_abi3, + "{}_none_any{}".format(pycp, suffix): _flags.whl_pycp3x, + "{}_abi3_any{}".format(pycp, suffix): _flags.whl_pycp3x_abi3, + "{}_cp_any{}".format(pycp, suffix): _flags.whl_pycp3x_abicp, + }.items(): + if f and f in flag_values: + fail("BUG") + elif f: + flag_values[f] = "" + + _whl_config_setting( + name = n, + flag_values = flag_values, + **kwargs + ) + + generic_flag_values = flag_values + + for (suffix, flag_values) in plat_flag_values: + flag_values = flag_values | generic_flag_values + + for n, f in { + "{}_none_{}".format(py, suffix): _flags.whl_plat, + "{}3_none_{}".format(py, suffix): _flags.whl_plat_py3, + "{}3_abi3_{}".format(py, suffix): _flags.whl_plat_py3_abi3, + "{}_none_{}".format(pycp, suffix): _flags.whl_plat_pycp3x, + "{}_abi3_{}".format(pycp, suffix): _flags.whl_plat_pycp3x_abi3, + "{}_cp_{}".format(pycp, suffix): _flags.whl_plat_pycp3x_abicp, + }.items(): + if f and f in flag_values: + fail("BUG") + elif f: + flag_values[f] = "" + + _whl_config_setting( + name = n, + flag_values = flag_values, + **kwargs + ) + +def _to_version_string(version, sep = "."): + if not version: + return "" + + return "{}{}{}".format(version[0], sep, version[1]) + +def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): + ret = [] + if os == "": + return [] + elif os == "windows": + ret.append(("{}_{}".format(os, cpu), {})) + elif os == "osx": + for cpu_, arch in { + cpu: UniversalWhlFlag.ARCH, + cpu + "_universal2": UniversalWhlFlag.UNIVERSAL, + }.items(): + for osx_version in osx_versions: + flags = { + FLAGS.pip_whl_osx_version: _to_version_string(osx_version), + } + if arch == UniversalWhlFlag.ARCH: + flags[FLAGS.pip_whl_osx_arch] = arch + + if not osx_version: + suffix = "{}_{}".format(os, cpu_) + else: + suffix = "{}_{}_{}".format(os, _to_version_string(osx_version, "_"), cpu_) + + ret.append((suffix, flags)) + + elif os == "linux": + for os_prefix, linux_libc in { + os: WhlLibcFlag.GLIBC, + "many" + os: WhlLibcFlag.GLIBC, + "musl" + os: WhlLibcFlag.MUSL, + }.items(): + if linux_libc == WhlLibcFlag.GLIBC: + libc_versions = glibc_versions + libc_flag = FLAGS.pip_whl_glibc_version + elif linux_libc == WhlLibcFlag.MUSL: + libc_versions = muslc_versions + libc_flag = FLAGS.pip_whl_muslc_version + else: + fail("Unsupported libc type: {}".format(linux_libc)) + + for libc_version in libc_versions: + if libc_version and os_prefix == os: + continue + elif libc_version: + suffix = "{}_{}_{}".format(os_prefix, _to_version_string(libc_version, "_"), cpu) + else: + suffix = "{}_{}".format(os_prefix, cpu) + + ret.append(( + suffix, + { + FLAGS.py_linux_libc: linux_libc, + libc_flag: _to_version_string(libc_version), + }, + )) + else: + fail("Unsupported os: {}".format(os)) + + return ret + +def _whl_config_setting(*, name, flag_values, visibility, **kwargs): + _config_setting_or( + name = name, + flag_values = flag_values | { + FLAGS.pip_whl: UseWhlFlag.ONLY, + }, + default = flag_values | { + _flags.whl_py2_py3: "", + FLAGS.pip_whl: UseWhlFlag.AUTO, + }, + visibility = visibility, + **kwargs + ) + +def _sdist_config_setting(*, name, visibility, **kwargs): + _config_setting_or( + name = name, + flag_values = {FLAGS.pip_whl: UseWhlFlag.NO}, + default = {FLAGS.pip_whl: UseWhlFlag.AUTO}, + visibility = visibility, + **kwargs + ) + +def _config_setting_or(*, name, flag_values, default, visibility, **kwargs): + name = "is_" + name + match_name = "_{}".format(name) + default_name = "_{}_default".format(name) + + native.alias( + name = name, + actual = select({ + "//conditions:default": default_name, + match_name: match_name, + }), + visibility = visibility, + ) + + _config_setting( + name = match_name, + flag_values = flag_values, + visibility = visibility, + **kwargs + ) + _config_setting( + name = default_name, + flag_values = default, + visibility = visibility, + **kwargs + ) + +def _config_setting(python_version = "", **kwargs): + if python_version: + # NOTE @aignas 2024-05-26: with this we are getting about 24k internal + # config_setting targets in our unit tests. Whilst the number of the + # external dependencies does not dictate this number, it does mean that + # bazel will take longer to parse stuff. This would be especially + # noticeable in repos, which use multiple hub repos within a single + # workspace. + # + # A way to reduce the number of targets would be: + # * put them to a central location and teach this code to just alias them, + # maybe we should create a pip_config_settings repo within the pip + # extension, which would collect config settings for all hub_repos. + # * put them in rules_python - this has the drawback of exposing things like + # is_cp3.10_linux and users may start depending upon the naming + # convention and this API is very unstable. + is_python_config_setting( + python_version = python_version, + **kwargs + ) + else: + # We need this to ensure that there are no ambiguous matches when python_version + # is unset, which usually happens when we are not using the python version aware + # rules. + flag_values = kwargs.pop("flag_values", {}) | { + FLAGS.python_version: "", + } + native.config_setting( + flag_values = flag_values, + **kwargs + ) diff --git a/python/private/pip_flags.bzl b/python/private/pip_flags.bzl new file mode 100644 index 0000000000..c8154ff383 --- /dev/null +++ b/python/private/pip_flags.bzl @@ -0,0 +1,69 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Values and helpers for pip_repository related flags. + +NOTE: The transitive loads of this should be kept minimal. This avoids loading +unnecessary files when all that are needed are flag definitions. +""" + +load(":enum.bzl", "enum") + +# Determines if we should use whls for third party +# +# buildifier: disable=name-conventions +UseWhlFlag = enum( + # Automatically decide the effective value based on environment, target + # platform and the presence of distributions for a particular package. + AUTO = "auto", + # Do not use `sdist` and fail if there are no available whls suitable for the target platform. + ONLY = "only", + # Do not use whl distributions and instead build the whls from `sdist`. + NO = "no", +) + +# Determines whether universal wheels should be preferred over arch platform specific ones. +# +# buildifier: disable=name-conventions +UniversalWhlFlag = enum( + # Prefer platform-specific wheels over universal wheels. + ARCH = "arch", + # Prefer universal wheels over platform-specific wheels. + UNIVERSAL = "universal", +) + +# Determines which libc flavor is preferred when selecting the linux whl distributions. +# +# buildifier: disable=name-conventions +WhlLibcFlag = enum( + # Prefer glibc wheels (e.g. manylinux_2_17_x86_64 or linux_x86_64) + GLIBC = "glibc", + # Prefer musl wheels (e.g. musllinux_2_17_x86_64) + MUSL = "musl", +) + +INTERNAL_FLAGS = [ + "whl_plat", + "whl_plat_py3", + "whl_plat_py3_abi3", + "whl_plat_pycp3x", + "whl_plat_pycp3x_abi3", + "whl_plat_pycp3x_abicp", + "whl_py2_py3", + "whl_py3", + "whl_py3_abi3", + "whl_pycp3x", + "whl_pycp3x_abi3", + "whl_pycp3x_abicp", +] diff --git a/tests/private/pip_config_settings/BUILD.bazel b/tests/private/pip_config_settings/BUILD.bazel new file mode 100644 index 0000000000..c3752e0ac3 --- /dev/null +++ b/tests/private/pip_config_settings/BUILD.bazel @@ -0,0 +1,5 @@ +load(":pip_config_settings_tests.bzl", "pip_config_settings_test_suite") + +pip_config_settings_test_suite( + name = "pip_config_settings", +) diff --git a/tests/private/pip_config_settings/pip_config_settings_tests.bzl b/tests/private/pip_config_settings/pip_config_settings_tests.bzl new file mode 100644 index 0000000000..a66e7f47d5 --- /dev/null +++ b/tests/private/pip_config_settings/pip_config_settings_tests.bzl @@ -0,0 +1,544 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for construction of Python version matching config settings.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:truth.bzl", "subjects") +load("@rules_testing//lib:util.bzl", test_util = "util") +load("//python/private:pip_config_settings.bzl", "pip_config_settings") # buildifier: disable=bzl-visibility + +def _subject_impl(ctx): + _ = ctx # @unused + return [DefaultInfo()] + +_subject = rule( + implementation = _subject_impl, + attrs = { + "dist": attr.string(), + }, +) + +_flag = struct( + platform = lambda x: ("//command_line_option:platforms", str(Label("//tests/support:" + x))), + pip_whl = lambda x: (str(Label("//python/config_settings:pip_whl")), str(x)), + pip_whl_glibc_version = lambda x: (str(Label("//python/config_settings:pip_whl_glibc_version")), str(x)), + pip_whl_muslc_version = lambda x: (str(Label("//python/config_settings:pip_whl_muslc_version")), str(x)), + pip_whl_osx_version = lambda x: (str(Label("//python/config_settings:pip_whl_osx_version")), str(x)), + pip_whl_osx_arch = lambda x: (str(Label("//python/config_settings:pip_whl_osx_arch")), str(x)), + py_linux_libc = lambda x: (str(Label("//python/config_settings:py_linux_libc")), str(x)), + python_version = lambda x: (str(Label("//python/config_settings:python_version")), str(x)), +) + +def _analysis_test(*, name, dist, want, config_settings = [_flag.platform("linux_aarch64")]): + subject_name = name + "_subject" + test_util.helper_target( + _subject, + name = subject_name, + dist = select( + dist | { + "//conditions:default": "no_match", + }, + ), + ) + config_settings = dict(config_settings) + if not config_settings: + fail("For reproducibility on different platforms, the config setting must be specified") + + analysis_test( + name = name, + target = subject_name, + impl = lambda env, target: _match(env, target, want), + config_settings = config_settings, + ) + +def _match(env, target, want): + target = env.expect.that_target(target) + target.attr("dist", factory = subjects.str).equals(want) + +_tests = [] + +# Tests when we only have an `sdist` present. + +def _test_sdist_default(name): + _analysis_test( + name = name, + dist = { + "is_sdist": "sdist", + }, + want = "sdist", + ) + +_tests.append(_test_sdist_default) + +def _test_sdist_no_whl(name): + _analysis_test( + name = name, + dist = { + "is_sdist": "sdist", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("no"), + ], + want = "sdist", + ) + +_tests.append(_test_sdist_no_whl) + +def _test_sdist_no_sdist(name): + _analysis_test( + name = name, + dist = { + "is_sdist": "sdist", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("only"), + ], + # We will use `no_match_error` in the real case to indicate that `sdist` is not + # allowed to be used. + want = "no_match", + ) + +_tests.append(_test_sdist_no_sdist) + +def _test_basic_whl_default(name): + _analysis_test( + name = name, + dist = { + "is_py_none_any": "whl", + "is_sdist": "sdist", + }, + want = "whl", + ) + +_tests.append(_test_basic_whl_default) + +def _test_basic_whl_nowhl(name): + _analysis_test( + name = name, + dist = { + "is_py_none_any": "whl", + "is_sdist": "sdist", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("no"), + ], + want = "sdist", + ) + +_tests.append(_test_basic_whl_nowhl) + +def _test_basic_whl_nosdist(name): + _analysis_test( + name = name, + dist = { + "is_py_none_any": "whl", + "is_sdist": "sdist", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("only"), + ], + want = "whl", + ) + +_tests.append(_test_basic_whl_nosdist) + +def _test_whl_default(name): + _analysis_test( + name = name, + dist = { + "is_py3_none_any": "whl", + "is_py_none_any": "basic_whl", + }, + want = "whl", + ) + +_tests.append(_test_whl_default) + +def _test_whl_nowhl(name): + _analysis_test( + name = name, + dist = { + "is_py3_none_any": "whl", + "is_py_none_any": "basic_whl", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("no"), + ], + want = "no_match", + ) + +_tests.append(_test_whl_nowhl) + +def _test_whl_nosdist(name): + _analysis_test( + name = name, + dist = { + "is_py3_none_any": "whl", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("only"), + ], + want = "whl", + ) + +_tests.append(_test_whl_nosdist) + +def _test_abi_whl_is_prefered(name): + _analysis_test( + name = name, + dist = { + "is_py3_abi3_any": "abi_whl", + "is_py3_none_any": "whl", + }, + want = "abi_whl", + ) + +_tests.append(_test_abi_whl_is_prefered) + +def _test_whl_with_constraints_is_prefered(name): + _analysis_test( + name = name, + dist = { + "is_py3_none_any": "default_whl", + "is_py3_none_any_linux_aarch64": "whl", + "is_py3_none_any_linux_x86_64": "amd64_whl", + }, + want = "whl", + ) + +_tests.append(_test_whl_with_constraints_is_prefered) + +def _test_cp_whl_is_prefered_over_py3(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_none_any": "cp", + "is_py3_abi3_any": "py3_abi3", + "is_py3_none_any": "py3", + }, + want = "cp", + ) + +_tests.append(_test_cp_whl_is_prefered_over_py3) + +def _test_cp_abi_whl_is_prefered_over_py3(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_abi3_any": "cp", + "is_py3_abi3_any": "py3", + }, + want = "cp", + ) + +_tests.append(_test_cp_abi_whl_is_prefered_over_py3) + +def _test_cp_version_is_selected_when_python_version_is_specified(name): + _analysis_test( + name = name, + dist = { + "is_cp3.10_cp3x_none_any": "cp310", + "is_cp3.8_cp3x_none_any": "cp38", + "is_cp3.9_cp3x_none_any": "cp39", + "is_cp3x_none_any": "cp_default", + }, + want = "cp310", + config_settings = [ + _flag.python_version("3.10.9"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_cp_version_is_selected_when_python_version_is_specified) + +def _test_py_none_any_versioned(name): + _analysis_test( + name = name, + dist = { + "is_cp3.10_py_none_any": "whl", + "is_cp3.9_py_none_any": "too-low", + }, + want = "whl", + config_settings = [ + _flag.python_version("3.10.9"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_py_none_any_versioned) + +def _test_cp_cp_whl(name): + _analysis_test( + name = name, + dist = { + "is_cp3.10_cp3x_cp_linux_aarch64": "whl", + }, + want = "whl", + config_settings = [ + _flag.python_version("3.10.9"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_cp_cp_whl) + +def _test_cp_version_sdist_is_selected(name): + _analysis_test( + name = name, + dist = { + "is_cp3.10_sdist": "sdist", + }, + want = "sdist", + config_settings = [ + _flag.python_version("3.10.9"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_cp_version_sdist_is_selected) + +def _test_platform_whl_is_prefered_over_any_whl_with_constraints(name): + _analysis_test( + name = name, + dist = { + "is_py3_abi3_any": "better_default_whl", + "is_py3_abi3_any_linux_aarch64": "better_default_any_whl", + "is_py3_none_any": "default_whl", + "is_py3_none_any_linux_aarch64": "whl", + "is_py3_none_linux_aarch64": "platform_whl", + }, + want = "platform_whl", + ) + +_tests.append(_test_platform_whl_is_prefered_over_any_whl_with_constraints) + +def _test_abi3_platform_whl_preference(name): + _analysis_test( + name = name, + dist = { + "is_py3_abi3_linux_aarch64": "abi3_platform", + "is_py3_none_linux_aarch64": "platform", + }, + want = "abi3_platform", + ) + +_tests.append(_test_abi3_platform_whl_preference) + +def _test_glibc(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_cp_manylinux_aarch64": "glibc", + "is_py3_abi3_linux_aarch64": "abi3_platform", + }, + want = "glibc", + ) + +_tests.append(_test_glibc) + +def _test_glibc_versioned(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_cp_manylinux_2_14_aarch64": "glibc", + "is_cp3x_cp_manylinux_2_17_aarch64": "glibc", + "is_py3_abi3_linux_aarch64": "abi3_platform", + }, + want = "glibc", + config_settings = [ + _flag.py_linux_libc("glibc"), + _flag.pip_whl_glibc_version("2.17"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_glibc_versioned) + +def _test_glibc_compatible_exists(name): + _analysis_test( + name = name, + dist = { + # Code using the conditions will need to construct selects, which + # do the version matching correctly. + "is_cp3x_cp_manylinux_2_14_aarch64": "2_14_whl_via_2_14_branch", + "is_cp3x_cp_manylinux_2_17_aarch64": "2_14_whl_via_2_17_branch", + }, + want = "2_14_whl_via_2_17_branch", + config_settings = [ + _flag.py_linux_libc("glibc"), + _flag.pip_whl_glibc_version("2.17"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_glibc_compatible_exists) + +def _test_musl(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_cp_musllinux_aarch64": "musl", + }, + want = "musl", + config_settings = [ + _flag.py_linux_libc("musl"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_musl) + +def _test_windows(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_cp_windows_x86_64": "whl", + }, + want = "whl", + config_settings = [ + _flag.platform("windows_x86_64"), + ], + ) + +_tests.append(_test_windows) + +def _test_osx(name): + _analysis_test( + name = name, + dist = { + # We prefer arch specific whls over universal + "is_cp3x_cp_osx_x86_64": "whl", + "is_cp3x_cp_osx_x86_64_universal2": "universal_whl", + }, + want = "whl", + config_settings = [ + _flag.platform("mac_x86_64"), + ], + ) + +_tests.append(_test_osx) + +def _test_osx_universal_default(name): + _analysis_test( + name = name, + dist = { + # We default to universal if only that exists + "is_cp3x_cp_osx_x86_64_universal2": "whl", + }, + want = "whl", + config_settings = [ + _flag.platform("mac_x86_64"), + ], + ) + +_tests.append(_test_osx_universal_default) + +def _test_osx_universal_only(name): + _analysis_test( + name = name, + dist = { + # If we prefer universal, then we use that + "is_cp3x_cp_osx_x86_64": "whl", + "is_cp3x_cp_osx_x86_64_universal2": "universal", + }, + want = "universal", + config_settings = [ + _flag.pip_whl_osx_arch("universal"), + _flag.platform("mac_x86_64"), + ], + ) + +_tests.append(_test_osx_universal_only) + +def _test_osx_os_version(name): + _analysis_test( + name = name, + dist = { + # Similarly to the libc version, the user of the config settings will have to + # construct the select so that the version selection is correct. + "is_cp3x_cp_osx_10_9_x86_64": "whl", + }, + want = "whl", + config_settings = [ + _flag.pip_whl_osx_version("10.9"), + _flag.platform("mac_x86_64"), + ], + ) + +_tests.append(_test_osx_os_version) + +def _test_all(name): + _analysis_test( + name = name, + dist = { + "is_" + f: f + for f in [ + "{py}_{abi}_{plat}".format(py = valid_py, abi = valid_abi, plat = valid_plat) + # we have py2.py3, py3, cp3x + for valid_py in ["py", "py3", "cp3x"] + # cp abi usually comes with a version and we only need one + # config setting variant for all of them because the python + # version will discriminate between different versions. + for valid_abi in ["none", "abi3", "cp"] + for valid_plat in [ + "any", + "manylinux_2_17_x86_64", + "manylinux_2_17_aarch64", + "osx_x86_64", + "windows_x86_64", + ] + if not ( + valid_abi == "abi3" and valid_py == "py" or + valid_abi == "cp" and valid_py != "cp3x" + ) + ] + }, + want = "cp3x_cp_manylinux_2_17_x86_64", + config_settings = [ + _flag.pip_whl_glibc_version("2.17"), + _flag.platform("linux_x86_64"), + ], + ) + +_tests.append(_test_all) + +def pip_config_settings_test_suite(name): # buildifier: disable=function-docstring + test_suite( + name = name, + tests = _tests, + ) + + pip_config_settings( + name = "dummy", + python_versions = ["3.8", "3.9", "3.10"], + glibc_versions = [(2, 14), (2, 17)], + muslc_versions = [(1, 1)], + osx_versions = [(10, 9), (11, 0)], + target_platforms = [ + "windows_x86_64", + "windows_aarch64", + "linux_x86_64", + "linux_ppc", + "linux_aarch64", + "osx_x86_64", + "osx_aarch64", + ], + ) diff --git a/tests/support/BUILD.bazel b/tests/support/BUILD.bazel index 3b77cde0c5..e5d5189a3b 100644 --- a/tests/support/BUILD.bazel +++ b/tests/support/BUILD.bazel @@ -52,6 +52,14 @@ platform( ], ) +platform( + name = "linux_aarch64", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], +) + platform( name = "mac_x86_64", constraint_values = [ @@ -68,6 +76,14 @@ platform( ], ) +platform( + name = "win_aarch64", + constraint_values = [ + "@platforms//os:windows", + "@platforms//cpu:aarch64", + ], +) + py_runtime( name = "platform_runtime", implementation_name = "fakepy", From 2d206708200838b287b47ba44802d3cb791cd2e5 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:19:54 +0900 Subject: [PATCH 08/18] feat(internal): support inferring the config settings from dist filenames This code will use the config settings generated by `pip_config_settings` in all cases and when the `whl_alias` instances are generated by passing `filename` values, then we will also use the platform-specific config settings from the `//_config` package in the hub repo. --- python/private/bzlmod/pip_repository.bzl | 9 +- python/private/render_pkg_aliases.bzl | 412 ++++++++++++++- python/private/whl_target_platforms.bzl | 11 + .../render_pkg_aliases_test.bzl | 495 ++++++++++++++++-- .../whl_target_platforms_tests.bzl | 28 +- 5 files changed, 890 insertions(+), 65 deletions(-) diff --git a/python/private/bzlmod/pip_repository.bzl b/python/private/bzlmod/pip_repository.bzl index 3a09766f65..0f962031d6 100644 --- a/python/private/bzlmod/pip_repository.bzl +++ b/python/private/bzlmod/pip_repository.bzl @@ -14,7 +14,11 @@ "" -load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") +load( + "//python/private:render_pkg_aliases.bzl", + "render_multiplatform_pkg_aliases", + "whl_alias", +) load("//python/private:text_util.bzl", "render") _BUILD_FILE_CONTENTS = """\ @@ -26,12 +30,13 @@ exports_files(["requirements.bzl"]) def _pip_repository_impl(rctx): bzl_packages = rctx.attr.whl_map.keys() - aliases = render_pkg_aliases( + aliases = render_multiplatform_pkg_aliases( aliases = { key: [whl_alias(**v) for v in json.decode(values)] for key, values in rctx.attr.whl_map.items() }, default_version = rctx.attr.default_version, + default_config_setting = "//_config:is_python_" + rctx.attr.default_version, requirement_cycles = rctx.attr.groups, ) for path, contents in aliases.items(): diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl index bc1bab2049..2ddd65d1fc 100644 --- a/python/private/render_pkg_aliases.bzl +++ b/python/private/render_pkg_aliases.bzl @@ -30,7 +30,9 @@ load( "WHEEL_FILE_PUBLIC_LABEL", ) load(":normalize_name.bzl", "normalize_name") +load(":parse_whl_name.bzl", "parse_whl_name") load(":text_util.bzl", "render") +load(":whl_target_platforms.bzl", "whl_target_platforms") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -51,10 +53,27 @@ If the value is missing, then the "default" Python version is being used, which has a "null" version value and will not match version constraints. """ +NO_MATCH_ERROR_MESSAGE_TEMPLATE_V2 = """\ +No matching wheel for current configuration's Python version. + +The current build configuration's Python version doesn't match any of the Python +wheels available for this wheel. This wheel supports the following Python +configuration settings: + {config_settings} + +To determine the current configuration's Python version, run: + `bazel config ` (shown further below) +and look for + {rules_python}//python/config_settings:python_version + +If the value is missing, then the "default" Python version is being used, +which has a "null" version value and will not match version constraints. +""" + def _render_whl_library_alias( *, name, - default_version, + default_config_setting, aliases, target_name, **kwargs): @@ -78,7 +97,7 @@ def _render_whl_library_alias( for alias in sorted(aliases, key = lambda x: x.version): actual = "@{repo}//:{name}".format(repo = alias.repo, name = target_name) selects.setdefault(actual, []).append(alias.config_setting) - if alias.version == default_version: + if alias.config_setting == default_config_setting: selects[actual].append("//conditions:default") no_match_error = None @@ -102,21 +121,23 @@ def _render_whl_library_alias( **kwargs ) -def _render_common_aliases(*, name, aliases, default_version = None, group_name = None): +def _render_common_aliases(*, name, aliases, default_config_setting = None, group_name = None): lines = [ """load("@bazel_skylib//lib:selects.bzl", "selects")""", """package(default_visibility = ["//visibility:public"])""", ] - versions = None + config_settings = None if aliases: - versions = sorted([v.version for v in aliases if v.version]) + config_settings = sorted([v.config_setting for v in aliases if v.config_setting]) - if not versions or default_version in versions: + if not config_settings or default_config_setting in config_settings: pass else: - error_msg = NO_MATCH_ERROR_MESSAGE_TEMPLATE.format( - supported_versions = ", ".join(versions), + error_msg = NO_MATCH_ERROR_MESSAGE_TEMPLATE_V2.format( + config_settings = render.indent( + "\n".join(config_settings), + ).lstrip(), rules_python = "rules_python", ) @@ -126,7 +147,7 @@ def _render_common_aliases(*, name, aliases, default_version = None, group_name # This is to simplify the code in _render_whl_library_alias and to ensure # that we don't pass a 'default_version' that is not in 'versions'. - default_version = None + default_config_setting = None lines.append( render.alias( @@ -138,7 +159,7 @@ def _render_common_aliases(*, name, aliases, default_version = None, group_name [ _render_whl_library_alias( name = name, - default_version = default_version, + default_config_setting = default_config_setting, aliases = aliases, target_name = target_name, visibility = ["//_groups:__subpackages__"] if name.startswith("_") else None, @@ -167,7 +188,7 @@ def _render_common_aliases(*, name, aliases, default_version = None, group_name return "\n\n".join(lines) -def render_pkg_aliases(*, aliases, default_version = None, requirement_cycles = None): +def render_pkg_aliases(*, aliases, default_config_setting = None, requirement_cycles = None): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -177,7 +198,7 @@ def render_pkg_aliases(*, aliases, default_version = None, requirement_cycles = Args: aliases: dict, the keys are normalized distribution names and values are the whl_alias instances. - default_version: the default version to be used for the aliases. + default_config_setting: the default to be used for the aliases. requirement_cycles: any package groups to also add. Returns: @@ -206,16 +227,17 @@ def render_pkg_aliases(*, aliases, default_version = None, requirement_cycles = "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( name = normalize_name(name), aliases = pkg_aliases, - default_version = default_version, + default_config_setting = default_config_setting, group_name = whl_group_mapping.get(normalize_name(name)), ).strip() for name, pkg_aliases in aliases.items() } + if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files -def whl_alias(*, repo, version = None, config_setting = None, extra_targets = None): +def whl_alias(*, repo, version = None, config_setting = None, filename = None, target_platforms = None): """The bzl_packages value used by by the render_pkg_aliases function. This contains the minimum amount of information required to generate correct @@ -228,9 +250,10 @@ def whl_alias(*, repo, version = None, config_setting = None, extra_targets = No constructed. This is mainly used for better error messages when there is no match found during a select. config_setting: optional(Label or str), the config setting that we should use. Defaults - to "@rules_python//python/config_settings:is_python_{version}". - extra_targets: optional(list[str]), the extra targets that we need to create - aliases for. + to "//_config:is_python_{version}". + filename: optional(str), the distribution filename to derive the config_setting. + target_platforms: optional(list[str]), the list of target_platforms for this + distribution. Returns: a struct with the validated and parsed values. @@ -239,12 +262,363 @@ def whl_alias(*, repo, version = None, config_setting = None, extra_targets = No fail("'repo' must be specified") if version: - config_setting = config_setting or Label("//python/config_settings:is_python_" + version) + config_setting = config_setting or ("//_config:is_python_" + version) config_setting = str(config_setting) return struct( repo = repo, version = version, config_setting = config_setting, - extra_targets = extra_targets or [], + filename = filename, + target_platforms = target_platforms, + ) + +def render_multiplatform_pkg_aliases(*, aliases, default_version = None, **kwargs): + """Render the multi-platform pkg aliases. + + Args: + aliases: dict[str, list(whl_alias)] A list of aliases that will be + transformed from ones having `filename` to ones having `config_setting`. + default_version: str, the default python version. Defaults to None. + **kwargs: extra arguments passed to render_pkg_aliases. + + Returns: + A dict of file paths and their contents. + """ + + flag_versions = get_whl_flag_versions( + aliases = [ + a + for bunch in aliases.values() + for a in bunch + ], ) + + config_setting_aliases = { + pkg: multiplatform_whl_aliases( + aliases = pkg_aliases, + default_version = default_version, + glibc_versions = flag_versions.get("glibc_versions", []), + muslc_versions = flag_versions.get("muslc_versions", []), + osx_versions = flag_versions.get("osx_versions", []), + ) + for pkg, pkg_aliases in aliases.items() + } + + contents = render_pkg_aliases( + aliases = config_setting_aliases, + **kwargs + ) + contents["_config/BUILD.bazel"] = _render_pip_config_settings(**flag_versions) + return contents + +def multiplatform_whl_aliases(*, aliases, default_version = None, **kwargs): + """convert a list of aliases from filename to config_setting ones. + + Args: + aliases: list(whl_alias): The aliases to process. Any aliases that have + the filename set will be converted to a list of aliases, each with + an appropriate config_setting value. + default_version: string | None, the default python version to use. + **kwargs: Extra parameters passed to get_filename_config_settings. + + Returns: + A dict with aliases to be used in the hub repo. + """ + + ret = [] + versioned_additions = {} + for alias in aliases: + if not alias.filename: + ret.append(alias) + continue + + config_settings, all_versioned_settings = get_filename_config_settings( + # TODO @aignas 2024-05-27: pass the parsed whl to reduce the + # number of duplicate operations. + filename = alias.filename, + target_platforms = alias.target_platforms, + python_version = alias.version, + python_default = default_version == alias.version, + **kwargs + ) + + for setting in config_settings: + ret.append(whl_alias( + repo = alias.repo, + version = alias.version, + config_setting = "//_config" + setting, + )) + + # Now for the versioned platform config settings, we need to select one + # that best fits the bill and if there are multiple wheels, e.g. + # manylinux_2_17_x86_64 and manylinux_2_28_x86_64, then we need to select + # the former when the glibc is in the range of [2.17, 2.28) and then chose + # the later if it is [2.28, ...). If the 2.28 wheel was not present in + # the hub, then we would need to use 2.17 for all the glibc version + # configurations. + # + # Here we add the version settings to a dict where we key the range of + # versions that the whl spans. If the wheel supports musl and glibc at + # the same time, we do this for each supported platform, hence the + # double dict. + for default_setting, versioned in all_versioned_settings.items(): + versions = sorted(versioned) + min_version = versions[0] + max_version = versions[-1] + + versioned_additions.setdefault(default_setting, {})[(min_version, max_version)] = struct( + repo = alias.repo, + python_version = alias.version, + settings = versioned, + ) + + versioned = {} + for default_setting, candidates in versioned_additions.items(): + # Sort the candidates by the range of versions the span, so that we + # start with the lowest version. + for _, candidate in sorted(candidates.items()): + # Set the default with the first candidate, which gives us the highest + # compatibility. If the users want to use a higher-version than the default + # they can configure the glibc_version flag. + versioned.setdefault(default_setting, whl_alias( + version = candidate.python_version, + config_setting = "//_config" + default_setting, + repo = candidate.repo, + )) + + # We will be overwriting previously added entries, but that is intended. + for _, setting in sorted(candidate.settings.items()): + versioned[setting] = whl_alias( + version = candidate.python_version, + config_setting = "//_config" + setting, + repo = candidate.repo, + ) + + ret.extend(versioned.values()) + return ret + +def _render_pip_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []): + return """\ +load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings") + +pip_config_settings( + name = "pip_config_settings", + glibc_versions = {glibc_versions}, + muslc_versions = {muslc_versions}, + osx_versions = {osx_versions}, + python_versions = {python_versions}, + target_platforms = {target_platforms}, + visibility = ["//:__subpackages__"], +) + +""".format( + glibc_versions = render.indent(render.list(glibc_versions)).lstrip(), + muslc_versions = render.indent(render.list(muslc_versions)).lstrip(), + osx_versions = render.indent(render.list(osx_versions)).lstrip(), + python_versions = render.indent(render.list(python_versions)).lstrip(), + target_platforms = render.indent(render.list(target_platforms)).lstrip(), + ) + +def get_whl_flag_versions(aliases): + """Return all of the flag versions that is used by the aliases + + Args: + aliases: list[whl_alias] + + Returns: + dict, which may have keys: + * python_versions + """ + python_versions = {} + glibc_versions = {} + target_platforms = {} + muslc_versions = {} + osx_versions = {} + + for a in aliases: + if not a.version and not a.filename: + continue + + if a.version: + python_versions[a.version] = None + + if not a.filename: + continue + + if a.filename.endswith(".whl") and not a.filename.endswith("-any.whl"): + parsed = parse_whl_name(a.filename) + else: + for plat in a.target_platforms or []: + target_platforms[plat] = None + continue + + for platform_tag in parsed.platform_tag.split("."): + parsed = whl_target_platforms(platform_tag) + + for p in parsed: + target_platforms[p.target_platform] = None + + if platform_tag.startswith("win") or platform_tag.startswith("linux"): + continue + + head, _, tail = platform_tag.partition("_") + major, _, tail = tail.partition("_") + minor, _, tail = tail.partition("_") + if tail: + version = (int(major), int(minor)) + if "many" in head: + glibc_versions[version] = None + elif "musl" in head: + muslc_versions[version] = None + elif "mac" in head: + osx_versions[version] = None + else: + fail(platform_tag) + + return { + k: sorted(v) + for k, v in { + "glibc_versions": glibc_versions, + "muslc_versions": muslc_versions, + "osx_versions": osx_versions, + "python_versions": python_versions, + "target_platforms": target_platforms, + }.items() + if v + } + +def get_filename_config_settings( + *, + filename, + target_platforms, + glibc_versions, + muslc_versions, + osx_versions, + python_version = "", + python_default = True): + """Get the filename config settings. + + Args: + filename: the distribution filename (can be a whl or an sdist). + target_platforms: list[str], target platforms in "{os}_{cpu}" format. + glibc_versions: list[tuple[int, int]], list of versions. + muslc_versions: list[tuple[int, int]], list of versions. + osx_versions: list[tuple[int, int]], list of versions. + python_version: the python version to generate the config_settings for. + python_default: if we should include the setting when python_version is not set. + + Returns: + A tuple: + * A list of config settings that are generated by ./pip_config_settings.bzl + * The list of default version settings. + """ + prefixes = [] + suffixes = [] + if (0, 0) in glibc_versions: + fail("Invalid version in 'glibc_versions': cannot specify (0, 0) as a value") + if (0, 0) in muslc_versions: + fail("Invalid version in 'muslc_versions': cannot specify (0, 0) as a value") + if (0, 0) in osx_versions: + fail("Invalid version in 'osx_versions': cannot specify (0, 0) as a value") + + glibc_versions = sorted(glibc_versions) + muslc_versions = sorted(muslc_versions) + osx_versions = sorted(osx_versions) + setting_supported_versions = {} + + if filename.endswith(".whl"): + parsed = parse_whl_name(filename) + if parsed.python_tag == "py2.py3": + py = "py" + elif parsed.python_tag.startswith("cp"): + py = "cp3x" + else: + py = parsed.python_tag + + if parsed.abi_tag.startswith("cp"): + abi = "cp" + else: + abi = parsed.abi_tag + + if parsed.platform_tag == "any": + prefixes = ["{}_{}_any".format(py, abi)] + suffixes = target_platforms + else: + prefixes = ["{}_{}".format(py, abi)] + suffixes = _whl_config_setting_sufixes( + platform_tag = parsed.platform_tag, + glibc_versions = glibc_versions, + muslc_versions = muslc_versions, + osx_versions = osx_versions, + setting_supported_versions = setting_supported_versions, + ) + else: + prefixes = ["sdist"] + suffixes = target_platforms + + if python_default and python_version: + prefixes += ["cp{}_{}".format(python_version, p) for p in prefixes] + elif python_version: + prefixes = ["cp{}_{}".format(python_version, p) for p in prefixes] + elif python_default: + pass + else: + fail("BUG: got no python_version and it is not default") + + versioned = { + ":is_{}_{}".format(p, suffix): { + version: ":is_{}_{}".format(p, setting) + for version, setting in versions.items() + } + for p in prefixes + for suffix, versions in setting_supported_versions.items() + } + + if suffixes or versioned: + return [":is_{}_{}".format(p, s) for p in prefixes for s in suffixes], versioned + else: + return [":is_{}".format(p) for p in prefixes], setting_supported_versions + +def _whl_config_setting_sufixes( + platform_tag, + glibc_versions, + muslc_versions, + osx_versions, + setting_supported_versions): + suffixes = [] + for platform_tag in platform_tag.split("."): + for p in whl_target_platforms(platform_tag): + prefix = p.os + suffix = p.cpu + if "manylinux" in platform_tag: + prefix = "manylinux" + versions = glibc_versions + elif "musllinux" in platform_tag: + prefix = "musllinux" + versions = muslc_versions + elif p.os in ["linux", "windows"]: + versions = [(0, 0)] + elif p.os == "osx": + versions = osx_versions + if "universal2" in platform_tag: + suffix += "_universal2" + else: + fail("Unsupported whl os: {}".format(p.os)) + + default_version_setting = "{}_{}".format(prefix, suffix) + supported_versions = {} + for v in versions: + if v == (0, 0): + suffixes.append(default_version_setting) + elif v >= p.version: + supported_versions[v] = "{}_{}_{}_{}".format( + prefix, + v[0], + v[1], + suffix, + ) + if supported_versions: + setting_supported_versions[default_version_setting] = supported_versions + + return suffixes diff --git a/python/private/whl_target_platforms.bzl b/python/private/whl_target_platforms.bzl index 678c841feb..08177ca1b3 100644 --- a/python/private/whl_target_platforms.bzl +++ b/python/private/whl_target_platforms.bzl @@ -265,6 +265,16 @@ def whl_target_platforms(platform_tag, abi_tag = ""): if abi_tag not in ["", "none", "abi3"]: abi = abi_tag + # TODO @aignas 2024-05-29: this code is present in many places, I think + _, _, tail = platform_tag.partition("_") + maybe_arch = tail + major, _, tail = tail.partition("_") + minor, _, tail = tail.partition("_") + if not tail or not major.isdigit() or not minor.isdigit(): + tail = maybe_arch + major = 0 + minor = 0 + for prefix, os in _OS_PREFIXES.items(): if platform_tag.startswith(prefix): return [ @@ -272,6 +282,7 @@ def whl_target_platforms(platform_tag, abi_tag = ""): os = os, cpu = cpu, abi = abi, + version = (int(major), int(minor)), target_platform = "_".join([abi, os, cpu] if abi else [os, cpu]), ) for cpu in cpus diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl index a38d657962..653cf466c7 100644 --- a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -16,7 +16,14 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility -load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") # buildifier: disable=bzl-visibility +load( + "//python/private:render_pkg_aliases.bzl", + "get_filename_config_settings", + "get_whl_flag_versions", + "multiplatform_whl_aliases", + "render_pkg_aliases", + "whl_alias", +) # buildifier: disable=bzl-visibility def _normalize_label_strings(want): """normalize expected strings. @@ -100,7 +107,7 @@ _tests.append(_test_legacy_aliases) def _test_bzlmod_aliases(env): actual = render_pkg_aliases( - default_version = "3.2", + default_config_setting = "//:my_config_setting", aliases = { "bar-baz": [ whl_alias(version = "3.2", repo = "pypi_32_bar_baz", config_setting = "//:my_config_setting"), @@ -174,7 +181,7 @@ _tests.append(_test_bzlmod_aliases) def _test_bzlmod_aliases_with_no_default_version(env): actual = render_pkg_aliases( - default_version = None, + default_config_setting = None, aliases = { "bar-baz": [ whl_alias( @@ -198,11 +205,10 @@ _NO_MATCH_ERROR = \"\"\"\\ No matching wheel for current configuration's Python version. The current build configuration's Python version doesn't match any of the Python -versions available for this wheel. This wheel supports the following Python versions: - 3.1, 3.2 - -As matched by the `@rules_python//python/config_settings:is_python_` -configuration settings. +wheels available for this wheel. This wheel supports the following Python +configuration settings: + //_config:is_python_3.1 + @@//python/config_settings:is_python_3.2 To determine the current configuration's Python version, run: `bazel config ` (shown further below) @@ -222,7 +228,7 @@ alias( name = "pkg", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:pkg", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:pkg", "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:pkg", }, no_match_error = _NO_MATCH_ERROR, @@ -233,7 +239,7 @@ alias( name = "whl", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:whl", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:whl", "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:whl", }, no_match_error = _NO_MATCH_ERROR, @@ -244,7 +250,7 @@ alias( name = "data", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:data", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:data", "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:data", }, no_match_error = _NO_MATCH_ERROR, @@ -255,7 +261,7 @@ alias( name = "dist_info", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:dist_info", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:dist_info", "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:dist_info", }, no_match_error = _NO_MATCH_ERROR, @@ -274,9 +280,10 @@ def _test_bzlmod_aliases_for_non_root_modules(env): # as _test_bzlmod_aliases. # # However, if the root module uses a different default version than the - # non-root module, then we will have a no-match-error because the default_version - # is not in the list of the versions in the whl_map. - default_version = "3.3", + # non-root module, then we will have a no-match-error because the + # default_config_setting is not in the list of the versions in the + # whl_map. + default_config_setting = "//_config:is_python_3.3", aliases = { "bar-baz": [ whl_alias(version = "3.2", repo = "pypi_32_bar_baz"), @@ -295,11 +302,10 @@ _NO_MATCH_ERROR = \"\"\"\\ No matching wheel for current configuration's Python version. The current build configuration's Python version doesn't match any of the Python -versions available for this wheel. This wheel supports the following Python versions: - 3.1, 3.2 - -As matched by the `@rules_python//python/config_settings:is_python_` -configuration settings. +wheels available for this wheel. This wheel supports the following Python +configuration settings: + //_config:is_python_3.1 + //_config:is_python_3.2 To determine the current configuration's Python version, run: `bazel config ` (shown further below) @@ -319,8 +325,8 @@ alias( name = "pkg", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:pkg", - "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:pkg", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:pkg", + "//_config:is_python_3.2": "@pypi_32_bar_baz//:pkg", }, no_match_error = _NO_MATCH_ERROR, ), @@ -330,8 +336,8 @@ alias( name = "whl", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:whl", - "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:whl", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:whl", + "//_config:is_python_3.2": "@pypi_32_bar_baz//:whl", }, no_match_error = _NO_MATCH_ERROR, ), @@ -341,8 +347,8 @@ alias( name = "data", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:data", - "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:data", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:data", + "//_config:is_python_3.2": "@pypi_32_bar_baz//:data", }, no_match_error = _NO_MATCH_ERROR, ), @@ -352,21 +358,21 @@ alias( name = "dist_info", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:dist_info", - "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:dist_info", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:dist_info", + "//_config:is_python_3.2": "@pypi_32_bar_baz//:dist_info", }, no_match_error = _NO_MATCH_ERROR, ), )""" env.expect.that_collection(actual.keys()).contains_exactly([want_key]) - env.expect.that_str(actual[want_key]).equals(_normalize_label_strings(want_content)) + env.expect.that_str(actual[want_key]).equals(want_content) _tests.append(_test_bzlmod_aliases_for_non_root_modules) def _test_aliases_are_created_for_all_wheels(env): actual = render_pkg_aliases( - default_version = "3.2", + default_config_setting = "//_config:is_python_3.2", aliases = { "bar": [ whl_alias(version = "3.1", repo = "pypi_31_bar"), @@ -390,7 +396,7 @@ _tests.append(_test_aliases_are_created_for_all_wheels) def _test_aliases_with_groups(env): actual = render_pkg_aliases( - default_version = "3.2", + default_config_setting = "//_config:is_python_3.2", aliases = { "bar": [ whl_alias(version = "3.1", repo = "pypi_31_bar"), @@ -432,6 +438,433 @@ def _test_aliases_with_groups(env): _tests.append(_test_aliases_with_groups) +def _test_empty_flag_versions(env): + got = get_whl_flag_versions( + aliases = [], + ) + want = {} + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_empty_flag_versions) + +def _test_get_python_versions(env): + got = get_whl_flag_versions( + aliases = [ + whl_alias(repo = "foo", version = "3.3"), + whl_alias(repo = "foo", version = "3.2"), + ], + ) + want = { + "python_versions": ["3.2", "3.3"], + } + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_get_python_versions) + +def _test_get_python_versions_from_filenames(env): + got = get_whl_flag_versions( + aliases = [ + whl_alias( + repo = "foo", + version = "3.3", + filename = "foo-0.0.0-py3-none-" + plat + ".whl", + ) + for plat in [ + "linux_x86_64", + "manylinux_2_17_x86_64", + "manylinux_2_14_aarch64.musllinux_1_1_aarch64", + "musllinux_1_0_x86_64", + "manylinux2014_x86_64.manylinux_2_17_x86_64", + "macosx_11_0_arm64", + "macosx_10_9_x86_64", + "macosx_10_9_universal2", + "windows_x86_64", + ] + ], + ) + want = { + "glibc_versions": [(2, 14), (2, 17)], + "muslc_versions": [(1, 0), (1, 1)], + "osx_versions": [(10, 9), (11, 0)], + "python_versions": ["3.3"], + "target_platforms": [ + "linux_aarch64", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], + } + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_get_python_versions_from_filenames) + +def _test_target_platforms_from_alias_target_platforms(env): + got = get_whl_flag_versions( + aliases = [ + whl_alias( + repo = "foo", + version = "3.3", + filename = "foo-0.0.0-py3-none-" + plat + ".whl", + ) + for plat in [ + "windows_x86_64", + ] + ] + [ + whl_alias( + repo = "foo", + version = "3.3", + filename = "foo-0.0.0-py3-none-any.whl", + target_platforms = [ + "linux_x86_64", + ], + ), + ], + ) + want = { + "python_versions": ["3.3"], + "target_platforms": [ + "linux_x86_64", + "windows_x86_64", + ], + } + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_target_platforms_from_alias_target_platforms) + +def _test_config_settings( + env, + *, + filename, + want, + want_versions = {}, + target_platforms = [], + glibc_versions = [], + muslc_versions = [], + osx_versions = [], + python_version = "", + python_default = True): + got, got_default_version_settings = get_filename_config_settings( + filename = filename, + target_platforms = target_platforms, + glibc_versions = glibc_versions, + muslc_versions = muslc_versions, + osx_versions = osx_versions, + python_version = python_version, + python_default = python_default, + ) + env.expect.that_collection(got).contains_exactly(want) + env.expect.that_dict(got_default_version_settings).contains_exactly(want_versions) + +def _test_sdist(env): + # Do the first test for multiple extensions + for ext in [".tar.gz", ".zip"]: + _test_config_settings( + env, + filename = "foo-0.0.1" + ext, + want = [":is_sdist"], + ) + + ext = ".zip" + _test_config_settings( + env, + filename = "foo-0.0.1" + ext, + target_platforms = [ + "linux_aarch64", + ], + want = [":is_sdist_linux_aarch64"], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1" + ext, + python_version = "3.2", + want = [ + ":is_sdist", + ":is_cp3.2_sdist", + ], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1" + ext, + python_version = "3.2", + python_default = True, + target_platforms = [ + "linux_aarch64", + "linux_x86_64", + ], + want = [ + ":is_sdist_linux_aarch64", + ":is_cp3.2_sdist_linux_aarch64", + ":is_sdist_linux_x86_64", + ":is_cp3.2_sdist_linux_x86_64", + ], + ) + +_tests.append(_test_sdist) + +def _test_py2_py3_none_any(env): + _test_config_settings( + env, + filename = "foo-0.0.1-py2.py3-none-any.whl", + want = [":is_py_none_any"], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1-py2.py3-none-any.whl", + target_platforms = [ + "linux_aarch64", + ], + want = [":is_py_none_any_linux_aarch64"], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1-py2.py3-none-any.whl", + python_version = "3.2", + python_default = True, + want = [ + ":is_py_none_any", + ":is_cp3.2_py_none_any", + ], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1-py2.py3-none-any.whl", + python_version = "3.2", + python_default = False, + target_platforms = [ + "osx_x86_64", + ], + want = [ + ":is_cp3.2_py_none_any_osx_x86_64", + ], + ) + +_tests.append(_test_py2_py3_none_any) + +def _test_py3_none_any(env): + _test_config_settings( + env, + filename = "foo-0.0.1-py3-none-any.whl", + want = [":is_py3_none_any"], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1-py3-none-any.whl", + target_platforms = ["linux_x86_64"], + want = [":is_py3_none_any_linux_x86_64"], + ) + +_tests.append(_test_py3_none_any) + +def _test_py3_none_macosx_10_9_universal2(env): + _test_config_settings( + env, + filename = "foo-0.0.1-py3-none-macosx_10_9_universal2.whl", + osx_versions = [ + (10, 9), + (11, 0), + ], + want = [], + want_versions = { + ":is_py3_none_osx_aarch64_universal2": { + (10, 9): ":is_py3_none_osx_10_9_aarch64_universal2", + (11, 0): ":is_py3_none_osx_11_0_aarch64_universal2", + }, + ":is_py3_none_osx_x86_64_universal2": { + (10, 9): ":is_py3_none_osx_10_9_x86_64_universal2", + (11, 0): ":is_py3_none_osx_11_0_x86_64_universal2", + }, + }, + ) + +_tests.append(_test_py3_none_macosx_10_9_universal2) + +def _test_cp37_abi3_linux_x86_64(env): + _test_config_settings( + env, + filename = "foo-0.0.1-cp37-abi3-linux_x86_64.whl", + want = [ + ":is_cp3x_abi3_linux_x86_64", + ], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1-cp37-abi3-linux_x86_64.whl", + python_version = "3.2", + python_default = True, + want = [ + ":is_cp3x_abi3_linux_x86_64", + # TODO @aignas 2024-05-29: update the pip_config_settings to generate this + ":is_cp3.2_cp3x_abi3_linux_x86_64", + ], + ) + +_tests.append(_test_cp37_abi3_linux_x86_64) + +def _test_cp37_abi3_windows_x86_64(env): + _test_config_settings( + env, + filename = "foo-0.0.1-cp37-abi3-windows_x86_64.whl", + want = [ + ":is_cp3x_abi3_windows_x86_64", + ], + ) + +_tests.append(_test_cp37_abi3_windows_x86_64) + +def _test_cp37_abi3_manylinux_2_17_x86_64(env): + _test_config_settings( + env, + filename = "foo-0.0.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + glibc_versions = [ + (2, 16), + (2, 17), + (2, 18), + ], + want = [], + want_versions = { + ":is_cp3x_abi3_manylinux_x86_64": { + (2, 17): ":is_cp3x_abi3_manylinux_2_17_x86_64", + (2, 18): ":is_cp3x_abi3_manylinux_2_18_x86_64", + }, + }, + ) + +_tests.append(_test_cp37_abi3_manylinux_2_17_x86_64) + +def _test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64(env): + # I've seen such a wheel being built for `uv` + _test_config_settings( + env, + filename = "foo-0.0.1-cp37-cp37-manylinux_2_17_arm64.musllinux_1_1_arm64.whl", + glibc_versions = [ + (2, 16), + (2, 17), + (2, 18), + ], + muslc_versions = [ + (1, 1), + ], + want = [], + want_versions = { + ":is_cp3x_cp_manylinux_aarch64": { + (2, 17): ":is_cp3x_cp_manylinux_2_17_aarch64", + (2, 18): ":is_cp3x_cp_manylinux_2_18_aarch64", + }, + ":is_cp3x_cp_musllinux_aarch64": { + (1, 1): ":is_cp3x_cp_musllinux_1_1_aarch64", + }, + }, + ) + +_tests.append(_test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64) + +def _test_multiplatform_whl_aliases_empty(env): + # Check that we still work with an empty requirements.txt + got = multiplatform_whl_aliases(aliases = [], default_version = None) + env.expect.that_collection(got).contains_exactly([]) + +_tests.append(_test_multiplatform_whl_aliases_empty) + +def _test_multiplatform_whl_aliases_nofilename(env): + aliases = [ + whl_alias( + repo = "foo", + config_setting = "//:label", + version = "3.1", + ), + ] + got = multiplatform_whl_aliases(aliases = aliases, default_version = None) + env.expect.that_collection(got).contains_exactly(aliases) + +_tests.append(_test_multiplatform_whl_aliases_nofilename) + +def _test_multiplatform_whl_aliases_filename(env): + aliases = [ + whl_alias( + repo = "foo-py3-0.0.3", + filename = "foo-0.0.3-py3-none-any.whl", + version = "3.2", + ), + whl_alias( + repo = "foo-py3-0.0.1", + filename = "foo-0.0.1-py3-none-any.whl", + version = "3.1", + ), + whl_alias( + repo = "foo-0.0.2", + filename = "foo-0.0.2-py3-none-any.whl", + version = "3.1", + target_platforms = [ + "linux_x86_64", + "linux_aarch64", + ], + ), + ] + got = multiplatform_whl_aliases( + aliases = aliases, + default_version = "3.1", + glibc_versions = [], + muslc_versions = [], + osx_versions = [], + ) + want = [ + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_any", repo = "foo-py3-0.0.1", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_any_linux_aarch64", repo = "foo-0.0.2", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_any_linux_x86_64", repo = "foo-0.0.2", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.2_py3_none_any", repo = "foo-py3-0.0.3", version = "3.2"), + whl_alias(config_setting = "//_config:is_py3_none_any", repo = "foo-py3-0.0.1", version = "3.1"), + whl_alias(config_setting = "//_config:is_py3_none_any_linux_aarch64", repo = "foo-0.0.2", version = "3.1"), + whl_alias(config_setting = "//_config:is_py3_none_any_linux_x86_64", repo = "foo-0.0.2", version = "3.1"), + ] + env.expect.that_collection(got).contains_exactly(want) + +_tests.append(_test_multiplatform_whl_aliases_filename) + +def _test_multiplatform_whl_aliases_filename_versioned(env): + aliases = [ + whl_alias( + repo = "glibc-2.17", + filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", + version = "3.1", + ), + whl_alias( + repo = "glibc-2.18", + filename = "foo-0.0.1-py3-none-manylinux_2_18_x86_64.whl", + version = "3.1", + ), + whl_alias( + repo = "musl", + filename = "foo-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + version = "3.1", + ), + ] + got = multiplatform_whl_aliases( + aliases = aliases, + default_version = None, + glibc_versions = [(2, 17), (2, 18)], + muslc_versions = [(1, 1), (1, 2)], + osx_versions = [], + ) + want = [ + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_manylinux_2_17_x86_64", repo = "glibc-2.17", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_manylinux_2_18_x86_64", repo = "glibc-2.18", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_manylinux_x86_64", repo = "glibc-2.17", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_musllinux_1_1_x86_64", repo = "musl", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_musllinux_1_2_x86_64", repo = "musl", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_musllinux_x86_64", repo = "musl", version = "3.1"), + ] + env.expect.that_collection(got).contains_exactly(want) + +_tests.append(_test_multiplatform_whl_aliases_filename_versioned) + def render_pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl b/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl index a06147b946..07f3158b31 100644 --- a/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl +++ b/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl @@ -22,20 +22,20 @@ _tests = [] def _test_simple(env): tests = { "macosx_10_9_arm64": [ - struct(os = "osx", cpu = "aarch64", abi = None, target_platform = "osx_aarch64"), + struct(os = "osx", cpu = "aarch64", abi = None, target_platform = "osx_aarch64", version = (10, 9)), ], "macosx_10_9_universal2": [ - struct(os = "osx", cpu = "x86_64", abi = None, target_platform = "osx_x86_64"), - struct(os = "osx", cpu = "aarch64", abi = None, target_platform = "osx_aarch64"), + struct(os = "osx", cpu = "x86_64", abi = None, target_platform = "osx_x86_64", version = (10, 9)), + struct(os = "osx", cpu = "aarch64", abi = None, target_platform = "osx_aarch64", version = (10, 9)), ], - "manylinux1_i686.manylinux_2_17_i686": [ - struct(os = "linux", cpu = "x86_32", abi = None, target_platform = "linux_x86_32"), + "manylinux_2_17_i686": [ + struct(os = "linux", cpu = "x86_32", abi = None, target_platform = "linux_x86_32", version = (2, 17)), ], "musllinux_1_1_ppc64le": [ - struct(os = "linux", cpu = "ppc", abi = None, target_platform = "linux_ppc"), + struct(os = "linux", cpu = "ppc", abi = None, target_platform = "linux_ppc", version = (1, 1)), ], "win_amd64": [ - struct(os = "windows", cpu = "x86_64", abi = None, target_platform = "windows_x86_64"), + struct(os = "windows", cpu = "x86_64", abi = None, target_platform = "windows_x86_64", version = (0, 0)), ], } @@ -49,20 +49,22 @@ _tests.append(_test_simple) def _test_with_abi(env): tests = { "macosx_10_9_arm64": [ - struct(os = "osx", cpu = "aarch64", abi = "cp39", target_platform = "cp39_osx_aarch64"), + struct(os = "osx", cpu = "aarch64", abi = "cp39", target_platform = "cp39_osx_aarch64", version = (10, 9)), ], "macosx_10_9_universal2": [ - struct(os = "osx", cpu = "x86_64", abi = "cp310", target_platform = "cp310_osx_x86_64"), - struct(os = "osx", cpu = "aarch64", abi = "cp310", target_platform = "cp310_osx_aarch64"), + struct(os = "osx", cpu = "x86_64", abi = "cp310", target_platform = "cp310_osx_x86_64", version = (10, 9)), + struct(os = "osx", cpu = "aarch64", abi = "cp310", target_platform = "cp310_osx_aarch64", version = (10, 9)), ], + # This should use version 0 because there are two platform_tags. This is + # just to ensure that the code is robust "manylinux1_i686.manylinux_2_17_i686": [ - struct(os = "linux", cpu = "x86_32", abi = "cp38", target_platform = "cp38_linux_x86_32"), + struct(os = "linux", cpu = "x86_32", abi = "cp38", target_platform = "cp38_linux_x86_32", version = (0, 0)), ], "musllinux_1_1_ppc64le": [ - struct(os = "linux", cpu = "ppc", abi = "cp311", target_platform = "cp311_linux_ppc"), + struct(os = "linux", cpu = "ppc", abi = "cp311", target_platform = "cp311_linux_ppc", version = (1, 1)), ], "win_amd64": [ - struct(os = "windows", cpu = "x86_64", abi = "cp311", target_platform = "cp311_windows_x86_64"), + struct(os = "windows", cpu = "x86_64", abi = "cp311", target_platform = "cp311_windows_x86_64", version = (0, 0)), ], } From c3dc7b059e3c9e88507e6667ddfb1f103627bfde Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:23:35 +0900 Subject: [PATCH 09/18] feat(toolchain): use `py_linux_libc` flag value With this we start using the `py_linux_libc` flag value in toolchains as well and ensure that no hermetic toolchain will be matched when we specify to pull the `musl` wheels. In that case the user will need to supply their own toolchain that is the expected behaviour. However, I am not sure how we could fail due to no available toolchain in that case because the autodetecting toolchain will just use the host interpreter. --- python/private/BUILD.bazel | 2 + python/private/py_toolchain_suite.bzl | 59 +++++++++++++++++++++------ python/private/toolchains_repo.bzl | 8 +++- python/versions.bzl | 21 ++++++++++ 4 files changed, 76 insertions(+), 14 deletions(-) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 24b4840266..ebefe8b000 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -252,6 +252,8 @@ bzl_library( name = "py_toolchain_suite_bzl", srcs = ["py_toolchain_suite.bzl"], deps = [ + ":config_settings_bzl", + ":text_util_bzl", ":toolchain_types_bzl", "@bazel_skylib//lib:selects", ], diff --git a/python/private/py_toolchain_suite.bzl b/python/private/py_toolchain_suite.bzl index 9971a8a4c3..69d9a7999e 100644 --- a/python/private/py_toolchain_suite.bzl +++ b/python/private/py_toolchain_suite.bzl @@ -15,6 +15,7 @@ """Create the toolchain defs in a BUILD.bazel file.""" load("@bazel_skylib//lib:selects.bzl", "selects") +load(":config_settings.bzl", "is_python_config_setting") load( ":toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE", @@ -22,7 +23,7 @@ load( "TARGET_TOOLCHAIN_TYPE", ) -def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_python_version_constraint, **kwargs): +def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_python_version_constraint, flag_values, **kwargs): """For internal use only. Args: @@ -30,6 +31,7 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth user_repository_name: The name of the user repository. python_version: The full (X.Y.Z) version of the interpreter. set_python_version_constraint: True or False as a string. + flag_values: Extra flag values to match for this toolchain. **kwargs: extra args passed to the `toolchain` calls. """ @@ -38,23 +40,54 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth # string as we cannot have list of bools in build rule attribues. # This if statement does not appear to work unless it is in the # toolchain file. - if set_python_version_constraint == "True": + if set_python_version_constraint in ["True", "False"]: major_minor, _, _ = python_version.rpartition(".") + match_any = [] + for i, v in enumerate([major_minor, python_version]): + name = "{prefix}_{python_version}_{i}".format( + prefix = prefix, + python_version = python_version, + i = i, + ) + match_any.append(name) + if flag_values: + is_python_config_setting( + name = name, + python_version = v, + flag_values = flag_values, + visibility = ["//visibility:private"], + ) + else: + native.alias( + name = name, + actual = Label("//python/config_settings:is_python_%s" % v), + visibility = ["//visibility:private"], + ) + + if set_python_version_constraint == "False": + name = "{prefix}_version_setting_no_python_version".format(prefix = prefix) + match_any.append(name) + native.config_setting( + name = name, + flag_values = flag_values | { + str(Label("//python/config_settings:python_version")): "", + }, + visibility = ["//visibility:private"], + ) + + name = "{prefix}_version_setting_{python_version}".format( + prefix = prefix, + python_version = python_version, + visibility = ["//visibility:private"], + ) selects.config_setting_group( - name = prefix + "_version_setting", - match_any = [ - Label("//python/config_settings:is_python_%s" % v) - for v in [ - major_minor, - python_version, - ] - ], + name = name, + match_any = match_any, visibility = ["//visibility:private"], ) - target_settings = [prefix + "_version_setting"] - elif set_python_version_constraint == "False": - target_settings = [] + + target_settings = [name] else: fail(("Invalid set_python_version_constraint value: got {} {}, wanted " + "either the string 'True' or the string 'False'; " + diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 018c4433ed..dd59e477d8 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -31,6 +31,7 @@ load( "WINDOWS_NAME", ) load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load("//python/private:text_util.bzl", "render") def get_repository_name(repository_workspace): dummy_label = "//:_" @@ -64,10 +65,15 @@ py_toolchain_suite( user_repository_name = "{user_repository_name}_{platform}", prefix = "{prefix}{platform}", target_compatible_with = {compatible_with}, + flag_values = {flag_values}, python_version = "{python_version}", set_python_version_constraint = "{set_python_version_constraint}", )""".format( - compatible_with = meta.compatible_with, + compatible_with = render.indent(render.list(meta.compatible_with)).lstrip(), + flag_values = render.indent(render.dict( + meta.flag_values, + key_repr = lambda x: repr(str(x)), # this is to correctly display labels + )).lstrip(), platform = platform, set_python_version_constraint = set_python_version_constraint, user_repository_name = user_repository_name, diff --git a/python/versions.bzl b/python/versions.bzl index 08882d3ade..26b975d068 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -501,6 +501,7 @@ PLATFORMS = { "@platforms//os:macos", "@platforms//cpu:aarch64", ], + flag_values = {}, os_name = MACOS_NAME, # Matches the value returned from: # repository_ctx.execute(["uname", "-m"]).stdout.strip() @@ -511,6 +512,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:aarch64", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, # Note: this string differs between OSX and Linux # Matches the value returned from: @@ -522,6 +526,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:armv7", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, arch = "armv7", ), @@ -530,6 +537,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:ppc", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, # Note: this string differs between OSX and Linux # Matches the value returned from: @@ -541,6 +551,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:riscv64", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, arch = "riscv64", ), @@ -549,6 +562,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:s390x", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, # Note: this string differs between OSX and Linux # Matches the value returned from: @@ -560,6 +576,7 @@ PLATFORMS = { "@platforms//os:macos", "@platforms//cpu:x86_64", ], + flag_values = {}, os_name = MACOS_NAME, arch = "x86_64", ), @@ -568,6 +585,7 @@ PLATFORMS = { "@platforms//os:windows", "@platforms//cpu:x86_64", ], + flag_values = {}, os_name = WINDOWS_NAME, arch = "x86_64", ), @@ -576,6 +594,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:x86_64", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, arch = "x86_64", ), From bf4c35c1c22de0e1463735ec4484f130e12c3bfd Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:26:25 +0900 Subject: [PATCH 10/18] feat(bzlmod): fetch and configure platform specific wheels This feature leverages all of the foundational work from previous commits and updates the tests and documentation. --- CHANGELOG.md | 16 +- docs/sphinx/pip.md | 166 +------------- docs/sphinx/pypi-dependencies.md | 278 +++++++++++++++++++----- examples/bzlmod/libs/my_lib/__init__.py | 8 +- python/private/bzlmod/BUILD.bazel | 1 + python/private/bzlmod/pip.bzl | 124 ++++++----- 6 files changed, 320 insertions(+), 273 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c16501019b..f0af77836d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,10 +30,16 @@ A brief description of the categories of changes: * (bzlmod): The `python` and internal `rules_python` extensions have been marked as `reproducible` and will not include any lock file entries from now on. - * (gazelle): Remove gazelle plugin's python deps and make it hermetic. Introduced a new Go-based helper leveraging tree-sitter for syntax analysis. Implemented the use of `pypi/stdlib-list` for standard library module verification. +* (pip.parse): Do not ignore yanked packages when using `experimental_index_url`. + This is to mimic what `uv` is doing. We will print a warning instead. +* (pip.parse): Add references to all supported wheels when using `experimental_index_url` + to allowing to correctly fetch the wheels for the right platform. See the + updated docs on how to use the feature. This is work towards addressing + [#735](https://github.com/bazelbuild/rules_python/issues/735) and + [#260](https://github.com/bazelbuild/rules_python/issues/260). ### Fixed * (gazelle) Remove `visibility` from `NonEmptyAttr`. @@ -88,6 +94,14 @@ A brief description of the categories of changes: invalid usage previously but we were not failing the build. From now on this is explicitly disallowed. * (toolchains) Added riscv64 platform definition for python toolchains. +* (pip): Support fetching and using the wheels for other platforms. This + supports customizing whether the linux wheels are pulled for `musl` or + `glibc`, whether `universal2` or arch-specific MacOS wheels are preferred and + it also allows to select a particular `libc` version. All of this is done via + the `string_flags` in `@rules_python//python/config_settings`. If there are + no wheels that are supported for the target platform, `rules_python` will + fallback onto building the `sdist` from source. This behaviour can be + disabled if desired using one of the available string flags as well. [precompile-docs]: /precompiling diff --git a/docs/sphinx/pip.md b/docs/sphinx/pip.md index e1c8e343f0..43d8fc4978 100644 --- a/docs/sphinx/pip.md +++ b/docs/sphinx/pip.md @@ -1,168 +1,4 @@ (pip-integration)= # Pip Integration -To pull in dependencies from PyPI, the `pip_parse` function is used, which -invokes `pip` to download and install dependencies from PyPI. - -In your WORKSPACE file: - -```starlark -load("@rules_python//python:pip.bzl", "pip_parse") - -pip_parse( - name = "pip_deps", - requirements_lock = ":requirements.txt", -) - -load("@pip_deps//:requirements.bzl", "install_deps") - -install_deps() -``` - -For `bzlmod` an equivalent `MODULE.bazel` would look like: -```starlark -pip = use_extension("//python/extensions:pip.bzl", "pip") -pip.parse( - hub_name = "pip_deps", - requirements_lock = ":requirements.txt", -) -use_repo(pip, "pip_deps") -``` - -You can then reference installed dependencies from a `BUILD` file with: - -```starlark -load("@pip_deps//:requirements.bzl", "requirement") - -py_library( - name = "bar", - ... - deps = [ - "//my/other:dep", - "@pip_deps//requests", - "@pip_deps//numpy", - ], -) -``` - -The rules also provide a convenience macro for translating the entries in the -`requirements.txt` file (e.g. `opencv-python`) to the right bazel label (e.g. -`@pip_deps//opencv_python`). The convention of bazel labels is lowercase -`snake_case`, but you can use the helper to avoid depending on this convention -as follows: - -```starlark -load("@pip_deps//:requirements.bzl", "requirement") - -py_library( - name = "bar", - ... - deps = [ - "//my/other:dep", - requirement("requests"), - requirement("numpy"), - ], -) -``` - -If you would like to access [entry points][whl_ep], see the `py_console_script_binary` rule documentation. - -[whl_ep]: https://packaging.python.org/specifications/entry-points/ - -(per-os-arch-requirements)= -## Requirements for a specific OS/Architecture - -In some cases you may need to use different requirements files for different OS, Arch combinations. This is enabled via the `requirements_by_platform` attribute in `pip.parse` extension and the `pip_parse` repository rule. The keys of the dictionary are labels to the file and the values are a list of comma separated target (os, arch) tuples. - -For example: -```starlark - # ... - requirements_by_platform = { - "requirements_linux_x86_64.txt": "linux_x86_64", - "requirements_osx.txt": "osx_*", - "requirements_linux_exotic.txt": "linux_exotic", - "requirements_some_platforms.txt": "linux_aarch64,windows_*", - }, - # For the list of standard platforms that the rules_python has toolchains for, default to - # the following requirements file. - requirements_lock = "requirements_lock.txt", -``` - -In case of duplicate platforms, `rules_python` will raise an error as there has -to be unambiguous mapping of the requirement files to the (os, arch) tuples. - -An alternative way is to use per-OS requirement attributes. -```starlark - # ... - requirements_windows = "requirements_windows.txt", - requirements_darwin = "requirements_darwin.txt", - # For the remaining platforms (which is basically only linux OS), use this file. - requirements_lock = "requirements_lock.txt", -) -``` - -(vendoring-requirements)= -## Vendoring the requirements.bzl file - -In some cases you may not want to generate the requirements.bzl file as a repository rule -while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module -such as a ruleset, you may want to include the requirements.bzl file rather than make your users -install the WORKSPACE setup to generate it. -See https://github.com/bazelbuild/rules_python/issues/608 - -This is the same workflow as Gazelle, which creates `go_repository` rules with -[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) - -To do this, use the "write to source file" pattern documented in -https://blog.aspect.dev/bazel-can-write-to-the-source-folder -to put a copy of the generated requirements.bzl into your project. -Then load the requirements.bzl file directly rather than from the generated repository. -See the example in rules_python/examples/pip_parse_vendored. - - -(credential-helper)= -## Credential Helper - -The "use Bazel downloader for python wheels" experimental feature includes support for the Bazel -[Credential Helper][cred-helper-design]. - -Your python artifact registry may provide a credential helper for you. Refer to your index's docs -to see if one is provided. - -See the [Credential Helper Spec][cred-helper-spec] for details. - -[cred-helper-design]: https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md -[cred-helper-spec]: https://github.com/EngFlow/credential-helper-spec/blob/main/spec.md - - -### Basic Example: - -The simplest form of a credential helper is a bash script that accepts an arg and spits out JSON to -stdout. For a service like Google Artifact Registry that uses ['Basic' HTTP Auth][rfc7617] and does -not provide a credential helper that conforms to the [spec][cred-helper-spec], the script might -look like: - -```bash -#!/bin/bash -# cred_helper.sh -ARG=$1 # but we don't do anything with it as it's always "get" - -# formatting is optional -echo '{' -echo ' "headers": {' -echo ' "Authorization": ["Basic dGVzdDoxMjPCow=="] -echo ' }' -echo '}' -``` - -Configure Bazel to use this credential helper for your python index `example.com`: - -``` -# .bazelrc -build --credential_helper=example.com=/full/path/to/cred_helper.sh -``` - -Bazel will call this file like `cred_helper.sh get` and use the returned JSON to inject headers -into whatever HTTP(S) request it performs against `example.com`. - -[rfc7617]: https://datatracker.ietf.org/doc/html/rfc7617 +See [PyPI dependencies](./pypi-dependencies). diff --git a/docs/sphinx/pypi-dependencies.md b/docs/sphinx/pypi-dependencies.md index f08f7fb7a7..db017d249f 100644 --- a/docs/sphinx/pypi-dependencies.md +++ b/docs/sphinx/pypi-dependencies.md @@ -1,3 +1,6 @@ +:::{default-domain} bzl +::: + # Using dependencies from PyPI Using PyPI packages (aka "pip install") involves two main steps. @@ -25,19 +28,21 @@ pip.parse( use_repo(pip, "my_deps") ``` For more documentation, including how the rules can update/create a requirements -file, see the bzlmod examples under the {gh-path}`examples` folder. +file, see the bzlmod examples under the {gh-path}`examples` folder or the documentation +for the {obj}`@rules_python//python/extensions:pip.bzl` extension. +```{note} We are using a host-platform compatible toolchain by default to setup pip dependencies. During the setup phase, we create some symlinks, which may be inefficient on Windows by default. In that case use the following `.bazelrc` options to improve performance if you have admin privileges: -``` -startup --windows_enable_symlinks -``` + + startup --windows_enable_symlinks This will enable symlinks on Windows and help with bootstrap performance of setting up the hermetic host python interpreter on this platform. Linux and OSX users should see no difference. +``` ### Using a WORKSPACE file @@ -59,16 +64,67 @@ load("@my_deps//:requirements.bzl", "install_deps") install_deps() ``` +(vendoring-requirements)= +#### Vendoring the requirements.bzl file + +In some cases you may not want to generate the requirements.bzl file as a repository rule +while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module +such as a ruleset, you may want to include the requirements.bzl file rather than make your users +install the WORKSPACE setup to generate it. +See https://github.com/bazelbuild/rules_python/issues/608 + +This is the same workflow as Gazelle, which creates `go_repository` rules with +[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) + +To do this, use the "write to source file" pattern documented in +https://blog.aspect.dev/bazel-can-write-to-the-source-folder +to put a copy of the generated requirements.bzl into your project. +Then load the requirements.bzl file directly rather than from the generated repository. +See the example in rules_python/examples/pip_parse_vendored. + +(per-os-arch-requirements)= +### Requirements for a specific OS/Architecture + +In some cases you may need to use different requirements files for different OS, Arch combinations. This is enabled via the `requirements_by_platform` attribute in `pip.parse` extension and the `pip_parse` repository rule. The keys of the dictionary are labels to the file and the values are a list of comma separated target (os, arch) tuples. + +For example: +```starlark + # ... + requirements_by_platform = { + "requirements_linux_x86_64.txt": "linux_x86_64", + "requirements_osx.txt": "osx_*", + "requirements_linux_exotic.txt": "linux_exotic", + "requirements_some_platforms.txt": "linux_aarch64,windows_*", + }, + # For the list of standard platforms that the rules_python has toolchains for, default to + # the following requirements file. + requirements_lock = "requirements_lock.txt", +``` + +In case of duplicate platforms, `rules_python` will raise an error as there has +to be unambiguous mapping of the requirement files to the (os, arch) tuples. + +An alternative way is to use per-OS requirement attributes. +```starlark + # ... + requirements_windows = "requirements_windows.txt", + requirements_darwin = "requirements_darwin.txt", + # For the remaining platforms (which is basically only linux OS), use this file. + requirements_lock = "requirements_lock.txt", +) +``` + ### pip rules -Note that since `pip_parse` is a repository rule and therefore executes pip at -WORKSPACE-evaluation time, Bazel has no information about the Python toolchain -and cannot enforce that the interpreter used to invoke pip matches the -interpreter used to run `py_binary` targets. By default, `pip_parse` uses the -system command `"python3"`. To override this, pass in the `python_interpreter` -attribute or `python_interpreter_target` attribute to `pip_parse`. +Note that since `pip_parse` and `pip.parse` are executed at evaluation time, +Bazel has no information about the Python toolchain and cannot enforce that the +interpreter used to invoke `pip` matches the interpreter used to run +`py_binary` targets. By default, `pip_parse` uses the system command +`"python3"`. To override this, pass in the `python_interpreter` attribute or +`python_interpreter_target` attribute to `pip_parse`. The `pip.parse` `bzlmod` extension +by default uses the hermetic python toolchain for the host platform. -You can have multiple `pip_parse`s in the same workspace. Or use the pip +You can have multiple `pip_parse`s in the same workspace, or use the pip extension multiple times when using bzlmod. This configuration will create multiple external repos that have no relation to one another and may result in downloading the same wheels numerous times. @@ -111,7 +167,7 @@ want to use `requirement()`, you can use the library labels directly instead. For `pip_parse`, the labels are of the following form: ```starlark -@{name}_{package}//:pkg +@{name}//{package} ``` Here `name` is the `name` attribute that was passed to `pip_parse` and @@ -121,30 +177,67 @@ update `name` from "old" to "new", then you can run the following buildozer command: ```shell -buildozer 'substitute deps @old_([^/]+)//:pkg @new_${1}//:pkg' //...:* +buildozer 'substitute deps @old//([^/]+) @new//${1}' //...:* ``` [requirements-drawbacks]: https://github.com/bazelbuild/rules_python/issues/414 +### Entry points + +If you would like to access [entry points][whl_ep], see the `py_console_script_binary` rule documentation, +which can help you create a `py_binary` target for a particular console script exposed by a package. + +[whl_ep]: https://packaging.python.org/specifications/entry-points/ + ### 'Extras' dependencies Any 'extras' specified in the requirements lock file will be automatically added as transitive dependencies of the package. In the example above, you'd just put -`requirement("useful_dep")`. +`requirement("useful_dep")` or `@pypi//useful_dep`. -### Packaging cycles +### Consuming Wheel Dists Directly -Sometimes PyPi packages contain dependency cycles -- for instance `sphinx` -depends on `sphinxcontrib-serializinghtml`. When using them as `requirement()`s, -ala +If you need to depend on the wheel dists themselves, for instance, to pass them +to some other packaging tool, you can get a handle to them with the +`whl_requirement` macro. For example: + +```starlark +load("@pypi//:requirements.bzl", "whl_requirement") + +filegroup( + name = "whl_files", + data = [ + # This is equivalent to "@pypi//boto3:whl" + whl_requirement("boto3"), + ] +) +``` + +### Creating a filegroup of files within a whl + +The rule {obj}`whl_filegroup` exists as an easy way to extract the necessary files +from a whl file without the need to modify the `BUILD.bazel` contents of the +whl repositories generated via `pip_repository`. Use it similarly to the `filegroup` +above. See the API docs for more information. + +(advance-topics)= +## Advanced topics + +(circular-deps)= +### Circular dependencies + +Sometimes PyPi packages contain dependency cycles -- for instance a particular +version `sphinx` (this is no longer the case in the latest version as of +2024-06-02) depends on `sphinxcontrib-serializinghtml`. When using them as +`requirement()`s, ala ``` py_binary( - name = "doctool", - ... - deps = [ - requirement("sphinx"), - ] + name = "doctool", + ... + deps = [ + requirement("sphinx"), + ], ) ``` @@ -166,15 +259,15 @@ issues by specifying groups of packages which form cycles. `pip_parse` will transparently fix the cycles for you and provide the cyclic dependencies simultaneously. -``` +```starlark pip_parse( - ... - experimental_requirement_cycles = { - "sphinx": [ - "sphinx", - "sphinxcontrib-serializinghtml", - ] - }, + ... + experimental_requirement_cycles = { + "sphinx": [ + "sphinx", + "sphinxcontrib-serializinghtml", + ] + }, ) ``` @@ -183,17 +276,17 @@ be distinct. `apache-airflow` for instance has dependency cycles with a number of its optional dependencies, which means those optional dependencies must all be a part of the `airflow` cycle. For instance -- -``` +```starlark pip_parse( - ... - experimental_requirement_cycles = { - "airflow": [ - "apache-airflow", - "apache-airflow-providers-common-sql", - "apache-airflow-providers-postgres", - "apache-airflow-providers-sqlite", - ] - } + ... + experimental_requirement_cycles = { + "airflow": [ + "apache-airflow", + "apache-airflow-providers-common-sql", + "apache-airflow-providers-postgres", + "apache-airflow-providers-sqlite", + ] + } ) ``` @@ -213,17 +306,98 @@ leg of the dependency manually. For instance by making `apache-airflow-providers-postgres` not explicitly depend on `apache-airflow` or perhaps `apache-airflow-providers-common-sql`. -## Consuming Wheel Dists Directly -If you need to depend on the wheel dists themselves, for instance, to pass them -to some other packaging tool, you can get a handle to them with the -`whl_requirement` macro. For example: +(bazel-downloader)= +### Bazel downloader and multi-platform wheel hub repository. -```starlark -filegroup( - name = "whl_files", - data = [ - whl_requirement("boto3"), - ] -) +The `bzlmod` `pip.parse` call supports pulling information from `PyPI` (or a +compatible mirror) and it will ensure that the [bazel +downloader][bazel_downloader] is used for downloading the wheels. This allows +the users to use the [credential helper](#credential-helper) to authenticate +with the mirror and it also ensures that the distribution downloads are cached. +It also avoids using `pip` altogether and results in much faster dependency +fetching. + +This can be enabled by `experimental_index_url` and related flags as shown in +the {gh-path}`examples/bzlmod/MODULE.bazel` example. + +When using this feature during the `pip` extension evaluation you will see the accessed indexes similar to below: +```console +Loading: 0 packages loaded + currently loading: docs/sphinx + Fetching module extension pip in @@//python/extensions:pip.bzl; starting + Fetching https://pypi.org/simple/twine/ ``` + +This does not mean that `rules_python` is fetching the wheels eagerly, but it +rather means that it is calling the PyPI server to get the Simple API response +to get the list of all available source and wheel distributions. Once it has +got all of the available distributions, it will select the right ones depending +on the `sha256` values in your `requirements_lock.txt` file. The compatible +distribution URLs will be then written to the `MODULE.bazel.lock` file. Currently +users wishing to use the lock file with `rules_python` with this feature have +to set an environment variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will +become default in the next release. + +Fetching the distribution information from the PyPI allows `rules_python` to +know which `whl` should be used on which target platform and it will determine +that by parsing the `whl` filename based on [PEP600], [PEP656] standards. This +allows the user to configure the behaviour by using the following publicly +available flags: +* {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant. +* {obj}`--@rules_python//python/config_settings:pip_whl` for selecting `whl` distribution preference. +* {obj}`--@rules_python//python/config_settings:pip_whl_osx_arch` for selecting MacOS wheel preference. +* {obj}`--@rules_python//python/config_settings:pip_whl_glibc_version` for selecting the GLIBC version compatibility. +* {obj}`--@rules_python//python/config_settings:pip_whl_muslc_version` for selecting the musl version compatibility. +* {obj}`--@rules_python//python/config_settings:pip_whl_osx_version` for selecting MacOS version compatibility. + +[bazel_downloader]: https://bazel.build/rules/lib/builtins/repository_ctx#download +[pep600]: https://peps.python.org/pep-0600/ +[pep656]: https://peps.python.org/pep-0656/ + +(credential-helper)= +### Credential Helper + +The "use Bazel downloader for python wheels" experimental feature includes support for the Bazel +[Credential Helper][cred-helper-design]. + +Your python artifact registry may provide a credential helper for you. Refer to your index's docs +to see if one is provided. + +See the [Credential Helper Spec][cred-helper-spec] for details. + +[cred-helper-design]: https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md +[cred-helper-spec]: https://github.com/EngFlow/credential-helper-spec/blob/main/spec.md + + +#### Basic Example: + +The simplest form of a credential helper is a bash script that accepts an arg and spits out JSON to +stdout. For a service like Google Artifact Registry that uses ['Basic' HTTP Auth][rfc7617] and does +not provide a credential helper that conforms to the [spec][cred-helper-spec], the script might +look like: + +```bash +#!/bin/bash +# cred_helper.sh +ARG=$1 # but we don't do anything with it as it's always "get" + +# formatting is optional +echo '{' +echo ' "headers": {' +echo ' "Authorization": ["Basic dGVzdDoxMjPCow=="]' +echo ' }' +echo '}' +``` + +Configure Bazel to use this credential helper for your python index `example.com`: + +``` +# .bazelrc +build --credential_helper=example.com=/full/path/to/cred_helper.sh +``` + +Bazel will call this file like `cred_helper.sh get` and use the returned JSON to inject headers +into whatever HTTP(S) request it performs against `example.com`. + +[rfc7617]: https://datatracker.ietf.org/doc/html/rfc7617 diff --git a/examples/bzlmod/libs/my_lib/__init__.py b/examples/bzlmod/libs/my_lib/__init__.py index 8ce96ea207..271e933417 100644 --- a/examples/bzlmod/libs/my_lib/__init__.py +++ b/examples/bzlmod/libs/my_lib/__init__.py @@ -19,4 +19,10 @@ def websockets_is_for_python_version(sanitized_version_check): # We are checking that the name of the repository folders # match the expected generated names. If we update the folder # structure or naming we will need to modify this test. - return f"_{sanitized_version_check}_websockets" in websockets.__file__ + want = f"_{sanitized_version_check}_websockets" + got_full = websockets.__file__ + if want not in got_full: + print(f"Failed, expected '{want}' to be a substring of '{got_full}'.") + return False + + return True diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel index 2eab575726..3362f34ffd 100644 --- a/python/private/bzlmod/BUILD.bazel +++ b/python/private/bzlmod/BUILD.bazel @@ -36,6 +36,7 @@ bzl_library( "//python/private:normalize_name_bzl", "//python/private:parse_requirements_bzl", "//python/private:parse_whl_name_bzl", + "//python/private:pip_repo_name_bzl", "//python/private:version_label_bzl", ":bazel_features_bzl", ] + [ diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl index 8702f1fbe7..122debd2a2 100644 --- a/python/private/bzlmod/pip.bzl +++ b/python/private/bzlmod/pip.bzl @@ -26,11 +26,11 @@ load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load("//python/private:parse_whl_name.bzl", "parse_whl_name") +load("//python/private:pip_repo_name.bzl", "pip_repo_name") load("//python/private:pypi_index.bzl", "simpleapi_download") load("//python/private:render_pkg_aliases.bzl", "whl_alias") load("//python/private:repo_utils.bzl", "repo_utils") load("//python/private:version_label.bzl", "version_label") -load("//python/private:whl_target_platforms.bzl", "select_whl") load(":pip_repository.bzl", "pip_repository") def _parse_version(version): @@ -199,19 +199,6 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s repository_platform = host_platform(module_ctx.os) for whl_name, requirements in requirements_by_platform.items(): - requirement = select_requirement( - requirements, - platform = repository_platform, - ) - if not requirement: - # Sometimes the package is not present for host platform if there - # are whls specified only in particular requirements files, in that - # case just continue, however, if the download_only flag is set up, - # then the user can also specify the target platform of the wheel - # packages they want to download, in that case there will be always - # a requirement here, so we will not be in this code branch. - continue - # We are not using the "sanitized name" because the user # would need to guess what name we modified the whl name # to. @@ -223,11 +210,9 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s # Construct args separately so that the lock file can be smaller and does not include unused # attrs. - repo_name = "{}_{}".format(pip_name, whl_name) whl_library_args = dict( repo = pip_name, dep_template = "@{}//{{name}}:{{target}}".format(hub_name), - requirement = requirement.requirement_line, ) maybe_args = dict( # The following values are safe to omit if they have false like values @@ -237,7 +222,6 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s environment = pip_attr.environment, envsubst = pip_attr.envsubst, experimental_target_platforms = pip_attr.experimental_target_platforms, - extra_pip_args = requirement.extra_pip_args, group_deps = group_deps, group_name = group_name, pip_data_exclude = pip_attr.pip_data_exclude, @@ -257,51 +241,83 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s ) whl_library_args.update({k: v for k, (v, default) in maybe_args_with_default.items() if v == default}) - if requirement.whls or requirement.sdist: - logger.debug(lambda: "Selecting a compatible dist for {} from dists:\n{}".format( - repository_platform, - json.encode( - struct( - whls = requirement.whls, - sdist = requirement.sdist, - ), - ), - )) - distribution = select_whl( - whls = requirement.whls, - want_platform = repository_platform, - ) or requirement.sdist - - logger.debug(lambda: "Selected: {}".format(distribution)) - - if distribution: - is_hub_reproducible = False - whl_library_args["requirement"] = requirement.srcs.requirement - whl_library_args["urls"] = [distribution.url] - whl_library_args["sha256"] = distribution.sha256 - whl_library_args["filename"] = distribution.filename - if pip_attr.netrc: - whl_library_args["netrc"] = pip_attr.netrc - if pip_attr.auth_patterns: - whl_library_args["auth_patterns"] = pip_attr.auth_patterns - - # pip is not used to download wheels and the python `whl_library` helpers are only extracting things - whl_library_args.pop("extra_pip_args", None) - - # This is no-op because pip is not used to download the wheel. - whl_library_args.pop("download_only", None) - else: - logger.warn("falling back to pip for installing the right file for {}".format(requirement.requirement_line)) + if get_index_urls: + # TODO @aignas 2024-05-26: move to a separate function + found_something = False + for requirement in requirements: + for distribution in requirement.whls + [requirement.sdist]: + if not distribution: + # sdist may be None + continue + + found_something = True + is_hub_reproducible = False + + if pip_attr.netrc: + whl_library_args["netrc"] = pip_attr.netrc + if pip_attr.auth_patterns: + whl_library_args["auth_patterns"] = pip_attr.auth_patterns + + # pip is not used to download wheels and the python `whl_library` helpers are only extracting things + whl_library_args.pop("extra_pip_args", None) + + # This is no-op because pip is not used to download the wheel. + whl_library_args.pop("download_only", None) + + repo_name = pip_repo_name(pip_name, distribution.filename, distribution.sha256) + whl_library_args["requirement"] = requirement.srcs.requirement + whl_library_args["urls"] = [distribution.url] + whl_library_args["sha256"] = distribution.sha256 + whl_library_args["filename"] = distribution.filename + whl_library_args["experimental_target_platforms"] = requirement.target_platforms + + # Pure python wheels or sdists may need to have a platform here + target_platforms = None + if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"): + if len(requirements) > 1: + target_platforms = requirement.target_platforms + + whl_library(name = repo_name, **dict(sorted(whl_library_args.items()))) + + whl_map[hub_name].setdefault(whl_name, []).append( + whl_alias( + repo = repo_name, + version = major_minor, + filename = distribution.filename, + target_platforms = target_platforms, + ), + ) + + if found_something: + continue + + requirement = select_requirement( + requirements, + platform = repository_platform, + ) + if not requirement: + # Sometimes the package is not present for host platform if there + # are whls specified only in particular requirements files, in that + # case just continue, however, if the download_only flag is set up, + # then the user can also specify the target platform of the wheel + # packages they want to download, in that case there will be always + # a requirement here, so we will not be in this code branch. + continue + elif get_index_urls: + logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line)) + + whl_library_args["requirement"] = requirement.requirement_line + if requirement.extra_pip_args: + whl_library_args["extra_pip_args"] = requirement.extra_pip_args # We sort so that the lock-file remains the same no matter the order of how the # args are manipulated in the code going before. + repo_name = "{}_{}".format(pip_name, whl_name) whl_library(name = repo_name, **dict(sorted(whl_library_args.items()))) whl_map[hub_name].setdefault(whl_name, []).append( whl_alias( repo = repo_name, version = major_minor, - # Call Label() to canonicalize because its used in a different context - config_setting = Label("//python/config_settings:is_python_" + major_minor), ), ) From 8a6bda59e95466c4922ca7c32392200d5c19c008 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 4 Jun 2024 11:29:41 +0900 Subject: [PATCH 11/18] fixup! feat(toolchain): use `py_linux_libc` flag value --- python/config_settings/transition.bzl | 19 +++++++++----- python/private/py_toolchain_suite.bzl | 26 ++++--------------- .../transition/multi_version_tests.bzl | 13 +++++++--- 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/python/config_settings/transition.bzl b/python/config_settings/transition.bzl index cd54b21956..48b0447ede 100644 --- a/python/config_settings/transition.bzl +++ b/python/config_settings/transition.bzl @@ -53,12 +53,14 @@ def _transition_py_impl(ctx): for file in target[DefaultInfo].default_runfiles.files.to_list(): if file.short_path == expected_target_path: zipfile = file - zipfile_symlink = ctx.actions.declare_file(ctx.attr.name + ".zip") - ctx.actions.symlink( - is_executable = True, - output = zipfile_symlink, - target_file = zipfile, - ) + + if zipfile: + zipfile_symlink = ctx.actions.declare_file(ctx.attr.name + ".zip") + ctx.actions.symlink( + is_executable = True, + output = zipfile_symlink, + target_file = zipfile, + ) env = {} for k, v in ctx.attr.env.items(): env[k] = ctx.expand_location(v) @@ -75,7 +77,10 @@ def _transition_py_impl(ctx): elif BuiltinPyRuntimeInfo in target: py_runtime_info = target[BuiltinPyRuntimeInfo] else: - fail("target {} does not have rules_python PyRuntimeInfo or builtin PyRuntimeInfo".format(target)) + fail( + "target {} does not have rules_python PyRuntimeInfo or builtin PyRuntimeInfo. ".format(target) + + "There is likely no toolchain being matched to your configuration, use --toolchain_resolution_debug parameter to get more information", + ) providers = [ DefaultInfo( diff --git a/python/private/py_toolchain_suite.bzl b/python/private/py_toolchain_suite.bzl index 69d9a7999e..174c36f782 100644 --- a/python/private/py_toolchain_suite.bzl +++ b/python/private/py_toolchain_suite.bzl @@ -15,7 +15,6 @@ """Create the toolchain defs in a BUILD.bazel file.""" load("@bazel_skylib//lib:selects.bzl", "selects") -load(":config_settings.bzl", "is_python_config_setting") load( ":toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE", @@ -42,36 +41,22 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth # toolchain file. if set_python_version_constraint in ["True", "False"]: major_minor, _, _ = python_version.rpartition(".") + python_versions = [major_minor, python_version] + if set_python_version_constraint == "False": + python_versions.append("") match_any = [] - for i, v in enumerate([major_minor, python_version]): + for i, v in enumerate(python_versions): name = "{prefix}_{python_version}_{i}".format( prefix = prefix, python_version = python_version, i = i, ) match_any.append(name) - if flag_values: - is_python_config_setting( - name = name, - python_version = v, - flag_values = flag_values, - visibility = ["//visibility:private"], - ) - else: - native.alias( - name = name, - actual = Label("//python/config_settings:is_python_%s" % v), - visibility = ["//visibility:private"], - ) - - if set_python_version_constraint == "False": - name = "{prefix}_version_setting_no_python_version".format(prefix = prefix) - match_any.append(name) native.config_setting( name = name, flag_values = flag_values | { - str(Label("//python/config_settings:python_version")): "", + Label("@rules_python//python/config_settings:python_version"): v, }, visibility = ["//visibility:private"], ) @@ -86,7 +71,6 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth match_any = match_any, visibility = ["//visibility:private"], ) - target_settings = [name] else: fail(("Invalid set_python_version_constraint value: got {} {}, wanted " + diff --git a/tests/config_settings/transition/multi_version_tests.bzl b/tests/config_settings/transition/multi_version_tests.bzl index 32f7209c9f..f3707dba20 100644 --- a/tests/config_settings/transition/multi_version_tests.bzl +++ b/tests/config_settings/transition/multi_version_tests.bzl @@ -16,9 +16,16 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:util.bzl", rt_util = "util") -load("//python:versions.bzl", "TOOL_VERSIONS") load("//python/config_settings:transition.bzl", py_binary_transitioned = "py_binary", py_test_transitioned = "py_test") +# NOTE @aignas 2024-06-04: we are using here something that is registered in the MODULE.Bazel +# and if you find tests failing, it could be because of the toolchain resolution issues here. +# +# If the toolchain is not resolved then you will have a weird message telling +# you that your transition target does not have a PyRuntime provider, which is +# caused by there not being a toolchain detected for the target. +_PYTHON_VERSION = "3.11" + _tests = [] def _test_py_test_with_transition(name): @@ -26,7 +33,7 @@ def _test_py_test_with_transition(name): py_test_transitioned, name = name + "_subject", srcs = [name + "_subject.py"], - python_version = TOOL_VERSIONS.keys()[0], + python_version = _PYTHON_VERSION, ) analysis_test( @@ -46,7 +53,7 @@ def _test_py_binary_with_transition(name): py_binary_transitioned, name = name + "_subject", srcs = [name + "_subject.py"], - python_version = TOOL_VERSIONS.keys()[0], + python_version = _PYTHON_VERSION, ) analysis_test( From 319409d75d04b0f3df3055e92fe37ec24b6829ed Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 4 Jun 2024 11:36:47 +0900 Subject: [PATCH 12/18] fixup! fixup! feat(toolchain): use `py_linux_libc` flag value --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84b7cc6443..a9680bf9b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,9 @@ A brief description of the categories of changes: [#1643](https://github.com/bazelbuild/rules_python/issues/1643). * (pip.parse): Install `yanked` packages and print a warning instead of ignoring them. This better matches the behaviour of `uv pip install`. +* (toolchains): Now matching of the default toolchain is more robust and explicit + and should fix rare edge-cases where the host toolchain autodetection would fail + due to some reasons. ### Added * (rules) Precompiling Python source at build time is available. but is From 65d8278e191d14c7d2dc0a36ad75ddd3bb123dfd Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:51:07 +0900 Subject: [PATCH 13/18] fixup! feat(internal): add a function to generate config settings for all whls --- python/private/pip_config_settings.bzl | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/python/private/pip_config_settings.bzl b/python/private/pip_config_settings.bzl index 211faabc6a..c930f3c4f8 100644 --- a/python/private/pip_config_settings.bzl +++ b/python/private/pip_config_settings.bzl @@ -81,7 +81,8 @@ def pip_config_settings( osx_versions = [], target_platforms = [], name = None, - visibility = None): + visibility = None, + config_setting_rule = None): """Generate all of the pip config settings. Args: @@ -98,6 +99,8 @@ def pip_config_settings( constraint values for each condition. visibility (list[str], optional): The visibility to be passed to the exposed labels. All other labels will be private. + config_setting_rule (rule): The config setting rule to use for creating the + objects. Can be overridden for unit tests reasons. """ glibc_versions = [""] + glibc_versions @@ -130,6 +133,7 @@ def pip_config_settings( name = "sdist" + suffix, constraint_values = constraint_values, visibility = visibility, + config_setting_rule = config_setting_rule, ) for python_version in python_versions: _sdist_config_setting( @@ -137,6 +141,7 @@ def pip_config_settings( python_version = python_version, constraint_values = constraint_values, visibility = visibility, + config_setting_rule = config_setting_rule, ) for python_version in [""] + python_versions: @@ -152,6 +157,7 @@ def pip_config_settings( constraint_values = constraint_values, python_version = python_version, visibility = visibility, + config_setting_rule = config_setting_rule, ) def _whl_config_settings(*, suffix, plat_flag_values, **kwargs): @@ -271,8 +277,9 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): return ret -def _whl_config_setting(*, name, flag_values, visibility, **kwargs): - _config_setting_or( +def _whl_config_setting(*, name, flag_values, visibility, config_setting_rule = None, **kwargs): + config_setting_rule = config_setting_rule or _config_setting_or + config_setting_rule( name = name, flag_values = flag_values | { FLAGS.pip_whl: UseWhlFlag.ONLY, @@ -285,8 +292,9 @@ def _whl_config_setting(*, name, flag_values, visibility, **kwargs): **kwargs ) -def _sdist_config_setting(*, name, visibility, **kwargs): - _config_setting_or( +def _sdist_config_setting(*, name, visibility, config_setting_rule = None, **kwargs): + config_setting_rule = config_setting_rule or _config_setting_or + config_setting_rule( name = name, flag_values = {FLAGS.pip_whl: UseWhlFlag.NO}, default = {FLAGS.pip_whl: UseWhlFlag.AUTO}, From bc1cdef455205d4602a296c9509f3a1a4d72d890 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:19:16 +0900 Subject: [PATCH 14/18] fixup! feat(internal): add a function to generate config settings for all whls --- python/private/pip_config_settings.bzl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/private/pip_config_settings.bzl b/python/private/pip_config_settings.bzl index c930f3c4f8..2fe3c87a10 100644 --- a/python/private/pip_config_settings.bzl +++ b/python/private/pip_config_settings.bzl @@ -82,6 +82,7 @@ def pip_config_settings( target_platforms = [], name = None, visibility = None, + alias_rule = None, config_setting_rule = None): """Generate all of the pip config settings. @@ -99,6 +100,8 @@ def pip_config_settings( constraint values for each condition. visibility (list[str], optional): The visibility to be passed to the exposed labels. All other labels will be private. + alias_rule (rule): The alias rule to use for creating the + objects. Can be overridden for unit tests reasons. config_setting_rule (rule): The config setting rule to use for creating the objects. Can be overridden for unit tests reasons. """ @@ -111,9 +114,11 @@ def pip_config_settings( for t in target_platforms ] + alias_rule = alias_rule or native.alias + for version in python_versions: is_python = "is_python_{}".format(version) - native.alias( + alias_rule( name = is_python, actual = Label("//python/config_settings:" + is_python), visibility = visibility, @@ -280,7 +285,7 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): def _whl_config_setting(*, name, flag_values, visibility, config_setting_rule = None, **kwargs): config_setting_rule = config_setting_rule or _config_setting_or config_setting_rule( - name = name, + name = "is_" + name, flag_values = flag_values | { FLAGS.pip_whl: UseWhlFlag.ONLY, }, @@ -295,7 +300,7 @@ def _whl_config_setting(*, name, flag_values, visibility, config_setting_rule = def _sdist_config_setting(*, name, visibility, config_setting_rule = None, **kwargs): config_setting_rule = config_setting_rule or _config_setting_or config_setting_rule( - name = name, + name = "is_" + name, flag_values = {FLAGS.pip_whl: UseWhlFlag.NO}, default = {FLAGS.pip_whl: UseWhlFlag.AUTO}, visibility = visibility, @@ -303,7 +308,6 @@ def _sdist_config_setting(*, name, visibility, config_setting_rule = None, **kwa ) def _config_setting_or(*, name, flag_values, default, visibility, **kwargs): - name = "is_" + name match_name = "_{}".format(name) default_name = "_{}_default".format(name) From 6cdb5348da9d2fd3abe5bc794b5d99cec0b6048f Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:19:51 +0900 Subject: [PATCH 15/18] fixup! feat(internal): support inferring the config settings from dist filenames --- .../render_pkg_aliases_test.bzl | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl index 653cf466c7..bb4b86fd77 100644 --- a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -16,6 +16,10 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +load( + "//python/private:pip_config_settings.bzl", + "pip_config_settings", +) # buildifier: disable=bzl-visibility load( "//python/private:render_pkg_aliases.bzl", "get_filename_config_settings", @@ -865,6 +869,56 @@ def _test_multiplatform_whl_aliases_filename_versioned(env): _tests.append(_test_multiplatform_whl_aliases_filename_versioned) +def _test_config_settings_exist(env): + for py_tag in ["py2.py3", "py3", "py311", "cp311"]: + if py_tag == "py2.py3": + abis = ["none"] + elif py_tag.startswith("py"): + abis = ["none", "abi3"] + else: + abis = ["none", "abi3", "cp311"] + + for abi_tag in abis: + for platform_tag, kwargs in { + "any": {}, + "manylinux_2_17_x86_64": { + "glibc_versions": [(2, 17), (2, 18)], + "target_platforms": ["linux_x86_64"], + }, + "manylinux_2_18_x86_64": { + "glibc_versions": [(2, 17), (2, 18)], + "target_platforms": ["linux_x86_64"], + }, + }.items(): + aliases = [ + whl_alias( + repo = "repo", + filename = "foo-0.0.1-{}-{}-{}.whl".format(py_tag, abi_tag, platform_tag), + version = "3.11", + ), + ] + available_config_settings = [] + mock_rule = lambda name, **kwargs: available_config_settings.append(name) + pip_config_settings( + python_versions = ["3.11"], + alias_rule = mock_rule, + config_setting_rule = mock_rule, + **kwargs + ) + + got_aliases = multiplatform_whl_aliases( + aliases = aliases, + default_version = None, + glibc_versions = kwargs.get("glibc_versions", []), + muslc_versions = kwargs.get("muslc_versions", []), + osx_versions = kwargs.get("osx_versions", []), + ) + got = [a.config_setting.partition(":")[-1] for a in got_aliases] + + env.expect.that_collection(available_config_settings).contains_at_least(got) + +_tests.append(_test_config_settings_exist) + def render_pkg_aliases_test_suite(name): """Create the test suite. From 445337e7d8d89936a69a0ae3eaffbcf9e53b8746 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:23:58 +0900 Subject: [PATCH 16/18] fixup! fixup! feat(internal): support inferring the config settings from dist filenames --- python/private/render_pkg_aliases.bzl | 2 +- .../render_pkg_aliases/render_pkg_aliases_test.bzl | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl index 2ddd65d1fc..4028a9a114 100644 --- a/python/private/render_pkg_aliases.bzl +++ b/python/private/render_pkg_aliases.bzl @@ -534,7 +534,7 @@ def get_filename_config_settings( elif parsed.python_tag.startswith("cp"): py = "cp3x" else: - py = parsed.python_tag + py = "py3" if parsed.abi_tag.startswith("cp"): abi = "cp" diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl index bb4b86fd77..bd9147c75e 100644 --- a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -881,6 +881,10 @@ def _test_config_settings_exist(env): for abi_tag in abis: for platform_tag, kwargs in { "any": {}, + "macosx_11_0_arm64": { + "osx_versions": [(11, 0)], + "target_platforms": ["osx_aarch64"], + }, "manylinux_2_17_x86_64": { "glibc_versions": [(2, 17), (2, 18)], "target_platforms": ["linux_x86_64"], @@ -889,6 +893,10 @@ def _test_config_settings_exist(env): "glibc_versions": [(2, 17), (2, 18)], "target_platforms": ["linux_x86_64"], }, + "musllinux_1_1_aarch64": { + "muslc_versions": [(1, 2), (1, 1), (1, 0)], + "target_platforms": ["linux_aarch64"], + }, }.items(): aliases = [ whl_alias( From 495758c8774f2d033769923feca8a7c902f44506 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Thu, 6 Jun 2024 16:57:22 +0900 Subject: [PATCH 17/18] doc: add a not to CHANGELOG on the naming of the repos --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdd1addaf2..81340160a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,11 @@ A brief description of the categories of changes: to allowing to correctly fetch the wheels for the right platform. See the updated docs on how to use the feature. This is work towards addressing [#735](https://github.com/bazelbuild/rules_python/issues/735) and - [#260](https://github.com/bazelbuild/rules_python/issues/260). + [#260](https://github.com/bazelbuild/rules_python/issues/260). The spoke + repository names when using this flag will have a structure of + `{pip_hub_prefix}_{wheel_name}_{py_tag}_{abi_tag}_{platform_tag}_{sha256}`, + which is an implementation detail which should not be relied on and is there + purely for better debugging experience. ### Fixed * (gazelle) Remove `visibility` from `NonEmptyAttr`. From 6dde58718011f15904487daa60e4fc67f532499d Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Thu, 6 Jun 2024 17:10:58 +0900 Subject: [PATCH 18/18] test: add an additional test to ensure that the aliases are the same --- python/private/render_pkg_aliases.bzl | 4 +--- .../render_pkg_aliases_test.bzl | 21 +++++++++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl index 4028a9a114..a37c32882e 100644 --- a/python/private/render_pkg_aliases.bzl +++ b/python/private/render_pkg_aliases.bzl @@ -410,9 +410,7 @@ pip_config_settings( python_versions = {python_versions}, target_platforms = {target_platforms}, visibility = ["//:__subpackages__"], -) - -""".format( +)""".format( glibc_versions = render.indent(render.list(glibc_versions)).lstrip(), muslc_versions = render.indent(render.list(muslc_versions)).lstrip(), osx_versions = render.indent(render.list(osx_versions)).lstrip(), diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl index bd9147c75e..a0689f70b9 100644 --- a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -25,6 +25,7 @@ load( "get_filename_config_settings", "get_whl_flag_versions", "multiplatform_whl_aliases", + "render_multiplatform_pkg_aliases", "render_pkg_aliases", "whl_alias", ) # buildifier: disable=bzl-visibility @@ -110,7 +111,8 @@ alias( _tests.append(_test_legacy_aliases) def _test_bzlmod_aliases(env): - actual = render_pkg_aliases( + # Use this function as it is used in pip_repository + actual = render_multiplatform_pkg_aliases( default_config_setting = "//:my_config_setting", aliases = { "bar-baz": [ @@ -178,13 +180,27 @@ alias( ), )""" + env.expect.that_str(actual.pop("_config/BUILD.bazel")).equals( + """\ +load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings") + +pip_config_settings( + name = "pip_config_settings", + glibc_versions = [], + muslc_versions = [], + osx_versions = [], + python_versions = ["3.2"], + target_platforms = [], + visibility = ["//:__subpackages__"], +)""", + ) env.expect.that_collection(actual.keys()).contains_exactly([want_key]) env.expect.that_str(actual[want_key]).equals(want_content) _tests.append(_test_bzlmod_aliases) def _test_bzlmod_aliases_with_no_default_version(env): - actual = render_pkg_aliases( + actual = render_multiplatform_pkg_aliases( default_config_setting = None, aliases = { "bar-baz": [ @@ -272,6 +288,7 @@ alias( ), )""" + actual.pop("_config/BUILD.bazel") env.expect.that_collection(actual.keys()).contains_exactly([want_key]) env.expect.that_str(actual[want_key]).equals(_normalize_label_strings(want_content))