diff --git a/CHANGELOG.md b/CHANGELOG.md index b65e52658e..6612da58d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,11 @@ A brief description of the categories of changes: Implemented the use of `pypi/stdlib-list` for standard library module verification. * (pip.parse): Do not ignore yanked packages when using `experimental_index_url`. This is to mimic what `uv` is doing. We will print a warning instead. -* (pip.parse): Add references to all supported wheels when using `experimental_index_url`. +* (pip.parse): Add references to all supported wheels when using `experimental_index_url` + to allowing to correctly fetch the wheels for the right platform. See the + updated docs on how to use the feature. This is work towards addressing + [#735](https://github.com/bazelbuild/rules_python/issues/735) and + [#260](https://github.com/bazelbuild/rules_python/issues/260). ### Fixed * (gazelle) Remove `visibility` from `NonEmptyAttr`. @@ -53,6 +57,12 @@ A brief description of the categories of changes: replaced by whl_modifications. * (pip) Correctly select wheels when the python tag includes minor versions. See ([#1930](https://github.com/bazelbuild/rules_python/issues/1930)) +* (pip.parse): The lock file is now reproducible on any host platform if the + `experimental_index_url` is not used by any of the modules in the dependency + chain. To make the lock file identical on each `os` and `arch`, please use + the `experimental_index_url` feature which will fetch metadata from PyPI or a + different private index and write the contents to the lock file. Fixes + [#1643](https://github.com/bazelbuild/rules_python/issues/1643). ### Added * (rules) Precompiling Python source at build time is available. but is diff --git a/MODULE.bazel b/MODULE.bazel index a9f405071f..7aa053ff17 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -56,9 +56,10 @@ register_toolchains("@pythons_hub//:all") ##################### # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it. -pip = use_extension("//python/extensions:pip.bzl", "pip") +pip = use_extension("//python/private/bzlmod:pip.bzl", "pip_internal") pip.parse( - experimental_index_url = "https://pypi.org/simple", + envsubst = ["PIP_INDEX_URL"], + experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}", hub_name = "rules_python_publish_deps", python_version = "3.11", requirements_by_platform = { @@ -80,8 +81,8 @@ bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_nam bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle") dev_pip = use_extension( - "//python/extensions:pip.bzl", - "pip", + "//python/private/bzlmod:pip.bzl", + "pip_internal", dev_dependency = True, ) dev_pip.parse( diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl index 75dbc47505..8ce01ef165 100644 --- a/python/private/bzlmod/pip.bzl +++ b/python/private/bzlmod/pip.bzl @@ -103,6 +103,7 @@ You cannot use both the additive_build_content and additive_build_content_file a def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache): logger = repo_utils.logger(module_ctx) python_interpreter_target = pip_attr.python_interpreter_target + is_hub_reproducible = True # if we do not have the python_interpreter set in the attributes # we programmatically find it. @@ -240,7 +241,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s ) whl_library_args.update({k: v for k, (v, default) in maybe_args_with_default.items() if v == default}) - if pip_attr.experimental_index_url: + if get_index_urls: # TODO @aignas 2024-05-26: move to a separate function found_something = False for requirement in requirements: @@ -250,6 +251,8 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s continue found_something = True + is_hub_reproducible = False + if pip_attr.netrc: whl_library_args["netrc"] = pip_attr.netrc if pip_attr.auth_patterns: @@ -300,7 +303,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s # packages they want to download, in that case there will be always # a requirement here, so we will not be in this code branch. continue - elif pip_attr.experimental_index_url: + elif get_index_urls: logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line)) whl_library_args["requirement"] = requirement.requirement_line @@ -318,6 +321,8 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s ), ) + return is_hub_reproducible + def _pip_impl(module_ctx): """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories. @@ -427,6 +432,7 @@ def _pip_impl(module_ctx): hub_group_map = {} simpleapi_cache = {} + is_extension_reproducible = True for mod in module_ctx.modules: for pip_attr in mod.tags.parse: @@ -463,7 +469,8 @@ def _pip_impl(module_ctx): else: pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) - _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache) + is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache) + is_extension_reproducible = is_extension_reproducible and is_hub_reproducible for hub_name, whl_map in hub_whl_map.items(): pip_repository( @@ -477,6 +484,20 @@ def _pip_impl(module_ctx): groups = hub_group_map.get(hub_name), ) + if bazel_features.external_deps.extension_metadata_has_reproducible: + return module_ctx.extension_metadata(reproducible = is_extension_reproducible) + else: + return None + +def _pip_non_reproducible(module_ctx): + _pip_impl(module_ctx) + + if bazel_features.external_deps.extension_metadata_has_reproducible: + # We allow for calling the PyPI index and that will go into the MODULE.bazel.lock file + return module_ctx.extension_metadata(reproducible = False) + else: + return None + def _pip_parse_ext_attrs(): attrs = dict({ "experimental_extra_index_urls": attr.string_list( @@ -676,17 +697,6 @@ Apply any overrides (e.g. patches) to a given Python distribution defined by other tags in this extension.""", ) -def _extension_extra_args(): - args = {} - - if bazel_features.external_deps.module_extension_has_os_arch_dependent: - args = args | { - "arch_dependent": True, - "os_dependent": True, - } - - return args - pip = module_extension( doc = """\ This extension is used to make dependencies from pip available. @@ -729,7 +739,54 @@ extension. """, ), }, - **_extension_extra_args() +) + +pip_internal = module_extension( + doc = """\ +This extension is used to make dependencies from pypi available. + +For now this is intended to be used internally so that usage of the `pip` +extension in `rules_python` does not affect the evaluations of the extension +for the consumers. + +pip.parse: +To use, call `pip.parse()` and specify `hub_name` and your requirements file. +Dependencies will be downloaded and made available in a repo named after the +`hub_name` argument. + +Each `pip.parse()` call configures a particular Python version. Multiple calls +can be made to configure different Python versions, and will be grouped by +the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy` +to automatically resolve to different, Python version-specific, libraries. + +pip.whl_mods: +This tag class is used to help create JSON files to describe modifications to +the BUILD files for wheels. +""", + implementation = _pip_non_reproducible, + tag_classes = { + "override": _override_tag, + "parse": tag_class( + attrs = _pip_parse_ext_attrs(), + doc = """\ +This tag class is used to create a pypi hub and all of the spokes that are part of that hub. +This tag class reuses most of the pypi attributes that are found in +@rules_python//python/pip_install:pip_repository.bzl. +The exception is it does not use the arg 'repo_prefix'. We set the repository +prefix for the user and the alias arg is always True in bzlmod. +""", + ), + "whl_mods": tag_class( + attrs = _whl_mod_attrs(), + doc = """\ +This tag class is used to create JSON file that are used when calling wheel_builder.py. These +JSON files contain instructions on how to modify a wheel's project. Each of the attributes +create different modifications based on the type of attribute. Previously to bzlmod these +JSON files where referred to as annotations, and were renamed to whl_modifications in this +extension. +""", + ), + }, ) def _whl_mods_repo_impl(rctx):