Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(pip.parse): make the pip extension reproducible if PyPI is not called #1937

Merged
merged 2 commits into from
Jun 6, 2024
Merged

fix(pip.parse): make the pip extension reproducible if PyPI is not called #1937

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9d35b35
fix(pip.parse): make the pip extension reproducible if PyPI is not ca…
aignas Jun 2, 2024
3fd6821
comment: address comments and set the internal pip extension to use P…
aignas Jun 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ A brief description of the categories of changes:
replaced by whl_modifications.
* (pip) Correctly select wheels when the python tag includes minor versions.
See ([#1930](https://github.com/bazelbuild/rules_python/issues/1930))
* (pip.parse): The lock file is now reproducible on any host platform if the
`experimental_index_url` is not used by any of the modules in the dependency
chain. To make the lock file identical on each `os` and `arch`, please use
the `experimental_index_url` feature which will fetch metadata from PyPI or a
different private index and write the contents to the lock file. Fixes
[#1643](https://github.com/bazelbuild/rules_python/issues/1643).

### Added
* (rules) Precompiling Python source at build time is available. but is
Expand Down
11 changes: 3 additions & 8 deletions MODULE.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,8 @@ register_toolchains("@pythons_hub//:all")
#####################
# Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.

pip = use_extension("//python/extensions:pip.bzl", "pip")
pip = use_extension("//python/private/bzlmod:pip.bzl", "pip_internal")
pip.parse(
experimental_index_url = "https://pypi.org/simple",
hub_name = "rules_python_publish_deps",
python_version = "3.11",
requirements_by_platform = {
Expand All @@ -80,13 +79,11 @@ bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_nam
bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle")

dev_pip = use_extension(
"//python/extensions:pip.bzl",
"pip",
"//python/private/bzlmod:pip.bzl",
"pip_internal",
dev_dependency = True,
)
dev_pip.parse(
envsubst = ["PIP_INDEX_URL"],
experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
experimental_requirement_cycles = {
"sphinx": [
"sphinx",
Expand All @@ -102,8 +99,6 @@ dev_pip.parse(
requirements_lock = "//docs/sphinx:requirements.txt",
)
dev_pip.parse(
envsubst = ["PIP_INDEX_URL"],
experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
hub_name = "pypiserver",
python_version = "3.11",
requirements_lock = "//examples/wheel:requirements_server.txt",
Expand Down
100 changes: 86 additions & 14 deletions python/private/bzlmod/pip.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache):
logger = repo_utils.logger(module_ctx)
python_interpreter_target = pip_attr.python_interpreter_target
is_hub_reproducible = True

# if we do not have the python_interpreter set in the attributes
# we programmatically find it.
Expand Down Expand Up @@ -274,6 +275,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s
logger.debug(lambda: "Selected: {}".format(distribution))

if distribution:
is_hub_reproducible = False
whl_library_args["requirement"] = requirement.srcs.requirement
whl_library_args["urls"] = [distribution.url]
whl_library_args["sha256"] = distribution.sha256
Expand Down Expand Up @@ -303,6 +305,8 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s
),
)

return is_hub_reproducible

def _pip_impl(module_ctx):
"""Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.

Expand Down Expand Up @@ -412,6 +416,7 @@ def _pip_impl(module_ctx):
hub_group_map = {}

simpleapi_cache = {}
is_extension_reproducible = True

for mod in module_ctx.modules:
for pip_attr in mod.tags.parse:
Expand Down Expand Up @@ -448,7 +453,8 @@ def _pip_impl(module_ctx):
else:
pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)

_create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
is_extension_reproducible = is_extension_reproducible and is_hub_reproducible

for hub_name, whl_map in hub_whl_map.items():
pip_repository(
Expand All @@ -462,7 +468,34 @@ def _pip_impl(module_ctx):
groups = hub_group_map.get(hub_name),
)

def _pip_parse_ext_attrs():
if bazel_features.external_deps.extension_metadata_has_reproducible:
# If we are not using the `experimental_index_url feature, the extension is fully
# deterministic and we don't need to create a lock entry for it.
#
# In order to be able to dogfood the `experimental_index_url` feature before it gets
# stabilized, we have created the `_pip_non_reproducible` function, that will result
# in extra entries in the lock file.
return module_ctx.extension_metadata(reproducible = is_extension_reproducible)
else:
return None

def _pip_non_reproducible(module_ctx):
_pip_impl(module_ctx)

# We default to calling the PyPI index and that will go into the
# MODULE.bazel.lock file, hence return nothing here.
return None

def _pip_parse_ext_attrs(**kwargs):
"""Get the attributes for the pip extension.

Args:
**kwargs: A kwarg for setting defaults for the specific attributes. The
key is expected to be the same as the attribute key.

Returns:
A dict of attributes.
"""
attrs = dict({
"experimental_extra_index_urls": attr.string_list(
doc = """\
Expand All @@ -477,6 +510,7 @@ This is equivalent to `--extra-index-urls` `pip` option.
default = [],
),
"experimental_index_url": attr.string(
default = kwargs.get("experimental_index_url", ""),
doc = """\
The index URL to use for downloading wheels using bazel downloader. This value is going
to be subject to `envsubst` substitutions if necessary.
Expand Down Expand Up @@ -661,17 +695,6 @@ Apply any overrides (e.g. patches) to a given Python distribution defined by
other tags in this extension.""",
)

def _extension_extra_args():
args = {}

if bazel_features.external_deps.module_extension_has_os_arch_dependent:
args = args | {
"arch_dependent": True,
"os_dependent": True,
}

return args

pip = module_extension(
doc = """\
This extension is used to make dependencies from pip available.
Expand Down Expand Up @@ -714,7 +737,56 @@ extension.
""",
),
},
**_extension_extra_args()
)

pip_internal = module_extension(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This needs to declare os/arch dependence.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not necessarily, since we are calling PyPI to get the right dependencies, the lock file will be the same on all platforms, so this can be removed, I'll change it so that it is the default behaviour.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice!

doc = """\
This extension is used to make dependencies from pypi available.

For now this is intended to be used internally so that usage of the `pip`
extension in `rules_python` does not affect the evaluations of the extension
for the consumers.

pip.parse:
To use, call `pip.parse()` and specify `hub_name` and your requirements file.
Dependencies will be downloaded and made available in a repo named after the
`hub_name` argument.

Each `pip.parse()` call configures a particular Python version. Multiple calls
can be made to configure different Python versions, and will be grouped by
the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy`
to automatically resolve to different, Python version-specific, libraries.

pip.whl_mods:
This tag class is used to help create JSON files to describe modifications to
the BUILD files for wheels.
""",
implementation = _pip_non_reproducible,
tag_classes = {
"override": _override_tag,
"parse": tag_class(
attrs = _pip_parse_ext_attrs(
experimental_index_url = "https://pypi.org/simple",
),
doc = """\
This tag class is used to create a pypi hub and all of the spokes that are part of that hub.
This tag class reuses most of the pypi attributes that are found in
@rules_python//python/pip_install:pip_repository.bzl.
The exception is it does not use the arg 'repo_prefix'. We set the repository
prefix for the user and the alias arg is always True in bzlmod.
""",
),
"whl_mods": tag_class(
attrs = _whl_mod_attrs(),
doc = """\
This tag class is used to create JSON file that are used when calling wheel_builder.py. These
JSON files contain instructions on how to modify a wheel's project. Each of the attributes
create different modifications based on the type of attribute. Previously to bzlmod these
JSON files where referred to as annotations, and were renamed to whl_modifications in this
extension.
""",
),
},
)

def _whl_mods_repo_impl(rctx):
Expand Down
Loading