Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Nasty rebase I had to merge
Browse files Browse the repository at this point in the history
chrislovecnm committed Jun 20, 2023
2 parents 4a7b954 + 00962c4 commit 07dd164
Showing 9 changed files with 234 additions and 63 deletions.
2 changes: 1 addition & 1 deletion docs/pip_repository.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 8 additions & 6 deletions examples/build_file_generation/WORKSPACE
Original file line number Diff line number Diff line change
@@ -17,22 +17,24 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

# Define an http_archive rule that will download the below ruleset,
# test the sha, and extract the ruleset to you local bazel cache.

http_archive(
name = "io_bazel_rules_go",
sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa",
sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
"https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip",
"https://github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip",
],
)

# Download the bazel_gazelle ruleset.

http_archive(
name = "bazel_gazelle",
sha256 = "448e37e0dbf61d6fa8f00aaa12d191745e14f07c31cabfa731f0c8e8a4f41b97",
sha256 = "727f3e4edd96ea20c29e8c2ca9e8d2af724d8c7778e7923a854b2c80952bc405",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz",
"https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.30.0/bazel-gazelle-v0.30.0.tar.gz",
"https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.30.0/bazel-gazelle-v0.30.0.tar.gz",
],
)

16 changes: 9 additions & 7 deletions examples/bzlmod/MODULE.bazel
Original file line number Diff line number Diff line change
@@ -34,7 +34,7 @@ python.toolchain(
# You only need to load this repositories if you are using multiple Python versions.
# See the tests folder for various examples on using multiple Python versions.
# The names "python_3_9" and "python_3_10" are autmatically created by the repo
# rules based on the python_versions.
# rules based on the `python_version` arg values.
use_repo(python, "python_3_10", "python_3_9", "python_aliases")

# This extension allows a user to create modifications to how rules_python
@@ -77,11 +77,12 @@ write_file(
)
use_repo(pip, "whl_mods_hub")

# For each pip setup we call pip.parse. We can pass in various options
# but typically we are passing in a requirements and an interpreter.
# If you provide the python_version we will attempt to determine
# the interpreter target automatically. Otherwise use python_interpreter_target
# to override the lookup.
# To fetch pip dependencies, use pip.parse. We can pass in various options,
# but typically we pass requirements and the Python version. The Python
# version must have been configured by a corresponding `python.toolchain()`
# call.
# Alternatively, `python_interpreter_target` can be used to directly specify
# the Python interpreter to run to resolve dependencies.
pip.parse(
hub_name = "pip",
python_version = "3.9",
@@ -109,7 +110,8 @@ pip.parse(
},
)

# we use the pip_39 repo because entry points are not yet supported.
# NOTE: The pip_39 repo is only used because the plain `@pip` repo doesn't
# yet support entry points; see https://github.com/bazelbuild/rules_python/issues/1262
use_repo(pip, "pip", "pip_39")

bazel_dep(name = "other_module", version = "", repo_name = "our_other_module")
12 changes: 6 additions & 6 deletions internal_deps.bzl
Original file line number Diff line number Diff line change
@@ -63,20 +63,20 @@ def rules_python_internal_deps():
maybe(
http_archive,
name = "io_bazel_rules_go",
sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa",
sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
"https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip",
"https://github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip",
],
)

maybe(
http_archive,
name = "bazel_gazelle",
sha256 = "efbbba6ac1a4fd342d5122cbdfdb82aeb2cf2862e35022c752eaddffada7c3f3",
sha256 = "727f3e4edd96ea20c29e8c2ca9e8d2af724d8c7778e7923a854b2c80952bc405",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.27.0/bazel-gazelle-v0.27.0.tar.gz",
"https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.27.0/bazel-gazelle-v0.27.0.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.30.0/bazel-gazelle-v0.30.0.tar.gz",
"https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.30.0/bazel-gazelle-v0.30.0.tar.gz",
],
)

122 changes: 81 additions & 41 deletions python/extensions/pip.bzl
Original file line number Diff line number Diff line change
@@ -74,21 +74,25 @@ You cannot use both the additive_build_content and additive_build_content_file a
whl_mods = whl_mods,
)

def _create_pip(module_ctx, pip_attr, whl_map):
def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map):
python_interpreter_target = pip_attr.python_interpreter_target

# if we do not have the python_interpreter set in the attributes
# we programtically find it.
hub_name = pip_attr.hub_name
if python_interpreter_target == None:
python_name = "python_{}".format(pip_attr.python_version.replace(".", "_"))
if python_name not in INTERPRETER_LABELS.keys():
fail("""
Unable to find '{}' in the list of interpreters please update your pip.parse call with the correct python name
""".format(pip_attr.python_name))

fail((
"Unable to find interpreter for pip hub '{hub_name}' for " +
"python_version={version}: Make sure a corresponding " +
'`python.toolchain(python_version="{version}")` call exists'
).format(
hub_name = hub_name,
version = pip_attr.python_version,
))
python_interpreter_target = INTERPRETER_LABELS[python_name]

hub_name = pip_attr.hub_name
pip_name = hub_name + "_{}".format(pip_attr.python_version.replace(".", ""))
requrements_lock = locked_requirements_label(module_ctx, pip_attr)

@@ -149,10 +153,10 @@ Unable to find '{}' in the list of interpreters please update your pip.parse cal
whl_map[hub_name][whl_name][pip_attr.python_version] = pip_name + "_"

def _pip_impl(module_ctx):
"""Implmentation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories.
"""Implementation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories.
This implmentation iterates through all of the "pip.parse" calls and creates
different pip hubs repositories based on the "hub_name". Each of the
This implmentation iterates through all of the `pip.parse` calls and creates
different pip hub repositories based on the "hub_name". Each of the
pip calls create spoke repos that uses a specific Python interpreter.
In a MODULES.bazel file we have:
@@ -170,13 +174,13 @@ def _pip_impl(module_ctx):
requirements_windows = "//:requirements_windows_3_10.txt",
)
For instance we have a hub with the name of "pip".
For instance, we have a hub with the name of "pip".
A repository named the following is created. It is actually called last when
all of the pip spokes are collected.
- @@rules_python~override~pip~pip
As show in the example code above we have the following.
As shown in the example code above we have the following.
Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip".
These definitions create two different pip spoke repositories that are
related to the hub "pip".
@@ -247,22 +251,32 @@ def _pip_impl(module_ctx):

for mod in module_ctx.modules:
for pip_attr in mod.tags.parse:
if pip_attr.hub_name in pip_hub_map:
hub_name = pip_attr.hub_name
if hub_name in pip_hub_map:
# We cannot have two hubs with the same name in different
# modules.
if pip_hub_map[pip_attr.hub_name].module_name != mod.name:
fail("""Unable to create pip with the hub_name '{}', same hub name
in a different module found.""".format(pip_attr.hub_name))

if pip_attr.python_version in pip_hub_map[pip_attr.hub_name].python_versions:
fail(
"""Unable to create pip with the hub_name '{}', same hub name
using the same Python repo name '{}' found in module '{}'.""".format(
pip_attr.hub_name,
pip_attr.python_version,
mod.name,
),
)
if pip_hub_map[hub_name].module_name != mod.name:
fail((
"Duplicate cross-module pip hub named '{hub}': pip hub " +
"names must be unique across modules. First defined " +
"by module '{first_module}', second attempted by " +
"module '{second_module}'"
).format(
hub = hub_name,
first_module = pip_hub_map[hub_name].module_name,
second_module = mod.name,
))

if pip_attr.python_version in pip_hub_map[hub_name].python_versions:
fail((
"Duplicate pip python version '{version}' for hub " +
"'{hub}' in module '{module}': the Python versions " +
"used for a hub must be unique"
).format(
hub = hub_name,
module = mod.name,
version = pip_attr.python_version,
))
else:
pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
else:
@@ -271,17 +285,19 @@ def _pip_impl(module_ctx):
python_versions = [pip_attr.python_version],
)

_create_pip(module_ctx, pip_attr, hub_whl_map)
_create_versioned_pip_and_whl_repos(module_ctx, pip_attr, hub_whl_map)

for hub_name, whl_map in hub_whl_map.items():
for whl_name, version_map in whl_map.items():
if DEFAULT_PYTHON_VERSION not in version_map:
fail(
"""
Unable to find the default python version in the version map, please update your requirements files
to include Python '{}'.
""".format(DEFAULT_PYTHON_VERSION),
)
fail((
"Default python version '{version}' missing in pip " +
"hub '{hub}': update your pip.parse() calls so that " +
'includes `python_version = "{version}"`'
).format(
version = DEFAULT_PYTHON_VERSION,
hub = hub_name,
))

# Create the alias repositories which contains different select
# statements These select statements point to the different pip
@@ -309,14 +325,34 @@ def _pip_parse_ext_attrs():
"hub_name": attr.string(
mandatory = True,
doc = """
The unique hub name. Mulitple pip.parse calls that contain the same hub name,
create spokes for specific Python versions.
The name of the repo pip dependencies will be accessible from.
This name must be unique between modules; unless your module is guaranteed to
always be the root module, it's highly recommended to include your module name
in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can
be used for shorter local names within your module.
Within a module, the same `hub_name` can be specified to group different Python
versions of pip dependencies under one repository name. This allows using a
Python version-agnostic name when referring to pip dependencies; the
correct version will be automatically selected.
Typically, a module will only have a single hub of pip dependencies, but this
is not required. Each hub is a separate resolution of pip dependencies. This
means if different programs need different versions of some library, separate
hubs can be created, and each program can use its respective hub's targets.
Targets from different hubs should not be used together.
""",
),
"python_version": attr.string(
mandatory = True,
doc = """
The Python version for the pip spoke.
The Python version to use for resolving the pip dependencies. If not specified,
then the default Python version (as set by the root module or rules_python)
will be used.
The version specified here must have a corresponding `python.toolchain()`
configured.
""",
),
"whl_modifications": attr.label_keyed_string_dict(
@@ -390,13 +426,17 @@ otherwise it is best practice to just use one.""",

pip = module_extension(
doc = """\
This extension is used to make dependencies from pip available.
pip.parse:
This tag class is used to create a pip hub and all of the spokes that are part of that hub.
We can have multiple different hubs, but we cannot have hubs that have the same name in
different modules. Each hub needs one or more spokes. A spoke contains a specific version
of Python, and the requirement(s) files that are unquie to that Python version.
In order to add more spokes you call this extension mulitiple times using the same hub
name.
To use, call `pip.parse()` and specify `hub_name` and your requirements file.
Dependencies will be downloaded and made available in a repo named after the
`hub_name` argument.
Each `pip.parse()` call configures a particular Python version. Multiple calls
can be made to configure different Python versions, and will be grouped by
the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy`
to automatically resolve to different, Python version-specific, libraries.
pip.whl_mods:
This tag class is used to create different wheel modification JSON files. These files
2 changes: 1 addition & 1 deletion python/pip_install/pip_repository.bzl
Original file line number Diff line number Diff line change
@@ -363,7 +363,7 @@ pip_hub_repository_bzlmod_attrs = {

pip_hub_repository_bzlmod = repository_rule(
attrs = pip_hub_repository_bzlmod_attrs,
doc = """A rule for bzlmod mulitple pip repository creation. Intended for private use only.""",
doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""",
implementation = _pip_hub_repository_bzlmod_impl,
)

12 changes: 11 additions & 1 deletion python/private/proto/py_proto_library.bzl
Original file line number Diff line number Diff line change
@@ -75,12 +75,22 @@ def _py_proto_aspect_impl(target, ctx):
name_mapper = lambda name: name.replace("-", "_").replace(".", "/"),
)

# Handles multiple repository and virtual import cases
proto_root = proto_info.proto_source_root
if proto_root.startswith(ctx.bin_dir.path):
plugin_output = proto_root
else:
plugin_output = ctx.bin_dir.path + "/" + proto_root

if plugin_output == ".":
plugin_output = ctx.bin_dir.path

proto_common.compile(
actions = ctx.actions,
proto_info = proto_info,
proto_lang_toolchain_info = proto_lang_toolchain_info,
generated_files = generated_sources,
plugin_output = ctx.bin_dir.path,
plugin_output = plugin_output,
)

# Generated sources == Python sources
18 changes: 18 additions & 0 deletions tests/py_wheel/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Copyright 2023 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for py_wheel."""

load(":py_wheel_tests.bzl", "py_wheel_test_suite")

py_wheel_test_suite(name = "py_wheel_tests")
99 changes: 99 additions & 0 deletions tests/py_wheel/py_wheel_tests.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
# Copyright 2023 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test for py_wheel."""

load("@rules_testing//lib:analysis_test.bzl", "analysis_test", "test_suite")
load("@rules_testing//lib:util.bzl", rt_util = "util")
load("//python:packaging.bzl", "py_wheel")

_tests = []

def _test_metadata(name):
rt_util.helper_target(
py_wheel,
name = name + "_subject",
distribution = "mydist_" + name,
version = "0.0.0",
)
analysis_test(
name = name,
impl = _test_metadata_impl,
target = name + "_subject",
)

def _test_metadata_impl(env, target):
action = env.expect.that_target(target).action_generating(
"{package}/{name}.metadata.txt",
)
action.content().split("\n").contains_exactly([
env.expect.meta.format_str("Name: mydist_{test_name}"),
"Metadata-Version: 2.1",
"",
])

_tests.append(_test_metadata)

def _test_content_type_from_attr(name):
rt_util.helper_target(
py_wheel,
name = name + "_subject",
distribution = "mydist_" + name,
version = "0.0.0",
description_content_type = "text/x-rst",
)
analysis_test(
name = name,
impl = _test_content_type_from_attr_impl,
target = name + "_subject",
)

def _test_content_type_from_attr_impl(env, target):
action = env.expect.that_target(target).action_generating(
"{package}/{name}.metadata.txt",
)
action.content().split("\n").contains(
"Description-Content-Type: text/x-rst",
)

_tests.append(_test_content_type_from_attr)

def _test_content_type_from_description(name):
rt_util.helper_target(
py_wheel,
name = name + "_subject",
distribution = "mydist_" + name,
version = "0.0.0",
description_file = "desc.md",
)
analysis_test(
name = name,
impl = _test_content_type_from_description_impl,
target = name + "_subject",
)

def _test_content_type_from_description_impl(env, target):
action = env.expect.that_target(target).action_generating(
"{package}/{name}.metadata.txt",
)
action.content().split("\n").contains(
"Description-Content-Type: text/markdown",
)

_tests.append(_test_content_type_from_description)

def py_wheel_test_suite(name):
test_suite(
name = name,
tests = _tests,
)

0 comments on commit 07dd164

Please sign in to comment.