From 89bec57f443b160433d2c6e15ddd874ba21d2ecd Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 20 Jun 2023 08:41:09 -0700 Subject: [PATCH 1/4] test: basic analysis tests for py_wheel (#1279) This adds some basic analysis tests of py_wheel to cover the functionality it's recently introduced. --- tests/py_wheel/BUILD.bazel | 18 ++++++ tests/py_wheel/py_wheel_tests.bzl | 99 +++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 tests/py_wheel/BUILD.bazel create mode 100644 tests/py_wheel/py_wheel_tests.bzl diff --git a/tests/py_wheel/BUILD.bazel b/tests/py_wheel/BUILD.bazel new file mode 100644 index 0000000000..d925bb9801 --- /dev/null +++ b/tests/py_wheel/BUILD.bazel @@ -0,0 +1,18 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for py_wheel.""" + +load(":py_wheel_tests.bzl", "py_wheel_test_suite") + +py_wheel_test_suite(name = "py_wheel_tests") diff --git a/tests/py_wheel/py_wheel_tests.bzl b/tests/py_wheel/py_wheel_tests.bzl new file mode 100644 index 0000000000..e580732aac --- /dev/null +++ b/tests/py_wheel/py_wheel_tests.bzl @@ -0,0 +1,99 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test for py_wheel.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python:packaging.bzl", "py_wheel") + +_tests = [] + +def _test_metadata(name): + rt_util.helper_target( + py_wheel, + name = name + "_subject", + distribution = "mydist_" + name, + version = "0.0.0", + ) + analysis_test( + name = name, + impl = _test_metadata_impl, + target = name + "_subject", + ) + +def _test_metadata_impl(env, target): + action = env.expect.that_target(target).action_generating( + "{package}/{name}.metadata.txt", + ) + action.content().split("\n").contains_exactly([ + env.expect.meta.format_str("Name: mydist_{test_name}"), + "Metadata-Version: 2.1", + "", + ]) + +_tests.append(_test_metadata) + +def _test_content_type_from_attr(name): + rt_util.helper_target( + py_wheel, + name = name + "_subject", + distribution = "mydist_" + name, + version = "0.0.0", + description_content_type = "text/x-rst", + ) + analysis_test( + name = name, + impl = _test_content_type_from_attr_impl, + target = name + "_subject", + ) + +def _test_content_type_from_attr_impl(env, target): + action = env.expect.that_target(target).action_generating( + "{package}/{name}.metadata.txt", + ) + action.content().split("\n").contains( + "Description-Content-Type: text/x-rst", + ) + +_tests.append(_test_content_type_from_attr) + +def _test_content_type_from_description(name): + rt_util.helper_target( + py_wheel, + name = name + "_subject", + distribution = "mydist_" + name, + version = "0.0.0", + description_file = "desc.md", + ) + analysis_test( + name = name, + impl = _test_content_type_from_description_impl, + target = name + "_subject", + ) + +def _test_content_type_from_description_impl(env, target): + action = env.expect.that_target(target).action_generating( + "{package}/{name}.metadata.txt", + ) + action.content().split("\n").contains( + "Description-Content-Type: text/markdown", + ) + +_tests.append(_test_content_type_from_description) + +def py_wheel_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) From 1a333cecd73af9916e89b1e1a33fed73d913eb49 Mon Sep 17 00:00:00 2001 From: Ivo List Date: Tue, 20 Jun 2023 19:36:39 +0200 Subject: [PATCH 2/4] fix: plugin_output in py_proto_library rule (#1280) plugin_output was wrong in case multiple repositories are involved and/or _virtual_imports. The code is taken from `cc_proto_library` and has been verified in practice. --- python/private/proto/py_proto_library.bzl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/private/proto/py_proto_library.bzl b/python/private/proto/py_proto_library.bzl index 988558500d..9377c8513c 100644 --- a/python/private/proto/py_proto_library.bzl +++ b/python/private/proto/py_proto_library.bzl @@ -75,12 +75,22 @@ def _py_proto_aspect_impl(target, ctx): name_mapper = lambda name: name.replace("-", "_").replace(".", "/"), ) + # Handles multiple repository and virtual import cases + proto_root = proto_info.proto_source_root + if proto_root.startswith(ctx.bin_dir.path): + plugin_output = proto_root + else: + plugin_output = ctx.bin_dir.path + "/" + proto_root + + if plugin_output == ".": + plugin_output = ctx.bin_dir.path + proto_common.compile( actions = ctx.actions, proto_info = proto_info, proto_lang_toolchain_info = proto_lang_toolchain_info, generated_files = generated_sources, - plugin_output = ctx.bin_dir.path, + plugin_output = plugin_output, ) # Generated sources == Python sources From cc435943e92a32200e7620e234f058cc828d6f6e Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 20 Jun 2023 12:58:30 -0700 Subject: [PATCH 3/4] cleanup: Typos, doc, and formatting cleanup in pip extension et al (#1275) * Corrects some typos in docs * Expands the user-facing documentation * Fixes errors having newlines in the middle of them * Renames some internal functions to be more self-describing. --- docs/pip_repository.md | 2 +- examples/bzlmod/MODULE.bazel | 16 ++-- python/extensions/pip.bzl | 122 +++++++++++++++++--------- python/pip_install/pip_repository.bzl | 2 +- 4 files changed, 92 insertions(+), 50 deletions(-) diff --git a/docs/pip_repository.md b/docs/pip_repository.md index 0e0fea358f..853605276f 100644 --- a/docs/pip_repository.md +++ b/docs/pip_repository.md @@ -10,7 +10,7 @@ pip_hub_repository_bzlmod(name, repo_mapping, repo_name, whl_library_alias_names) -A rule for bzlmod mulitple pip repository creation. Intended for private use only. +A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY. **ATTRIBUTES** diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index e09e029c3b..a61e09481d 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -34,16 +34,17 @@ python.toolchain( # You only need to load this repositories if you are using multiple Python versions. # See the tests folder for various examples on using multiple Python versions. # The names "python_3_9" and "python_3_10" are autmatically created by the repo -# rules based on the python_versions. +# rules based on the `python_version` arg values. use_repo(python, "python_3_10", "python_3_9", "python_aliases") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") -# For each pip setup we call pip.parse. We can pass in various options -# but typically we are passing in a requirements and an interpreter. -# If you provide the python_version we will attempt to determine -# the interpreter target automatically. Otherwise use python_interpreter_target -# to override the lookup. +# To fetch pip dependencies, use pip.parse. We can pass in various options, +# but typically we pass requirements and the Python version. The Python +# version must have been configured by a corresponding `python.toolchain()` +# call. +# Alternatively, `python_interpreter_target` can be used to directly specify +# the Python interpreter to run to resolve dependencies. pip.parse( hub_name = "pip", python_version = "3.9", @@ -57,7 +58,8 @@ pip.parse( requirements_windows = "//:requirements_windows_3_10.txt", ) -# we use the pip_39 repo because entry points are not yet supported. +# NOTE: The pip_39 repo is only used because the plain `@pip` repo doesn't +# yet support entry points; see https://github.com/bazelbuild/rules_python/issues/1262 use_repo(pip, "pip", "pip_39") bazel_dep(name = "other_module", version = "", repo_name = "our_other_module") diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index e4eb9b5236..5cad3b1ac0 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -27,21 +27,25 @@ load( ) load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") -def _create_pip(module_ctx, pip_attr, whl_map): +def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): python_interpreter_target = pip_attr.python_interpreter_target # if we do not have the python_interpreter set in the attributes # we programtically find it. + hub_name = pip_attr.hub_name if python_interpreter_target == None: python_name = "python_{}".format(pip_attr.python_version.replace(".", "_")) if python_name not in INTERPRETER_LABELS.keys(): - fail(""" -Unable to find '{}' in the list of interpreters please update your pip.parse call with the correct python name -""".format(pip_attr.python_name)) - + fail(( + "Unable to find interpreter for pip hub '{hub_name}' for " + + "python_version={version}: Make sure a corresponding " + + '`python.toolchain(python_version="{version}")` call exists' + ).format( + hub_name = hub_name, + version = pip_attr.python_version, + )) python_interpreter_target = INTERPRETER_LABELS[python_name] - hub_name = pip_attr.hub_name pip_name = hub_name + "_{}".format(pip_attr.python_version.replace(".", "")) requrements_lock = locked_requirements_label(module_ctx, pip_attr) @@ -93,10 +97,10 @@ Unable to find '{}' in the list of interpreters please update your pip.parse cal whl_map[hub_name][whl_name][pip_attr.python_version] = pip_name + "_" def _pip_impl(module_ctx): - """Implmentation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories. + """Implementation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories. - This implmentation iterates through all of the "pip.parse" calls and creates - different pip hubs repositories based on the "hub_name". Each of the + This implmentation iterates through all of the `pip.parse` calls and creates + different pip hub repositories based on the "hub_name". Each of the pip calls create spoke repos that uses a specific Python interpreter. In a MODULES.bazel file we have: @@ -115,13 +119,13 @@ def _pip_impl(module_ctx): ) - For instance we have a hub with the name of "pip". + For instance, we have a hub with the name of "pip". A repository named the following is created. It is actually called last when all of the pip spokes are collected. - @@rules_python~override~pip~pip - As show in the example code above we have the following. + As shown in the example code above we have the following. Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip". These definitions create two different pip spoke repositories that are related to the hub "pip". @@ -185,22 +189,32 @@ def _pip_impl(module_ctx): for mod in module_ctx.modules: for pip_attr in mod.tags.parse: - if pip_attr.hub_name in pip_hub_map: + hub_name = pip_attr.hub_name + if hub_name in pip_hub_map: # We cannot have two hubs with the same name in different # modules. - if pip_hub_map[pip_attr.hub_name].module_name != mod.name: - fail("""Unable to create pip with the hub_name '{}', same hub name - in a different module found.""".format(pip_attr.hub_name)) - - if pip_attr.python_version in pip_hub_map[pip_attr.hub_name].python_versions: - fail( - """Unable to create pip with the hub_name '{}', same hub name - using the same Python repo name '{}' found in module '{}'.""".format( - pip_attr.hub_name, - pip_attr.python_version, - mod.name, - ), - ) + if pip_hub_map[hub_name].module_name != mod.name: + fail(( + "Duplicate cross-module pip hub named '{hub}': pip hub " + + "names must be unique across modules. First defined " + + "by module '{first_module}', second attempted by " + + "module '{second_module}'" + ).format( + hub = hub_name, + first_module = pip_hub_map[hub_name].module_name, + second_module = mod.name, + )) + + if pip_attr.python_version in pip_hub_map[hub_name].python_versions: + fail(( + "Duplicate pip python version '{version}' for hub " + + "'{hub}' in module '{module}': the Python versions " + + "used for a hub must be unique" + ).format( + hub = hub_name, + module = mod.name, + version = pip_attr.python_version, + )) else: pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) else: @@ -209,17 +223,19 @@ def _pip_impl(module_ctx): python_versions = [pip_attr.python_version], ) - _create_pip(module_ctx, pip_attr, hub_whl_map) + _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, hub_whl_map) for hub_name, whl_map in hub_whl_map.items(): for whl_name, version_map in whl_map.items(): if DEFAULT_PYTHON_VERSION not in version_map: - fail( - """ -Unable to find the default python version in the version map, please update your requirements files -to include Python '{}'. -""".format(DEFAULT_PYTHON_VERSION), - ) + fail(( + "Default python version '{version}' missing in pip " + + "hub '{hub}': update your pip.parse() calls so that " + + 'includes `python_version = "{version}"`' + ).format( + version = DEFAULT_PYTHON_VERSION, + hub = hub_name, + )) # Create the alias repositories which contains different select # statements These select statements point to the different pip @@ -247,14 +263,34 @@ def _pip_parse_ext_attrs(): "hub_name": attr.string( mandatory = True, doc = """ -The unique hub name. Mulitple pip.parse calls that contain the same hub name, -create spokes for specific Python versions. +The name of the repo pip dependencies will be accessible from. + +This name must be unique between modules; unless your module is guaranteed to +always be the root module, it's highly recommended to include your module name +in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can +be used for shorter local names within your module. + +Within a module, the same `hub_name` can be specified to group different Python +versions of pip dependencies under one repository name. This allows using a +Python version-agnostic name when referring to pip dependencies; the +correct version will be automatically selected. + +Typically, a module will only have a single hub of pip dependencies, but this +is not required. Each hub is a separate resolution of pip dependencies. This +means if different programs need different versions of some library, separate +hubs can be created, and each program can use its respective hub's targets. +Targets from different hubs should not be used together. """, ), "python_version": attr.string( mandatory = True, doc = """ -The Python version for the pip spoke. +The Python version to use for resolving the pip dependencies. If not specified, +then the default Python version (as set by the root module or rules_python) +will be used. + +The version specified here must have a corresponding `python.toolchain()` +configured. """, ), }, **pip_repository_attrs) @@ -270,12 +306,16 @@ The Python version for the pip spoke. pip = module_extension( doc = """\ -This extension is used to create a pip hub and all of the spokes that are part of that hub. -We can have multiple different hubs, but we cannot have hubs that have the same name in -different modules. Each hub needs one or more spokes. A spoke contains a specific version -of Python, and the requirement(s) files that are unquie to that Python version. -In order to add more spokes you call this extension mulitiple times using the same hub -name. +This extension is used to make dependencies from pip available. + +To use, call `pip.parse()` and specify `hub_name` and your requirements file. +Dependencies will be downloaded and made available in a repo named after the +`hub_name` argument. + +Each `pip.parse()` call configures a particular Python version. Multiple calls +can be made to configure different Python versions, and will be grouped by +the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy` +to automatically resolve to different, Python version-specific, libraries. """, implementation = _pip_impl, tag_classes = { diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index f18b69bd97..866a834a72 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -363,7 +363,7 @@ pip_hub_repository_bzlmod_attrs = { pip_hub_repository_bzlmod = repository_rule( attrs = pip_hub_repository_bzlmod_attrs, - doc = """A rule for bzlmod mulitple pip repository creation. Intended for private use only.""", + doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", implementation = _pip_hub_repository_bzlmod_impl, ) From 00962c44d95c325abdd5abab0773feebfdbc13e2 Mon Sep 17 00:00:00 2001 From: Chris Love <335402+chrislovecnm@users.noreply.github.com> Date: Tue, 20 Jun 2023 15:38:38 -0600 Subject: [PATCH 4/4] feat: Upgrading gazelle and rules_go (#1283) Upgrading to the latest version of gazelle and rules_go. This should address `--incompatible_config_setting_private_default_visibility` flag. --- examples/build_file_generation/WORKSPACE | 14 ++++++++------ internal_deps.bzl | 12 ++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/examples/build_file_generation/WORKSPACE b/examples/build_file_generation/WORKSPACE index 65e0a6e5f3..7c74835870 100644 --- a/examples/build_file_generation/WORKSPACE +++ b/examples/build_file_generation/WORKSPACE @@ -17,22 +17,24 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # Define an http_archive rule that will download the below ruleset, # test the sha, and extract the ruleset to you local bazel cache. + http_archive( name = "io_bazel_rules_go", - sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa", + sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip", ], ) # Download the bazel_gazelle ruleset. + http_archive( name = "bazel_gazelle", - sha256 = "448e37e0dbf61d6fa8f00aaa12d191745e14f07c31cabfa731f0c8e8a4f41b97", + sha256 = "727f3e4edd96ea20c29e8c2ca9e8d2af724d8c7778e7923a854b2c80952bc405", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.30.0/bazel-gazelle-v0.30.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.30.0/bazel-gazelle-v0.30.0.tar.gz", ], ) diff --git a/internal_deps.bzl b/internal_deps.bzl index e4d2f69d41..dfaea3b139 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -63,20 +63,20 @@ def rules_python_internal_deps(): maybe( http_archive, name = "io_bazel_rules_go", - sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa", + sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip", ], ) maybe( http_archive, name = "bazel_gazelle", - sha256 = "efbbba6ac1a4fd342d5122cbdfdb82aeb2cf2862e35022c752eaddffada7c3f3", + sha256 = "727f3e4edd96ea20c29e8c2ca9e8d2af724d8c7778e7923a854b2c80952bc405", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.27.0/bazel-gazelle-v0.27.0.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.27.0/bazel-gazelle-v0.27.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.30.0/bazel-gazelle-v0.30.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.30.0/bazel-gazelle-v0.30.0.tar.gz", ], )