From 426ca8a34a2871cbc777450b6d0c347a158217d7 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 13 Sep 2024 22:56:59 -0700 Subject: [PATCH] Move ProtoInfo and ProtoLangToolchainInfo from Bazel The change is no-op for Bazel < 8, it always falls back to native providers. When we cherry-pick --incompatible_autoload_externally to older Bazels, ProtoInfo can be replaced with Starlark implementation, providing that users set the flag so that there is no second implementation exposed from Bazel. PiperOrigin-RevId: 674561141 --- MODULE.bazel | 2 +- bazel/common/proto_info.bzl | 6 +- bazel/common/proto_lang_toolchain_info.bzl | 25 ++- bazel/private/native.bzl | 2 - bazel/private/proto_bazel_features.bzl | 3 + bazel/private/proto_info.bzl | 186 +++++++++++++++++++++ bazel/tests/testdata/BUILD | 2 + build_defs/java_opts.bzl | 1 + conformance/BUILD.bazel | 1 + conformance/test_protos/BUILD.bazel | 3 +- editions/BUILD | 2 + upb_generator/c/BUILD | 1 + upb_generator/minitable/BUILD | 1 + upb_generator/reflection/BUILD | 1 + 14 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 bazel/private/proto_info.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 4f6bc7e7c4dd..32d4b65d542f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -85,7 +85,7 @@ bazel_dep( bazel_dep( name = "bazel_features", - version = "1.16.0", + version = "1.17.0", repo_name = "proto_bazel_features", ) diff --git a/bazel/common/proto_info.bzl b/bazel/common/proto_info.bzl index 19fb7294e56c..887e1cef4e79 100644 --- a/bazel/common/proto_info.bzl +++ b/bazel/common/proto_info.bzl @@ -1,5 +1,7 @@ """ProtoInfo""" -load("//bazel/private:native.bzl", "NativeProtoInfo") +load("@proto_bazel_features//:features.bzl", "bazel_features") +load("//bazel/private:proto_info.bzl", _ProtoInfo = "ProtoInfo") # buildifier: disable=bzl-visibility -ProtoInfo = NativeProtoInfo +# This resolves to Starlark ProtoInfo in Bazel 8 or with --incompatible_enable_autoload flag +ProtoInfo = getattr(bazel_features.globals, "ProtoInfo", None) or _ProtoInfo diff --git a/bazel/common/proto_lang_toolchain_info.bzl b/bazel/common/proto_lang_toolchain_info.bzl index 1eac535551e0..bc083468ad53 100644 --- a/bazel/common/proto_lang_toolchain_info.bzl +++ b/bazel/common/proto_lang_toolchain_info.bzl @@ -1,5 +1,26 @@ """ProtoLangToolchainInfo""" -load("//bazel/private:native.bzl", "native_proto_common") +load("//bazel/private:native.bzl", "native_proto_common") # buildifier: disable=bzl-visibility -ProtoLangToolchainInfo = native_proto_common.ProtoLangToolchainInfo +# Use Starlark implementation only if native_proto_common.ProtoLangToolchainInfo doesn't exist +ProtoLangToolchainInfo = getattr(native_proto_common, "ProtoLangToolchainInfo", provider( + doc = """Specifies how to generate language-specific code from .proto files. + Used by LANG_proto_library rules.""", + fields = dict( + out_replacement_format_flag = """(str) Format string used when passing output to the plugin + used by proto compiler.""", + output_files = """("single","multiple","legacy") Format out_replacement_format_flag with + a path to single file or a directory in case of multiple files.""", + plugin_format_flag = "(str) Format string used when passing plugin to proto compiler.", + plugin = "(FilesToRunProvider) Proto compiler plugin.", + runtime = "(Target) Runtime.", + provided_proto_sources = "(list[File]) Proto sources provided by the toolchain.", + proto_compiler = "(FilesToRunProvider) Proto compiler.", + protoc_opts = "(list[str]) Options to pass to proto compiler.", + progress_message = "(str) Progress message to set on the proto compiler action.", + mnemonic = "(str) Mnemonic to set on the proto compiler action.", + allowlist_different_package = """(Target) Allowlist to create lang_proto_library in a + different package than proto_library""", + toolchain_type = """(Label) Toolchain type that was used to obtain this info""", + ), +)) diff --git a/bazel/private/native.bzl b/bazel/private/native.bzl index 56856e3720c8..87d980c62484 100644 --- a/bazel/private/native.bzl +++ b/bazel/private/native.bzl @@ -1,5 +1,3 @@ """Renames toplevel symbols so they can be exported in Starlark under the same name""" -NativeProtoInfo = ProtoInfo - native_proto_common = proto_common_do_not_use diff --git a/bazel/private/proto_bazel_features.bzl b/bazel/private/proto_bazel_features.bzl index 3b9e22f20878..f839a10c5fd4 100644 --- a/bazel/private/proto_bazel_features.bzl +++ b/bazel/private/proto_bazel_features.bzl @@ -16,6 +16,7 @@ _PROTO_BAZEL_FEATURES = """bazel_features = struct( ), globals = struct( PackageSpecificationInfo = {PackageSpecificationInfo}, + ProtoInfo = getattr(getattr(native, 'legacy_globals', None), 'ProtoInfo', {ProtoInfo}) ), ) """ @@ -33,6 +34,7 @@ def _proto_bazel_features_impl(rctx): PackageSpecificationInfo = major_version_int > 6 or (major_version_int == 6 and minor_version_int >= 4) protobuf_on_allowlist = major_version_int > 7 + ProtoInfo = "ProtoInfo" if major_version_int < 8 else "None" rctx.file("BUILD.bazel", """ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") @@ -47,6 +49,7 @@ exports_files(["features.bzl"]) starlark_proto_info = repr(starlark_proto_info), PackageSpecificationInfo = "PackageSpecificationInfo" if PackageSpecificationInfo else "None", protobuf_on_allowlist = repr(protobuf_on_allowlist), + ProtoInfo = ProtoInfo, )) proto_bazel_features = repository_rule( diff --git a/bazel/private/proto_info.bzl b/bazel/private/proto_info.bzl new file mode 100644 index 000000000000..aa6f60bed7b7 --- /dev/null +++ b/bazel/private/proto_info.bzl @@ -0,0 +1,186 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2024 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +# +""" +Definition of ProtoInfo provider. +""" + +_warning = """ Don't use this field. It's intended for internal use and will be changed or removed + without warning.""" + +def _uniq(iterable): + unique_elements = {element: None for element in iterable} + return list(unique_elements.keys()) + +def _join(*path): + return "/".join([p for p in path if p != ""]) + +def _empty_to_dot(path): + return path if path else "." + +def _from_root(root, repo, relpath): + """Constructs an exec path from root to relpath""" + if not root: + # `relpath` is a directory with an input source file, the exec path is one of: + # - when in main repo: `package/path` + # - when in a external repository: `external/repo/package/path` + # - with sibling layout: `../repo/package/path` + return _join(repo, relpath) + else: + # `relpath` is a directory with a generated file or an output directory: + # - when in main repo: `{root}/package/path` + # - when in an external repository: `{root}/external/repo/package/path` + # - with sibling layout: `{root}/package/path` + return _join(root, "" if repo.startswith("../") else repo, relpath) + +def _create_proto_info(*, srcs, deps, descriptor_set, proto_path = "", workspace_root = "", bin_dir = None, allow_exports = None): + """Constructs ProtoInfo. + + Args: + srcs: ([File]) List of .proto files (possibly under _virtual path) + deps: ([ProtoInfo]) List of dependencies + descriptor_set: (File) Descriptor set for this Proto + proto_path: (str) Path that should be stripped from files in srcs. When + stripping is needed, the files should be symlinked into `_virtual_imports/target_name` + directory. Only such paths are accepted. + workspace_root: (str) Set to ctx.workspace_root if this is not the main repository. + bin_dir: (str) Set to ctx.bin_dir if _virtual_imports are used. + allow_exports: (Target) The packages where this proto_library can be exported. + + Returns: + (ProtoInfo) + """ + + # Validate parameters + src_prefix = _join(workspace_root.replace("external/", "../"), proto_path) + for src in srcs: + if type(src) != "File": + fail("srcs parameter expects a list of Files") + if src.owner.workspace_root != workspace_root: + fail("srcs parameter expects all files to have the same workspace_root: ", workspace_root) + if not src.short_path.startswith(src_prefix): + fail("srcs parameter expects all files start with %s" % src_prefix) + if type(descriptor_set) != "File": + fail("descriptor_set parameter expected to be a File") + if proto_path: + if "_virtual_imports/" not in proto_path: + fail("proto_path needs to contain '_virtual_imports' directory") + if proto_path.split("/")[-2] != "_virtual_imports": + fail("proto_path needs to be formed like '_virtual_imports/target_name'") + if not bin_dir: + fail("bin_dir parameter should be set when _virtual_imports are used") + + direct_proto_sources = srcs + transitive_proto_sources = depset( + direct = direct_proto_sources, + transitive = [dep._transitive_proto_sources for dep in deps], + order = "preorder", + ) + transitive_sources = depset( + direct = srcs, + transitive = [dep.transitive_sources for dep in deps], + order = "preorder", + ) + + # There can be up more than 1 direct proto_paths, for example when there's + # a generated and non-generated .proto file in srcs + root_paths = _uniq([src.root.path for src in srcs]) + transitive_proto_path = depset( + direct = [_empty_to_dot(_from_root(root, workspace_root, proto_path)) for root in root_paths], + transitive = [dep.transitive_proto_path for dep in deps], + ) + + if srcs: + check_deps_sources = depset(direct = srcs) + else: + check_deps_sources = depset(transitive = [dep.check_deps_sources for dep in deps]) + + transitive_descriptor_sets = depset( + direct = [descriptor_set], + transitive = [dep.transitive_descriptor_sets for dep in deps], + ) + + # Layering checks. + if srcs: + exported_sources = depset(direct = direct_proto_sources) + else: + exported_sources = depset(transitive = [dep._exported_sources for dep in deps]) + + if "_virtual_imports/" in proto_path: + #TODO: remove bin_dir from proto_source_root (when users assuming it's there are migrated) + proto_source_root = _empty_to_dot(_from_root(bin_dir, workspace_root, proto_path)) + elif workspace_root.startswith("../"): + proto_source_root = proto_path + else: + proto_source_root = _empty_to_dot(_join(workspace_root, proto_path)) + + proto_info = dict( + direct_sources = srcs, + transitive_sources = transitive_sources, + direct_descriptor_set = descriptor_set, + transitive_descriptor_sets = transitive_descriptor_sets, + proto_source_root = proto_source_root, + transitive_proto_path = transitive_proto_path, + check_deps_sources = check_deps_sources, + transitive_imports = transitive_sources, + _direct_proto_sources = direct_proto_sources, + _transitive_proto_sources = transitive_proto_sources, + _exported_sources = exported_sources, + ) + if allow_exports: + proto_info["allow_exports"] = allow_exports + return proto_info + +ProtoInfo, _ = provider( + doc = "Encapsulates information provided by a `proto_library.`", + fields = { + "direct_sources": "(list[File]) The `.proto` source files from the `srcs` attribute.", + "transitive_sources": """(depset[File]) The `.proto` source files from this rule and all + its dependent protocol buffer rules.""", + "direct_descriptor_set": """(File) The descriptor set of the direct sources. If no srcs, + contains an empty file.""", + "transitive_descriptor_sets": """(depset[File]) A set of descriptor set files of all + dependent `proto_library` rules, and this one's. This is not the same as passing + --include_imports to proto-compiler. Will be empty if no dependencies.""", + "proto_source_root": """(str) The directory relative to which the `.proto` files defined in + the `proto_library` are defined. For example, if this is `a/b` and the rule has the + file `a/b/c/d.proto` as a source, that source file would be imported as + `import c/d.proto` + + In principle, the `proto_source_root` directory itself should always + be relative to the output directory (`ctx.bin_dir`). + + This is at the moment not true for `proto_libraries` using (additional and/or strip) + import prefixes. `proto_source_root` is in this case prefixed with the output + directory. For example, the value is similar to + `bazel-out/k8-fastbuild/bin/a/_virtual_includes/b` for an input file in + `a/_virtual_includes/b/c.proto` that should be imported as `c.proto`. + + When using the value please account for both cases in a general way. + That is assume the value is either prefixed with the output directory or not. + This will make it possible to fix `proto_library` in the future. + """, + "transitive_proto_path": """(depset(str) A set of `proto_source_root`s collected from the + transitive closure of this rule.""", + "check_deps_sources": """(depset[File]) The `.proto` sources from the 'srcs' attribute. + If the library is a proxy library that has no sources, it contains the + `check_deps_sources` from this library's direct deps.""", + "allow_exports": """(Target) The packages where this proto_library can be exported.""", + + # Deprecated fields: + "transitive_imports": """(depset[File]) Deprecated: use `transitive_sources` instead.""", + + # Internal fields: + "_direct_proto_sources": """(list[File]) The `ProtoSourceInfo`s from the `srcs` + attribute.""" + _warning, + "_transitive_proto_sources": """(depset[File]) The `ProtoSourceInfo`s from this + rule and all its dependent protocol buffer rules.""" + _warning, + "_exported_sources": """(depset[File]) A set of `ProtoSourceInfo`s that may be + imported by another `proto_library` depending on this one.""" + _warning, + }, + init = _create_proto_info, +) diff --git a/bazel/tests/testdata/BUILD b/bazel/tests/testdata/BUILD index d5cdd6bebf0b..a49aa84393c6 100644 --- a/bazel/tests/testdata/BUILD +++ b/bazel/tests/testdata/BUILD @@ -1,3 +1,5 @@ +load("//bazel/toolchains:proto_lang_toolchain.bzl", "proto_lang_toolchain") + package( default_applicable_licenses = ["//:license"], default_visibility = ["//visibility:public"], diff --git a/build_defs/java_opts.bzl b/build_defs/java_opts.bzl index b5d338494ec6..8d9e92df2894 100644 --- a/build_defs/java_opts.bzl +++ b/build_defs/java_opts.bzl @@ -17,6 +17,7 @@ BUNDLE_LICENSE = "https://opensource.org/licenses/BSD-3-Clause" def protobuf_java_export(**kwargs): java_export( javacopts = JAVA_OPTS, + # https://github.com/bazelbuild/rules_jvm_external/issues/1245 javadocopts = [ "-notimestamp", "-use", diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel index 386a1c9c7984..9d7938d75db9 100644 --- a/conformance/BUILD.bazel +++ b/conformance/BUILD.bazel @@ -12,6 +12,7 @@ load("//:protobuf.bzl", "internal_csharp_proto_library", "internal_objc_proto_li load("//bazel:cc_proto_library.bzl", "cc_proto_library") load("//bazel:java_lite_proto_library.bzl", "java_lite_proto_library") load("//bazel:java_proto_library.bzl", "java_proto_library") +load("//bazel:proto_library.bzl", "proto_library") load("//build_defs:internal_shell.bzl", "inline_sh_binary") load("//ruby:defs.bzl", "internal_ruby_proto_library") diff --git a/conformance/test_protos/BUILD.bazel b/conformance/test_protos/BUILD.bazel index 37819f372d55..c3033bda1fd6 100644 --- a/conformance/test_protos/BUILD.bazel +++ b/conformance/test_protos/BUILD.bazel @@ -1,8 +1,9 @@ -load("@rules_cc//cc:defs.bzl", "cc_proto_library", "objc_library") +load("@rules_cc//cc:defs.bzl", "objc_library") load("//:protobuf.bzl", "internal_csharp_proto_library", "internal_objc_proto_library", "internal_py_proto_library") load("//bazel:cc_proto_library.bzl", "cc_proto_library") load("//bazel:java_lite_proto_library.bzl", "java_lite_proto_library") load("//bazel:java_proto_library.bzl", "java_proto_library") +load("//bazel:proto_library.bzl", "proto_library") load("//ruby:defs.bzl", "internal_ruby_proto_library") package( diff --git a/editions/BUILD b/editions/BUILD index 500d6eb86401..015efc728e01 100644 --- a/editions/BUILD +++ b/editions/BUILD @@ -3,6 +3,8 @@ load("//:protobuf.bzl", "internal_objc_proto_library", "internal_php_proto_libra load("//bazel:cc_proto_library.bzl", "cc_proto_library") load("//bazel:java_lite_proto_library.bzl", "java_lite_proto_library") load("//bazel:java_proto_library.bzl", "java_proto_library") +load("//bazel:proto_library.bzl", "proto_library") +load("//bazel:py_proto_library.bzl", "py_proto_library") load("//bazel:upb_proto_library.bzl", "upb_c_proto_library", "upb_proto_reflection_library") load(":defaults.bzl", "compile_edition_defaults", "embed_edition_defaults") diff --git a/upb_generator/c/BUILD b/upb_generator/c/BUILD index f1987752642e..c55c9c143599 100644 --- a/upb_generator/c/BUILD +++ b/upb_generator/c/BUILD @@ -5,6 +5,7 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +load("//bazel/toolchains:proto_lang_toolchain.bzl", "proto_lang_toolchain") load( "//upb/bazel:build_defs.bzl", "UPB_DEFAULT_CPPOPTS", diff --git a/upb_generator/minitable/BUILD b/upb_generator/minitable/BUILD index bd4695f5f65d..b1b05b97bac5 100644 --- a/upb_generator/minitable/BUILD +++ b/upb_generator/minitable/BUILD @@ -5,6 +5,7 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +load("//bazel/toolchains:proto_lang_toolchain.bzl", "proto_lang_toolchain") load("//upb/bazel:build_defs.bzl", "UPB_DEFAULT_CPPOPTS") load( "//upb_generator:bootstrap_compiler.bzl", diff --git a/upb_generator/reflection/BUILD b/upb_generator/reflection/BUILD index 645463baea52..db8bf892d6b9 100644 --- a/upb_generator/reflection/BUILD +++ b/upb_generator/reflection/BUILD @@ -5,6 +5,7 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +load("//bazel/toolchains:proto_lang_toolchain.bzl", "proto_lang_toolchain") load("//upb/bazel:build_defs.bzl", "UPB_DEFAULT_CPPOPTS") package(default_applicable_licenses = ["//:license"])