Initial support for upb proto generation to bazel_to_cmake

This reworks how the C++ native rules and protocol buffer invocations
work allowing the same CMake generation pathways to be used for upb, gRpc,
and standard proto generation mechanisms.

PiperOrigin-RevId: 493069228
Change-Id: I582c671a25be4061b761921225aa5e42604e5f3c

bazel/cc_grpc_library.bzl

@@ -51,10 +51,11 @@ def cc_grpc_library(name, srcs, deps, visibility = None, service_namespace = Non
     # TODO: Derive the output names here?
     # TODO: Support mock generation.
     flags = []
-    if service_namespace is not None:
+    if service_namespace:
         flags.append("services_namespace=" + service_namespace)
 
     generate_cc(
+        well_known_protos = False,
         name = codegen_grpc_target,
         visibility = ["//visibility:private"],
         srcs = srcs,

third_party/com_github_cares_cares/cares.BUILD.bazel

@@ -1,4 +1,5 @@
 load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 
 config_setting(
     name = "darwin",
@@ -92,7 +93,7 @@ WIN_SUBSTITUTIONS = {
     "#cmakedefine CARES_HAVE_WINDOWS_H": "#define CARES_HAVE_WINDOWS_H 1",
     "#cmakedefine CARES_HAVE_WINSOCK2_H": "#define CARES_HAVE_WINSOCK2_H 1",
     "#cmakedefine CARES_HAVE_WS2TCPIP_H": "#define CARES_HAVE_WS2TCPIP_H 1",
-    "#endif /* __CARES_BUILD_H */": _OVERRIDES,
+    "#endif /* __CARES_BUILD_H */": OVERRIDES,
 }
 
 DEFAULT_SUBSTITUTIONS = {
@@ -105,7 +106,7 @@ DEFAULT_SUBSTITUTIONS = {
     "#cmakedefine CARES_HAVE_WINDOWS_H": "// #define CARES_HAVE_WINDOWS_H 0",
     "#cmakedefine CARES_HAVE_WINSOCK2_H": "// #define CARES_HAVE_WINSOCK2_H 0",
     "#cmakedefine CARES_HAVE_WS2TCPIP_H": "// #define CARES_HAVE_WS2TCPIP_H 0",
-    "#endif /* __CARES_BUILD_H */": _OVERRIDES,
+    "#endif /* __CARES_BUILD_H */": OVERRIDES,
 }
 
 expand_template(

third_party/com_google_absl/workspace.bzl

@@ -74,6 +74,7 @@ ABSL_CMAKE_MAPPING = {
     "@com_google_absl//absl/functional:bind_front": "absl::bind_front",
     "@com_google_absl//absl/functional:function_ref": "absl::function_ref",
     "@com_google_absl//absl/hash:hash": "absl::hash",
+    "@com_google_absl//absl/log:log": "absl::log",
     "@com_google_absl//absl/log:die_if_null": "absl::die_if_null",
     "@com_google_absl//absl/log:check": "absl::check",
     "@com_google_absl//absl/memory:memory": "absl::memory",

third_party/com_google_protobuf/workspace.bzl

@@ -39,25 +39,32 @@ def repo():
         # https://github.com/protocolbuffers/protobuf/blob/master/CMakeLists.txt
         cmake_name = "Protobuf",
         bazel_to_cmake = {
-            "include": ["", "build_defs"],
-            "aliased_targets_only": True,
+            "args": [
+                "--target=//:protoc",
+                "--target=//:protobuf",
+                "--target=//:protobuf_lite",
+                "--target=//:protobuf_headers",
+            ],
+            "exclude": [
+                "cmake/**",
+                "conformance/**",
+                "csharp/**",
+                "docs/**",
+                "editors/**",
+                "examples/**",
+                "java/**",
+                "kokoro/**",
+                "objectivec/**",
+                "php/**",
+                "pkg/**",
+                "ruby/**",
+                "ruby/**",
+                "toolchain/**",
+            ],
         },
         cmake_target_mapping = {
             "@com_google_protobuf//:protoc": "protobuf::protoc",
             "@com_google_protobuf//:protobuf": "protobuf::libprotobuf",
             "@com_google_protobuf//:protobuf_lite": "protobuf::libprotobuf-lite",
-            # Well-known protos
-            "@com_google_protobuf//:any_proto": "protobuf::any_proto",
-            "@com_google_protobuf//:api_proto": "protobuf::api_proto",  #
-            "@com_google_protobuf//:compiler_plugin_proto": "protobuf::compiler_plugin_proto",  #
-            "@com_google_protobuf//:descriptor_proto": "protobuf::descriptor_proto",
-            "@com_google_protobuf//:duration_proto": "protobuf::duration_proto",
-            "@com_google_protobuf//:empty_proto": "protobuf::empty_proto",  #
-            "@com_google_protobuf//:field_mask_proto": "protobuf::field_mask_proto",  #
-            "@com_google_protobuf//:source_context_proto": "protobuf::source_context_proto",  #
-            "@com_google_protobuf//:struct_proto": "protobuf::struct_proto",  #
-            "@com_google_protobuf//:timestamp_proto": "protobuf::timestamp_proto",
-            "@com_google_protobuf//:type_proto": "protobuf::type_proto",  #
-            "@com_google_protobuf//:wrappers_proto": "protobuf::wrappers_proto",
         },
     )

third_party/repo.bzl

@@ -144,6 +144,7 @@ def third_party_http_archive(
         cmakelists_prefix = None,
         cmakelists_suffix = None,
         cmake_package_aliases = None,
+        cmake_extra_build_file = None,
         **kwargs):
     _third_party_http_archive(name = name, **kwargs)
 

tools/cmake/bazel_to_cmake/bzl_library/bazel_skylib.py

@@ -38,6 +38,7 @@
 from ..starlark.invocation_context import RelativeLabel
 from ..starlark.provider import TargetInfo
 from ..starlark.select import Configurable
+from ..starlark.select import Select
 from ..util import cmake_is_true
 from ..util import cmake_is_windows
 from ..util import write_file_if_not_already_equal
@@ -49,6 +50,24 @@ class BazelSelectsWrapper:
   def __init__(self, context: InvocationContext):
     self._context = context
 
+  def with_or_dict(self, input_dict):
+    output_dict = {}
+    for (key, value) in input_dict.items():
+      if isinstance(key, tuple):
+        for config_setting in key:
+          if config_setting in output_dict.keys():
+            raise ValueError(f"key {config_setting} appears multiple times")
+          output_dict[config_setting] = value
+      else:
+        if key in output_dict.keys():
+          raise ValueError(f"key {key} appears multiple times")
+        output_dict[key] = value
+    return output_dict
+
+  def with_or(self, input_dict, no_match_error=None):
+    del no_match_error
+    return Select(self.with_or_dict(input_dict))
+
   def config_setting_group(
       self,
       name: str,

tools/cmake/bazel_to_cmake/bzl_library/grpc_generate_cc.py

@@ -16,9 +16,10 @@
 
 # pylint: disable=invalid-name,missing-function-docstring,relative-beyond-top-level,g-long-lambda
 
-from typing import List, Optional, cast
+from typing import List, Optional, cast, Any
 
-from ..protoc_helper import protoc_compile_protos_impl
+from ..native_rules_proto import PluginSettings
+from ..native_rules_proto import protoc_compile_protos_impl
 from ..starlark.bazel_globals import BazelGlobals
 from ..starlark.bazel_globals import register_bzl_library
 from ..starlark.bazel_target import TargetId
@@ -28,20 +29,26 @@
 from ..starlark.label import as_target_id
 from ..starlark.select import Configurable
 
+_GRPC = PluginSettings(
+    TargetId("@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin"), "grpc",
+    ".grpc.pb", [TargetId("@com_github_grpc_grpc//:grpc++_codegen_proto")])
+
 
 @register_bzl_library(
     "@com_github_grpc_grpc//bazel:generate_cc.bzl", build=True)
 class GrpcGenerateCcLibrary(BazelGlobals):
 
   def bazel_generate_cc(self,
+                        well_known_protos: Any,
                         name: str,
                         visibility: Optional[List[RelativeLabel]] = None,
                         **kwargs):
     context = self._context.snapshot()
     target = context.resolve_target(name)
     context.add_rule(
         target,
-        lambda: _generate_cc_impl(context, target, **kwargs),
+        lambda: _generate_cc_impl(
+            context, target, well_known_protos=well_known_protos, **kwargs),
         analyze_by_default=False)
 
 
@@ -62,15 +69,18 @@ def _generate_cc_impl(_context: InvocationContext,
   assert len(info.srcs) == 1
   proto_src: TargetId = as_target_id(next(iter(info.srcs)))
 
-  resolved_plugin = None
+  plugin_settings = _GRPC
   if plugin is not None:
     resolved_plugin = _context.resolve_target_or_label(
         cast(RelativeLabel, _context.evaluate_configurable(plugin)))
+    plugin_settings = PluginSettings(
+        resolved_plugin, "grpc", ".grpc.pb",
+        [TargetId("@com_github_grpc_grpc//:grpc++_codegen_proto")])
 
   protoc_compile_protos_impl(
       _context,
-      _target,
-      proto_src,
-      plugin=resolved_plugin,
+      target=_target,
+      proto_src=proto_src,
+      plugin_settings=plugin_settings,
       add_files_provider=True,
       flags=flags)

tools/cmake/bazel_to_cmake/bzl_library/helpers.py

@@ -67,15 +67,31 @@ def write_bazel_to_cmake_cmakelists(
     bazel_to_cmake: Dict[str, Any],
     cmake_target_mapping: Optional[Dict[str, str]] = None,
     build_file: Optional[RelativeLabel] = None,
+    cmake_extra_build_file: Optional[RelativeLabel] = None,
     repo_mapping: Optional[Dict[str, str]] = None,
     **kwargs):
   """Writes a nested CMakeLists.txt which invokes bazel_to_cmake.py."""
+  if kwargs.get("build_file_content") is not None:
+    raise ValueError("build_file_content not allowed.")
   del kwargs
+
   workspace = _context.access(EvaluationState).workspace
+  cmake_command = workspace.cmake_vars["CMAKE_COMMAND"]
+  bazel_to_cmake_args = []
 
-  if build_file is not None:
-    cmake_command = workspace.cmake_vars["CMAKE_COMMAND"]
+  if cmake_extra_build_file is not None:
+    # Labelize build file.
+    build_file_path = _context.get_source_file_path(
+        _context.resolve_target_or_label(cmake_extra_build_file))
+
+    assert build_file_path is not None
+    quoted_build_path = quote_path(build_file_path)
+    _patch_commands.append(
+        f"""{quote_path(cmake_command)} -E copy {quoted_build_path} extraBUILD.bazel"""
+    )
+    bazel_to_cmake_args.append("--extra-build=extraBUILD.bazel")
 
+  if build_file is not None:
     # Labelize build file.
     build_file_path = _context.get_source_file_path(
         _context.resolve_target_or_label(build_file))
@@ -85,17 +101,19 @@ def write_bazel_to_cmake_cmakelists(
     _patch_commands.append(
         f"""{quote_path(cmake_command)} -E copy {quoted_build_path} BUILD.bazel"""
     )
+
   bazel_to_cmake_path = os.path.abspath(sys.argv[0])
   assert workspace.save_workspace is not None
-  bazel_to_cmake_args = [
+  bazel_to_cmake_args.extend([
       f"--load-workspace {quote_path(workspace.save_workspace)}",
       f"--cmake-project-name {cmake_name}",
       '--cmake-binary-dir "${CMAKE_CURRENT_BINARY_DIR}"',
       f"--bazel-repo-name {name}",
       '--build-rules-output "${CMAKE_CURRENT_BINARY_DIR}/build_rules.cmake"'
-  ]
+  ])
   for mapped, orig in (repo_mapping or {}).items():
-    bazel_to_cmake_args.append(f"--repo-mapping {mapped} {orig}")
+    bazel_to_cmake_args.append(
+        f"--repo-mapping {quote_string(mapped)} {quote_string(orig)}")
   for include_package in bazel_to_cmake.get("include", []):
     bazel_to_cmake_args.append(
         quote_string("--include-package=" + include_package))

tools/cmake/bazel_to_cmake/bzl_library/local_mirror.py

@@ -104,7 +104,7 @@ def _local_mirror_impl(_context: InvocationContext, **kwargs):
       section=FETCH_CONTENT_DECLARE_SECTION)
   builder.addtext(out.getvalue(), section=FETCH_CONTENT_DECLARE_SECTION)
   builder.addtext(
-      f"add_subdirectory({quote_string(str(cmaketxt_path.parent))} _local_mirror_configs EXCLUDE_FROM_ALL)\n",
+      f"add_subdirectory({quote_string(str(cmaketxt_path.parent))} EXCLUDE_FROM_ALL)\n",
       section=FETCH_CONTENT_DECLARE_SECTION)
 
   # Now write the nested CMakeLists.txt file
@@ -121,3 +121,23 @@ def _local_mirror_impl(_context: InvocationContext, **kwargs):
     out.write(str(kwargs.get("cmakelists_suffix")))
 
   cmaketxt_path.write_text(out.getvalue(), encoding="utf-8")
+
+  # Clients rely on find_package; provide a -config.cmake file
+  # for that.
+  cmake_find_package_redirects_dir = state.workspace.cmake_vars[
+      "CMAKE_FIND_PACKAGE_REDIRECTS_DIR"]
+  if (kwargs.get("cmake_package_redirect_extra") is not None or
+      kwargs.get("cmake_package_aliases") is not None or
+      kwargs.get("cmake_package_redirect_libraries") is not None):
+    # No aliases, etc. allowed for local_mirror.
+    raise ValueError("CMake options not supported by local_mirror")
+
+  config_path = os.path.join(cmake_find_package_redirects_dir,
+                             f"{cmake_name.lower()}-config.cmake")
+  pathlib.Path(config_path).write_text(
+      f"""
+set({cmake_name.lower()}_ROOT_DIR {local_mirror_dir})
+set({cmake_name.lower()}_FOUND ON)
+set({cmake_name.upper()}_FOUND ON)
+""",
+      encoding="utf-8")

tools/cmake/bazel_to_cmake/bzl_library/rules_cc.py

@@ -16,6 +16,7 @@
 # pylint: disable=relative-beyond-top-level,missing-class-docstring
 
 from .. import native_rules_cc
+from .. import native_rules_proto
 from ..starlark.bazel_globals import BazelGlobals
 from ..starlark.bazel_globals import register_bzl_library
 
@@ -33,4 +34,4 @@ def bazel_cc_test(self, **kwargs):
     return native_rules_cc.cc_test(self._context, **kwargs)
 
   def bazel_cc_proto_library(self, **kwargs):
-    return native_rules_cc.cc_proto_library(self._context, **kwargs)
+    return native_rules_proto.cc_proto_library(self._context, **kwargs)

tools/cmake/bazel_to_cmake/bzl_library/rules_proto.py

@@ -15,7 +15,7 @@
 
 # pylint: disable=relative-beyond-top-level
 
-from .. import native_rules
+from .. import native_rules_proto
 from ..starlark.bazel_globals import BazelGlobals
 from ..starlark.bazel_globals import register_bzl_library
 from ..starlark.ignored import IgnoredObject
@@ -25,7 +25,7 @@
 class RulesCcDefsLibrary(BazelGlobals):
 
   def bazel_proto_library(self, **kwargs):
-    return native_rules.proto_library(self._context, **kwargs)
+    return native_rules_proto.proto_library(self._context, **kwargs)
 
   @property
   def bazel_proto_lang_toolchain(self):