api: organize go_proto_libraries (#8003)

Fixes #7982

Defines a package level proto library and its associated internal go_proto_library.

Deletes all existing api_go_proto_library, api_go_grpc_library, and go_package annotations in protos (they are not required and pollute the sources).

I deliberately avoided touching anything under udpa since it's being moved to another repository.

Risk Level: low
Testing: build completes

Signed-off-by: Kuat Yessenov <kuat@google.com>

api/bazel/BUILD

@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler")
+
+licenses(["notice"])  # Apache 2
+
+go_proto_compiler(
+    name = "pgv_plugin_go",
+    options = ["lang=go"],
+    plugin = "@com_envoyproxy_protoc_gen_validate//:protoc-gen-validate",
+    suffix = ".pb.validate.go",
+    valid_archive = False,
+    visibility = ["//visibility:public"],
+)

api/bazel/api_build_system.bzl

@@ -7,9 +7,23 @@ _PY_SUFFIX = "_py"
 _CC_SUFFIX = "_cc"
 _CC_EXPORT_SUFFIX = "_export_cc"
 _GO_PROTO_SUFFIX = "_go_proto"
-_GO_GRPC_SUFFIX = "_go_grpc"
 _GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/data-plane-api/api/"
 
+_COMMON_PROTO_DEPS = [
+    "@com_google_protobuf//:any_proto",
+    "@com_google_protobuf//:descriptor_proto",
+    "@com_google_protobuf//:duration_proto",
+    "@com_google_protobuf//:empty_proto",
+    "@com_google_protobuf//:struct_proto",
+    "@com_google_protobuf//:timestamp_proto",
+    "@com_google_protobuf//:wrappers_proto",
+    "@com_google_googleapis//google/api:http_proto",
+    "@com_google_googleapis//google/api:annotations_proto",
+    "@com_google_googleapis//google/rpc:status_proto",
+    "@com_github_gogo_protobuf//:gogo_proto",
+    "@com_envoyproxy_protoc_gen_validate//validate:validate_proto",
+]
+
 def _Suffix(d, suffix):
     return d + suffix
 
@@ -61,41 +75,6 @@ def py_proto_library(name, deps = []):
         visibility = ["//visibility:public"],
     )
 
-def api_go_proto_library(name, proto, deps = []):
-    go_proto_library(
-        name = _Suffix(name, _GO_PROTO_SUFFIX),
-        importpath = _Suffix(_GO_IMPORTPATH_PREFIX, name),
-        proto = proto,
-        visibility = ["//visibility:public"],
-        deps = deps + [
-            "@com_github_gogo_protobuf//:gogo_proto_go",
-            "@io_bazel_rules_go//proto/wkt:any_go_proto",
-            "@io_bazel_rules_go//proto/wkt:duration_go_proto",
-            "@io_bazel_rules_go//proto/wkt:struct_go_proto",
-            "@io_bazel_rules_go//proto/wkt:timestamp_go_proto",
-            "@io_bazel_rules_go//proto/wkt:wrappers_go_proto",
-            "@com_envoyproxy_protoc_gen_validate//validate:go_default_library",
-            "@com_google_googleapis//google/rpc:status_go_proto",
-        ],
-    )
-
-def api_go_grpc_library(name, proto, deps = []):
-    go_grpc_library(
-        name = _Suffix(name, _GO_GRPC_SUFFIX),
-        importpath = _Suffix(_GO_IMPORTPATH_PREFIX, name),
-        proto = proto,
-        visibility = ["//visibility:public"],
-        deps = deps + [
-            "@com_github_gogo_protobuf//:gogo_proto_go",
-            "@io_bazel_rules_go//proto/wkt:any_go_proto",
-            "@io_bazel_rules_go//proto/wkt:duration_go_proto",
-            "@io_bazel_rules_go//proto/wkt:struct_go_proto",
-            "@io_bazel_rules_go//proto/wkt:wrappers_go_proto",
-            "@com_envoyproxy_protoc_gen_validate//validate:go_default_library",
-            "@com_google_googleapis//google/api:annotations_go_proto",
-        ],
-    )
-
 # This is api_proto_library plus some logic internal to //envoy/api.
 def api_proto_library_internal(visibility = ["//visibility:private"], **kwargs):
     # //envoy/docs/build.sh needs visibility in order to generate documents.
@@ -108,8 +87,6 @@ def api_proto_library_internal(visibility = ["//visibility:private"], **kwargs):
 
 # TODO(htuch): has_services is currently ignored but will in future support
 # gRPC stub generation.
-# TODO(htuch): Automatically generate go_proto_library and go_grpc_library
-# from api_proto_library.
 def api_proto_library(
         name,
         visibility = ["//visibility:private"],
@@ -124,20 +101,7 @@ def api_proto_library(
     native.proto_library(
         name = name,
         srcs = srcs,
-        deps = deps + external_proto_deps + [
-            "@com_google_protobuf//:any_proto",
-            "@com_google_protobuf//:descriptor_proto",
-            "@com_google_protobuf//:duration_proto",
-            "@com_google_protobuf//:empty_proto",
-            "@com_google_protobuf//:struct_proto",
-            "@com_google_protobuf//:timestamp_proto",
-            "@com_google_protobuf//:wrappers_proto",
-            "@com_google_googleapis//google/api:http_proto",
-            "@com_google_googleapis//google/api:annotations_proto",
-            "@com_google_googleapis//google/rpc:status_proto",
-            "@com_github_gogo_protobuf//:gogo_proto",
-            "@com_envoyproxy_protoc_gen_validate//validate:validate_proto",
-        ],
+        deps = deps + external_proto_deps + _COMMON_PROTO_DEPS,
         visibility = visibility,
     )
     pgv_cc_proto_library(
@@ -181,3 +145,50 @@ def api_go_test(name, size, importpath, srcs = [], deps = []):
         importpath = importpath,
         deps = deps,
     )
+
+_GO_BAZEL_RULE_MAPPING = {
+    "@opencensus_proto//opencensus/proto/trace/v1:trace_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_go",
+    "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_and_config_proto_go",
+    "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto": "@com_google_googleapis//google/api/expr/v1alpha1:cel_go_proto",
+}
+
+def go_proto_mapping(dep):
+    mapped = _GO_BAZEL_RULE_MAPPING.get(dep)
+    if mapped == None:
+        return _Suffix("@" + Label(dep).workspace_name + "//" + Label(dep).package + ":" + Label(dep).name, _GO_PROTO_SUFFIX)
+    return mapped
+
+def api_proto_package(name = "pkg", srcs = [], deps = [], has_services = False, visibility = ["//visibility:public"]):
+    if srcs == []:
+        srcs = native.glob(["*.proto"])
+
+    native.proto_library(
+        name = name,
+        srcs = srcs,
+        deps = deps + _COMMON_PROTO_DEPS,
+        visibility = visibility,
+    )
+
+    compilers = ["@io_bazel_rules_go//proto:go_proto", "//bazel:pgv_plugin_go"]
+    if has_services:
+        compilers = ["@io_bazel_rules_go//proto:go_grpc", "//bazel:pgv_plugin_go"]
+
+    go_proto_library(
+        name = _Suffix(name, _GO_PROTO_SUFFIX),
+        compilers = compilers,
+        importpath = _Suffix(_GO_IMPORTPATH_PREFIX, native.package_name()),
+        proto = name,
+        visibility = ["//visibility:public"],
+        deps = [go_proto_mapping(dep) for dep in deps] + [
+            "@com_github_gogo_protobuf//:gogo_proto_go",
+            "@com_github_golang_protobuf//ptypes:go_default_library",
+            "@com_github_golang_protobuf//ptypes/any:go_default_library",
+            "@com_github_golang_protobuf//ptypes/duration:go_default_library",
+            "@com_github_golang_protobuf//ptypes/struct:go_default_library",
+            "@com_github_golang_protobuf//ptypes/timestamp:go_default_library",
+            "@com_github_golang_protobuf//ptypes/wrappers:go_default_library",
+            "@com_envoyproxy_protoc_gen_validate//validate:go_default_library",
+            "@com_google_googleapis//google/api:annotations_go_proto",
+            "@com_google_googleapis//google/rpc:status_go_proto",
+        ],
+    )

api/bazel/repositories.bzl

@@ -161,7 +161,7 @@ filegroup(
 
 ZIPKINAPI_BUILD_CONTENT = """
 
-load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_go_proto_library")
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library")
 load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
 
 api_proto_library(
@@ -173,8 +173,9 @@ api_proto_library(
     visibility = ["//visibility:public"],
 )
 
-api_go_proto_library(
-    name = "zipkin",
+go_proto_library(
+    name = "zipkin_go_proto",
     proto = ":zipkin",
+    visibility = ["//visibility:public"],
 )
 """

api/envoy/admin/v2alpha/BUILD

@@ -1,7 +1,18 @@
-load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal")
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package")
 
 licenses(["notice"])  # Apache 2
 
+api_proto_package(
+    deps = [
+        "//envoy/api/v2",
+        "//envoy/api/v2/auth",
+        "//envoy/api/v2/core",
+        "//envoy/config/bootstrap/v2:pkg",
+        "//envoy/service/tap/v2alpha:pkg",
+        "//envoy/type",
+    ],
+)
+
 api_proto_library_internal(
     name = "config_dump",
     srcs = ["config_dump.proto"],

api/envoy/admin/v3alpha/BUILD

@@ -1,7 +1,18 @@
-load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal")
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package")
 
 licenses(["notice"])  # Apache 2
 
+api_proto_package(
+    deps = [
+        "//envoy/api/v3alpha",
+        "//envoy/api/v3alpha/auth",
+        "//envoy/api/v3alpha/core",
+        "//envoy/config/bootstrap/v3alpha:pkg",
+        "//envoy/service/tap/v3alpha:pkg",
+        "//envoy/type",
+    ],
+)
+
 api_proto_library_internal(
     name = "config_dump",
     srcs = ["config_dump.proto"],

api/envoy/api/v2/BUILD

@@ -1,4 +1,4 @@
-load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal")
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package")
 
 licenses(["notice"])  # Apache 2
 
@@ -16,19 +16,28 @@ package_group(
     ],
 )
 
+api_proto_package(
+    name = "v2",
+    has_services = True,
+    deps = [
+        "//envoy/api/v2/auth",
+        "//envoy/api/v2/cluster",
+        "//envoy/api/v2/core",
+        "//envoy/api/v2/endpoint:pkg",
+        "//envoy/api/v2/listener:pkg",
+        "//envoy/api/v2/ratelimit:pkg",
+        "//envoy/api/v2/route:pkg",
+        "//envoy/type",
+    ],
+)
+
 api_proto_library_internal(
     name = "discovery",
     srcs = ["discovery.proto"],
     visibility = [":friends"],
     deps = ["//envoy/api/v2/core:base"],
 )
 
-api_go_proto_library(
-    name = "discovery",
-    proto = ":discovery",
-    deps = ["//envoy/api/v2/core:base_go_proto"],
-)
-
 api_proto_library_internal(
     name = "eds",
     srcs = ["eds.proto"],
@@ -44,19 +53,6 @@ api_proto_library_internal(
     ],
 )
 
-api_go_grpc_library(
-    name = "eds",
-    proto = ":eds",
-    deps = [
-        ":discovery_go_proto",
-        "//envoy/api/v2/core:address_go_proto",
-        "//envoy/api/v2/core:base_go_proto",
-        "//envoy/api/v2/core:health_check_go_proto",
-        "//envoy/api/v2/endpoint:endpoint_go_proto",
-        "//envoy/type:percent_go_proto",
-    ],
-)
-
 api_proto_library_internal(
     name = "cds",
     srcs = ["cds.proto"],
@@ -79,26 +75,6 @@ api_proto_library_internal(
     ],
 )
 
-api_go_grpc_library(
-    name = "cds",
-    proto = ":cds",
-    deps = [
-        ":discovery_go_proto",
-        ":eds_go_grpc",
-        "//envoy/api/v2/auth:cert_go_proto",
-        "//envoy/api/v2/cluster:circuit_breaker_go_proto",
-        "//envoy/api/v2/cluster:filter_go_proto",
-        "//envoy/api/v2/cluster:outlier_detection_go_proto",
-        "//envoy/api/v2/core:address_go_proto",
-        "//envoy/api/v2/core:base_go_proto",
-        "//envoy/api/v2/core:config_source_go_proto",
-        "//envoy/api/v2/core:health_check_go_proto",
-        "//envoy/api/v2/core:protocol_go_proto",
-        "//envoy/api/v2/endpoint:endpoint_go_proto",
-        "//envoy/type:percent_go_proto",
-    ],
-)
-
 api_proto_library_internal(
     name = "lds",
     srcs = ["lds.proto"],
@@ -113,18 +89,6 @@ api_proto_library_internal(
     ],
 )
 
-api_go_grpc_library(
-    name = "lds",
-    proto = ":lds",
-    deps = [
-        ":discovery_go_proto",
-        "//envoy/api/v2/core:address_go_proto",
-        "//envoy/api/v2/core:base_go_proto",
-        "//envoy/api/v2/listener:listener_go_proto",
-        "//envoy/api/v2/listener:udp_listener_config_go_proto",
-    ],
-)
-
 api_proto_library_internal(
     name = "rds",
     srcs = ["rds.proto"],
@@ -138,17 +102,6 @@ api_proto_library_internal(
     ],
 )
 
-api_go_grpc_library(
-    name = "rds",
-    proto = ":rds",
-    deps = [
-        ":discovery_go_proto",
-        "//envoy/api/v2/core:base_go_proto",
-        "//envoy/api/v2/core:config_source_go_proto",
-        "//envoy/api/v2/route:route_go_proto",
-    ],
-)
-
 api_proto_library_internal(
     name = "srds",
     srcs = ["srds.proto"],
@@ -160,12 +113,3 @@ api_proto_library_internal(
         "//envoy/api/v2/route",
     ],
 )
-
-api_go_grpc_library(
-    name = "srds",
-    proto = ":srds",
-    deps = [
-        ":discovery_go_proto",
-        "//envoy/api/v2/core:base_go_proto",
-    ],
-)

api/envoy/api/v2/auth/BUILD

@@ -1,4 +1,4 @@
-load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal")
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package")
 
 licenses(["notice"])  # Apache 2
 
@@ -15,6 +15,13 @@ package_group(
     ],
 )
 
+api_proto_package(
+    name = "auth",
+    deps = [
+        "//envoy/api/v2/core",
+    ],
+)
+
 api_proto_library_internal(
     name = "cert",
     srcs = ["cert.proto"],
@@ -24,12 +31,3 @@ api_proto_library_internal(
         "//envoy/api/v2/core:config_source",
     ],
 )
-
-api_go_proto_library(
-    name = "cert",
-    proto = ":cert",
-    deps = [
-        "//envoy/api/v2/core:base_go_proto",
-        "//envoy/api/v2/core:config_source_go_proto",
-    ],
-)