diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 3e4510226c5e..359897306793 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -47,10 +47,9 @@ def _repository_locations(): if "project_url" not in location: _fail_missing_attribute("project_url", key) - s = location["project_url"] - if not s.startswith("https://") and not s.startswith("http://"): - fail("project_url must start with https:// or http://: " + s) - mutable_location.pop("project_url") + project_url = mutable_location.pop("project_url") + if not project_url.startswith("https://") and not project_url.startswith("http://"): + fail("project_url must start with https:// or http://: " + project_url) if "version" not in location: _fail_missing_attribute("version", key) @@ -58,24 +57,28 @@ def _repository_locations(): if "use_category" not in location: _fail_missing_attribute("use_category", key) - mutable_location.pop("use_category") + use_category = mutable_location.pop("use_category") + + if "dataplane_ext" in use_category or "observability_ext" in use_category: + if "extensions" not in location: + _fail_missing_attribute("extensions", key) + mutable_location.pop("extensions") if "last_updated" not in location: _fail_missing_attribute("last_updated", key) - s = location["last_updated"] + last_updated = mutable_location.pop("last_updated") # Starlark doesn't have regexes. - if len(s) != 10 or s[4] != "-" or s[7] != "-": - fail("last_updated must match YYYY-DD-MM: " + s) - mutable_location.pop("last_updated") + if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-": + fail("last_updated must match YYYY-DD-MM: " + last_updated) if "cpe" in location: - s = location["cpe"] + cpe = mutable_location.pop("cpe") # Starlark doesn't have regexes. - if s != "N/A" and (not s.startswith("cpe:2.3:a:") or not s.endswith(":*") and len(s.split(":")) != 6): - fail("CPE must match cpe:2.3:a:::*: " + s) - mutable_location.pop("cpe") + cpe_matches = (cpe != "N/A" and (not cpe.startswith("cpe:2.3:a:") or not cpe.endswith(":*") and len(cpe.split(":")) != 6)) + if cpe_matches: + fail("CPE must match cpe:2.3:a:::*: " + cpe) elif not [category for category in USE_CATEGORIES_WITH_CPE_OPTIONAL if category in location["use_category"]]: _fail_missing_attribute("cpe", key) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index b57403b7824b..bacd6cebc7e3 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -27,19 +27,23 @@ USE_CATEGORIES = [ "build", # This dependency is used to process xDS requests. "controlplane", - # This dependency is used in processing downstream or upstream requests. - "dataplane", - # This dependecy is used for logging, metrics or tracing. It may process unstrusted input. - "observability", + # This dependency is used in processing downstream or upstream requests (core). + "dataplane_core", + # This dependency is used in processing downstream or upstream requests (extensions). + "dataplane_ext", + # This dependecy is used for logging, metrics or tracing (core). It may process unstrusted input. + "observability_core", + # This dependecy is used for logging, metrics or tracing (extensions). It may process unstrusted input. + "observability_ext", # This dependency does not handle untrusted data and is used for various utility purposes. "other", - # This dependency is used for unit tests. - "test", + # This dependency is used only in tests. + "test_only", ] # Components with these use categories are not required to specify the 'cpe' # and 'last_updated' annotation. -USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test"] +USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test_only"] DEPENDENCY_REPOSITORIES_SPEC = dict( bazel_compdb = dict( @@ -112,7 +116,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( # # chromium-85.0.4183.83 urls = ["https://github.com/google/boringssl/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["controlplane", "dataplane_core"], last_updated = "2020-06-23", cpe = "cpe:2.3:a:google:boringssl:*", ), @@ -123,7 +127,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( version = "fips-20190808", sha256 = "3b5fdf23274d4179c2077b5e8fa625d9debd7a390aac1d165b7e47234f648bb8", urls = ["https://commondatastorage.googleapis.com/chromium-boringssl-fips/boringssl-ae223d6138807a13006342edfeef32e813246b39.tar.xz"], - use_category = ["dataplane"], + use_category = ["controlplane", "dataplane_core"], last_updated = "2019-08-08", cpe = "cpe:2.3:a:google:boringssl:*", ), @@ -135,7 +139,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "573baccd67aa591b8c7209bfb0c77e0d15633d77ced39d1ccbb1232828f7f7d9", strip_prefix = "abseil-cpp-{version}", urls = ["https://github.com/abseil/abseil-cpp/archive/{version}.tar.gz"], - use_category = ["dataplane", "controlplane"], + use_category = ["dataplane_core", "controlplane"], last_updated = "2020-08-08", cpe = "N/A", ), @@ -147,7 +151,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "d08312d0ecc3bd48eee0a4cc0d2137c9f194e0a28de2028928c0f6cae85f86ce", strip_prefix = "c-ares-{version}", urls = ["https://github.com/c-ares/c-ares/releases/download/cares-{underscore_version}/c-ares-{version}.tar.gz"], - use_category = ["dataplane", "controlplane"], + use_category = ["dataplane_core", "controlplane"], last_updated = "2020-05-11", cpe = "cpe:2.3:a:c-ares_project:c-ares:*", ), @@ -159,7 +163,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "8165aa25e529d7d4b9ae849d3bf30371255a99d6db0421516abcff23214cdc2c", strip_prefix = "libcircllhist-{version}", urls = ["https://github.com/circonus-labs/libcircllhist/archive/{version}.tar.gz"], - use_category = ["observability"], + use_category = ["controlplane", "observability_core", "dataplane_core"], last_updated = "2019-02-11", cpe = "N/A", ), @@ -171,7 +175,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "952ebbf5b11fbf59ae5d760a562d1e9112278f244340ad7714e8556cbe54f7f7", strip_prefix = "xxHash-{version}", urls = ["https://github.com/Cyan4973/xxHash/archive/v{version}.tar.gz"], - use_category = ["dataplane", "controlplane"], + use_category = ["dataplane_core", "controlplane"], last_updated = "2020-03-04", cpe = "N/A", ), @@ -183,7 +187,11 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "96c10c8e950a141a32034f19b19cdeb1da48fe859cf96ae5e19f894f36c62c71", strip_prefix = "sql-parser-{version}", urls = ["https://github.com/envoyproxy/sql-parser/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = [ + "envoy.filters.network.mysql_proxy", + "envoy.filters.network.postgres_proxy", + ], last_updated = "2020-06-10", cpe = "N/A", ), @@ -206,7 +214,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "decfdf9ad274070fa85f26407b816f5a4d82205ae86bac1990be658d0795ea4d", strip_prefix = "fmt-{version}", urls = ["https://github.com/fmtlib/fmt/releases/download/{version}/fmt-{version}.zip"], - use_category = ["dataplane", "controlplane"], + use_category = ["dataplane_core", "controlplane"], last_updated = "2020-08-07", cpe = "cpe:2.3:a:fmt:fmt:*", ), @@ -218,7 +226,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "f0114a4d3c88be9e696762f37a7c379619443ce9d668546c61b21d41affe5b62", strip_prefix = "spdlog-{version}", urls = ["https://github.com/gabime/spdlog/archive/v{version}.tar.gz"], - use_category = ["dataplane", "controlplane"], + use_category = ["dataplane_core", "controlplane"], last_updated = "2020-07-09", cpe = "N/A", ), @@ -231,7 +239,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( strip_prefix = "libprotobuf-mutator-{version}", urls = ["https://github.com/google/libprotobuf-mutator/archive/{version}.tar.gz"], last_updated = "2020-08-18", - use_category = ["test"], + use_category = ["test_only"], ), com_github_gperftools_gperftools = dict( project_name = "gperftools", @@ -242,7 +250,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( strip_prefix = "gperftools-{version}", urls = ["https://github.com/gperftools/gperftools/releases/download/gperftools-{version}/gperftools-{version}.tar.gz"], last_updated = "2020-07-06", - use_category = ["dataplane", "controlplane"], + use_category = ["dataplane_core", "controlplane"], cpe = "cpe:2.3:a:gperftools_project:gperftools:*", ), com_github_grpc_grpc = dict( @@ -255,7 +263,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "bbc8f020f4e85ec029b047fab939b8c81f3d67254b5c724e1003a2bc49ddd123", strip_prefix = "grpc-{version}", urls = ["https://github.com/grpc/grpc/archive/{version}.tar.gz"], - use_category = ["dataplane", "controlplane"], + use_category = ["dataplane_core", "controlplane"], last_updated = "2020-02-11", cpe = "cpe:2.3:a:grpc:grpc:*", ), @@ -267,7 +275,8 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "409f7fe570d3c16558e594421c47bdd130238323c9d6fd6c83dedd2aaeb082a8", strip_prefix = "LuaJIT-{version}", urls = ["https://github.com/LuaJIT/LuaJIT/archive/v{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = ["envoy.filters.http.lua"], last_updated = "2017-11-07", cpe = "cpe:2.3:a:luajit:luajit:*", ), @@ -279,7 +288,8 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "83deb2c880488dfe7dd8ebf09e3b1e7613ef4b8420de53de6f712f01aabca2b6", strip_prefix = "moonjit-{version}", urls = ["https://github.com/moonjit/moonjit/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = ["envoy.filters.http.lua"], last_updated = "2020-01-14", cpe = "cpe:2.3:a:moonjit_project:moonjit:*", ), @@ -291,7 +301,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "eacc6f0f8543583ecd659faf0a3f906ed03826f1d4157b536b4b385fe47c5bb8", strip_prefix = "nghttp2-{version}", urls = ["https://github.com/nghttp2/nghttp2/releases/download/v{version}/nghttp2-{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["controlplane", "dataplane_core"], last_updated = "2020-06-02", cpe = "cpe:2.3:a:nghttp2:nghttp2:*", ), @@ -303,7 +313,12 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "015c4187f7a6426a2b5196f0ccd982aa87f010cf61f507ae3ce5c90523f92301", strip_prefix = "opentracing-cpp-{version}", urls = ["https://github.com/opentracing/opentracing-cpp/archive/v{version}.tar.gz"], - use_category = ["observability"], + use_category = ["observability_ext"], + extensions = [ + "envoy.tracers.datadog", + "envoy.tracers.dynamic_ot", + "envoy.tracers.lightstep", + ], last_updated = "2019-01-16", cpe = "N/A", ), @@ -315,7 +330,8 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "3238921a8f578beb26c2215cd277e8f6752f3d29b020b881d60d96a240a38aed", strip_prefix = "lightstep-tracer-cpp-{version}", urls = ["https://github.com/lightstep/lightstep-tracer-cpp/archive/{version}.tar.gz"], - use_category = ["observability"], + use_category = ["observability_ext"], + extensions = ["envoy.tracers.lightstep"], last_updated = "2020-08-24", cpe = "N/A", ), @@ -327,7 +343,8 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "b84fd2fb0bb0578af4901db31d1c0ae909b532a1016fe6534cbe31a6c3ad6924", strip_prefix = "dd-opentracing-cpp-{version}", urls = ["https://github.com/DataDog/dd-opentracing-cpp/archive/v{version}.tar.gz"], - use_category = ["observability"], + use_category = ["observability_ext"], + extensions = ["envoy.tracers.datadog"], last_updated = "2020-05-15", cpe = "N/A", ), @@ -339,7 +356,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "23082937d1663a53b90cb5b61df4bcc312f6dee7018da78ba00dd6bd669dfef2", strip_prefix = "benchmark-{version}", urls = ["https://github.com/google/benchmark/archive/v{version}.tar.gz"], - use_category = ["test"], + use_category = ["test_only"], last_updated = "2020-06-09", ), com_github_libevent_libevent = dict( @@ -360,7 +377,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "4c80e5fe044ce5f8055b20a2f141ee32ec2614000f3e95d2aa81611a4c8f5213", strip_prefix = "libevent-{version}", urls = ["https://github.com/libevent/libevent/archive/{version}.tar.gz"], - use_category = ["dataplane", "controlplane"], + use_category = ["dataplane_core", "controlplane"], last_updated = "2020-07-31", cpe = "cpe:2.3:a:libevent_project:libevent:*", ), @@ -376,7 +393,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "155a8f8c1a753fb05b16a1b0cc0a0a9f61a78e245f9e0da483d13043b3bcbf2e", strip_prefix = "zlib-{version}", urls = ["https://github.com/madler/zlib/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["controlplane", "dataplane_core"], last_updated = "2019-04-14", cpe = "cpe:2.3:a:gnu:zlib:*", ), @@ -386,9 +403,9 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( project_url = "https://github.com/zlib-ng/zlib-ng", version = "193d8fd7dfb7927facab7a3034daa27ad5b9df1c", sha256 = "5fe543e8d007b9e7b729f3d6b3a5ee1f9b68d0eef5f6af1393745a4dcd472a98", - strip_prefix = "zlib-ng-193d8fd7dfb7927facab7a3034daa27ad5b9df1c", - urls = ["https://github.com/zlib-ng/zlib-ng/archive/193d8fd7dfb7927facab7a3034daa27ad5b9df1c.tar.gz"], - use_category = ["dataplane"], + strip_prefix = "zlib-ng-{version}", + urls = ["https://github.com/zlib-ng/zlib-ng/archive/{version}.tar.gz"], + use_category = ["controlplane", "dataplane_core"], last_updated = "2020-08-16", cpe = "N/A", ), @@ -400,19 +417,22 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "79ab7069ef1c7c3632e7ffe095f7185d4c77b64d8035db3c085c239d4fe96d5f", strip_prefix = "yaml-cpp-{version}", urls = ["https://github.com/jbeder/yaml-cpp/archive/{version}.tar.gz"], - use_category = ["controlplane"], + # YAML is also used for runtime as well as controlplane. It shouldn't appear on the + # dataplane but we can't verify this automatically due to code structure today. + use_category = ["controlplane", "dataplane_core"], last_updated = "2020-07-28", cpe = "cpe:2.3:a:yaml-cpp_project:yaml-cpp:*", ), com_github_msgpack_msgpack_c = dict( project_name = "msgpack for C/C++", - project_desc = "MessagePack is an efficient binary serialization format (Datadog tracing dependency)", + project_desc = "MessagePack is an efficient binary serialization format", project_url = "https://github.com/msgpack/msgpack-c", version = "3.3.0", sha256 = "6e114d12a5ddb8cb11f669f83f32246e484a8addd0ce93f274996f1941c1f07b", strip_prefix = "msgpack-{version}", urls = ["https://github.com/msgpack/msgpack-c/releases/download/cpp-{version}/msgpack-{version}.tar.gz"], - use_category = ["observability"], + use_category = ["observability_ext"], + extensions = ["envoy.tracers.datadog"], last_updated = "2020-06-05", cpe = "N/A", ), @@ -424,7 +444,8 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "f1fde4f3ebb3b2d841332c7a02a4b50e0529a19709934c63bc6208d1bbe28fb1", strip_prefix = "jwt_verify_lib-{version}", urls = ["https://github.com/google/jwt_verify_lib/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = ["envoy.filters.http.jwt_authn"], last_updated = "2020-07-09", cpe = "N/A", ), @@ -438,7 +459,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "6a12896313ce1ca630cf516a0ee43a79b5f13f5a5d8143f56560ac0b21c98fac", strip_prefix = "http-parser-{version}", urls = ["https://github.com/nodejs/http-parser/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["controlplane", "dataplane_core"], last_updated = "2020-07-10", cpe = "cpe:2.3:a:nodejs:node.js:*", ), @@ -450,7 +471,9 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "a2faafbc402394df0fa94602df4b5e4befd734aad6bb55dfef46f62fcaf1090b", strip_prefix = "rapidjson-{version}", urls = ["https://github.com/Tencent/rapidjson/archive/{version}.tar.gz"], - use_category = ["dataplane"], + # We're mostly using com_google_protobuf for JSON, but there are some extensions and hard to + # disentangle uses on the dataplane, e.g. header_formatter, Squash filter. + use_category = ["controlplane", "dataplane_core"], last_updated = "2019-12-02", cpe = "cpe:2.3:a:tencent:rapidjson:*", ), @@ -463,7 +486,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( strip_prefix = "twitter.common.lang-{version}/src", urls = ["https://files.pythonhosted.org/packages/08/bc/d6409a813a9dccd4920a6262eb6e5889e90381453a5f58938ba4cf1d9420/twitter.common.lang-{version}.tar.gz"], last_updated = "2018-06-26", - use_category = ["test"], + use_category = ["test_only"], ), com_github_twitter_common_rpc = dict( project_name = "twitter.common.rpc (Thrift)", @@ -474,7 +497,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( strip_prefix = "twitter.common.rpc-{version}/src", urls = ["https://files.pythonhosted.org/packages/be/97/f5f701b703d0f25fbf148992cd58d55b4d08d3db785aad209255ee67e2d0/twitter.common.rpc-{version}.tar.gz"], last_updated = "2018-06-26", - use_category = ["test"], + use_category = ["test_only"], ), com_github_twitter_common_finagle_thrift = dict( project_name = "twitter.common.finagle-thrift", @@ -485,7 +508,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( strip_prefix = "twitter.common.finagle-thrift-{version}/src", urls = ["https://files.pythonhosted.org/packages/f9/e7/4f80d582578f8489226370762d2cf6bc9381175d1929eba1754e03f70708/twitter.common.finagle-thrift-{version}.tar.gz"], last_updated = "2018-06-26", - use_category = ["test"], + use_category = ["test_only"], ), com_google_googletest = dict( project_name = "Google Test", @@ -498,7 +521,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( strip_prefix = "googletest-{version}", urls = ["https://github.com/google/googletest/archive/{version}.tar.gz"], last_updated = "2020-09-10", - use_category = ["test"], + use_category = ["test_only"], ), com_google_protobuf = dict( project_name = "Protocol Buffers", @@ -508,7 +531,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "d7cfd31620a352b2ee8c1ed883222a0d77e44346643458e062e86b1d069ace3e", strip_prefix = "protobuf-{version}", urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v{version}/protobuf-all-{version}.tar.gz"], - use_category = ["dataplane", "controlplane"], + use_category = ["dataplane_core", "controlplane"], last_updated = "2020-10-24", cpe = "cpe:2.3:a:google:protobuf:*", ), @@ -520,7 +543,8 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "62c8cb5ea2cca1142cde9d4a0778c52c6022345c3268c60ef81666946b958ad5", strip_prefix = "grpc-httpjson-transcoding-{version}", urls = ["https://github.com/grpc-ecosystem/grpc-httpjson-transcoding/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = ["envoy.filters.http.grpc_json_transcoder"], last_updated = "2020-03-02", cpe = "N/A", ), @@ -587,20 +611,27 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "12ff300fa804f97bd07e2ff071d969e09d5f3d7bbffeac438c725fa52a51a212", strip_prefix = "opencensus-cpp-{version}", urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/{version}.tar.gz"], - use_category = ["observability"], + use_category = ["observability_ext"], + extensions = ["envoy.tracers.opencensus"], last_updated = "2020-06-01", cpe = "N/A", ), # This should be removed, see https://github.com/envoyproxy/envoy/issues/11816. com_github_curl = dict( project_name = "curl", - project_desc = "Library for transferring data with URLs (OpenCensus and AWS X-Ray dependency)", + project_desc = "Library for transferring data with URLs", project_url = "https://curl.haxx.se", version = "7.72.0", sha256 = "d4d5899a3868fbb6ae1856c3e55a32ce35913de3956d1973caccd37bd0174fa2", strip_prefix = "curl-{version}", urls = ["https://github.com/curl/curl/releases/download/curl-{underscore_version}/curl-{version}.tar.gz"], - use_category = ["dataplane", "observability"], + use_category = ["dataplane_ext", "observability_ext"], + extensions = [ + "envoy.filters.http.aws_lambda", + "envoy.filters.http.aws_request_signing", + "envoy.grpc_credentials.aws_iam", + "envoy.tracers.opencensus", + ], last_updated = "2020-08-19", cpe = "cpe:2.3:a:haxx:curl:*", ), @@ -613,7 +644,14 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( # and contains complete checkout of V8 with all dependencies necessary to build wee8. sha256 = "ef404643d7da6854b76b9fb9950a79a1acbd037b7a26f02c585ac379b0f7dee1", urls = ["https://storage.googleapis.com/envoyproxy-wee8/wee8-{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = [ + "envoy.access_loggers.wasm", + "envoy.bootstrap.wasm", + "envoy.filters.http.wasm", + "envoy.filters.network.wasm", + "envoy.stat_sinks.wasm", + ], last_updated = "2020-08-31", cpe = "cpe:2.3:a:google:v8:*", ), @@ -625,7 +663,8 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( version = "f555d99a084cdd086a349548c70fb558ac5847cf", sha256 = "1833f08e7b0f18b49d7498b029b7f3e6559a82113ec82a98a9e945553756e351", urls = ["https://storage.googleapis.com/quiche-envoy-integration/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = ["envoy.transport_sockets.quic"], last_updated = "2020-09-18", cpe = "N/A", ), @@ -637,7 +676,8 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( version = "ef0d23689e240e6c8de4c3a5296b209128c87373", sha256 = "d769283fed1319bca68bae8bdd47fbc3a7933999329eee850eff1f1ea61ce176", urls = ["https://storage.googleapis.com/quiche-envoy-integration/googleurl_{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = [], last_updated = "2020-08-05", cpe = "N/A", ), @@ -649,19 +689,30 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "cad7d01139947d78e413d112cb8f7431fbb33cf66b0adf9c280824803fc2a72e", strip_prefix = "cel-cpp-{version}", urls = ["https://github.com/google/cel-cpp/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = [ + "envoy.filters.http.rbac", + "envoy.filters.network.rbac", + ], last_updated = "2020-07-14", cpe = "N/A", ), com_github_google_flatbuffers = dict( project_name = "FlatBuffers", - project_desc = "Cross platform serialization library architected for maximum memory efficiency (Wasm dependency)", + project_desc = "Cross platform serialization library architected for maximum memory efficiency", project_url = "https://github.com/google/flatbuffers", version = "a83caf5910644ba1c421c002ef68e42f21c15f9f", sha256 = "b8efbc25721e76780752bad775a97c3f77a0250271e2db37fc747b20e8b0f24a", strip_prefix = "flatbuffers-{version}", urls = ["https://github.com/google/flatbuffers/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = [ + "envoy.access_loggers.wasm", + "envoy.bootstrap.wasm", + "envoy.filters.http.wasm", + "envoy.filters.network.wasm", + "envoy.stat_sinks.wasm", + ], last_updated = "2020-07-29", cpe = "N/A", ), @@ -673,7 +724,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "2e9489a31ae007c81e90e8ec8a15d62d58a9c18d4fd1603f6441ef248556b41f", strip_prefix = "re2-{version}", urls = ["https://github.com/google/re2/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["controlplane", "dataplane_core"], last_updated = "2020-07-06", cpe = "N/A", ), @@ -690,7 +741,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( strip_prefix = "compiler-rt-{version}.src", urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/compiler-rt-{version}.src.tar.xz"], last_updated = "2020-03-24", - use_category = ["test"], + use_category = ["test_only"], ), upb = dict( project_name = "upb", @@ -700,7 +751,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "e9f281c56ab1eb1f97a80ca8a83bb7ef73d230eabb8591f83876f4e7b85d9b47", strip_prefix = "upb-{version}", urls = ["https://github.com/protocolbuffers/upb/archive/{version}.tar.gz"], - use_category = ["dataplane", "controlplane"], + use_category = ["controlplane"], last_updated = "2019-11-19", cpe = "N/A", ), @@ -712,7 +763,8 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "740236f44d66e33ea83382383b4fb7eabdab7093a644b525dd5ec90207f933bd", strip_prefix = "kafka-{version}/clients/src/main/resources/common/message", urls = ["https://github.com/apache/kafka/archive/{version}.zip"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = ["envoy.filters.network.kafka_broker"], last_updated = "2020-08-26", cpe = "cpe:2.3:a:apache:kafka:*", ), @@ -725,7 +777,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( strip_prefix = "kafka_2.12-{version}", urls = ["https://mirrors.gigenet.com/apache/kafka/{version}/kafka_2.12-{version}.tgz"], last_updated = "2020-08-26", - use_category = ["test"], + use_category = ["test_only"], ), kafka_python_client = dict( project_name = "Kafka (Python client)", @@ -736,7 +788,7 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( strip_prefix = "kafka-python-{version}", urls = ["https://github.com/dpkp/kafka-python/archive/{version}.tar.gz"], last_updated = "2020-08-26", - use_category = ["test"], + use_category = ["test_only"], ), proxy_wasm_cpp_sdk = dict( project_name = "WebAssembly for Proxies (C++ SDK)", @@ -746,7 +798,14 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "7d9e1f2e299215ed3e5fa8c8149740872b1100cfe3230fc639f967d9dcfd812e", strip_prefix = "proxy-wasm-cpp-sdk-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-sdk/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = [ + "envoy.access_loggers.wasm", + "envoy.bootstrap.wasm", + "envoy.filters.http.wasm", + "envoy.filters.network.wasm", + "envoy.stat_sinks.wasm", + ], last_updated = "2020-07-29", cpe = "N/A", ), @@ -758,7 +817,14 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "494d3f81156b92bac640c26000497fbf3a7b1bc35f9789594280450c6e5d8129", strip_prefix = "proxy-wasm-cpp-host-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-host/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = [ + "envoy.access_loggers.wasm", + "envoy.bootstrap.wasm", + "envoy.filters.http.wasm", + "envoy.filters.network.wasm", + "envoy.stat_sinks.wasm", + ], last_updated = "2020-07-29", cpe = "N/A", ), @@ -796,7 +862,14 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( sha256 = "4d0714f441333a63e50031c9e8e4890c78f3d21e053d46416949803e122a6574", strip_prefix = "antlr4-{version}", urls = ["https://github.com/antlr/antlr4/archive/{version}.tar.gz"], - use_category = ["dataplane"], + use_category = ["dataplane_ext"], + extensions = [ + "envoy.access_loggers.wasm", + "envoy.bootstrap.wasm", + "envoy.filters.http.wasm", + "envoy.filters.network.wasm", + "envoy.stat_sinks.wasm", + ], last_updated = "2020-07-29", cpe = "N/A", ), diff --git a/ci/do_ci.sh b/ci/do_ci.sh index a83bf818e5cb..aebbb4490fdc 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -402,6 +402,10 @@ elif [[ "$CI_TARGET" == "fix_spelling_pedantic" ]]; then exit 0 elif [[ "$CI_TARGET" == "docs" ]]; then echo "generating docs..." + # Validate dependency relationships between core/extensions and external deps. + tools/dependency/validate_test.py + tools/dependency/validate.py + # Build docs. docs/build.sh exit 0 elif [[ "$CI_TARGET" == "verify_examples" ]]; then diff --git a/docs/generate_external_dep_rst.py b/docs/generate_external_dep_rst.py index 5abc0c4c58af..3ce299b69641 100755 --- a/docs/generate_external_dep_rst.py +++ b/docs/generate_external_dep_rst.py @@ -52,11 +52,16 @@ def RenderVersion(version): return version +def RenderTitle(title): + underline = '~' * len(title) + return f'\n{title}\n{underline}\n\n' + + if __name__ == '__main__': security_rst_root = sys.argv[1] Dep = namedtuple('Dep', ['name', 'sort_name', 'version', 'cpe', 'last_updated']) - use_categories = defaultdict(list) + use_categories = defaultdict(lambda: defaultdict(list)) # Bin rendered dependencies into per-use category lists. for k, v in repository_locations.DEPENDENCY_REPOSITORIES.items(): cpe = v.get('cpe', '') @@ -71,14 +76,19 @@ def RenderVersion(version): last_updated = v['last_updated'] dep = Dep(name, project_name.lower(), version, cpe, last_updated) for category in v['use_category']: - use_categories[category].append(dep) + for ext in v.get('extensions', ['core']): + use_categories[category][ext].append(dep) def CsvRow(dep): return [dep.name, dep.version, dep.last_updated, dep.cpe] # Generate per-use category RST with CSV tables. - for category, deps in use_categories.items(): - output_path = pathlib.Path(security_rst_root, f'external_dep_{category}.rst') - content = CsvTable(['Name', 'Version', 'Last updated', 'CPE'], [2, 1, 1, 2], - [CsvRow(dep) for dep in sorted(deps, key=lambda d: d.sort_name)]) + for category, exts in use_categories.items(): + content = '' + for ext_name, deps in sorted(exts.items()): + if ext_name != 'core': + content += RenderTitle(ext_name) + output_path = pathlib.Path(security_rst_root, f'external_dep_{category}.rst') + content += CsvTable(['Name', 'Version', 'Last updated', 'CPE'], [2, 1, 1, 2], + [CsvRow(dep) for dep in sorted(deps, key=lambda d: d.sort_name)]) output_path.write_text(content) diff --git a/docs/root/intro/arch_overview/security/external_deps.rst b/docs/root/intro/arch_overview/security/external_deps.rst index 05ac71b0d96c..e033bb2fc51e 100644 --- a/docs/root/intro/arch_overview/security/external_deps.rst +++ b/docs/root/intro/arch_overview/security/external_deps.rst @@ -6,25 +6,35 @@ External dependencies Below we enumerate the external dependencies that may be linked into the Envoy binary. We exclude dependencies that only are used in CI or developer tooling above. -Data plane ----------- +Data plane (core) +----------------- -.. include:: external_dep_dataplane.rst +.. include:: external_dep_dataplane_core.rst + +Data plane (extensions) +----------------------- + +.. include:: external_dep_dataplane_ext.rst Control plane ------------- .. include:: external_dep_controlplane.rst -Observability -------------- +Observability (core) +-------------------- + +.. include:: external_dep_observability_core.rst + +Observability (extensions) +-------------------------- -.. include:: external_dep_observability.rst +.. include:: external_dep_observability_ext.rst -Test ----- +Test only +--------- -.. include:: external_dep_test.rst +.. include:: external_dep_test_only.rst Build ----- diff --git a/source/common/config/utility.h b/source/common/config/utility.h index 61b629f927f1..5ffbe637d3d7 100644 --- a/source/common/config/utility.h +++ b/source/common/config/utility.h @@ -9,7 +9,6 @@ #include "envoy/config/endpoint/v3/endpoint.pb.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" -#include "envoy/json/json_object.h" #include "envoy/local_info/local_info.h" #include "envoy/registry/registry.h" #include "envoy/server/filter_config.h" diff --git a/source/common/http/BUILD b/source/common/http/BUILD index 17071796bd6e..726c33322614 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -389,7 +389,6 @@ envoy_cc_library( "//source/common/common:enum_to_int", "//source/common/common:utility_lib", "//source/common/grpc:status_lib", - "//source/common/json:json_loader_lib", "//source/common/network:utility_lib", "//source/common/protobuf:utility_lib", "//source/common/runtime:runtime_features_lib", @@ -409,7 +408,6 @@ envoy_cc_library( ":utility_lib", "//include/envoy/common:regex_interface", "//include/envoy/http:header_map_interface", - "//include/envoy/json:json_object_interface", "//source/common/common:regex_lib", "//source/common/common:utility_lib", "//source/common/protobuf:utility_lib", diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h index cd9669efcf14..73dbd2d4a953 100644 --- a/source/common/http/header_utility.h +++ b/source/common/http/header_utility.h @@ -6,7 +6,6 @@ #include "envoy/config/route/v3/route_components.pb.h" #include "envoy/http/header_map.h" #include "envoy/http/protocol.h" -#include "envoy/json/json_object.h" #include "envoy/type/v3/range.pb.h" #include "common/protobuf/protobuf.h" diff --git a/source/common/http/utility.h b/source/common/http/utility.h index 98e93a99644f..d677b097d86c 100644 --- a/source/common/http/utility.h +++ b/source/common/http/utility.h @@ -15,7 +15,6 @@ #include "common/http/exception.h" #include "common/http/status.h" -#include "common/json/json_loader.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" diff --git a/source/common/json/json_loader.cc b/source/common/json/json_loader.cc index 083f7b64f409..bb4ccf808662 100644 --- a/source/common/json/json_loader.cc +++ b/source/common/json/json_loader.cc @@ -25,7 +25,6 @@ #include "rapidjson/writer.h" #include "absl/strings/match.h" -#include "yaml-cpp/yaml.h" namespace Envoy { namespace Json { diff --git a/source/common/router/config_utility.h b/source/common/router/config_utility.h index 3734040c9582..c7a8dc455c90 100644 --- a/source/common/router/config_utility.h +++ b/source/common/router/config_utility.h @@ -7,7 +7,6 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/route/v3/route_components.pb.h" #include "envoy/http/codes.h" -#include "envoy/json/json_object.h" #include "envoy/upstream/resource_manager.h" #include "common/common/empty_string.h" diff --git a/source/exe/BUILD b/source/exe/BUILD index a88969a6ae21..4fc1048ef2d1 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -104,9 +104,10 @@ envoy_cc_library( envoy_cc_library( name = "envoy_main_common_with_core_extensions_lib", + srcs = ["main_common.cc"], + hdrs = ["main_common.h"], deps = [ ":envoy_common_with_core_extensions_lib", - ":main_common_lib", ":platform_impl_lib", ":process_wide_lib", "//source/common/api:os_sys_calls_lib", diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h index a0fabc85bfdd..2de7578c944e 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h @@ -5,7 +5,6 @@ #include "envoy/extensions/filters/http/grpc_json_transcoder/v3/transcoder.pb.h" #include "envoy/http/filter.h" #include "envoy/http/header_map.h" -#include "envoy/json/json_object.h" #include "common/buffer/buffer_impl.h" #include "common/common/logger.h" diff --git a/source/extensions/filters/http/squash/BUILD b/source/extensions/filters/http/squash/BUILD index ea2bdcd1242b..e89a1c25d8b5 100644 --- a/source/extensions/filters/http/squash/BUILD +++ b/source/extensions/filters/http/squash/BUILD @@ -27,6 +27,7 @@ envoy_cc_library( "//source/common/http:headers_lib", "//source/common/http:message_lib", "//source/common/http:utility_lib", + "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], diff --git a/source/extensions/filters/http/squash/squash_filter.cc b/source/extensions/filters/http/squash/squash_filter.cc index fbb7ab2adc7d..ae1d4206f2bd 100644 --- a/source/extensions/filters/http/squash/squash_filter.cc +++ b/source/extensions/filters/http/squash/squash_filter.cc @@ -11,6 +11,7 @@ #include "common/http/headers.h" #include "common/http/message_impl.h" #include "common/http/utility.h" +#include "common/json/json_loader.h" #include "common/protobuf/protobuf.h" #include "common/protobuf/utility.h" diff --git a/source/extensions/filters/network/client_ssl_auth/client_ssl_auth.cc b/source/extensions/filters/network/client_ssl_auth/client_ssl_auth.cc index 4892e8107c61..3f9a342d6d45 100644 --- a/source/extensions/filters/network/client_ssl_auth/client_ssl_auth.cc +++ b/source/extensions/filters/network/client_ssl_auth/client_ssl_auth.cc @@ -14,6 +14,7 @@ #include "common/http/headers.h" #include "common/http/message_impl.h" #include "common/http/utility.h" +#include "common/json/json_loader.h" #include "common/network/utility.h" namespace Envoy { diff --git a/source/extensions/quic_listeners/quiche/platform/BUILD b/source/extensions/quic_listeners/quiche/platform/BUILD index ca08744cdaf7..f53e07b58a33 100644 --- a/source/extensions/quic_listeners/quiche/platform/BUILD +++ b/source/extensions/quic_listeners/quiche/platform/BUILD @@ -142,7 +142,6 @@ envoy_cc_library( "abseil_node_hash_map", "abseil_node_hash_set", "abseil_optional", - "googletest", ], tags = ["nofips"], visibility = ["//visibility:public"], diff --git a/source/server/configuration_impl.h b/source/server/configuration_impl.h index f7e471f9785e..dc7a893ab629 100644 --- a/source/server/configuration_impl.h +++ b/source/server/configuration_impl.h @@ -18,7 +18,6 @@ #include "envoy/server/instance.h" #include "common/common/logger.h" -#include "common/json/json_loader.h" #include "common/network/resolver_impl.h" #include "common/network/utility.h" diff --git a/test/common/formatter/BUILD b/test/common/formatter/BUILD index 8e873cfe24d9..c0203c1f1f00 100644 --- a/test/common/formatter/BUILD +++ b/test/common/formatter/BUILD @@ -40,6 +40,7 @@ envoy_cc_test( "//source/common/common:utility_lib", "//source/common/formatter:substitution_formatter_lib", "//source/common/http:header_map_lib", + "//source/common/json:json_loader_lib", "//source/common/network:address_lib", "//source/common/router:string_accessor_lib", "//test/mocks/api:api_mocks", diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index b84cfce84194..e6f7a99e46f6 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -9,6 +9,7 @@ #include "common/common/utility.h" #include "common/formatter/substitution_formatter.h" #include "common/http/header_map_impl.h" +#include "common/json/json_loader.h" #include "common/network/address_impl.h" #include "common/protobuf/utility.h" #include "common/router/string_accessor_impl.h" diff --git a/test/extensions/filters/http/gzip/BUILD b/test/extensions/filters/http/gzip/BUILD index b4b638ff40d1..649baaa34546 100644 --- a/test/extensions/filters/http/gzip/BUILD +++ b/test/extensions/filters/http/gzip/BUILD @@ -16,6 +16,7 @@ envoy_extension_cc_test( srcs = ["gzip_filter_test.cc"], extension_name = "envoy.filters.http.gzip", deps = [ + "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", "//source/extensions/compression/gzip/compressor:compressor_lib", "//source/extensions/compression/gzip/decompressor:zlib_decompressor_impl_lib", diff --git a/test/extensions/filters/http/gzip/gzip_filter_test.cc b/test/extensions/filters/http/gzip/gzip_filter_test.cc index 7f92d1c06e46..b7877bc183aa 100644 --- a/test/extensions/filters/http/gzip/gzip_filter_test.cc +++ b/test/extensions/filters/http/gzip/gzip_filter_test.cc @@ -3,6 +3,7 @@ #include "envoy/extensions/filters/http/gzip/v3/gzip.pb.h" #include "common/common/hex.h" +#include "common/json/json_loader.h" #include "common/protobuf/utility.h" #include "extensions/compression/gzip/compressor/zlib_compressor_impl.h" diff --git a/test/extensions/stats_sinks/hystrix/BUILD b/test/extensions/stats_sinks/hystrix/BUILD index 7137983d1810..766094848f2a 100644 --- a/test/extensions/stats_sinks/hystrix/BUILD +++ b/test/extensions/stats_sinks/hystrix/BUILD @@ -32,6 +32,7 @@ envoy_extension_cc_test( srcs = ["hystrix_test.cc"], extension_name = "envoy.stat_sinks.hystrix", deps = [ + "//source/common/json:json_loader_lib", "//source/common/stats:stats_lib", "//source/extensions/stat_sinks/hystrix:hystrix_lib", "//test/mocks/server:admin_mocks", diff --git a/test/extensions/stats_sinks/hystrix/hystrix_test.cc b/test/extensions/stats_sinks/hystrix/hystrix_test.cc index c45faec28341..7d480908a3a5 100644 --- a/test/extensions/stats_sinks/hystrix/hystrix_test.cc +++ b/test/extensions/stats_sinks/hystrix/hystrix_test.cc @@ -2,6 +2,8 @@ #include #include +#include "common/json/json_loader.h" + #include "extensions/stat_sinks/hystrix/hystrix.h" #include "test/mocks/network/mocks.h" diff --git a/tools/dependency/validate.py b/tools/dependency/validate.py new file mode 100755 index 000000000000..273c3117918c --- /dev/null +++ b/tools/dependency/validate.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Validate the relationship between Envoy dependencies and core/extensions. + +This script verifies that bazel query of the build graph is consistent with +the use_category metadata in bazel/repository_locations.bzl. +""" + +import re +import subprocess +import sys + +from importlib.machinery import SourceFileLoader +from importlib.util import spec_from_loader, module_from_spec + +# bazel/repository_locations.bzl must have a .bzl suffix for Starlark import, so +# we are forced to do this workaround. +_repository_locations_spec = spec_from_loader( + 'repository_locations', + SourceFileLoader('repository_locations', 'bazel/repository_locations.bzl')) +repository_locations = module_from_spec(_repository_locations_spec) +_repository_locations_spec.loader.exec_module(repository_locations) + +# source/extensions/extensions_build_config.bzl must have a .bzl suffix for Starlark +# import, so we are forced to do this workaround. +_extensions_build_config_spec = spec_from_loader( + 'extensions_build_config', + SourceFileLoader('extensions_build_config', 'source/extensions/extensions_build_config.bzl')) +extensions_build_config = module_from_spec(_extensions_build_config_spec) +_extensions_build_config_spec.loader.exec_module(extensions_build_config) + +BAZEL_QUERY_EXTERNAL_DEP_RE = re.compile('@(\w+)//') +EXTENSION_LABEL_RE = re.compile('(//source/extensions/.*):') + +# TODO(htuch): Add API dependencies to metadata, shrink this set. +UNKNOWN_DEPS = [ + 'org_golang_x_tools', 'com_github_cncf_udpa', 'org_golang_google_protobuf', + 'io_bazel_rules_nogo', 'com_envoyproxy_protoc_gen_validate', 'opencensus_proto', + 'io_bazel_rules_go', 'foreign_cc_platform_utils', 'com_github_golang_protobuf', + 'com_google_googleapis' +] +IGNORE_DEPS = set(['envoy', 'envoy_api', 'platforms', 'bazel_tools', 'local_config_cc'] + + UNKNOWN_DEPS) + + +class DependencyError(Exception): + """Error in dependency relationships.""" + pass + + +class DependencyInfo(object): + """Models dependency info in bazel/repositories.bzl.""" + + def DepsByUseCategory(self, use_category): + """Find the set of external dependencies in a given use_category. + + Args: + use_category: string providing use_category. + + Returns: + Set of dependency identifiers that match use_category. + """ + return set(name for name, metadata in repository_locations.DEPENDENCY_REPOSITORIES.items() + if use_category in metadata['use_category']) + + def GetMetadata(self, dependency): + """Obtain repository metadata for a dependency. + + Args: + dependency: string providing dependency identifier. + + Returns: + A dictionary with the repository metadata as defined in + bazel/repository_locations.bzl. + """ + return repository_locations.DEPENDENCY_REPOSITORIES.get(dependency) + + +class BuildGraph(object): + """Models the Bazel build graph.""" + + def QueryExternalDeps(self, *targets): + """Query the build graph for transitive external dependencies. + + Args: + targets: Bazel targets. + + Returns: + A set of dependency identifiers that are reachable from targets. + """ + deps_query = ' union '.join(f'deps({l})' for l in targets) + deps = subprocess.check_output(['bazel', 'query', deps_query], + stderr=subprocess.PIPE).decode().splitlines() + ext_deps = set() + for d in deps: + match = BAZEL_QUERY_EXTERNAL_DEP_RE.match(d) + if match: + ext_dep = match.group(1) + if ext_dep not in IGNORE_DEPS: + ext_deps.add(ext_dep) + return set(ext_deps) + + def ListExtensions(self): + """List all extensions. + + Returns: + Dictionary items from source/extensions/extensions_build_config.bzl. + """ + return extensions_build_config.EXTENSIONS.items() + + +class Validator(object): + """Collection of validation methods.""" + + def __init__(self, dep_info, build_graph): + self._dep_info = dep_info + self._build_graph = build_graph + self._queried_core_deps = build_graph.QueryExternalDeps( + '//source/exe:envoy_main_common_with_core_extensions_lib') + + def ValidateBuildGraphStructure(self): + """Validate basic assumptions about dependency relationship in the build graph. + + Raises: + DependencyError: on a dependency validation error. + """ + print('Validating build dependency structure...') + queried_core_ext_deps = self._build_graph.QueryExternalDeps( + '//source/exe:envoy_main_common_with_core_extensions_lib', '//source/extensions/...') + queried_all_deps = self._build_graph.QueryExternalDeps('//source/...') + if queried_all_deps != queried_core_ext_deps: + raise DependencyError('Invalid build graph structure. deps(//source/...) != ' + 'deps(//source/exe:envoy_main_common_with_core_extensions_lib) ' + 'union deps(//source/extensions/...)') + + def ValidateTestOnlyDeps(self): + """Validate that test-only dependencies aren't included in //source/... + + Raises: + DependencyError: on a dependency validation error. + """ + print('Validating test-only dependencies...') + queried_source_deps = self._build_graph.QueryExternalDeps('//source/...') + expected_test_only_deps = self._dep_info.DepsByUseCategory('test_only') + bad_test_only_deps = expected_test_only_deps.intersection(queried_source_deps) + if len(bad_test_only_deps) > 0: + raise DependencyError(f'//source depends on test-only dependencies: {bad_test_only_deps}') + + def ValidateDataPlaneCoreDeps(self): + """Validate dataplane_core dependencies. + + Check that we at least tag as dataplane_core dependencies that match some + well-known targets for the data-plane. + + Raises: + DependencyError: on a dependency validation error. + """ + print('Validating data-plane dependencies...') + # Necessary but not sufficient for dataplane. With some refactoring we could + # probably have more precise tagging of dataplane/controlplane/other deps in + # these paths. + queried_dataplane_core_min_deps = self._build_graph.QueryExternalDeps( + '//source/common/api/...', '//source/common/buffer/...', '//source/common/chromium_url/...', + '//source/common/crypto/...', '//source/common/conn_pool/...', + '//source/common/formatter/...', '//source/common/http/...', '//source/common/ssl/...', + '//source/common/tcp/...', '//source/common/tcp_proxy/...', '//source/common/network/...') + expected_dataplane_core_deps = self._dep_info.DepsByUseCategory('dataplane_core') + bad_dataplane_core_deps = queried_dataplane_core_min_deps.difference( + expected_dataplane_core_deps) + if len(bad_dataplane_core_deps) > 0: + raise DependencyError( + f'Observed dataplane core deps {queried_dataplane_core_min_deps} is not covered by ' + '"use_category" implied core deps {expected_dataplane_core_deps}: {bad_dataplane_core_deps} ' + 'are missing') + + def ValidateControlPlaneDeps(self): + """Validate controlplane dependencies. + + Check that we at least tag as controlplane dependencies that match some + well-known targets for + the control-plane. + + Raises: + DependencyError: on a dependency validation error. + """ + print('Validating control-plane dependencies...') + # Necessary but not sufficient for controlplane. With some refactoring we could + # probably have more precise tagging of dataplane/controlplane/other deps in + # these paths. + queried_controlplane_core_min_deps = self._build_graph.QueryExternalDeps( + '//source/common/config/...') + expected_controlplane_core_deps = self._dep_info.DepsByUseCategory('controlplane') + bad_controlplane_core_deps = queried_controlplane_core_min_deps.difference( + expected_controlplane_core_deps) + if len(bad_controlplane_core_deps) > 0: + raise DependencyError( + f'Observed controlplane core deps {queried_controlplane_core_min_deps} is not covered ' + 'by "use_category" implied core deps {expected_controlplane_core_deps}: ' + '{bad_controlplane_core_deps} are missing') + + def ValidateExtensionDeps(self, name, target): + """Validate that extensions are correctly declared for dataplane_ext and observability_ext. + + Args: + name: extension name. + target: extension Bazel target. + + Raises: + DependencyError: on a dependency validation error. + """ + print(f'Validating extension {name} dependencies...') + queried_deps = self._build_graph.QueryExternalDeps(target) + marginal_deps = queried_deps.difference(self._queried_core_deps) + expected_deps = [] + for d in marginal_deps: + # TODO(htuch): Ensure that queried deps are fully contained in + # repository_locations, i.e. that we're tracking with metadata all actual + # dependencies. Today, we are missing API and pip3 deps based on manual + # inspection. + metadata = self._dep_info.GetMetadata(d) + if metadata: + use_category = metadata['use_category'] + valid_use_category = any( + c in use_category for c in ['dataplane_ext', 'observability_ext', 'other']) + if not valid_use_category: + raise DependencyError( + f'Extensions {name} depends on {d} with "use_category" not including ' + '["dataplane_ext", "observability_ext", "other"]') + if 'extensions' in metadata: + allowed_extensions = metadata['extensions'] + if name not in allowed_extensions: + raise DependencyError( + f'Extension {name} depends on {d} but {d} does not list {name} in its allowlist') + + def ValidateAll(self): + """Collection of all validations. + + Raises: + DependencyError: on a dependency validation error. + """ + self.ValidateBuildGraphStructure() + self.ValidateTestOnlyDeps() + self.ValidateDataPlaneCoreDeps() + self.ValidateControlPlaneDeps() + # Validate the marginal dependencies introduced for each extension. + for name, target in sorted(build_graph.ListExtensions()): + target_all = EXTENSION_LABEL_RE.match(target).group(1) + '/...' + self.ValidateExtensionDeps(name, target_all) + + +if __name__ == '__main__': + dep_info = DependencyInfo() + build_graph = BuildGraph() + validator = Validator(dep_info, build_graph) + try: + validator.ValidateAll() + except DependencyError as e: + print('Dependency validation failed, please check metadata in bazel/repository_locations.bzl') + print(e) + sys.exit(1) diff --git a/tools/dependency/validate_test.py b/tools/dependency/validate_test.py new file mode 100755 index 000000000000..539504ba02f4 --- /dev/null +++ b/tools/dependency/validate_test.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Tests for validate.py""" + +import unittest + +import validate + + +class FakeDependencyInfo(object): + """validate.DependencyInfo fake.""" + + def __init__(self, deps): + self._deps = deps + + def DepsByUseCategory(self, use_category): + return set(n for n, m in self._deps.items() if use_category in m['use_category']) + + def GetMetadata(self, dependency): + return self._deps.get(dependency) + + +class FakeBuildGraph(object): + """validate.BuildGraph fake.""" + + def __init__(self, reachable_deps, extensions): + self._reachable_deps = reachable_deps + self._extensions = extensions + + def QueryExternalDeps(self, *targets): + return set(sum((self._reachable_deps.get(t, []) for t in targets), [])) + + def ListExtensions(self): + return self._extensions + + +def FakeDep(use_category, extensions=[]): + return {'use_category': use_category, 'extensions': extensions} + + +class ValidateTest(unittest.TestCase): + + def BuildValidator(self, deps, reachable_deps, extensions=[]): + return validate.Validator(FakeDependencyInfo(deps), FakeBuildGraph(reachable_deps, extensions)) + + def test_valid_build_graph_structure(self): + validator = self.BuildValidator({}, { + '//source/exe:envoy_main_common_with_core_extensions_lib': ['a'], + '//source/extensions/...': ['b'], + '//source/...': ['a', 'b'] + }) + validator.ValidateBuildGraphStructure() + + def test_invalid_build_graph_structure(self): + validator = self.BuildValidator({}, { + '//source/exe:envoy_main_common_with_core_extensions_lib': ['a'], + '//source/extensions/...': ['b'], + '//source/...': ['a', 'b', 'c'] + }) + self.assertRaises(validate.DependencyError, lambda: validator.ValidateBuildGraphStructure()) + + def test_valid_test_only_deps(self): + validator = self.BuildValidator({'a': FakeDep('dataplane_core')}, {'//source/...': ['a']}) + validator.ValidateTestOnlyDeps() + + def test_invalid_test_only_deps(self): + validator = self.BuildValidator({'a': FakeDep('test_only')}, {'//source/...': ['a']}) + self.assertRaises(validate.DependencyError, lambda: validator.ValidateTestOnlyDeps()) + + def test_valid_dataplane_core_deps(self): + validator = self.BuildValidator({'a': FakeDep('dataplane_core')}, + {'//source/common/http/...': ['a']}) + validator.ValidateDataPlaneCoreDeps() + + def test_invalid_dataplane_core_deps(self): + validator = self.BuildValidator({'a': FakeDep('controlplane')}, + {'//source/common/http/...': ['a']}) + self.assertRaises(validate.DependencyError, lambda: validator.ValidateDataPlaneCoreDeps()) + + def test_valid_controlplane_deps(self): + validator = self.BuildValidator({'a': FakeDep('controlplane')}, + {'//source/common/config/...': ['a']}) + validator.ValidateControlPlaneDeps() + + def test_invalid_controlplane_deps(self): + validator = self.BuildValidator({'a': FakeDep('other')}, {'//source/common/config/...': ['a']}) + self.assertRaises(validate.DependencyError, lambda: validator.ValidateControlPlaneDeps()) + + def test_valid_extension_deps(self): + validator = self.BuildValidator( + { + 'a': FakeDep('controlplane'), + 'b': FakeDep('dataplane_ext', ['foo']) + }, { + '//source/extensions/foo/...': ['a', 'b'], + '//source/exe:envoy_main_common_with_core_extensions_lib': ['a'] + }) + validator.ValidateExtensionDeps('foo', '//source/extensions/foo/...') + + def test_invalid_extension_deps_wrong_category(self): + validator = self.BuildValidator( + { + 'a': FakeDep('controlplane'), + 'b': FakeDep('controlplane', ['foo']) + }, { + '//source/extensions/foo/...': ['a', 'b'], + '//source/exe:envoy_main_common_with_core_extensions_lib': ['a'] + }) + self.assertRaises(validate.DependencyError, + lambda: validator.ValidateExtensionDeps('foo', '//source/extensions/foo/...')) + + def test_invalid_extension_deps_allowlist(self): + validator = self.BuildValidator( + { + 'a': FakeDep('controlplane'), + 'b': FakeDep('dataplane_ext', ['bar']) + }, { + '//source/extensions/foo/...': ['a', 'b'], + '//source/exe:envoy_main_common_with_core_extensions_lib': ['a'] + }) + self.assertRaises(validate.DependencyError, + lambda: validator.ValidateExtensionDeps('foo', '//source/extensions/foo/...')) + + +if __name__ == '__main__': + unittest.main()