dependencies: associate extensions with deps, validate use_category. (#…

…13340) This PR introduces a few related changes: * use_category is restructured to distinguish core/extension deps. There's also an extension allowlist added for each dependency in the dataplane_ext and observability_ext category. * tools/dependency/validate.py is introduced to validate a bunch of structural relationships implied by the bazel/repository_locations.bzl metadata. This includes that test-only deps aren't used in //source/.., that some obvious dataplane/controlplane packages taint the appropriate reachable deps and that the association between extensions/deps holds. * The CI docs jobs now runs tools/dependency/validate.py. * The dependency dashboard at https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/external_deps to include granular information on which dataplane/observability deps come from core/extensions. * Some misc. cleanup of source code dep that came up while working on this. Risk level: Low Testing: Pyunit tests added for validate.py. Part of #12673. Signed-off-by: Harvey Tuch <htuch@google.com>
envoyproxy · Oct 1, 2020 · 2550806 · 2550806
1 parent 2112092
commit 2550806
Show file tree

Hide file tree

Showing 26 changed files with 585 additions and 100 deletions.
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -47,35 +47,38 @@ def _repository_locations():
 
         if "project_url" not in location:
             _fail_missing_attribute("project_url", key)
-        s = location["project_url"]
-        if not s.startswith("https://") and not s.startswith("http://"):
-            fail("project_url must start with https:// or http://: " + s)
-        mutable_location.pop("project_url")
+        project_url = mutable_location.pop("project_url")
+        if not project_url.startswith("https://") and not project_url.startswith("http://"):
+            fail("project_url must start with https:// or http://: " + project_url)
 
         if "version" not in location:
             _fail_missing_attribute("version", key)
         mutable_location.pop("version")
 
         if "use_category" not in location:
             _fail_missing_attribute("use_category", key)
-        mutable_location.pop("use_category")
+        use_category = mutable_location.pop("use_category")
+
+        if "dataplane_ext" in use_category or "observability_ext" in use_category:
+            if "extensions" not in location:
+                _fail_missing_attribute("extensions", key)
+            mutable_location.pop("extensions")
 
         if "last_updated" not in location:
             _fail_missing_attribute("last_updated", key)
-        s = location["last_updated"]
+        last_updated = mutable_location.pop("last_updated")
 
         # Starlark doesn't have regexes.
-        if len(s) != 10 or s[4] != "-" or s[7] != "-":
-            fail("last_updated must match YYYY-DD-MM: " + s)
-        mutable_location.pop("last_updated")
+        if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-":
+            fail("last_updated must match YYYY-DD-MM: " + last_updated)
 
         if "cpe" in location:
-            s = location["cpe"]
+            cpe = mutable_location.pop("cpe")
 
             # Starlark doesn't have regexes.
-            if s != "N/A" and (not s.startswith("cpe:2.3:a:") or not s.endswith(":*") and len(s.split(":")) != 6):
-                fail("CPE must match cpe:2.3:a:<facet>:<facet>:*: " + s)
-            mutable_location.pop("cpe")
+            cpe_matches = (cpe != "N/A" and (not cpe.startswith("cpe:2.3:a:") or not cpe.endswith(":*") and len(cpe.split(":")) != 6))
+            if cpe_matches:
+                fail("CPE must match cpe:2.3:a:<facet>:<facet>:*: " + cpe)
         elif not [category for category in USE_CATEGORIES_WITH_CPE_OPTIONAL if category in location["use_category"]]:
             _fail_missing_attribute("cpe", key)
 

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
diff --git a/ci/do_ci.sh b/ci/do_ci.sh
@@ -402,6 +402,10 @@ elif [[ "$CI_TARGET" == "fix_spelling_pedantic" ]]; then
   exit 0
 elif [[ "$CI_TARGET" == "docs" ]]; then
   echo "generating docs..."
+  # Validate dependency relationships between core/extensions and external deps.
+  tools/dependency/validate_test.py
+  tools/dependency/validate.py
+  # Build docs.
   docs/build.sh
   exit 0
 elif [[ "$CI_TARGET" == "verify_examples" ]]; then

diff --git a/docs/generate_external_dep_rst.py b/docs/generate_external_dep_rst.py
@@ -52,11 +52,16 @@ def RenderVersion(version):
   return version
 
 
+def RenderTitle(title):
+  underline = '~' * len(title)
+  return f'\n{title}\n{underline}\n\n'
+
+
 if __name__ == '__main__':
   security_rst_root = sys.argv[1]
 
   Dep = namedtuple('Dep', ['name', 'sort_name', 'version', 'cpe', 'last_updated'])
-  use_categories = defaultdict(list)
+  use_categories = defaultdict(lambda: defaultdict(list))
   # Bin rendered dependencies into per-use category lists.
   for k, v in repository_locations.DEPENDENCY_REPOSITORIES.items():
     cpe = v.get('cpe', '')
@@ -71,14 +76,19 @@ def RenderVersion(version):
     last_updated = v['last_updated']
     dep = Dep(name, project_name.lower(), version, cpe, last_updated)
     for category in v['use_category']:
-      use_categories[category].append(dep)
+      for ext in v.get('extensions', ['core']):
+        use_categories[category][ext].append(dep)
 
   def CsvRow(dep):
     return [dep.name, dep.version, dep.last_updated, dep.cpe]
 
   # Generate per-use category RST with CSV tables.
-  for category, deps in use_categories.items():
-    output_path = pathlib.Path(security_rst_root, f'external_dep_{category}.rst')
-    content = CsvTable(['Name', 'Version', 'Last updated', 'CPE'], [2, 1, 1, 2],
-                       [CsvRow(dep) for dep in sorted(deps, key=lambda d: d.sort_name)])
+  for category, exts in use_categories.items():
+    content = ''
+    for ext_name, deps in sorted(exts.items()):
+      if ext_name != 'core':
+        content += RenderTitle(ext_name)
+      output_path = pathlib.Path(security_rst_root, f'external_dep_{category}.rst')
+      content += CsvTable(['Name', 'Version', 'Last updated', 'CPE'], [2, 1, 1, 2],
+                          [CsvRow(dep) for dep in sorted(deps, key=lambda d: d.sort_name)])
     output_path.write_text(content)
diff --git a/docs/root/intro/arch_overview/security/external_deps.rst b/docs/root/intro/arch_overview/security/external_deps.rst
@@ -6,25 +6,35 @@ External dependencies
 Below we enumerate the external dependencies that may be linked into the Envoy binary. We exclude
 dependencies that only are used in CI or developer tooling above.
 
-Data plane
-----------
+Data plane (core)
+-----------------
 
-.. include:: external_dep_dataplane.rst
+.. include:: external_dep_dataplane_core.rst
+
+Data plane (extensions)
+-----------------------
+
+.. include:: external_dep_dataplane_ext.rst
 
 Control plane
 -------------
 
 .. include:: external_dep_controlplane.rst
 
-Observability
--------------
+Observability (core)
+--------------------
+
+.. include:: external_dep_observability_core.rst
+
+Observability (extensions)
+--------------------------
 
-.. include:: external_dep_observability.rst
+.. include:: external_dep_observability_ext.rst
 
-Test
-----
+Test only
+---------
 
-.. include:: external_dep_test.rst
+.. include:: external_dep_test_only.rst
 
 Build
 -----

diff --git a/source/common/config/utility.h b/source/common/config/utility.h
@@ -9,7 +9,6 @@
 #include "envoy/config/endpoint/v3/endpoint.pb.h"
 #include "envoy/config/grpc_mux.h"
 #include "envoy/config/subscription.h"
-#include "envoy/json/json_object.h"
 #include "envoy/local_info/local_info.h"
 #include "envoy/registry/registry.h"
 #include "envoy/server/filter_config.h"

diff --git a/source/common/http/BUILD b/source/common/http/BUILD
@@ -389,7 +389,6 @@ envoy_cc_library(
         "//source/common/common:enum_to_int",
         "//source/common/common:utility_lib",
         "//source/common/grpc:status_lib",
-        "//source/common/json:json_loader_lib",
         "//source/common/network:utility_lib",
         "//source/common/protobuf:utility_lib",
         "//source/common/runtime:runtime_features_lib",
@@ -409,7 +408,6 @@ envoy_cc_library(
         ":utility_lib",
         "//include/envoy/common:regex_interface",
         "//include/envoy/http:header_map_interface",
-        "//include/envoy/json:json_object_interface",
         "//source/common/common:regex_lib",
         "//source/common/common:utility_lib",
         "//source/common/protobuf:utility_lib",

diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h
@@ -6,7 +6,6 @@
 #include "envoy/config/route/v3/route_components.pb.h"
 #include "envoy/http/header_map.h"
 #include "envoy/http/protocol.h"
-#include "envoy/json/json_object.h"
 #include "envoy/type/v3/range.pb.h"
 
 #include "common/protobuf/protobuf.h"

diff --git a/source/common/http/utility.h b/source/common/http/utility.h
@@ -15,7 +15,6 @@
 
 #include "common/http/exception.h"
 #include "common/http/status.h"
-#include "common/json/json_loader.h"
 
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"

diff --git a/source/common/json/json_loader.cc b/source/common/json/json_loader.cc
@@ -25,7 +25,6 @@
 #include "rapidjson/writer.h"
 
 #include "absl/strings/match.h"
-#include "yaml-cpp/yaml.h"
 
 namespace Envoy {
 namespace Json {

diff --git a/source/common/router/config_utility.h b/source/common/router/config_utility.h
@@ -7,7 +7,6 @@
 #include "envoy/config/core/v3/base.pb.h"
 #include "envoy/config/route/v3/route_components.pb.h"
 #include "envoy/http/codes.h"
-#include "envoy/json/json_object.h"
 #include "envoy/upstream/resource_manager.h"
 
 #include "common/common/empty_string.h"

diff --git a/source/exe/BUILD b/source/exe/BUILD
@@ -104,9 +104,10 @@ envoy_cc_library(
 
 envoy_cc_library(
     name = "envoy_main_common_with_core_extensions_lib",
+    srcs = ["main_common.cc"],
+    hdrs = ["main_common.h"],
     deps = [
         ":envoy_common_with_core_extensions_lib",
-        ":main_common_lib",
         ":platform_impl_lib",
         ":process_wide_lib",
         "//source/common/api:os_sys_calls_lib",

diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h
@@ -5,7 +5,6 @@
 #include "envoy/extensions/filters/http/grpc_json_transcoder/v3/transcoder.pb.h"
 #include "envoy/http/filter.h"
 #include "envoy/http/header_map.h"
-#include "envoy/json/json_object.h"
 
 #include "common/buffer/buffer_impl.h"
 #include "common/common/logger.h"

diff --git a/source/extensions/filters/http/squash/BUILD b/source/extensions/filters/http/squash/BUILD
@@ -27,6 +27,7 @@ envoy_cc_library(
         "//source/common/http:headers_lib",
         "//source/common/http:message_lib",
         "//source/common/http:utility_lib",
+        "//source/common/json:json_loader_lib",
         "//source/common/protobuf:utility_lib",
         "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto",
     ],

diff --git a/source/extensions/filters/http/squash/squash_filter.cc b/source/extensions/filters/http/squash/squash_filter.cc
@@ -11,6 +11,7 @@
 #include "common/http/headers.h"
 #include "common/http/message_impl.h"
 #include "common/http/utility.h"
+#include "common/json/json_loader.h"
 #include "common/protobuf/protobuf.h"
 #include "common/protobuf/utility.h"
 

diff --git a/source/extensions/filters/network/client_ssl_auth/client_ssl_auth.cc b/source/extensions/filters/network/client_ssl_auth/client_ssl_auth.cc
@@ -14,6 +14,7 @@
 #include "common/http/headers.h"
 #include "common/http/message_impl.h"
 #include "common/http/utility.h"
+#include "common/json/json_loader.h"
 #include "common/network/utility.h"
 
 namespace Envoy {

diff --git a/source/extensions/quic_listeners/quiche/platform/BUILD b/source/extensions/quic_listeners/quiche/platform/BUILD
@@ -142,7 +142,6 @@ envoy_cc_library(
         "abseil_node_hash_map",
         "abseil_node_hash_set",
         "abseil_optional",
-        "googletest",
     ],
     tags = ["nofips"],
     visibility = ["//visibility:public"],

diff --git a/source/server/configuration_impl.h b/source/server/configuration_impl.h
@@ -18,7 +18,6 @@
 #include "envoy/server/instance.h"
 
 #include "common/common/logger.h"
-#include "common/json/json_loader.h"
 #include "common/network/resolver_impl.h"
 #include "common/network/utility.h"
 

diff --git a/test/common/formatter/BUILD b/test/common/formatter/BUILD
@@ -40,6 +40,7 @@ envoy_cc_test(
         "//source/common/common:utility_lib",
         "//source/common/formatter:substitution_formatter_lib",
         "//source/common/http:header_map_lib",
+        "//source/common/json:json_loader_lib",
         "//source/common/network:address_lib",
         "//source/common/router:string_accessor_lib",
         "//test/mocks/api:api_mocks",

diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc
@@ -9,6 +9,7 @@
 #include "common/common/utility.h"
 #include "common/formatter/substitution_formatter.h"
 #include "common/http/header_map_impl.h"
+#include "common/json/json_loader.h"
 #include "common/network/address_impl.h"
 #include "common/protobuf/utility.h"
 #include "common/router/string_accessor_impl.h"

diff --git a/test/extensions/filters/http/gzip/BUILD b/test/extensions/filters/http/gzip/BUILD
@@ -16,6 +16,7 @@ envoy_extension_cc_test(
     srcs = ["gzip_filter_test.cc"],
     extension_name = "envoy.filters.http.gzip",
     deps = [
+        "//source/common/json:json_loader_lib",
         "//source/common/protobuf:utility_lib",
         "//source/extensions/compression/gzip/compressor:compressor_lib",
         "//source/extensions/compression/gzip/decompressor:zlib_decompressor_impl_lib",

diff --git a/test/extensions/filters/http/gzip/gzip_filter_test.cc b/test/extensions/filters/http/gzip/gzip_filter_test.cc
@@ -3,6 +3,7 @@
 #include "envoy/extensions/filters/http/gzip/v3/gzip.pb.h"
 
 #include "common/common/hex.h"
+#include "common/json/json_loader.h"
 #include "common/protobuf/utility.h"
 
 #include "extensions/compression/gzip/compressor/zlib_compressor_impl.h"

diff --git a/test/extensions/stats_sinks/hystrix/BUILD b/test/extensions/stats_sinks/hystrix/BUILD
@@ -32,6 +32,7 @@ envoy_extension_cc_test(
     srcs = ["hystrix_test.cc"],
     extension_name = "envoy.stat_sinks.hystrix",
     deps = [
+        "//source/common/json:json_loader_lib",
         "//source/common/stats:stats_lib",
         "//source/extensions/stat_sinks/hystrix:hystrix_lib",
         "//test/mocks/server:admin_mocks",

diff --git a/test/extensions/stats_sinks/hystrix/hystrix_test.cc b/test/extensions/stats_sinks/hystrix/hystrix_test.cc
@@ -2,6 +2,8 @@
 #include <memory>
 #include <sstream>
 
+#include "common/json/json_loader.h"
+
 #include "extensions/stat_sinks/hystrix/hystrix.h"
 
 #include "test/mocks/network/mocks.h"