dependencies: track untracked implied dependencies, wrapup dashboard. (…

…#13571) A few changes that wrapup #12673. * Python/Go dependencies that aren't part of the Envoy binary build don't make sense to track in repository_locations.bzl, since they have their own language specific metadata (e.g. requirements.txt) or are in many cases transitively implied. * Ensure that the full set of dependencies visible to bazel query is now validated. This requires that we explicitly call out transitive dependencies that are implied by direct dependencies in repository_locations.bzl. A new annotation `implied_untracked_deps` is used. Fixes #12673 Risk level: Low Testing: validate.py. Signed-off-by: Harvey Tuch <htuch@google.com>
envoyproxy · Oct 16, 2020 · 499f46a · 499f46a
1 parent e62c994
commit 499f46a
Show file tree

Hide file tree

Showing 6 changed files with 146 additions and 73 deletions.
diff --git a/api/bazel/external_deps.bzl b/api/bazel/external_deps.bzl
@@ -2,19 +2,38 @@ load("@envoy_api//bazel:repository_locations_utils.bzl", "load_repository_locati
 
 # Envoy dependencies may be annotated with the following attributes:
 DEPENDENCY_ANNOTATIONS = [
-    # List of the categories describing how the dependency is being used. This attribute is used
-    # for automatic tracking of security posture of Envoy's dependencies.
-    # Possible values are documented in the USE_CATEGORIES list below.
-    # This attribute is mandatory for each dependecy.
-    "use_category",
-
     # Attribute specifying CPE (Common Platform Enumeration, see https://nvd.nist.gov/products/cpe) ID
     # of the dependency. The ID may be in v2.3 or v2.2 format, although v2.3 is prefferred. See
     # https://nvd.nist.gov/products/cpe for CPE format. Use single wildcard '*' for version and vector elements
     # i.e. 'cpe:2.3:a:nghttp2:nghttp2:*'. Use "N/A" for dependencies without CPE assigned.
     # This attribute is optional for components with use categories listed in the
     # USE_CATEGORIES_WITH_CPE_OPTIONAL
     "cpe",
+
+    # A list of extensions when 'use_category' contains 'dataplane_ext' or 'observability_ext'.
+    "extensions",
+
+    # Additional dependencies loaded transitively via this dependency that are not tracked in
+    # Envoy (see the external dependency at the given version for information).
+    "implied_untracked_deps",
+
+    # When the dependency was last updated in Envoy.
+    "last_updated",
+
+    # Project metadata.
+    "project_desc",
+    "project_name",
+    "project_url",
+
+    # List of the categories describing how the dependency is being used. This attribute is used
+    # for automatic tracking of security posture of Envoy's dependencies.
+    # Possible values are documented in the USE_CATEGORIES list below.
+    # This attribute is mandatory for each dependecy.
+    "use_category",
+
+    # The dependency version. This may be either a tagged release (preferred)
+    # or git SHA (as an exception when no release tagged version is suitable).
+    "version",
 ]
 
 # NOTE: If a dependency use case is either dataplane or controlplane, the other uses are not needed
@@ -38,6 +57,10 @@ USE_CATEGORIES = [
     "other",
     # This dependency is used only in tests.
     "test_only",
+    # Documentation generation
+    "docs",
+    # Developer tools (not used in build or docs)
+    "devtools",
 ]
 
 # Components with these use categories are not required to specify the 'cpe'
@@ -62,41 +85,37 @@ def load_repository_locations(repository_locations_spec):
 
         if "project_name" not in location:
             _fail_missing_attribute("project_name", key)
-        mutable_location.pop("project_name")
 
         if "project_desc" not in location:
             _fail_missing_attribute("project_desc", key)
-        mutable_location.pop("project_desc")
 
         if "project_url" not in location:
             _fail_missing_attribute("project_url", key)
-        project_url = mutable_location.pop("project_url")
+        project_url = location["project_url"]
         if not project_url.startswith("https://") and not project_url.startswith("http://"):
             fail("project_url must start with https:// or http://: " + project_url)
 
         if "version" not in location:
             _fail_missing_attribute("version", key)
-        mutable_location.pop("version")
 
         if "use_category" not in location:
             _fail_missing_attribute("use_category", key)
-        use_category = mutable_location.pop("use_category")
+        use_category = location["use_category"]
 
         if "dataplane_ext" in use_category or "observability_ext" in use_category:
             if "extensions" not in location:
                 _fail_missing_attribute("extensions", key)
-            mutable_location.pop("extensions")
 
         if "last_updated" not in location:
             _fail_missing_attribute("last_updated", key)
-        last_updated = mutable_location.pop("last_updated")
+        last_updated = location["last_updated"]
 
         # Starlark doesn't have regexes.
         if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-":
             fail("last_updated must match YYYY-DD-MM: " + last_updated)
 
         if "cpe" in location:
-            cpe = mutable_location.pop("cpe")
+            cpe = location["cpe"]
 
             # Starlark doesn't have regexes.
             cpe_components = len(cpe.split(":"))
@@ -113,4 +132,9 @@ def load_repository_locations(repository_locations_spec):
             if category not in USE_CATEGORIES:
                 fail("Unknown use_category value '" + category + "' for dependecy " + key)
 
+        # Remove any extra annotations that we add, so that we don't confuse http_archive etc.
+        for annotation in DEPENDENCY_ANNOTATIONS:
+            if annotation in mutable_location:
+                mutable_location.pop(annotation)
+
     return locations
diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl
@@ -37,21 +37,20 @@ def envoy_dependency_imports(go_version = GO_VERSION):
         },
     )
 
+    # These dependencies, like most of the Go in this repository, exist only for the API.
     go_repository(
         name = "org_golang_google_grpc",
         build_file_proto_mode = "disable",
         importpath = "google.golang.org/grpc",
         sum = "h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=",
         version = "v1.29.1",
     )
-
     go_repository(
         name = "org_golang_x_net",
         importpath = "golang.org/x/net",
         sum = "h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=",
         version = "v0.0.0-20190813141303-74dc4d7220e7",
     )
-
     go_repository(
         name = "org_golang_x_text",
         importpath = "golang.org/x/text",

diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl
@@ -15,7 +15,7 @@ def _python_deps():
         # project_url = "https://github.com/yaml/pyyaml",
         # version = "5.3.1",
         # last_update = "2020-03-18"
-        # use_category = ["other"],
+        # use_category = ["devtools"],
         # cpe = "cpe:2.3:a:pyyaml:pyyaml:*",
     )
     pip3_import(
@@ -63,7 +63,7 @@ def _python_deps():
         # project_url = "https://clang.llvm.org/",
         # version = "10.0.1",
         # last_update = "2020-07-21"
-        # use_category = ["other"],
+        # use_category = ["devtools"],
         # cpe = "cpe:2.3:a:llvm:clang:*",
     )
     pip3_import(
@@ -75,7 +75,7 @@ def _python_deps():
         # project_url = "https://github.com/yaml/pyyaml",
         # version = "5.3.1",
         # last_update = "2020-03-18"
-        # use_category = ["other"],
+        # use_category = ["docs"],
         # cpe = "cpe:2.3:a:pyyaml:pyyaml:*",
     )
     pip3_import(
@@ -87,14 +87,14 @@ def _python_deps():
         # project_url = "http://thrift.apache.org/",
         # version = "0.11.0",
         # last_update = "2017-12-07"
-        # use_category = ["dataplane"],
+        # use_category = ["test"],
         # cpe = "cpe:2.3:a:apache:thrift:*",
 
         # project_name = "Six: Python 2 and 3 Compatibility Library",
         # project_url = "https://six.readthedocs.io/",
         # version = "1.15.0",
         # last_update = "2020-05-21"
-        # use_category = ["dataplane"],
+        # use_category = ["test"],
     )
 
 # Envoy deps that rely on a first stage of dependency loading in envoy_dependencies().

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
@@ -523,8 +523,14 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         version = "0.23.7",
         sha256 = "0310e837aed522875791750de44408ec91046c630374990edd51827cb169f616",
         urls = ["https://github.com/bazelbuild/rules_go/releases/download/v{version}/rules_go-v{version}.tar.gz"],
-        use_category = ["build"],
+        use_category = ["build", "api"],
         last_updated = "2020-08-06",
+        implied_untracked_deps = [
+            "com_github_golang_protobuf",
+            "io_bazel_rules_nogo",
+            "org_golang_google_protobuf",
+            "org_golang_x_tools",
+        ],
     ),
     rules_cc = dict(
         project_name = "C++ rules for Bazel",

diff --git a/generated_api_shadow/bazel/external_deps.bzl b/generated_api_shadow/bazel/external_deps.bzl