Significantly reduce the dependencies of RuleRunner (#15631)

The entire `init` package depended on the `plugins` target, but the underlying reason for that dependency was to include all plugins in published distributions and PEXes... which are defined in the `bin` package. As a side-effect, all `RuleRunner` tests depended on all plugins, which significantly increased the size of their cache keys. Before: ```console $ ./pants dependencies --transitive src/python/pants/testutil/rule_runner.py | wc -l 622 ``` After: ```console $ ./pants dependencies --transitive src/python/pants/testutil/rule_runner.py | wc -l 105 ``` Moral of the story: explicit deps are dangerous...? [ci skip-rust]
pantsbuild · May 24, 2022 · 1c1c0f1 · 1c1c0f1
1 parent 8f92f48
commit 1c1c0f1
Show file tree

Hide file tree

Showing 6 changed files with 108 additions and 65 deletions.
diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD
@@ -3,7 +3,69 @@
 
 python_sources(sources=["*.py", "!pants_loader.py"])
 
-python_sources(name="pants_loader", sources=["pants_loader.py"])
+python_sources(
+    name="pants_loader",
+    sources=["pants_loader.py"],
+    dependencies=[":plugins"],
+)
+
+# NOTE: When adding a new backend, add it to `generate_docs.py` too! (If stable enough for docs)
+target(
+    name="plugins",
+    dependencies=[
+        "src/python/pants/backend/awslambda/python",
+        "src/python/pants/backend/codegen/protobuf/lint/buf",
+        "src/python/pants/backend/codegen/protobuf/python",
+        "src/python/pants/backend/codegen/thrift/apache/python",
+        "src/python/pants/backend/docker",
+        "src/python/pants/backend/docker/lint/hadolint",
+        "src/python/pants/backend/experimental/cc",
+        "src/python/pants/backend/experimental/cc/lint/clangformat",
+        "src/python/pants/backend/experimental/codegen/avro/java",
+        "src/python/pants/backend/experimental/codegen/protobuf/go",
+        "src/python/pants/backend/experimental/codegen/protobuf/java",
+        "src/python/pants/backend/experimental/codegen/protobuf/scala",
+        "src/python/pants/backend/experimental/codegen/thrift/apache/java",
+        "src/python/pants/backend/experimental/codegen/thrift/scrooge/java",
+        "src/python/pants/backend/experimental/codegen/thrift/scrooge/scala",
+        "src/python/pants/backend/experimental/debian",
+        "src/python/pants/backend/experimental/go",
+        "src/python/pants/backend/experimental/go/lint/vet",
+        "src/python/pants/backend/experimental/helm",
+        "src/python/pants/backend/experimental/java",
+        "src/python/pants/backend/experimental/java/debug_goals",
+        "src/python/pants/backend/experimental/java/lint/google_java_format",
+        "src/python/pants/backend/experimental/kotlin",
+        "src/python/pants/backend/experimental/kotlin/debug_goals",
+        "src/python/pants/backend/experimental/kotlin/lint/ktlint",
+        "src/python/pants/backend/experimental/python",
+        "src/python/pants/backend/experimental/python/docs/sphinx",
+        "src/python/pants/backend/experimental/python/lint/autoflake",
+        "src/python/pants/backend/experimental/python/lint/pyupgrade",
+        "src/python/pants/backend/experimental/python/packaging/pyoxidizer",
+        "src/python/pants/backend/experimental/scala",
+        "src/python/pants/backend/experimental/scala/debug_goals",
+        "src/python/pants/backend/experimental/scala/lint/scalafmt",
+        "src/python/pants/backend/experimental/terraform",
+        "src/python/pants/backend/google_cloud_function/python",
+        "src/python/pants/backend/plugin_development",
+        "src/python/pants/backend/project_info",
+        "src/python/pants/backend/python",
+        "src/python/pants/backend/python/lint/bandit",
+        "src/python/pants/backend/python/lint/black",
+        "src/python/pants/backend/python/lint/docformatter",
+        "src/python/pants/backend/python/lint/flake8",
+        "src/python/pants/backend/python/lint/isort",
+        "src/python/pants/backend/python/lint/pylint",
+        "src/python/pants/backend/python/lint/yapf",
+        "src/python/pants/backend/python/mixed_interpreter_constraints",
+        "src/python/pants/backend/python/typecheck/mypy",
+        "src/python/pants/backend/shell",
+        "src/python/pants/backend/shell/lint/shellcheck",
+        "src/python/pants/backend/shell/lint/shfmt",
+        "src/python/pants/core",
+    ],
+)
 
 # This binary's entry_point is used by the pantsbuild.pants sdist to setup a binary for
 # pip installers, ie: it is why this works to get `pants` on your PATH:

diff --git a/src/python/pants/init/BUILD b/src/python/pants/init/BUILD
@@ -1,64 +1,17 @@
 # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-python_sources(dependencies=[":plugins"])
-
-# NOTE: When adding a new backend, add it to `generate_docs.py` too! (If stable enough for docs)
-target(
-    name="plugins",
-    dependencies=[
-        "src/python/pants/backend/awslambda/python",
-        "src/python/pants/backend/codegen/protobuf/lint/buf",
-        "src/python/pants/backend/codegen/protobuf/python",
-        "src/python/pants/backend/codegen/thrift/apache/python",
-        "src/python/pants/backend/docker",
-        "src/python/pants/backend/docker/lint/hadolint",
-        "src/python/pants/backend/experimental/cc",
-        "src/python/pants/backend/experimental/cc/lint/clangformat",
-        "src/python/pants/backend/experimental/codegen/avro/java",
-        "src/python/pants/backend/experimental/codegen/protobuf/go",
-        "src/python/pants/backend/experimental/codegen/protobuf/java",
-        "src/python/pants/backend/experimental/codegen/protobuf/scala",
-        "src/python/pants/backend/experimental/codegen/thrift/apache/java",
-        "src/python/pants/backend/experimental/codegen/thrift/scrooge/java",
-        "src/python/pants/backend/experimental/codegen/thrift/scrooge/scala",
-        "src/python/pants/backend/experimental/debian",
-        "src/python/pants/backend/experimental/go",
-        "src/python/pants/backend/experimental/go/lint/vet",
-        "src/python/pants/backend/experimental/helm",
-        "src/python/pants/backend/experimental/java",
-        "src/python/pants/backend/experimental/java/debug_goals",
-        "src/python/pants/backend/experimental/java/lint/google_java_format",
-        "src/python/pants/backend/experimental/kotlin",
-        "src/python/pants/backend/experimental/kotlin/debug_goals",
-        "src/python/pants/backend/experimental/kotlin/lint/ktlint",
-        "src/python/pants/backend/experimental/python",
-        "src/python/pants/backend/experimental/python/docs/sphinx",
-        "src/python/pants/backend/experimental/python/lint/autoflake",
-        "src/python/pants/backend/experimental/python/lint/pyupgrade",
-        "src/python/pants/backend/experimental/python/packaging/pyoxidizer",
-        "src/python/pants/backend/experimental/scala",
-        "src/python/pants/backend/experimental/scala/debug_goals",
-        "src/python/pants/backend/experimental/scala/lint/scalafmt",
-        "src/python/pants/backend/experimental/terraform",
-        "src/python/pants/backend/google_cloud_function/python",
-        "src/python/pants/backend/plugin_development",
-        "src/python/pants/backend/project_info",
-        "src/python/pants/backend/python",
-        "src/python/pants/backend/python/lint/bandit",
-        "src/python/pants/backend/python/lint/black",
-        "src/python/pants/backend/python/lint/docformatter",
-        "src/python/pants/backend/python/lint/flake8",
-        "src/python/pants/backend/python/lint/isort",
-        "src/python/pants/backend/python/lint/pylint",
-        "src/python/pants/backend/python/lint/yapf",
-        "src/python/pants/backend/python/mixed_interpreter_constraints",
-        "src/python/pants/backend/python/typecheck/mypy",
-        "src/python/pants/backend/shell",
-        "src/python/pants/backend/shell/lint/shellcheck",
-        "src/python/pants/backend/shell/lint/shfmt",
-        "src/python/pants/core",
-    ],
+python_sources(
+    overrides={
+        "extension_loader.py": {
+            "dependencies": [
+                # NB: These are explicitly loaded by default by the extension loader: see
+                # the corresponding note.
+                "src/python/pants/core",
+                "src/python/pants/backend/project_info",
+            ]
+        }
+    },
 )
 
 python_tests(

diff --git a/src/python/pants/init/extension_loader.py b/src/python/pants/init/extension_loader.py
@@ -111,6 +111,7 @@ def load_build_configuration_from_source(
     :raises: :class:``pants.base.exceptions.BuildConfigurationError`` if there is a problem loading
       the build configuration.
     """
+    # NB: Backends added here must be explicit dependencies of this module.
     backend_packages = FrozenOrderedSet(["pants.core", "pants.backend.project_info", *backends])
     for backend_package in backend_packages:
         load_backend(build_configuration, backend_package)

diff --git a/src/python/pants/option/BUILD b/src/python/pants/option/BUILD
@@ -8,13 +8,20 @@ python_tests(
     overrides={
         (
             "config_test.py",
-            "global_options_test.py",
             "options_test.py",
             "options_bootstrapper_test.py",
             "subsystem_test.py",
         ): {
             "dependencies": ["//BUILD_ROOT:files", "//pants.toml:files"],
         },
+        "global_options_test.py": {
+            "dependencies": [
+                "//BUILD_ROOT:files",
+                "//pants.toml:files",
+                # This test loads `pants.toml`, which references many of these plugins.
+                "src/python/pants/bin:plugins",
+            ],
+        },
         "options_integration_test.py": {"timeout": 150},
     },
 )
diff --git a/tests/python/pants_test/init/BUILD b/tests/python/pants_test/init/BUILD
@@ -4,10 +4,15 @@
 python_tests(
     name="tests",
     timeout=360,
-    dependencies=[
-        # Used by `test_options_initializer`.
-        "//BUILD_ROOT:files"
-    ],
+    overrides={
+        ("test_options_initializer.py", "test_plugin_resolver.py"): {
+            "dependencies": [
+                "//BUILD_ROOT:files",
+                # These tests load `pants.toml`, which references many of these plugins.
+                "src/python/pants/bin:plugins",
+            ],
+        }
+    },
 )
 
 python_sources()
diff --git a/tests/python/pants_test/pantsd/BUILD b/tests/python/pants_test/pantsd/BUILD
@@ -1,7 +1,22 @@
 # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-python_tests(sources=["test_*.py"], dependencies=["//BUILD_ROOT:files"])
+python_tests(
+    sources=["test_*.py"],
+    overrides={
+        "test_process_manager.py": {
+            "dependencies": [
+                "//BUILD_ROOT:files",
+            ],
+        },
+        "test_pants_daemon_core.py": {
+            "dependencies": [
+                "//BUILD_ROOT:files",
+                "src/python/pants/bin:plugins",
+            ]
+        },
+    },
+)
 
 python_sources(
     name="pantsd_integration_test_base",