Pass the __local__ environment target to all goals (#16786)

Closes #16770.

Now, we dynamically determine what environment target to use at the root of the graph, and inject the value. This will always be the `__local__` target, or `None` if no targets are defined. That is, it cannot be a `docker_environment` etc. Then, callers can change the value to be a different environment by using `Get()` to inject a different `EnvironmentName` `Param`.

To avoid a rule graph cycle, this adds `WrappedTargetForBootstrap`.

[ci skip-rust]

src/python/pants/backend/codegen/avro/java/BUILD

@@ -5,6 +5,10 @@ python_sources(dependencies=[":lockfile"])
 
 resource(name="lockfile", source="avro-tools.default.lockfile.txt")
 
-python_tests(name="tests", dependencies=[":test_lockfiles"])
+python_tests(
+    name="tests",
+    dependencies=[":test_lockfiles"],
+    overrides={"rules_integration_test.py": {"timeout": 120}},
+)
 
 resources(name="test_lockfiles", sources=["*.test.lock"])

src/python/pants/bin/local_pants_runner.py

@@ -11,7 +11,8 @@
 from pants.base.specs import Specs
 from pants.base.specs_parser import SpecsParser
 from pants.build_graph.build_configuration import BuildConfiguration
-from pants.engine.environment import CompleteEnvironment, EnvironmentName
+from pants.core.util_rules.environments import determine_bootstrap_environment
+from pants.engine.environment import CompleteEnvironment
 from pants.engine.internals import native_engine
 from pants.engine.internals.native_engine import PySessionCancellationLatch
 from pants.engine.internals.scheduler import ExecutionError
@@ -206,7 +207,10 @@ def _finish_run(self, code: ExitCode) -> None:
     def _get_workunits_callbacks(self) -> tuple[WorkunitsCallback, ...]:
         # Load WorkunitsCallbacks by requesting WorkunitsCallbackFactories, and then constructing
         # a per-run instance of each WorkunitsCallback.
-        params = Params(self.union_membership, EnvironmentName())
+        params = Params(
+            self.union_membership,
+            determine_bootstrap_environment(self.graph_session.scheduler_session),
+        )
         (workunits_callback_factories,) = self.graph_session.scheduler_session.product_request(
             WorkunitsCallbackFactories, [params]
         )

src/python/pants/core/subsystems/python_bootstrap.py

@@ -15,8 +15,6 @@
 from pants.core.util_rules import asdf
 from pants.core.util_rules.asdf import AsdfToolPathsRequest, AsdfToolPathsResult
 from pants.core.util_rules.environments import (
-    LOCAL_ENVIRONMENT_MATCHER,
-    EnvironmentRequest,
     EnvironmentsSubsystem,
     EnvironmentTarget,
     PythonBootstrapBinaryNamesField,
@@ -241,11 +239,9 @@ def get_pyenv_root(env: Environment) -> str | None:
 
 
 @rule
-async def python_bootstrap(python_bootstrap_subsystem: PythonBootstrapSubsystem) -> PythonBootstrap:
-    env_tgt = await Get(
-        EnvironmentTarget,
-        EnvironmentRequest(LOCAL_ENVIRONMENT_MATCHER, description_of_origin="<infallible>"),
-    )
+async def python_bootstrap(
+    python_bootstrap_subsystem: PythonBootstrapSubsystem, env_tgt: EnvironmentTarget
+) -> PythonBootstrap:
     if env_tgt.val is not None:
         interpreter_search_paths = env_tgt.val[PythonInterpreterSearchPathsField].value
         interpreter_names = env_tgt.val[PythonBootstrapBinaryNamesField].value

src/python/pants/core/util_rules/environments.py

@@ -5,17 +5,21 @@
 
 import dataclasses
 from dataclasses import dataclass
+from typing import cast
 
 from pants.build_graph.address import Address, AddressInput
 from pants.engine.engine_aware import EngineAwareParameter
+from pants.engine.environment import EnvironmentName as EnvironmentName
+from pants.engine.internals.graph import WrappedTargetForBootstrap
+from pants.engine.internals.scheduler import SchedulerSession
+from pants.engine.internals.selectors import Params
 from pants.engine.platform import Platform
-from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.engine.rules import Get, MultiGet, QueryRule, collect_rules, rule
 from pants.engine.target import (
     COMMON_TARGET_FIELDS,
     StringField,
     StringSequenceField,
     Target,
-    WrappedTarget,
     WrappedTargetRequest,
 )
 from pants.option.option_types import DictOption
@@ -172,6 +176,14 @@ class DockerEnvironmentTarget(Target):
 # -------------------------------------------------------------------------------------------
 
 
+def determine_bootstrap_environment(session: SchedulerSession) -> EnvironmentName:
+    local_env = cast(
+        ChosenLocalEnvironmentName,
+        session.product_request(ChosenLocalEnvironmentName, [Params()])[0],
+    )
+    return EnvironmentName(local_env.val)
+
+
 class NoCompatibleEnvironmentError(Exception):
     pass
 
@@ -190,28 +202,11 @@ class AllEnvironmentTargets(FrozenDict[str, Target]):
 
 @dataclass(frozen=True)
 class ChosenLocalEnvironmentName:
-    f"""Which environment name from `[environments-preview].names` that
-    {LOCAL_ENVIRONMENT_MATCHER} resolves to."""
+    """Which environment name from `[environments-preview].names` that __local__ resolves to."""
 
     val: str | None
 
 
-@dataclass(frozen=True)
-class EnvironmentName(EngineAwareParameter):
-    f"""The normalized name for an environment, from `[environments-preview].names`, after
-    applying things like {LOCAL_ENVIRONMENT_MATCHER}.
-
-    Note that we have this type, rather than only `EnvironmentTarget`, for a more efficient
-    rule graph. This node impacts the equality of many downstream nodes, so we want its identity
-    to only be a single string, rather than a Target instance.
-    """
-
-    val: str | None
-
-    def debug_hint(self) -> str:
-        return self.val or "<none>"
-
-
 @dataclass(frozen=True)
 class EnvironmentTarget:
     val: Target | None
@@ -245,8 +240,9 @@ async def determine_all_environments(
 
 @rule
 async def determine_local_environment(
-    platform: Platform, all_environment_targets: AllEnvironmentTargets
+    all_environment_targets: AllEnvironmentTargets,
 ) -> ChosenLocalEnvironmentName:
+    platform = Platform.create_for_localhost()
     if not all_environment_targets:
         return ChosenLocalEnvironmentName(None)
     compatible_name_and_targets = [
@@ -340,23 +336,23 @@ async def get_target_for_environment_name(
         ),
     )
     wrapped_target = await Get(
-        WrappedTarget,
+        WrappedTargetForBootstrap,
         WrappedTargetRequest(address, description_of_origin=_description_of_origin),
     )
-    tgt = wrapped_target.target
+    tgt = wrapped_target.val
     if not tgt.has_field(CompatiblePlatformsField) and not tgt.has_field(DockerImageField):
         raise ValueError(
             softwrap(
                 f"""
                 Expected to use the address to a `_local_environment` or `_docker_environment`
                 target in the option `[environments-preview].names`, but the name
                 `{env_name.val}` was set to the target {address.spec} with the target type
-                `{wrapped_target.target.alias}`.
+                `{tgt.alias}`.
                 """
             )
         )
     return EnvironmentTarget(tgt)
 
 
 def rules():
-    return collect_rules()
+    return (*collect_rules(), QueryRule(ChosenLocalEnvironmentName, []))

src/python/pants/core/util_rules/environments_integration_test.py

@@ -14,12 +14,18 @@ def test_unrecognized_build_file_symbols_during_bootstrap() -> None:
     # we special-case the bootstrap scheduler.
     build_file = dedent(
         """\
-        bad_tgt_type(name='bad')
-        _local_environment(name='env', bad_field=123)
+        # This target type's backend is not loaded during plugin resolution.
+        shell_sources(name='shell')
+
+        # TODO(#7735): Once we migrate the Shell backend to use environments, add one of its
+        #  plugin fields here
+        _local_environment(name='env')
         """
     )
     with setup_tmpdir({"BUILD": build_file}) as tmpdir:
-        args = [f"--environments-preview-names={{'env': '{tmpdir}:env'}}", "--plugins=ansicolors"]
-        run_pants([*args, "--version"]).assert_success()
-        # But then we should error after bootstrapping.
-        run_pants([*args, "list", tmpdir]).assert_failure()
+        args = [
+            "--backend-packages=pants.backend.shell",
+            f"--environments-preview-names={{'env': '{tmpdir}:env'}}",
+            "--plugins=ansicolors",
+        ]
+        run_pants([*args, "list", tmpdir]).assert_success()

src/python/pants/core/util_rules/environments_test.py

@@ -32,11 +32,11 @@ def rule_runner() -> RuleRunner:
         rules=[
             *environments.rules(),
             QueryRule(AllEnvironmentTargets, []),
-            QueryRule(ChosenLocalEnvironmentName, []),
             QueryRule(EnvironmentTarget, [EnvironmentName]),
             QueryRule(EnvironmentName, [EnvironmentRequest]),
         ],
         target_types=[LocalEnvironmentTarget, DockerEnvironmentTarget],
+        singleton_environment=None,
     )
 
 

src/python/pants/engine/environment.py

@@ -1,11 +1,13 @@
 # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
+from __future__ import annotations
 
 import logging
 import re
 from dataclasses import dataclass
 from typing import Dict, Optional, Sequence
 
+from pants.engine.engine_aware import EngineAwareParameter
 from pants.engine.internals.session import SessionValues
 from pants.engine.rules import collect_rules, rule
 from pants.util.frozendict import FrozenDict
@@ -28,10 +30,30 @@
 environment-variable matching APIs to remove ambiguity.
 """
 
+# ----------------------------------------------------------------------
+# Environments mechanism
+# ----------------------------------------------------------------------
+
 
 @dataclass(frozen=True)
-class EnvironmentName:
-    """TODO: Replace with `pants.core.util_rules.environments.EnvironmentName`."""
+class EnvironmentName(EngineAwareParameter):
+    """The normalized name for an environment, from `[environments-preview].names`, after applying
+    things like the __local__ matcher.
+
+    Note that we have this type, rather than only `EnvironmentTarget`, for a more efficient rule
+    graph. This node impacts the equality of many downstream nodes, so we want its identity to only
+    be a single string, rather than a Target instance.
+    """
+
+    val: str | None
+
+    def debug_hint(self) -> str:
+        return self.val or "<none>"
+
+
+# ----------------------------------------------------------------------
+# Environments variables
+# ----------------------------------------------------------------------
 
 
 class CompleteEnvironment(FrozenDict):

src/python/pants/engine/internals/build_files.py

@@ -31,11 +31,6 @@
 from pants.util.strutil import softwrap
 
 
-@dataclass(frozen=True)
-class IgnoreUnrecognizedBuildFileSymbols:
-    val: bool
-
-
 @dataclass(frozen=True)
 class BuildFileOptions:
     patterns: tuple[str, ...]

src/python/pants/engine/internals/graph.py

@@ -24,7 +24,6 @@
 from pants.engine.collection import Collection
 from pants.engine.fs import EMPTY_SNAPSHOT, GlobMatchErrorBehavior, PathGlobs, Paths, Snapshot
 from pants.engine.internals import native_engine
-from pants.engine.internals.build_files import IgnoreUnrecognizedBuildFileSymbols
 from pants.engine.internals.native_engine import AddressParseException
 from pants.engine.internals.parametrize import Parametrize, _TargetParametrization
 from pants.engine.internals.parametrize import (  # noqa: F401
@@ -170,20 +169,13 @@ def warn_deprecated_field_type(field_type: type[Field]) -> None:
     )
 
 
-@rule
-async def resolve_target_parametrizations(
-    request: _TargetParametrizationsRequest,
-    registered_target_types: RegisteredTargetTypes,
-    union_membership: UnionMembership,
-    target_types_to_generate_requests: TargetTypesToGenerateTargetsRequests,
-    unmatched_build_file_globs: UnmatchedBuildFileGlobs,
-    ignore_unrecognized_build_file_symbols: IgnoreUnrecognizedBuildFileSymbols,
-) -> _TargetParametrizations:
-    address = request.address
-
+@rule_helper
+async def _determine_target_adaptor_and_type(
+    address: Address, registered_target_types: RegisteredTargetTypes, *, description_of_origin: str
+) -> tuple[TargetAdaptor, type[Target]]:
     target_adaptor = await Get(
         TargetAdaptor,
-        TargetAdaptorRequest(address, description_of_origin=request.description_of_origin),
+        TargetAdaptorRequest(address, description_of_origin=description_of_origin),
     )
     target_type = registered_target_types.aliases_to_types.get(target_adaptor.type_alias, None)
     if target_type is None:
@@ -196,6 +188,21 @@ async def resolve_target_parametrizations(
         and not address.is_generated_target
     ):
         warn_deprecated_target_type(target_type)
+    return target_adaptor, target_type
+
+
+@rule
+async def resolve_target_parametrizations(
+    request: _TargetParametrizationsRequest,
+    registered_target_types: RegisteredTargetTypes,
+    union_membership: UnionMembership,
+    target_types_to_generate_requests: TargetTypesToGenerateTargetsRequests,
+    unmatched_build_file_globs: UnmatchedBuildFileGlobs,
+) -> _TargetParametrizations:
+    address = request.address
+    target_adaptor, target_type = await _determine_target_adaptor_and_type(
+        address, registered_target_types, description_of_origin=request.description_of_origin
+    )
 
     target = None
     parametrizations: list[_TargetParametrization] = []
@@ -300,7 +307,6 @@ async def resolve_target_parametrizations(
                         parameterized_address,
                         name_explicitly_set=target_adaptor.name_explicitly_set,
                         union_membership=union_membership,
-                        ignore_unrecognized_fields=ignore_unrecognized_build_file_symbols.val,
                     ),
                 )
                 for parameterized_address, parameterized_fields in (first, *rest)
@@ -313,7 +319,6 @@ async def resolve_target_parametrizations(
                 address,
                 name_explicitly_set=target_adaptor.name_explicitly_set,
                 union_membership=union_membership,
-                ignore_unrecognized_fields=ignore_unrecognized_build_file_symbols.val,
             )
             parametrizations.append(_TargetParametrization(target, FrozenDict()))
 
@@ -356,6 +361,41 @@ async def resolve_target(
     return WrappedTarget(target)
 
 
+@dataclass(frozen=True)
+class WrappedTargetForBootstrap:
+    """Used to avoid a rule graph cycle when evaluating bootstrap targets.
+
+    This does not work with target generation and parametrization. It also ignores any unrecognized
+    fields in the target, to accommodate plugin fields which are not yet registered during
+    bootstrapping.
+
+    This should only be used by bootstrapping code.
+    """
+
+    val: Target
+
+
+@rule
+async def resolve_target_for_bootstrapping(
+    request: WrappedTargetRequest,
+    registered_target_types: RegisteredTargetTypes,
+    union_membership: UnionMembership,
+) -> WrappedTargetForBootstrap:
+    target_adaptor, target_type = await _determine_target_adaptor_and_type(
+        request.address,
+        registered_target_types,
+        description_of_origin=request.description_of_origin,
+    )
+    target = target_type(
+        target_adaptor.kwargs,
+        request.address,
+        name_explicitly_set=target_adaptor.name_explicitly_set,
+        union_membership=union_membership,
+        ignore_unrecognized_fields=True,
+    )
+    return WrappedTargetForBootstrap(target)
+
+
 @rule
 async def resolve_targets(
     targets: UnexpandedTargets,

src/python/pants/engine/streaming_workunit_handler.py

@@ -10,6 +10,7 @@
 from typing import Any, Callable, Iterable, Sequence, Tuple
 
 from pants.base.specs import Specs
+from pants.core.util_rules.environments import determine_bootstrap_environment
 from pants.engine.addresses import Addresses
 from pants.engine.environment import EnvironmentName
 from pants.engine.fs import Digest, DigestContents, FileDigest, Snapshot
@@ -103,7 +104,11 @@ def get_expanded_specs(self) -> ExpandedSpecs:
         """Return a dict containing the canonicalized addresses of the specs for this run, and what
         files they expand to."""
 
-        params = Params(self._specs, self._options_bootstrapper, EnvironmentName())
+        params = Params(
+            self._specs,
+            self._options_bootstrapper,
+            determine_bootstrap_environment(self._scheduler),
+        )
         request = self._scheduler.execution_request([(Addresses, params), (Targets, params)])
         unexpanded_addresses, expanded_targets = self._scheduler.execute(request)