diff --git a/docs/docs/terraform/index.mdx b/docs/docs/terraform/index.mdx index 18dce02cc73..6f69c2fffe2 100644 --- a/docs/docs/terraform/index.mdx +++ b/docs/docs/terraform/index.mdx @@ -8,7 +8,8 @@ :::caution Terraform support is in alpha stage Pants is currently building support for developing and deploying Terraform. Simple use cases might be supported, but many options are missing. -Please share feedback for what you need to use Pants with your Terraform modules and deployments by either [opening a GitHub issue](https://github.com/pantsbuild/pants/issues/new/choose) or [joining our Slack](/community/getting-help)! +Terraform release progress is tracked in the [stability for release issue](https://github.com/pantsbuild/pants/issues/21119). +Please share feedback for what you need to use Pants with your Terraform modules and deployments by commenting on that issue, [opening a new GitHub issue](https://github.com/pantsbuild/pants/issues/new/choose) or [joining our Slack](/community/getting-help)! ::: ## Initial setup diff --git a/docs/notes/2.23.x.md b/docs/notes/2.23.x.md index 4b64c6ed481..a62bad38a9c 100644 --- a/docs/notes/2.23.x.md +++ b/docs/notes/2.23.x.md @@ -124,6 +124,8 @@ The `python_aws_lambda_function` and `python_aws_lambda_layer` targets now allow #### Terraform +Terraform supports caching providers. + The default version of terraform has been updated from 1.7.1 to 1.9.0. The `tfsec` linter now works on all supported platforms without extra config. diff --git a/src/python/pants/backend/terraform/dependencies.py b/src/python/pants/backend/terraform/dependencies.py index d1a2d4a74b8..c5e11a7f80e 100644 --- a/src/python/pants/backend/terraform/dependencies.py +++ b/src/python/pants/backend/terraform/dependencies.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations +import os import os.path from dataclasses import dataclass from typing import Optional @@ -88,12 +89,8 @@ async def get_terraform_providers( ), ) if fetched_deps.exit_code != 0: - raise ProcessExecutionFailure( - fetched_deps.exit_code, - fetched_deps.stdout, - fetched_deps.stderr, - init_process_description, - keep_sandboxes=keep_sandboxes, + raise ProcessExecutionFailure.from_result( + fetched_deps, init_process_description, keep_sandboxes ) return TerraformDependenciesResponse(fetched_deps.output_digest) diff --git a/src/python/pants/backend/terraform/dependencies_test.py b/src/python/pants/backend/terraform/dependencies_test.py index 8067e5902f2..63214064517 100644 --- a/src/python/pants/backend/terraform/dependencies_test.py +++ b/src/python/pants/backend/terraform/dependencies_test.py @@ -1,5 +1,7 @@ # Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + import dataclasses import json import textwrap @@ -14,7 +16,7 @@ standard_deployment, terraform_lockfile, ) -from pants.engine.fs import DigestContents, FileContent +from pants.engine.fs import DigestContents, DigestEntries, FileContent, SymlinkEntry from pants.engine.internals.native_engine import Address from pants.testutil.rule_runner import RuleRunner @@ -24,7 +26,7 @@ def _do_init_terraform( rule_runner: RuleRunner, standard_deployment: StandardDeployment, initialise_backend: bool -) -> DigestContents: +) -> tuple[DigestContents, DigestEntries]: rule_runner.write_files(standard_deployment.files) target = rule_runner.get_target(standard_deployment.target) field_set = DeployTerraformFieldSet.create(target) @@ -39,17 +41,30 @@ def _do_init_terraform( ], ) initialised_files = rule_runner.request(DigestContents, [result.sources_and_deps]) + initialised_entries = rule_runner.request(DigestEntries, [result.sources_and_deps]) assert isinstance(initialised_files, DigestContents) - return initialised_files + return initialised_files, initialised_entries def find_file(files: DigestContents, pattern: str) -> Optional[FileContent]: return next((file for file in files if Path(file.path).match(pattern)), None) +def find_link(entries: DigestEntries, pattern: str) -> Optional[SymlinkEntry]: + for entry in entries: + if not isinstance(entry, SymlinkEntry): + continue + + if Path(entry.path).match(pattern): + # allow any prefix to account for absolute targets + return entry + + return None + + def test_init_terraform(rule_runner: RuleRunner, standard_deployment: StandardDeployment) -> None: """Test for the happy path of initialising Terraform with a backend config.""" - initialised_files = _do_init_terraform( + initialised_files, initialised_links = _do_init_terraform( rule_runner, standard_deployment, initialise_backend=True ) @@ -60,9 +75,9 @@ def test_init_terraform(rule_runner: RuleRunner, standard_deployment: StandardDe assert stub_tfstate["backend"]["config"]["path"] == str(standard_deployment.state_file) # Assert dependencies are initialised by checking for the dependency itself - assert find_file( - initialised_files, - ".terraform/providers/registry.terraform.io/hashicorp/null/*/*/terraform-provider-null*", + assert find_link( + initialised_links, + ".terraform/providers/registry.terraform.io/hashicorp/null/*/*", ), "Did not find expected provider" # Assert lockfile is included @@ -80,7 +95,7 @@ def test_init_terraform_uses_lockfiles( files={**standard_deployment.files, **{"src/tf/.terraform.lock.hcl": terraform_lockfile}}, ) - initialised_files = _do_init_terraform( + initialised_files, initialised_entries = _do_init_terraform( rule_runner, deployment_with_lockfile, initialise_backend=True ) @@ -92,9 +107,9 @@ def test_init_terraform_uses_lockfiles( ), "version in lockfile has changed, we should not have regenerated the lockfile" # Assert dependencies are initialised to the older version - result_provider = find_file( - initialised_files, - ".terraform/providers/registry.terraform.io/hashicorp/null/*/*/terraform-provider-null*", + result_provider = find_link( + initialised_entries, + ".terraform/providers/registry.terraform.io/hashicorp/null/*/*", ) assert result_provider, "Did not find any providers, did we initialise them successfully?" assert ( @@ -105,7 +120,7 @@ def test_init_terraform_uses_lockfiles( def test_init_terraform_without_backends( rule_runner: RuleRunner, standard_deployment: StandardDeployment ) -> None: - initialised_files = _do_init_terraform( + initialised_files, initialised_entries = _do_init_terraform( rule_runner, standard_deployment, initialise_backend=False ) @@ -115,9 +130,9 @@ def test_init_terraform_without_backends( ), "Terraform state file should not be present if the request was to not initialise the backend" # The dependencies should still be present - assert find_file( - initialised_files, - ".terraform/providers/registry.terraform.io/hashicorp/null/*/*/terraform-provider-null*", + assert find_link( + initialised_entries, + ".terraform/providers/registry.terraform.io/hashicorp/null/*/*", ), "Did not find expected provider" @@ -157,7 +172,9 @@ def test_init_terraform_with_transitive_module(rule_runner: RuleRunner, tmpdir) Path(str(tmpdir.mkdir(".terraform").join("state.json"))), Address("src/tf/deployment", target_name="root"), ) - initialised_files = _do_init_terraform(rule_runner, deployment, initialise_backend=True) + initialised_files, initialised_entries = _do_init_terraform( + rule_runner, deployment, initialise_backend=True + ) assert initialised_files # Assert that init succeeded and created the modules mapping @@ -175,7 +192,7 @@ def test_init_terraform_with_transitive_module(rule_runner: RuleRunner, tmpdir) ) # Assert that the provider dependency was initialised - assert find_file( - initialised_files, - ".terraform/providers/registry.terraform.io/hashicorp/null/*/*/terraform-provider-null*", + assert find_link( + initialised_entries, + ".terraform/providers/registry.terraform.io/hashicorp/null/*/*", ), "Did not find expected provider contained in module, did we successfully include it in the files passed to `init`?" diff --git a/src/python/pants/backend/terraform/testutil.py b/src/python/pants/backend/terraform/testutil.py index e735ebdd60f..c38391a989e 100644 --- a/src/python/pants/backend/terraform/testutil.py +++ b/src/python/pants/backend/terraform/testutil.py @@ -23,7 +23,8 @@ from pants.core.register import rules as core_rules from pants.core.util_rules import source_files from pants.engine import process -from pants.engine.internals.native_engine import Address +from pants.engine.fs import DigestEntries +from pants.engine.internals.native_engine import Address, Digest from pants.engine.rules import QueryRule from pants.testutil.rule_runner import RuleRunner @@ -50,6 +51,7 @@ def rule_runner_with_auto_approve() -> RuleRunner: *process.rules(), QueryRule(DeployProcess, (DeployTerraformFieldSet,)), QueryRule(TerraformInitResponse, (TerraformInitRequest,)), + QueryRule(DigestEntries, (Digest,)), ], preserve_tmpdirs=True, ) diff --git a/src/python/pants/backend/terraform/tool.py b/src/python/pants/backend/terraform/tool.py index 228896353d9..399da6c6d30 100644 --- a/src/python/pants/backend/terraform/tool.py +++ b/src/python/pants/backend/terraform/tool.py @@ -1,7 +1,22 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +"""# Terraform + +## Caching: Pants uses the [provider cache](https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache) for caching providers. +These are the things that need to be downloaded, so this provides the most speedup. +We use the providers cache instead of identifying and caching the providers individually for a few reasons: +1. This leverages Terraform's existing caching mechanism +2. This is much simpler +3. This incurs almost no overhead, since it is done as part of `terraform init`. We don't need to run more analysers or separately download providers + +We didn't use `terraform providers lock` for a few reasons: +1. `terraform providers lock` isn't designed for this usecase, it's designed to create mirrors of providers. It does more work (to set up manifests) and would require us to set more config settings +2. `terraform providers lock` doesn't use itself as a cache. So every time we would want to refresh the cache, we need to download _everything_ again. Even if nothing has changed. +""" + from __future__ import annotations +import os import shlex from dataclasses import dataclass from pathlib import Path @@ -13,6 +28,7 @@ ExternalToolRequest, TemplatedExternalTool, ) +from pants.core.util_rules.system_binaries import BinaryShims, BinaryShimsRequest, GetentBinary from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest from pants.engine.fs import EMPTY_DIGEST, Digest from pants.engine.internals.selectors import Get @@ -374,6 +390,14 @@ def default_known_versions(cls): advanced=True, ) + @property + def plugin_cache_dir(self) -> str: + return "__terraform_filesystem_mirror" + + @property + def append_only_caches(self) -> dict[str, str]: + return {"terraform_plugins": self.plugin_cache_dir} + @dataclass(frozen=True) class TerraformProcess: @@ -389,7 +413,10 @@ class TerraformProcess: @rule async def setup_terraform_process( - request: TerraformProcess, terraform: TerraformTool, platform: Platform + request: TerraformProcess, + terraform: TerraformTool, + getent_binary: GetentBinary, + platform: Platform, ) -> Process: downloaded_terraform = await Get( DownloadedExternalTool, @@ -398,7 +425,30 @@ async def setup_terraform_process( ) env = await Get(EnvironmentVars, EnvironmentVarsRequest(terraform.extra_env_vars)) - immutable_input_digests = {"__terraform": downloaded_terraform.digest} + extra_bins = await Get( + BinaryShims, + BinaryShimsRequest, + BinaryShimsRequest.for_paths(getent_binary, rationale="download terraform providers"), + ) + + path = [] + user_path = env.get("PATH") + if user_path: + path.append(user_path) + path.append(extra_bins.path_component) + + env = EnvironmentVars( + { + **env, + "PATH": ":".join(path), + "TF_PLUGIN_CACHE_DIR": (os.path.join("{chroot}", terraform.plugin_cache_dir)), + } + ) + + immutable_input_digests = { + "__terraform": downloaded_terraform.digest, + **extra_bins.immutable_input_digests, + } def prepend_paths(paths: Tuple[str, ...]) -> Tuple[str, ...]: return tuple((Path(request.chdir) / path).as_posix() for path in paths) @@ -409,6 +459,7 @@ def prepend_paths(paths: Tuple[str, ...]) -> Tuple[str, ...]: immutable_input_digests=immutable_input_digests, output_files=prepend_paths(request.output_files), output_directories=prepend_paths(request.output_directories), + append_only_caches=terraform.append_only_caches, env=env, description=request.description, level=LogLevel.DEBUG, diff --git a/src/python/pants/core/util_rules/system_binaries.py b/src/python/pants/core/util_rules/system_binaries.py index 6e0a928ab93..f59c13e8326 100644 --- a/src/python/pants/core/util_rules/system_binaries.py +++ b/src/python/pants/core/util_rules/system_binaries.py @@ -414,6 +414,10 @@ class FindBinary(BinaryPath): pass +class GetentBinary(BinaryPath): + pass + + class GpgBinary(BinaryPath): pass @@ -907,6 +911,16 @@ async def find_git(system_binaries: SystemBinariesSubsystem.EnvironmentAware) -> return GitBinary(first_path.path, first_path.fingerprint) +@rule(desc="Finding the `getent` binary", level=LogLevel.DEBUG) +async def find_getent(system_binaries: SystemBinariesSubsystem.EnvironmentAware) -> GetentBinary: + request = BinaryPathRequest( + binary_name="getent", search_path=system_binaries.system_binary_paths + ) + paths = await Get(BinaryPaths, BinaryPathRequest, request) + first_path = paths.first_path_or_raise(request, rationale="getent file") + return GetentBinary(first_path.path, first_path.fingerprint) + + @rule(desc="Finding the `gpg` binary", level=LogLevel.DEBUG) async def find_gpg(system_binaries: SystemBinariesSubsystem.EnvironmentAware) -> GpgBinary: request = BinaryPathRequest(binary_name="gpg", search_path=system_binaries.system_binary_paths) diff --git a/src/python/pants/engine/process.py b/src/python/pants/engine/process.py index abd4dd820ec..855e1e38d68 100644 --- a/src/python/pants/engine/process.py +++ b/src/python/pants/engine/process.py @@ -287,6 +287,18 @@ def try_decode(content: bytes) -> str: ) super().__init__("\n".join(err_strings)) + @classmethod + def from_result( + cls, result: FallibleProcessResult, description: str, keep_sandboxes: KeepSandboxes + ) -> ProcessExecutionFailure: + return cls( + result.exit_code, + result.stdout, + result.stderr, + description, + keep_sandboxes=keep_sandboxes, + ) + @rule def get_multi_platform_request_description(req: Process) -> ProductDescription: