From 4ce45e1e8dd27bcfd679cc7c53d5f5ac709d897b Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 4 Oct 2023 21:23:27 -0700 Subject: [PATCH 01/11] Add support for OpenTofu --- .circleci/config.yml | 8 ++++++++ modules/terraform/cmd.go | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0520d799..53f7145e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,6 +62,14 @@ install_gruntwork_utils: &install_gruntwork_utils sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" sudo ln -s /usr/local/go/bin/go /usr/bin/go echo "The installed version of Go is now $(go version)" + + # Install OpenTofu + curl -L "https://github.com/opentofu/opentofu/releases/download/v1.6.0-alpha1/tofu_1.6.0-alpha1_linux_amd64.zip" -o tofu.zip + unzip -o tofu.zip + sudo install -m 0755 tofu /usr/local/bin/tofu + rm -rf tofu + rm -rf tofu.zip + tofu --version install_docker_buildx: &install_docker_buildx name: install docker buildx diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 3591c083a..39b3a8d99 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -33,7 +33,7 @@ var commandsWithParallelism = []string{ // GetCommonOptions extracts commons terraform options func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "" { - options.TerraformBinary = "terraform" + options.TerraformBinary = "tofu" } if options.TerraformBinary == "terragrunt" { From 86c5f64527a135347ae54df4635c65c4a11c28f1 Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 4 Oct 2023 23:27:49 -0700 Subject: [PATCH 02/11] Updated tests --- modules/terraform/plan_test.go | 5 ++++- modules/terraform/show_test.go | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/terraform/plan_test.go b/modules/terraform/plan_test.go index 33c627ea2..4f6517513 100644 --- a/modules/terraform/plan_test.go +++ b/modules/terraform/plan_test.go @@ -3,6 +3,7 @@ package terraform import ( "fmt" "path/filepath" + "strings" "testing" "github.com/gruntwork-io/terratest/modules/files" @@ -79,8 +80,10 @@ func TestInitAndPlanWithPlanFile(t *testing.T) { out, err := InitAndPlanE(t, options) require.NoError(t, err) + // clean output to be consistent in checks + out = strings.ReplaceAll(out, "\n", "") assert.Contains(t, out, "1 to add, 0 to change, 0 to destroy.") - assert.Contains(t, out, fmt.Sprintf("Saved the plan to: %s", planFilePath)) + assert.Contains(t, out, fmt.Sprintf("Saved the plan to:%s", planFilePath)) assert.FileExists(t, planFilePath, "Plan file was not saved to expected location:", planFilePath) } diff --git a/modules/terraform/show_test.go b/modules/terraform/show_test.go index d844fff09..f47ac4eeb 100644 --- a/modules/terraform/show_test.go +++ b/modules/terraform/show_test.go @@ -3,6 +3,7 @@ package terraform import ( "fmt" "path/filepath" + "strings" "testing" "github.com/gruntwork-io/terratest/modules/files" @@ -25,7 +26,8 @@ func TestShowWithInlinePlan(t *testing.T) { } out := InitAndPlan(t, options) - require.Contains(t, out, fmt.Sprintf("Saved the plan to: %s", planFilePath)) + out = strings.ReplaceAll(out, "\n", "") + require.Contains(t, out, fmt.Sprintf("Saved the plan to:%s", planFilePath)) require.FileExists(t, planFilePath, "Plan file was not saved to expected location:", planFilePath) // show command does not accept Vars @@ -55,7 +57,8 @@ func TestShowWithStructInlinePlan(t *testing.T) { } out := InitAndPlan(t, options) - require.Contains(t, out, fmt.Sprintf("Saved the plan to: %s", planFilePath)) + out = strings.ReplaceAll(out, "\n", "") + require.Contains(t, out, fmt.Sprintf("Saved the plan to:%s", planFilePath)) require.FileExists(t, planFilePath, "Plan file was not saved to expected location:", planFilePath) // show command does not accept Vars From 88d9be247e0f6a34e1bbc90ee343bd1c108ec599 Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 4 Oct 2023 23:44:20 -0700 Subject: [PATCH 03/11] Testing provider version update --- examples/terraform-aws-ec2-windows-example/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/terraform-aws-ec2-windows-example/main.tf b/examples/terraform-aws-ec2-windows-example/main.tf index 498d6fd73..402f04154 100644 --- a/examples/terraform-aws-ec2-windows-example/main.tf +++ b/examples/terraform-aws-ec2-windows-example/main.tf @@ -8,7 +8,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "< 4.0" + version = "< 5.0" } } } From a0e089cf0abaa2144980eb4af024a8dbd167001e Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 4 Oct 2023 23:59:50 -0700 Subject: [PATCH 04/11] Providers versions update --- examples/azure/terraform-azure-aci-example/main.tf | 4 ++-- examples/azure/terraform-azure-acr-example/main.tf | 2 +- .../azure/terraform-azure-availabilityset-example/main.tf | 2 +- examples/azure/terraform-azure-cosmosdb-example/main.tf | 2 +- examples/azure/terraform-azure-disk-example/main.tf | 2 +- examples/azure/terraform-azure-monitor-example/main.tf | 6 +++--- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/azure/terraform-azure-aci-example/main.tf b/examples/azure/terraform-azure-aci-example/main.tf index 7aa3d1026..cdc15196d 100644 --- a/examples/azure/terraform-azure-aci-example/main.tf +++ b/examples/azure/terraform-azure-aci-example/main.tf @@ -11,7 +11,7 @@ terraform { required_providers { azurerm = { - version = "~>2.29.0" + version = "~>3.75.0" source = "hashicorp/azurerm" } } @@ -39,7 +39,7 @@ resource "azurerm_container_group" "aci" { location = azurerm_resource_group.rg.location resource_group_name = azurerm_resource_group.rg.name - ip_address_type = "public" + ip_address_type = "Public" dns_name_label = "aci${var.postfix}" os_type = "Linux" diff --git a/examples/azure/terraform-azure-acr-example/main.tf b/examples/azure/terraform-azure-acr-example/main.tf index 431564466..6614779f5 100644 --- a/examples/azure/terraform-azure-acr-example/main.tf +++ b/examples/azure/terraform-azure-acr-example/main.tf @@ -11,7 +11,7 @@ terraform { required_providers { azurerm = { - version = "~>2.29.0" + version = "~>3.75.0" source = "hashicorp/azurerm" } } diff --git a/examples/azure/terraform-azure-availabilityset-example/main.tf b/examples/azure/terraform-azure-availabilityset-example/main.tf index e0353e2f8..20ae59d73 100644 --- a/examples/azure/terraform-azure-availabilityset-example/main.tf +++ b/examples/azure/terraform-azure-availabilityset-example/main.tf @@ -20,7 +20,7 @@ terraform { required_version = ">= 0.12.26" required_providers { azurerm = { - version = "~> 2.50" + version = "~>3.75.0" source = "hashicorp/azurerm" } } diff --git a/examples/azure/terraform-azure-cosmosdb-example/main.tf b/examples/azure/terraform-azure-cosmosdb-example/main.tf index 2c4531710..a00352914 100644 --- a/examples/azure/terraform-azure-cosmosdb-example/main.tf +++ b/examples/azure/terraform-azure-cosmosdb-example/main.tf @@ -21,7 +21,7 @@ terraform { required_version = ">= 0.12.26" required_providers { azurerm = { - version = "~> 2.29" + version = "~>3.75.0" source = "hashicorp/azurerm" } } diff --git a/examples/azure/terraform-azure-disk-example/main.tf b/examples/azure/terraform-azure-disk-example/main.tf index 700b93492..e97fc7b41 100644 --- a/examples/azure/terraform-azure-disk-example/main.tf +++ b/examples/azure/terraform-azure-disk-example/main.tf @@ -16,7 +16,7 @@ terraform { required_version = ">= 0.12.26" required_providers { azurerm = { - version = "~> 2.29" + version = "~>3.75.0" source = "hashicorp/azurerm" } } diff --git a/examples/azure/terraform-azure-monitor-example/main.tf b/examples/azure/terraform-azure-monitor-example/main.tf index f097a9b76..dedd389bf 100644 --- a/examples/azure/terraform-azure-monitor-example/main.tf +++ b/examples/azure/terraform-azure-monitor-example/main.tf @@ -21,11 +21,11 @@ terraform { required_version = ">= 0.12.26" required_providers { azurerm = { - version = "~> 2.29" + version = "~>3.75.0" source = "hashicorp/azurerm" } azuread = { - version = "=0.7.0" + version = "=2.43.0" source = "hashicorp/azuread" } } @@ -84,7 +84,7 @@ resource "azurerm_key_vault" "monitor" { resource_group_name = azurerm_resource_group.monitor.name enabled_for_disk_encryption = true tenant_id = data.azurerm_client_config.current.tenant_id - soft_delete_enabled = true + soft_delete_retention_days = 7 purge_protection_enabled = false sku_name = "standard" From a6df39cc43e1ca0ba8c88bf879805194a3c251b2 Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 5 Oct 2023 13:06:51 -0700 Subject: [PATCH 05/11] Cleanup broken modules --- examples/azure/terraform-azure-aci-example/main.tf | 4 ++-- examples/azure/terraform-azure-acr-example/main.tf | 2 +- examples/azure/terraform-azure-cosmosdb-example/main.tf | 2 +- examples/azure/terraform-azure-disk-example/main.tf | 2 +- examples/azure/terraform-azure-monitor-example/main.tf | 6 +++--- examples/terraform-aws-ec2-windows-example/main.tf | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/azure/terraform-azure-aci-example/main.tf b/examples/azure/terraform-azure-aci-example/main.tf index cdc15196d..7aa3d1026 100644 --- a/examples/azure/terraform-azure-aci-example/main.tf +++ b/examples/azure/terraform-azure-aci-example/main.tf @@ -11,7 +11,7 @@ terraform { required_providers { azurerm = { - version = "~>3.75.0" + version = "~>2.29.0" source = "hashicorp/azurerm" } } @@ -39,7 +39,7 @@ resource "azurerm_container_group" "aci" { location = azurerm_resource_group.rg.location resource_group_name = azurerm_resource_group.rg.name - ip_address_type = "Public" + ip_address_type = "public" dns_name_label = "aci${var.postfix}" os_type = "Linux" diff --git a/examples/azure/terraform-azure-acr-example/main.tf b/examples/azure/terraform-azure-acr-example/main.tf index 6614779f5..431564466 100644 --- a/examples/azure/terraform-azure-acr-example/main.tf +++ b/examples/azure/terraform-azure-acr-example/main.tf @@ -11,7 +11,7 @@ terraform { required_providers { azurerm = { - version = "~>3.75.0" + version = "~>2.29.0" source = "hashicorp/azurerm" } } diff --git a/examples/azure/terraform-azure-cosmosdb-example/main.tf b/examples/azure/terraform-azure-cosmosdb-example/main.tf index a00352914..2c4531710 100644 --- a/examples/azure/terraform-azure-cosmosdb-example/main.tf +++ b/examples/azure/terraform-azure-cosmosdb-example/main.tf @@ -21,7 +21,7 @@ terraform { required_version = ">= 0.12.26" required_providers { azurerm = { - version = "~>3.75.0" + version = "~> 2.29" source = "hashicorp/azurerm" } } diff --git a/examples/azure/terraform-azure-disk-example/main.tf b/examples/azure/terraform-azure-disk-example/main.tf index e97fc7b41..700b93492 100644 --- a/examples/azure/terraform-azure-disk-example/main.tf +++ b/examples/azure/terraform-azure-disk-example/main.tf @@ -16,7 +16,7 @@ terraform { required_version = ">= 0.12.26" required_providers { azurerm = { - version = "~>3.75.0" + version = "~> 2.29" source = "hashicorp/azurerm" } } diff --git a/examples/azure/terraform-azure-monitor-example/main.tf b/examples/azure/terraform-azure-monitor-example/main.tf index dedd389bf..f097a9b76 100644 --- a/examples/azure/terraform-azure-monitor-example/main.tf +++ b/examples/azure/terraform-azure-monitor-example/main.tf @@ -21,11 +21,11 @@ terraform { required_version = ">= 0.12.26" required_providers { azurerm = { - version = "~>3.75.0" + version = "~> 2.29" source = "hashicorp/azurerm" } azuread = { - version = "=2.43.0" + version = "=0.7.0" source = "hashicorp/azuread" } } @@ -84,7 +84,7 @@ resource "azurerm_key_vault" "monitor" { resource_group_name = azurerm_resource_group.monitor.name enabled_for_disk_encryption = true tenant_id = data.azurerm_client_config.current.tenant_id - soft_delete_retention_days = 7 + soft_delete_enabled = true purge_protection_enabled = false sku_name = "standard" diff --git a/examples/terraform-aws-ec2-windows-example/main.tf b/examples/terraform-aws-ec2-windows-example/main.tf index 402f04154..498d6fd73 100644 --- a/examples/terraform-aws-ec2-windows-example/main.tf +++ b/examples/terraform-aws-ec2-windows-example/main.tf @@ -8,7 +8,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "< 5.0" + version = "< 4.0" } } } From e9074908653d5ad2ed11d26a3593fbd5484f13c4 Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 5 Oct 2023 13:10:39 -0700 Subject: [PATCH 06/11] Updated default options --- examples/azure/terraform-azure-availabilityset-example/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/azure/terraform-azure-availabilityset-example/main.tf b/examples/azure/terraform-azure-availabilityset-example/main.tf index 20ae59d73..e0353e2f8 100644 --- a/examples/azure/terraform-azure-availabilityset-example/main.tf +++ b/examples/azure/terraform-azure-availabilityset-example/main.tf @@ -20,7 +20,7 @@ terraform { required_version = ">= 0.12.26" required_providers { azurerm = { - version = "~>3.75.0" + version = "~> 2.50" source = "hashicorp/azurerm" } } From f3158e0a1838321eecba563b909da92af43bcc7b Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 5 Oct 2023 13:43:35 -0700 Subject: [PATCH 07/11] Add configuration of default terraform executable --- modules/terraform/cmd.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 39b3a8d99..ab050fd7e 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -2,6 +2,7 @@ package terraform import ( "fmt" + "os/exec" "github.com/gruntwork-io/terratest/modules/collections" "github.com/gruntwork-io/terratest/modules/retry" @@ -30,10 +31,20 @@ var commandsWithParallelism = []string{ "destroy-all", } +const ( + // TofuDefaultPath command to run tofu + TofuDefaultPath = "tofu" + + // TerraformDefaultPath to run terraform + TerraformDefaultPath = "terraform" +) + +var defaultExecutable = defaultTerraformExecutable() + // GetCommonOptions extracts commons terraform options func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "" { - options.TerraformBinary = "tofu" + options.TerraformBinary = defaultExecutable } if options.TerraformBinary == "terragrunt" { @@ -112,3 +123,17 @@ func GetExitCodeForTerraformCommandE(t testing.TestingT, additionalOptions *Opti } return DefaultErrorExitCode, getExitCodeErr } + +func defaultTerraformExecutable() string { + cmd := exec.Command(TerraformDefaultPath, "-version") + cmd.Stdin = nil + cmd.Stdout = nil + cmd.Stderr = nil + + if err := cmd.Run(); err == nil { + return TerraformDefaultPath + } + + // fallback to Tofu if terraform is not available + return TofuDefaultPath +} From 3d4d4987ba6879013753527fd0748c7ddf7b33c3 Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 5 Oct 2023 13:52:09 -0700 Subject: [PATCH 08/11] Added separated integration test for tofu --- .circleci/config.yml | 66 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 53f7145e0..1705aecfe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,8 +62,10 @@ install_gruntwork_utils: &install_gruntwork_utils sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" sudo ln -s /usr/local/go/bin/go /usr/bin/go echo "The installed version of Go is now $(go version)" - - # Install OpenTofu + +install_tofu: &install_tofu + name: Install OpenTofu + command: | curl -L "https://github.com/opentofu/opentofu/releases/download/v1.6.0-alpha1/tofu_1.6.0-alpha1_linux_amd64.zip" -o tofu.zip unzip -o tofu.zip sudo install -m 0755 tofu /usr/local/bin/tofu @@ -166,7 +168,8 @@ jobs: paths: - project - test: + # run tests with terraform binary + test_terraform: <<: *defaults resource_class: large steps: @@ -203,6 +206,48 @@ jobs: - store_test_results: path: /tmp/logs + # run tests with tofu binary + test_tofu: + <<: *defaults + resource_class: large + steps: + - attach_workspace: + at: /home/circleci + + - run: + <<: *install_gruntwork_utils + - run: + <<: *install_docker_buildx + - run: + <<: *install_tofu + + # The weird way you have to set PATH in Circle 2.0 + - run: | + echo 'export PATH=$HOME/.local/bin:$HOME/terraform:$HOME/packer:$PATH' >> $BASH_ENV + # remove terraform binary so tofu will be used + sudo rm -f $(which terraform) + + # Run the tests. Note that we set the "-p 1" flag to tell Go to run tests in each package sequentially. Without + # this, Go buffers all log output until all packages are done, which with slower running tests can cause CircleCI + # to kill the build after more than 10 minutes without log output. + # NOTE: because this doesn't build with the kubernetes tag, it will not run the kubernetes tests. See + # kubernetes_test build steps. + - run: mkdir -p /tmp/logs + # check we can compile the azure code, but don't actually run the tests + - run: run-go-tests --packages "-p 1 -tags=azure -run IDontExist ./modules/azure" + - run: run-go-tests --packages "-p 1 ./..." | tee /tmp/logs/test_output.log + + - run: + command: | + ./cmd/bin/terratest_log_parser_linux_amd64 --testlog /tmp/logs/test_output.log --outputdir /tmp/logs + when: always + + # Store test result and log artifacts for browsing purposes + - store_artifacts: + path: /tmp/logs + - store_test_results: + path: /tmp/logs + # We run the GCP tests in a separate build step using the Docker executor for better isolation and resiliency. Using # The Docker executor ensures GCP tests do not erroneously make metadata network calls within CircleCI's private # environment. For more information see: https://github.com/gruntwork-io/terratest/pull/765. @@ -371,7 +416,20 @@ workflows: tags: only: /^v.*/ - - test: + - test_terraform: + context: + - AWS__PHXDEVOPS__circle-ci-test + - GITHUB__PAT__gruntwork-ci + - SLACK__TOKEN__refarch-deployer-test + - SLACK__WEBHOOK__refarch-deployer-test + - SLACK__CHANNEL__test-workflow-approvals + requires: + - setup + filters: + tags: + only: /^v.*/ + + - test_tofu: context: - AWS__PHXDEVOPS__circle-ci-test - GITHUB__PAT__gruntwork-ci From b2566ab94dc686525d3437fd8f4125faf07d5cdf Mon Sep 17 00:00:00 2001 From: Denis O Date: Fri, 6 Oct 2023 09:54:58 -0700 Subject: [PATCH 09/11] Terragrunt version update --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1705aecfe..9fbd8a198 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ env: &env MODULE_GCP_CI_VERSION: v0.1.1 TERRAFORM_VERSION: 1.1.4 PACKER_VERSION: 1.7.4 - TERRAGRUNT_VERSION: v0.36.0 + TERRAGRUNT_VERSION: v0.52.0 OPA_VERSION: v0.33.1 GO_VERSION: 1.21.1 GO111MODULE: auto From a7af9cda1f4b1db88a8d0b0f631b079fbe942455 Mon Sep 17 00:00:00 2001 From: Denis O Date: Fri, 6 Oct 2023 14:13:45 -0700 Subject: [PATCH 10/11] Fixed terraform version checking --- modules/terraform/cmd.go | 4 ++-- modules/version-checker/version_checker.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index ab050fd7e..ff7f425c1 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -39,12 +39,12 @@ const ( TerraformDefaultPath = "terraform" ) -var defaultExecutable = defaultTerraformExecutable() +var DefaultExecutable = defaultTerraformExecutable() // GetCommonOptions extracts commons terraform options func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "" { - options.TerraformBinary = defaultExecutable + options.TerraformBinary = DefaultExecutable } if options.TerraformBinary == "terragrunt" { diff --git a/modules/version-checker/version_checker.go b/modules/version-checker/version_checker.go index a6ac4cca7..73d22e194 100644 --- a/modules/version-checker/version_checker.go +++ b/modules/version-checker/version_checker.go @@ -4,6 +4,8 @@ import ( "fmt" "regexp" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/gruntwork-io/terratest/modules/shell" "github.com/gruntwork-io/terratest/modules/testing" "github.com/hashicorp/go-version" @@ -125,7 +127,7 @@ func getBinary(params CheckVersionParams) (string, error) { case Packer: return "packer", nil case Terraform: - return "terraform", nil + return terraform.DefaultExecutable, nil default: return "", fmt.Errorf("unsupported Binary for checking versions {%d}", params.Binary) } From df5dcc8c34954977d902d38e3b16291f3e0a98a3 Mon Sep 17 00:00:00 2001 From: Denis O Date: Mon, 9 Oct 2023 07:19:04 -0700 Subject: [PATCH 11/11] Updated terraform version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9fbd8a198..0061c2cb9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ env: &env GRUNTWORK_INSTALLER_VERSION: v0.0.36 MODULE_CI_VERSION: v0.46.0 MODULE_GCP_CI_VERSION: v0.1.1 - TERRAFORM_VERSION: 1.1.4 + TERRAFORM_VERSION: 1.5.7 PACKER_VERSION: 1.7.4 TERRAGRUNT_VERSION: v0.52.0 OPA_VERSION: v0.33.1