Fix build error due to deleted module. Add pause_before to packer-basic-example. Shave other yaks.

.circleci/config.yml

@@ -1,6 +1,6 @@
 env: &env
   environment:
-    GRUNTWORK_INSTALLER_VERSION: v0.0.30
+    GRUNTWORK_INSTALLER_VERSION: v0.0.33
     MODULE_CI_VERSION: v0.29.2
     TERRAFORM_VERSION: 0.14.0
     TERRAGRUNT_VERSION: v0.24.2
@@ -35,7 +35,10 @@ install_gruntwork_utils: &install_gruntwork_utils
     curl -Ls https://raw.githubusercontent.com/gruntwork-io/gruntwork-installer/master/bootstrap-gruntwork-installer.sh | bash /dev/stdin --version "${GRUNTWORK_INSTALLER_VERSION}"
     gruntwork-install --module-name "gruntwork-module-circleci-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}"
     gruntwork-install --module-name "kubernetes-circleci-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}"
-    gruntwork-install --module-name "gcp-helpers" --repo "https://github.com/gruntwork-io/module-ci" --branch "gcp-helpers"
+
+    # TODO: Switch to a release version when https://github.com/gruntwork-io/terraform-google-ci/pull/10 is merged!!
+    gruntwork-install --module-name "gcp-helpers" --repo "https://github.com/gruntwork-io/terraform-google-ci" --branch "gcp-helpers"
+
     configure-environment-for-gruntwork-module \
       --terraform-version ${TERRAFORM_VERSION} \
       --terragrunt-version ${TERRAGRUNT_VERSION} \

examples/packer-basic-example/build.json

@@ -9,13 +9,14 @@
     "oci_base_image_ocid": "",
     "oci_compartment_ocid": "",
     "oci_pass_phrase": "",
-    "oci_subnet_ocid": ""
+    "oci_subnet_ocid": "",
+    "instance_type": "t2.micro"
   },
   "builders": [{
     "type": "amazon-ebs",
     "ami_name": "{{user `ami_base_name`}}-terratest-packer-example",
     "ami_description": "An example of how to create a custom AMI on top of Ubuntu",
-    "instance_type": "t2.micro",
+    "instance_type": "{{user `instance_type`}}",
     "region": "{{user `aws_region`}}",
     "source_ami_filter": {
       "filters": {
@@ -54,6 +55,7 @@
     "inline": [
       "sudo DEBIAN_FRONTEND=noninteractive apt-get update",
       "sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y"
-    ]
+    ],
+    "pause_before": "30s"
   }]
 }

examples/packer-docker-example/build.json

@@ -1,13 +1,14 @@
 {
   "variables": {
     "aws_region": "us-east-1",
-    "ami_name_base": "terratest-packer-docker-example"
+    "ami_name_base": "terratest-packer-docker-example",
+    "instance_type": "t2.micro"
   },
   "builders": [{
     "name": "ubuntu-ami",
     "ami_name": "{{user `ami_name_base`}}-{{isotime | clean_resource_name}}",
     "ami_description": "An example of how to create a custom AMI with a simple web app on top of Ubuntu",
-    "instance_type": "t2.micro",
+    "instance_type": "{{user `instance_type`}}",
     "region": "{{user `aws_region`}}",
     "type": "amazon-ebs",
     "source_ami_filter": {

modules/terraform/apply_test.go

@@ -15,10 +15,10 @@ func TestApplyNoError(t *testing.T) {
 	testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-no-error", t.Name())
 	require.NoError(t, err)
 
-	options := &Options{
+	options := WithDefaultRetryableErrors(t, &Options{
 		TerraformDir: testFolder,
 		NoColor:      true,
-	}
+	})
 
 	out := InitAndApply(t, options)
 
@@ -34,9 +34,9 @@ func TestApplyWithErrorNoRetry(t *testing.T) {
 	testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-with-error", t.Name())
 	require.NoError(t, err)
 
-	options := &Options{
+	options := WithDefaultRetryableErrors(t, &Options{
 		TerraformDir: testFolder,
-	}
+	})
 
 	out, err := InitAndApplyE(t, options)
 
@@ -50,13 +50,13 @@ func TestApplyWithErrorWithRetry(t *testing.T) {
 	testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-with-error", t.Name())
 	require.NoError(t, err)
 
-	options := &Options{
+	options := WithDefaultRetryableErrors(t, &Options{
 		TerraformDir: testFolder,
 		MaxRetries:   1,
 		RetryableTerraformErrors: map[string]string{
 			"This is the first run, exiting with an error": "Intentional failure in test fixture",
 		},
-	}
+	})
 
 	out := InitAndApply(t, options)
 
@@ -68,10 +68,10 @@ func TestTgApplyAllTgError(t *testing.T) {
 	testFolder, err := files.CopyTerragruntFolderToTemp("../../test/fixtures/terragrunt/terragrunt-no-error", t.Name())
 	require.NoError(t, err)
 
-	options := &Options{
+	options := WithDefaultRetryableErrors(t, &Options{
 		TerraformDir:    testFolder,
 		TerraformBinary: "terragrunt",
-	}
+	})
 
 	out := TgApplyAll(t, options)
 
@@ -84,14 +84,14 @@ func TestTgApplyAllError(t *testing.T) {
 	testFolder, err := files.CopyTerragruntFolderToTemp("../../test/fixtures/terragrunt/terragrunt-with-error", t.Name())
 	require.NoError(t, err)
 
-	options := &Options{
+	options := WithDefaultRetryableErrors(t, &Options{
 		TerraformDir:    testFolder,
 		TerraformBinary: "terragrunt",
 		MaxRetries:      1,
 		RetryableTerraformErrors: map[string]string{
 			"This is the first run, exiting with an error": "Intentional failure in test fixture",
 		},
-	}
+	})
 
 	out := TgApplyAll(t, options)
 
@@ -101,10 +101,10 @@ func TestTgApplyAllError(t *testing.T) {
 func TestTgApplyOutput(t *testing.T) {
 	t.Parallel()
 
-	options := &Options{
+	options := WithDefaultRetryableErrors(t, &Options{
 		TerraformDir:    "../../test/fixtures/terragrunt/terragrunt-output",
 		TerraformBinary: "terragrunt",
-	}
+	})
 
 	Apply(t, options)
 
@@ -127,10 +127,10 @@ func TestIdempotentNoChanges(t *testing.T) {
 	testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-no-error", t.Name())
 	require.NoError(t, err)
 
-	options := &Options{
+	options := WithDefaultRetryableErrors(t, &Options{
 		TerraformDir: testFolder,
 		NoColor:      true,
-	}
+	})
 
 	InitAndApplyAndIdempotentE(t, options)
 }
@@ -141,10 +141,10 @@ func TestIdempotentWithChanges(t *testing.T) {
 	testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-not-idempotent", t.Name())
 	require.NoError(t, err)
 
-	options := &Options{
+	options := WithDefaultRetryableErrors(t, &Options{
 		TerraformDir: testFolder,
 		NoColor:      true,
-	}
+	})
 
 	out, err := InitAndApplyAndIdempotentE(t, options)
 
@@ -159,10 +159,10 @@ func TestParallelism(t *testing.T) {
 	testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-parallelism", t.Name())
 	require.NoError(t, err)
 
-	options := &Options{
+	options := WithDefaultRetryableErrors(t, &Options{
 		TerraformDir: testFolder,
 		NoColor:      true,
-	}
+	})
 
 	Init(t, options)
 

modules/terraform/options.go

@@ -27,6 +27,7 @@ var (
 		".*Failed to query available provider packages.*": "Failed to retrieve plugin due to transient network error.",
 		".*timeout while waiting for plugin to start.*":   "Failed to retrieve plugin due to transient network error.",
 		".*timed out waiting for server handshake.*":      "Failed to retrieve plugin due to transient network error.",
+		"could not query provider registry for":           "Failed to retrieve plugin due to transient network error.",
 
 		// Provider bugs where the data after apply is not propagated. This is usually an eventual consistency issue, so
 		// retrying should self resolve it.

test/packer_basic_example_test.go

@@ -34,6 +34,9 @@ func TestPackerBasicExample(t *testing.T) {
 	// Pick a random AWS region to test in. This helps ensure your code works in all regions.
 	awsRegion := terratest_aws.GetRandomStableRegion(t, nil, nil)
 
+	// Some AWS regions are missing certain instance types, so pick an available type based on the region we picked
+	instanceType := terratest_aws.GetRecommendedInstanceType(t, awsRegion, []string{"t2.micro", "t3.micro"})
+
 	// website::tag::1::Read Packer's template and set AWS Region variable.
 	packerOptions := &packer.Options{
 		// The path to where the Packer template is located
@@ -43,6 +46,7 @@ func TestPackerBasicExample(t *testing.T) {
 		Vars: map[string]string{
 			"aws_region":    awsRegion,
 			"ami_base_name": fmt.Sprintf("%s", random.UniqueId()),
+			"instance_type": instanceType,
 		},
 
 		// Only build the AWS AMI
@@ -87,15 +91,18 @@ func TestPackerBasicExampleWithVarFile(t *testing.T) {
 	// Pick a random AWS region to test in. This helps ensure your code works in all regions.
 	awsRegion := terratest_aws.GetRandomStableRegion(t, nil, nil)
 
+	// Some AWS regions are missing certain instance types, so pick an available type based on the region we picked
+	instanceType := terratest_aws.GetRecommendedInstanceType(t, awsRegion, []string{"t2.micro", "t3.micro"})
+
 	// Create temporary packer variable file to store aws region
 	varFile, err := ioutil.TempFile("", "*.json")
 	require.NoError(t, err, "Did not expect temp file creation to cause error")
 
 	// Be sure to clean up temp file
 	defer os.Remove(varFile.Name())
 
-	// Write random generated aws region to temporary json file
-	varFileContent := []byte(fmt.Sprintf("{ \"aws_region\": \"%s\" }", awsRegion))
+	// Write the vars we need to a temporary json file
+	varFileContent := []byte(fmt.Sprintf(`{"aws_region": "%s", "instance_type": "%s"}`, awsRegion, instanceType))
 	_, err = varFile.Write(varFileContent)
 	require.NoError(t, err, "Did not expect writing to temp file %s to cause error", varFile.Name())
 
@@ -149,6 +156,9 @@ func TestPackerMultipleConcurrentAmis(t *testing.T) {
 		// Pick a random AWS region to test in. This helps ensure your code works in all regions.
 		awsRegion := terratest_aws.GetRandomStableRegion(t, nil, nil)
 
+		// Some AWS regions are missing certain instance types, so pick an available type based on the region we picked
+		instanceType := terratest_aws.GetRecommendedInstanceType(t, awsRegion, []string{"t2.micro", "t3.micro"})
+
 		packerOptions := &packer.Options{
 			// The path to where the Packer template is located
 			Template: "../examples/packer-basic-example/build.json",
@@ -157,6 +167,7 @@ func TestPackerMultipleConcurrentAmis(t *testing.T) {
 			Vars: map[string]string{
 				"aws_region":    awsRegion,
 				"ami_base_name": fmt.Sprintf("%s", random.UniqueId()),
+				"instance_type": instanceType,
 			},
 
 			// Only build the AWS AMI

test/terraform_packer_example_test.go

@@ -66,6 +66,9 @@ func TestTerraformPackerExample(t *testing.T) {
 
 // Build the AMI in packer-docker-example
 func buildAMI(t *testing.T, awsRegion string, workingDir string) {
+	// Some AWS regions are missing certain instance types, so pick an available type based on the region we picked
+	instanceType := aws.GetRecommendedInstanceType(t, awsRegion, []string{"t2.micro", "t3.micro"})
+
 	packerOptions := &packer.Options{
 		// The path to where the Packer template is located
 		Template: "../examples/packer-docker-example/build.json",
@@ -75,7 +78,8 @@ func buildAMI(t *testing.T, awsRegion string, workingDir string) {
 
 		// Variables to pass to our Packer build using -var options
 		Vars: map[string]string{
-			"aws_region": awsRegion,
+			"aws_region":    awsRegion,
+			"instance_type": instanceType,
 		},
 
 		// Configure retries for intermittent errors