Add ability to test whether warnings are raised during test steps

.changes/unreleased/FEATURES-20230118-083204.yaml

@@ -0,0 +1,8 @@
+kind: FEATURES
+body: 'helper/resource: Added `TestStep` type `ExpectWarning` field, which enables
+  verifying that a warning has been emitted during plan, apply, refresh and destroy.
+  Requires that Terraform 0.15.3 or later is used as it makes use of the Terraform
+  [machine-readable UI](https://developer.hashicorp.com/terraform/internals/machine-readable-ui)'
+time: 2023-01-18T08:32:04.688764Z
+custom:
+  Issue: "16"

go.mod

@@ -12,8 +12,9 @@ require (
 	github.com/hashicorp/hc-install v0.4.0
 	github.com/hashicorp/hcl/v2 v2.15.0
 	github.com/hashicorp/logutils v1.0.0
-	github.com/hashicorp/terraform-exec v0.17.3
+	github.com/hashicorp/terraform-exec v0.17.4-0.20230124151948-5eada031c166
 	github.com/hashicorp/terraform-json v0.14.0
+	github.com/hashicorp/terraform-plugin-framework v1.0.1
 	github.com/hashicorp/terraform-plugin-go v0.14.3
 	github.com/hashicorp/terraform-plugin-log v0.7.0
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.24.1

go.sum

@@ -93,10 +93,12 @@ github.com/hashicorp/hcl/v2 v2.15.0 h1:CPDXO6+uORPjKflkWCCwoWc9uRp+zSIPcCQ+BrxV7
 github.com/hashicorp/hcl/v2 v2.15.0/go.mod h1:JRmR89jycNkrrqnMmvPDMd56n1rQJ2Q6KocSLCMCXng=
 github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/terraform-exec v0.17.3 h1:MX14Kvnka/oWGmIkyuyvL6POx25ZmKrjlaclkx3eErU=
-github.com/hashicorp/terraform-exec v0.17.3/go.mod h1:+NELG0EqQekJzhvikkeQsOAZpsw0cv/03rbeQJqscAI=
+github.com/hashicorp/terraform-exec v0.17.4-0.20230124151948-5eada031c166 h1:WCvC+VknMywOyDXTIzvHHYWknjNBKI0/dmIR9EZd0bw=
+github.com/hashicorp/terraform-exec v0.17.4-0.20230124151948-5eada031c166/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8=
 github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s=
 github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM=
+github.com/hashicorp/terraform-plugin-framework v1.0.1 h1:apX2jtaEKa15+do6H2izBJdl1dEH2w5BPVkDJ3Q3mKA=
+github.com/hashicorp/terraform-plugin-framework v1.0.1/go.mod h1:FV97t2BZOARkL7NNlsc/N25c84MyeSSz72uPp7Vq1lg=
 github.com/hashicorp/terraform-plugin-go v0.14.3 h1:nlnJ1GXKdMwsC8g1Nh05tK2wsC3+3BL/DBBxFEki+j0=
 github.com/hashicorp/terraform-plugin-go v0.14.3/go.mod h1:7ees7DMZ263q8wQ6E4RdIdR6nHHJtrdt4ogX5lPkX1A=
 github.com/hashicorp/terraform-plugin-log v0.7.0 h1:SDxJUyT8TwN4l5b5/VkiTIaQgY6R+Y2BQ0sRZftGKQs=

helper/resource/testing.go

@@ -510,6 +510,14 @@ type TestStep struct {
 	// test to pass.
 	ExpectError *regexp.Regexp
 
+	// ExpectWarning requires the use of Terraform 0.15.3 or later as it makes
+	// use of the Terraform [machine-readable UI](https://developer.hashicorp.com/terraform/internals/machine-readable-ui).
+	//
+	// ExpectWarning allows the construction of test cases that we expect to pass
+	// with a warning. The specified regexp must match against stdout for the
+	// test to pass.
+	ExpectWarning *regexp.Regexp
+
 	// PlanOnly can be set to only run `plan` with this configuration, and not
 	// actually apply it. This is useful for ensuring config changes result in
 	// no-op plans

helper/resource/testing_new.go

@@ -5,6 +5,7 @@ package resource
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -253,34 +254,107 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest
 		if step.RefreshState {
 			logging.HelperResourceTrace(ctx, "TestStep is RefreshState mode")
 
-			err := testStepNewRefreshState(ctx, t, wd, step, providers)
+			testStepNewRefreshStateResponse, err := testStepNewRefreshState(ctx, t, wd, step, providers)
+			refreshTfJSONDiags := testStepNewRefreshStateResponse.tfJSONDiags
+
 			if step.ExpectError != nil {
 				logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError")
+
 				if err == nil {
 					logging.HelperResourceError(ctx,
 						"Error running refresh: expected an error but got none",
 					)
 					t.Fatalf("Step %d/%d error running refresh: expected an error but got none", stepNumber, len(c.Steps))
 				}
-				if !step.ExpectError.MatchString(err.Error()) {
+
+				errorFound := step.ExpectError.MatchString(err.Error())
+				jsonErrorFound := refreshTfJSONDiags.Contains(step.ExpectError, tfjson.DiagnosticSeverityError)
+
+				errorOutput := []string{err.Error()}
+
+				for _, v := range refreshTfJSONDiags.Errors() {
+					b, err := json.Marshal(v)
+					if err != nil {
+						t.Errorf("could not marshal tfjson diagnostic: %s", err)
+					}
+
+					errorOutput = append(errorOutput, string(b))
+				}
+
+				if !errorFound && !jsonErrorFound {
 					logging.HelperResourceError(ctx,
 						fmt.Sprintf("Error running refresh: expected an error with pattern (%s)", step.ExpectError.String()),
-						map[string]interface{}{logging.KeyError: err},
+						map[string]interface{}{logging.KeyError: strings.Join(errorOutput, "")},
 					)
-					t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), err)
+					t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), strings.Join(errorOutput, "\n"))
 				}
 			} else {
 				if err != nil && c.ErrorCheck != nil {
 					logging.HelperResourceDebug(ctx, "Calling TestCase ErrorCheck")
+
+					// This is a "best effort" to supply ErrorCheck with the contents of the
+					// error diagnostics returned from running the Terraform command when
+					// executed with the -json flag. If ErrorCheck functions are using
+					// regexp to perform matches that include line breaks then these will
+					// likely no longer behave as expected and therefore potentially
+					// represent a breaking change.
+					var diagStrings []string
+
+					for _, v := range refreshTfJSONDiags.Errors() {
+						diagStrings = append(diagStrings, "Error: "+v.Summary)
+						diagStrings = append(diagStrings, v.Detail)
+					}
+
+					err = fmt.Errorf("%s\n"+strings.Join(diagStrings, "\n"), err)
+
 					err = c.ErrorCheck(err)
+
 					logging.HelperResourceDebug(ctx, "Called TestCase ErrorCheck")
 				}
 				if err != nil {
 					logging.HelperResourceError(ctx,
 						"Error running refresh",
 						map[string]interface{}{logging.KeyError: err},
 					)
-					t.Fatalf("Step %d/%d error running refresh: %s", stepNumber, len(c.Steps), err)
+
+					errorOutput := []string{err.Error()}
+
+					for _, v := range refreshTfJSONDiags {
+						b, err := json.Marshal(v)
+						if err != nil {
+							t.Errorf("could not marshal tfjson diagnostic: %s", err)
+						}
+
+						errorOutput = append(errorOutput, string(b))
+					}
+
+					t.Fatalf("Step %d/%d error: %s", stepNumber, len(c.Steps), strings.Join(errorOutput, "\n"))
+				}
+			}
+
+			if step.ExpectWarning != nil {
+				logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning")
+
+				warningFound := refreshTfJSONDiags.Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning)
+
+				if !warningFound {
+					logging.HelperResourceError(ctx,
+						fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()),
+						map[string]interface{}{logging.KeyError: err},
+					)
+
+					var warningDiags []string
+
+					for _, v := range refreshTfJSONDiags.Warnings() {
+						b, err := json.Marshal(v)
+						if err != nil {
+							t.Errorf("could not marshal tfjson diagnostic: %s", err)
+						}
+
+						warningDiags = append(warningDiags, string(b))
+					}
+
+					t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), strings.Join(warningDiags, "\n"))
 				}
 			}
 
@@ -292,7 +366,9 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest
 		if step.Config != "" {
 			logging.HelperResourceTrace(ctx, "TestStep is Config mode")
 
-			err := testStepNewConfig(ctx, t, c, wd, step, providers)
+			testStepNewConfigResponse, err := testStepNewConfig(ctx, t, c, wd, step, providers)
+			newConfigTfJSONDiags := testStepNewConfigResponse.tfJSONDiags
+
 			if step.ExpectError != nil {
 				logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError")
 
@@ -302,17 +378,47 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest
 					)
 					t.Fatalf("Step %d/%d, expected an error but got none", stepNumber, len(c.Steps))
 				}
-				if !step.ExpectError.MatchString(err.Error()) {
+
+				errorFound := step.ExpectError.MatchString(err.Error())
+				jsonErrorFound := newConfigTfJSONDiags.Contains(step.ExpectError, tfjson.DiagnosticSeverityError)
+
+				errorOutput := []string{err.Error()}
+
+				for _, v := range newConfigTfJSONDiags.Errors() {
+					b, err := json.Marshal(v)
+					if err != nil {
+						t.Errorf("could not marshal tfjson Diagnostic: %s", err)
+					}
+
+					errorOutput = append(errorOutput, string(b))
+				}
+
+				if !errorFound && !jsonErrorFound {
 					logging.HelperResourceError(ctx,
 						fmt.Sprintf("Expected an error with pattern (%s)", step.ExpectError.String()),
-						map[string]interface{}{logging.KeyError: err},
+						map[string]interface{}{logging.KeyError: strings.Join(errorOutput, "")},
 					)
-					t.Fatalf("Step %d/%d, expected an error with pattern, no match on: %s", stepNumber, len(c.Steps), err)
+					t.Fatalf("Step %d/%d, expected an error matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(errorOutput, ""))
 				}
 			} else {
 				if err != nil && c.ErrorCheck != nil {
 					logging.HelperResourceDebug(ctx, "Calling TestCase ErrorCheck")
 
+					// This is a "best effort" to supply ErrorCheck with the contents of the
+					// error diagnostics returned from running the Terraform command when
+					// executed with the -json flag. If ErrorCheck functions are using
+					// regexp to perform matches that include line breaks then these will
+					// likely no longer behave as expected and therefore potentially
+					// represent a breaking change.
+					var diagStrings []string
+
+					for _, v := range newConfigTfJSONDiags.Errors() {
+						diagStrings = append(diagStrings, "Error: "+v.Summary)
+						diagStrings = append(diagStrings, v.Detail)
+					}
+
+					err = fmt.Errorf("%s\n"+strings.Join(diagStrings, "\n"), err)
+
 					err = c.ErrorCheck(err)
 
 					logging.HelperResourceDebug(ctx, "Called TestCase ErrorCheck")
@@ -322,7 +428,45 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest
 						"Unexpected error",
 						map[string]interface{}{logging.KeyError: err},
 					)
-					t.Fatalf("Step %d/%d error: %s", stepNumber, len(c.Steps), err)
+
+					errorOutput := []string{err.Error()}
+
+					for _, v := range newConfigTfJSONDiags.Errors() {
+						b, err := json.Marshal(v)
+						if err != nil {
+							t.Errorf("could not marshall tfjson Diagnostic: %s", err)
+						}
+
+						errorOutput = append(errorOutput, string(b))
+					}
+
+					t.Fatalf("Step %d/%d error: %s", stepNumber, len(c.Steps), strings.Join(errorOutput, "\n"))
+				}
+			}
+
+			if step.ExpectWarning != nil {
+				logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning")
+
+				warningFound := newConfigTfJSONDiags.Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning)
+
+				if !warningFound {
+					logging.HelperResourceError(ctx,
+						fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()),
+						map[string]interface{}{logging.KeyError: err},
+					)
+
+					var warningDiags []string
+
+					for _, v := range newConfigTfJSONDiags.Warnings() {
+						b, err := json.Marshal(v)
+						if err != nil {
+							t.Errorf("could not marshal tfjson diagnostic: %s", err)
+						}
+
+						warningDiags = append(warningDiags, string(b))
+					}
+
+					t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), strings.Join(warningDiags, "\n"))
 				}
 			}
 
@@ -394,7 +538,7 @@ func testIDRefresh(ctx context.Context, t testing.T, c TestCase, wd *plugintest.
 
 	// Refresh!
 	err = runProviderCommand(ctx, t, func() error {
-		err = wd.Refresh(ctx)
+		_, err := wd.Refresh(ctx)
 		if err != nil {
 			t.Fatalf("Error running terraform refresh: %s", err)
 		}