From 569231b506246810ceefe58da84ee0a64f3960e0 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Wed, 21 Dec 2022 13:18:33 +0000 Subject: [PATCH 01/32] Piping output from terraform apply into stdout.txt in working directory and unmarshalling onto terraform-json diagnostic to verify warning matches expected regex (#16) --- helper/resource/testing.go | 5 ++++ helper/resource/testing_new.go | 45 ++++++++++++++++++++++++++++++ internal/plugintest/working_dir.go | 28 ++++++++++++++++++- 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/helper/resource/testing.go b/helper/resource/testing.go index 987654c6d..311224c94 100644 --- a/helper/resource/testing.go +++ b/helper/resource/testing.go @@ -501,6 +501,11 @@ type TestStep struct { // test to pass. ExpectError *regexp.Regexp + // ExpectWarning allows the construction of test cases that we expect to pass + // with a warning. The specified regexp must match against stdout for the + // test to pass. + ExpectWarning *regexp.Regexp + // PlanOnly can be set to only run `plan` with this configuration, and not // actually apply it. This is useful for ensuring config changes result in // no-op plans diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 86f653f8f..8bb6cf408 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -4,8 +4,13 @@ package resource import ( + "bufio" "context" + "encoding/json" "fmt" + "log" + "os" + "path/filepath" "reflect" "strings" @@ -319,6 +324,46 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } } + if step.ExpectWarning != nil { + logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") + + // TODO: Move file open and defer close outside of for ... {} loop. + file, err := os.Open(filepath.Join(wd.GetBaseDir(), "stdout.txt")) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + warningFound := false + + // TODO: Use MultiLineJSONDecode providing entire contents of stdout.txt is valid JSON. + scanner := bufio.NewScanner(file) + // optionally, resize scanner's capacity for lines over 64K, see next example + for scanner.Scan() { + var outer struct { + Diagnostic tfjson.Diagnostic + } + + if json.Unmarshal([]byte(scanner.Text()), &outer) == nil { + if step.ExpectWarning.MatchString(outer.Diagnostic.Summary) && outer.Diagnostic.Severity == tfjson.DiagnosticSeverityWarning { + warningFound = true + } + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + + if !warningFound { + //logging.HelperResourceError(ctx, + // fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), + // map[string]interface{}{logging.KeyError: err}, + //) + t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String()) + } + } + appliedCfg = step.mergedConfig(ctx, c) logging.HelperResourceDebug(ctx, "Finished TestStep") diff --git a/internal/plugintest/working_dir.go b/internal/plugintest/working_dir.go index 734b6c9b5..2934df269 100644 --- a/internal/plugintest/working_dir.go +++ b/internal/plugintest/working_dir.go @@ -47,6 +47,25 @@ type WorkingDir struct { reattachInfo tfexec.ReattachInfo } +func (wd *WorkingDir) SetTFStdout() error { + dst, err := os.Create(filepath.Join(wd.baseDir, "stdout.txt")) + if err != nil { + return fmt.Errorf("unable to create file for writing stdout: %w", err) + } + + wd.tf.SetStdout(dst) + + return nil +} + +func (wd *WorkingDir) UnsetTFStdout() { + wd.tf.SetStdout(nil) +} + +func (wd *WorkingDir) GetBaseDir() string { + return wd.baseDir +} + // Close deletes the directories and files created to represent the receiving // working directory. After this method is called, the working directory object // is invalid and may no longer be used. @@ -241,7 +260,14 @@ func (wd *WorkingDir) Apply(ctx context.Context) error { logging.HelperResourceTrace(ctx, "Calling Terraform CLI apply command") - err := wd.tf.Apply(context.Background(), args...) + err := wd.SetTFStdout() + if err != nil { + return err + } + + err = wd.tf.Apply(context.Background(), args...) + + wd.UnsetTFStdout() logging.HelperResourceTrace(ctx, "Called Terraform CLI apply command") From 1d9131574693a45f57db444e4c0d57a7aad778af Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Tue, 3 Jan 2023 10:56:42 +0000 Subject: [PATCH 02/32] Add test to verify that warnings are generated during test step execution (#16) --- helper/resource/teststep_providers_test.go | 43 ++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/helper/resource/teststep_providers_test.go b/helper/resource/teststep_providers_test.go index d96261214..47d61b220 100644 --- a/helper/resource/teststep_providers_test.go +++ b/helper/resource/teststep_providers_test.go @@ -6,6 +6,7 @@ package resource import ( "context" "fmt" + "regexp" "strings" "testing" "time" @@ -2132,6 +2133,48 @@ func TestTest_TestStep_ProviderFactories_Import_External_With_Data_Source(t *tes }) } +func TestTest_TestStep_ProviderFactories_ExpectWarning(t *testing.T) { + t.Parallel() + + Test(t, TestCase{ + ProviderFactories: map[string]func() (*schema.Provider, error){ + "random": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "random_password": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) (diags diag.Diagnostics) { + d.SetId("id") + return append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning diagnostic - summary", + }) + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, + }, + }, + }, nil + }, + }, + Steps: []TestStep{ + { + Config: `resource "random_password" "test" { }`, + ExpectWarning: regexp.MustCompile(`.*warning diagnostic - summary`), + }, + }, + }) +} + func setTimeForTest(t time.Time) func() { return func() { getTimeForTest = func() time.Time { From 5c1b67e95618aada8655c3a7f506f4dd7c5e3612 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Tue, 3 Jan 2023 12:10:16 +0000 Subject: [PATCH 03/32] Moving defer file.Close outside for loop (#16) --- helper/resource/testing_new.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 8bb6cf408..10572700f 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -125,6 +125,17 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // acts as default for import tests var appliedCfg string + var file *os.File + defer func() error { + if file != nil { + err := file.Close() + if err != nil { + return err + } + } + return nil + }() + for stepIndex, step := range c.Steps { stepNumber := stepIndex + 1 // 1-based indexing for humans ctx = logging.TestStepNumberContext(ctx, stepNumber) @@ -327,12 +338,10 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - // TODO: Move file open and defer close outside of for ... {} loop. - file, err := os.Open(filepath.Join(wd.GetBaseDir(), "stdout.txt")) + file, err = os.Open(filepath.Join(wd.GetBaseDir(), "stdout.txt")) if err != nil { log.Fatal(err) } - defer file.Close() warningFound := false From 71f08775c06ac96c7342b2e788eb4e4ac6c625ab Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Tue, 3 Jan 2023 13:45:47 +0000 Subject: [PATCH 04/32] Updating defer func to log error and fail test if closing file containing output from stdout returns an error (#16) --- helper/resource/testing_new.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 10572700f..2b6493d0e 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -125,15 +125,21 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // acts as default for import tests var appliedCfg string - var file *os.File - defer func() error { - if file != nil { - err := file.Close() + // stdoutFile is used as the handle to the file that contains + // json output from running Terraform commands. + var stdoutFile *os.File + defer func() { + if stdoutFile != nil { + err := stdoutFile.Close() if err != nil { - return err + logging.HelperResourceError(ctx, + "Error closing file containing json output from Terraform commands", + map[string]interface{}{logging.KeyError: err}, + ) + t.Fatalf("Error closing file containing json from Terraform commands: %s", err.Error()) + return } } - return nil }() for stepIndex, step := range c.Steps { @@ -338,7 +344,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - file, err = os.Open(filepath.Join(wd.GetBaseDir(), "stdout.txt")) + stdoutFile, err = os.Open(filepath.Join(wd.GetBaseDir(), "stdout.txt")) if err != nil { log.Fatal(err) } @@ -346,7 +352,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest warningFound := false // TODO: Use MultiLineJSONDecode providing entire contents of stdout.txt is valid JSON. - scanner := bufio.NewScanner(file) + scanner := bufio.NewScanner(stdoutFile) // optionally, resize scanner's capacity for lines over 64K, see next example for scanner.Scan() { var outer struct { @@ -356,6 +362,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if json.Unmarshal([]byte(scanner.Text()), &outer) == nil { if step.ExpectWarning.MatchString(outer.Diagnostic.Summary) && outer.Diagnostic.Severity == tfjson.DiagnosticSeverityWarning { warningFound = true + break } } } From 1061d084917520636384840e591455ed3144222d Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Thu, 12 Jan 2023 13:35:31 +0000 Subject: [PATCH 05/32] Inspecting streaming json output for warnings, and error and streaming json for errors (#16) --- helper/resource/testing_new.go | 96 ++++++++++------ helper/resource/testing_new_config.go | 5 +- helper/resource/teststep_providers_test.go | 128 ++++++++++++++++++++- internal/plugintest/working_dir.go | 39 +++---- 4 files changed, 210 insertions(+), 58 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 2b6493d0e..350ba4709 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "strings" "github.com/google/go-cmp/cmp" @@ -127,7 +128,10 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // stdoutFile is used as the handle to the file that contains // json output from running Terraform commands. - var stdoutFile *os.File + stdoutFile, err := os.Create(filepath.Join(wd.GetBaseDir(), "stdout.txt")) + if err != nil { + t.Fatalf("unable to create file for writing stdout: %w", err) + } defer func() { if stdoutFile != nil { err := stdoutFile.Close() @@ -307,7 +311,8 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.Config != "" { logging.HelperResourceTrace(ctx, "TestStep is Config mode") - err := testStepNewConfig(ctx, t, c, wd, step, providers) + err := testStepNewConfig(ctx, t, c, wd, step, providers, stdoutFile) + if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -317,12 +322,16 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest ) t.Fatalf("Step %d/%d, expected an error but got none", stepNumber, len(c.Steps)) } - if !step.ExpectError.MatchString(err.Error()) { + + errorFound := step.ExpectError.MatchString(err.Error()) + jsonErrorFound, jsonDiags := diagnosticFound(wd, step.ExpectError, tfjson.DiagnosticSeverityError) + + if !errorFound && !jsonErrorFound { logging.HelperResourceError(ctx, fmt.Sprintf("Expected an error with pattern (%s)", step.ExpectError.String()), map[string]interface{}{logging.KeyError: err}, ) - t.Fatalf("Step %d/%d, expected an error with pattern, no match on: %s", stepNumber, len(c.Steps), err) + t.Fatalf("Step %d/%d, expected an error matching pattern, no match on: %s", stepNumber, len(c.Steps), jsonDiags) } } else { if err != nil && c.ErrorCheck != nil { @@ -344,39 +353,14 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - stdoutFile, err = os.Open(filepath.Join(wd.GetBaseDir(), "stdout.txt")) - if err != nil { - log.Fatal(err) - } - - warningFound := false - - // TODO: Use MultiLineJSONDecode providing entire contents of stdout.txt is valid JSON. - scanner := bufio.NewScanner(stdoutFile) - // optionally, resize scanner's capacity for lines over 64K, see next example - for scanner.Scan() { - var outer struct { - Diagnostic tfjson.Diagnostic - } - - if json.Unmarshal([]byte(scanner.Text()), &outer) == nil { - if step.ExpectWarning.MatchString(outer.Diagnostic.Summary) && outer.Diagnostic.Severity == tfjson.DiagnosticSeverityWarning { - warningFound = true - break - } - } - } - - if err := scanner.Err(); err != nil { - log.Fatal(err) - } + warningFound, jsonDiags := diagnosticFound(wd, step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { - //logging.HelperResourceError(ctx, - // fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), - // map[string]interface{}{logging.KeyError: err}, - //) - t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String()) + logging.HelperResourceError(ctx, + fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), + map[string]interface{}{logging.KeyError: err}, + ) + t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), jsonDiags) } } @@ -391,6 +375,48 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } } +func diagnosticFound(wd *plugintest.WorkingDir, r *regexp.Regexp, severity tfjson.DiagnosticSeverity) (bool, []string) { + stdoutFile, err := os.Open(filepath.Join(wd.GetBaseDir(), "stdout.txt")) + if err != nil { + log.Fatal(err) + } + + scanner := bufio.NewScanner(stdoutFile) + var jsonOutput []string + + for scanner.Scan() { + var outer struct { + Diagnostic tfjson.Diagnostic + } + + txt := scanner.Text() + + if json.Unmarshal([]byte(txt), &outer) == nil { + if outer.Diagnostic.Severity == "" { + continue + } + + jsonOutput = append(jsonOutput, txt) + + if !r.MatchString(outer.Diagnostic.Summary) && !r.MatchString(outer.Diagnostic.Detail) { + continue + } + + if outer.Diagnostic.Severity != severity { + continue + } + + return true, jsonOutput + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + + return false, jsonOutput +} + func getState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir) (*terraform.State, error) { t.Helper() diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index 3ad3f0916..af7362aeb 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "io" tfjson "github.com/hashicorp/terraform-json" "github.com/mitchellh/go-testing-interface" @@ -17,7 +18,7 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) error { +func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, w io.Writer) error { t.Helper() err := wd.SetConfig(ctx, step.mergedConfig(ctx, c)) @@ -68,7 +69,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // Apply the diff, creating real resources err = runProviderCommand(ctx, t, func() error { - return wd.Apply(ctx) + return wd.ApplyJSON(ctx, w) }, wd, providers) if err != nil { if step.Destroy { diff --git a/helper/resource/teststep_providers_test.go b/helper/resource/teststep_providers_test.go index 47d61b220..2aac4a56c 100644 --- a/helper/resource/teststep_providers_test.go +++ b/helper/resource/teststep_providers_test.go @@ -2133,7 +2133,91 @@ func TestTest_TestStep_ProviderFactories_Import_External_With_Data_Source(t *tes }) } -func TestTest_TestStep_ProviderFactories_ExpectWarning(t *testing.T) { +func TestTest_TestStep_ProviderFactories_ExpectErrorSummary(t *testing.T) { + t.Parallel() + + Test(t, TestCase{ + ProviderFactories: map[string]func() (*schema.Provider, error){ + "random": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "random_password": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) (diags diag.Diagnostics) { + d.SetId("id") + return append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "error diagnostic - summary", + }) + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, + }, + }, + }, nil + }, + }, + Steps: []TestStep{ + { + Config: `resource "random_password" "test" { }`, + ExpectError: regexp.MustCompile(`.*error diagnostic - summary`), + }, + }, + }) +} + +func TestTest_TestStep_ProviderFactories_ExpectErrorDetail(t *testing.T) { + t.Parallel() + + Test(t, TestCase{ + ProviderFactories: map[string]func() (*schema.Provider, error){ + "random": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "random_password": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) (diags diag.Diagnostics) { + d.SetId("id") + return append(diags, diag.Diagnostic{ + Severity: diag.Error, + Detail: "error diagnostic - detail", + }) + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, + }, + }, + }, nil + }, + }, + Steps: []TestStep{ + { + Config: `resource "random_password" "test" { }`, + ExpectError: regexp.MustCompile(`.*error diagnostic - detail`), + }, + }, + }) +} + +func TestTest_TestStep_ProviderFactories_ExpectWarningSummary(t *testing.T) { t.Parallel() Test(t, TestCase{ @@ -2175,6 +2259,48 @@ func TestTest_TestStep_ProviderFactories_ExpectWarning(t *testing.T) { }) } +func TestTest_TestStep_ProviderFactories_ExpectWarningDetail(t *testing.T) { + t.Parallel() + + Test(t, TestCase{ + ProviderFactories: map[string]func() (*schema.Provider, error){ + "random": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "random_password": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) (diags diag.Diagnostics) { + d.SetId("id") + return append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Detail: "warning diagnostic - detail", + }) + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, + }, + }, + }, nil + }, + }, + Steps: []TestStep{ + { + Config: `resource "random_password" "test" { }`, + ExpectWarning: regexp.MustCompile(`.*warning diagnostic - detail`), + }, + }, + }) +} + func setTimeForTest(t time.Time) func() { return func() { getTimeForTest = func() time.Time { diff --git a/internal/plugintest/working_dir.go b/internal/plugintest/working_dir.go index 2934df269..b779135ea 100644 --- a/internal/plugintest/working_dir.go +++ b/internal/plugintest/working_dir.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "path/filepath" @@ -47,21 +48,6 @@ type WorkingDir struct { reattachInfo tfexec.ReattachInfo } -func (wd *WorkingDir) SetTFStdout() error { - dst, err := os.Create(filepath.Join(wd.baseDir, "stdout.txt")) - if err != nil { - return fmt.Errorf("unable to create file for writing stdout: %w", err) - } - - wd.tf.SetStdout(dst) - - return nil -} - -func (wd *WorkingDir) UnsetTFStdout() { - wd.tf.SetStdout(nil) -} - func (wd *WorkingDir) GetBaseDir() string { return wd.baseDir } @@ -260,14 +246,27 @@ func (wd *WorkingDir) Apply(ctx context.Context) error { logging.HelperResourceTrace(ctx, "Calling Terraform CLI apply command") - err := wd.SetTFStdout() - if err != nil { - return err + err := wd.tf.Apply(context.Background(), args...) + + logging.HelperResourceTrace(ctx, "Called Terraform CLI apply command") + + return err +} + +// ApplyJSON runs "terraform apply" with the `-json` flag. The streaming JSON output +// generated when `terraform apply` is executed is written to `w`. +// If CreatePlan has previously completed successfully and the saved plan has not +// been cleared in the meantime then this will apply the saved plan. Otherwise, +// it will implicitly create a new plan and apply it. +func (wd *WorkingDir) ApplyJSON(ctx context.Context, w io.Writer) error { + args := []tfexec.ApplyOption{tfexec.Reattach(wd.reattachInfo), tfexec.Refresh(false)} + if wd.HasSavedPlan() { + args = append(args, tfexec.DirOrPlan(PlanFileName)) } - err = wd.tf.Apply(context.Background(), args...) + logging.HelperResourceTrace(ctx, "Calling Terraform CLI apply command") - wd.UnsetTFStdout() + err := wd.tf.ApplyJSON(context.Background(), w, args...) logging.HelperResourceTrace(ctx, "Called Terraform CLI apply command") From 4cac99eb0ef4cd1fab7676731375f885459af829 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Thu, 12 Jan 2023 16:20:47 +0000 Subject: [PATCH 06/32] Switching to using streaming json for terraform refresh (#16) --- helper/resource/testing_new.go | 42 ++++++- helper/resource/testing_new_config.go | 2 +- helper/resource/testing_new_refresh_state.go | 5 +- helper/resource/teststep_providers_test.go | 110 +++++++++++++++++++ internal/plugintest/working_dir.go | 11 ++ 5 files changed, 162 insertions(+), 8 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 350ba4709..802d0afa4 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -272,19 +272,31 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.RefreshState { logging.HelperResourceTrace(ctx, "TestStep is RefreshState mode") - err := testStepNewRefreshState(ctx, t, wd, step, providers) + err := testStepNewRefreshState(ctx, t, wd, step, providers, stdoutFile) + if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") + if err == nil { logging.HelperResourceError(ctx, "Error running refresh: expected an error but got none", ) t.Fatalf("Step %d/%d error running refresh: expected an error but got none", stepNumber, len(c.Steps)) } - if !step.ExpectError.MatchString(err.Error()) { + + errorFound := step.ExpectError.MatchString(err.Error()) + jsonErrorFound, jsonDiags := diagnosticFound(wd, step.ExpectError, tfjson.DiagnosticSeverityError) + + errorOutput := []string{err.Error()} + + if jsonErrorFound { + errorOutput = jsonDiags + } + + if !errorFound && !jsonErrorFound { logging.HelperResourceError(ctx, fmt.Sprintf("Error running refresh: expected an error with pattern (%s)", step.ExpectError.String()), - map[string]interface{}{logging.KeyError: err}, + map[string]interface{}{logging.KeyError: strings.Join(errorOutput, "")}, ) t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), err) } @@ -303,6 +315,20 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } } + if step.ExpectWarning != nil { + logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") + + warningFound, jsonDiags := diagnosticFound(wd, step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + + if !warningFound { + logging.HelperResourceError(ctx, + fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), + map[string]interface{}{logging.KeyError: err}, + ) + t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(jsonDiags, "\n")) + } + } + logging.HelperResourceDebug(ctx, "Finished TestStep") continue @@ -326,10 +352,16 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest errorFound := step.ExpectError.MatchString(err.Error()) jsonErrorFound, jsonDiags := diagnosticFound(wd, step.ExpectError, tfjson.DiagnosticSeverityError) + errorOutput := []string{err.Error()} + + if jsonErrorFound { + errorOutput = jsonDiags + } + if !errorFound && !jsonErrorFound { logging.HelperResourceError(ctx, fmt.Sprintf("Expected an error with pattern (%s)", step.ExpectError.String()), - map[string]interface{}{logging.KeyError: err}, + map[string]interface{}{logging.KeyError: strings.Join(errorOutput, "")}, ) t.Fatalf("Step %d/%d, expected an error matching pattern, no match on: %s", stepNumber, len(c.Steps), jsonDiags) } @@ -360,7 +392,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), map[string]interface{}{logging.KeyError: err}, ) - t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), jsonDiags) + t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(jsonDiags, "\n")) } } diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index af7362aeb..9cc9001b6 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -29,7 +29,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // require a refresh before applying // failing to do this will result in data sources not being updated err = runProviderCommand(ctx, t, func() error { - return wd.Refresh(ctx) + return wd.RefreshJSON(ctx, w) }, wd, providers) if err != nil { return fmt.Errorf("Error running pre-apply refresh: %w", err) diff --git a/helper/resource/testing_new_refresh_state.go b/helper/resource/testing_new_refresh_state.go index fe7b28145..181544cd5 100644 --- a/helper/resource/testing_new_refresh_state.go +++ b/helper/resource/testing_new_refresh_state.go @@ -6,6 +6,7 @@ package resource import ( "context" "fmt" + "io" tfjson "github.com/hashicorp/terraform-json" "github.com/mitchellh/go-testing-interface" @@ -16,7 +17,7 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) error { +func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, w io.Writer) error { t.Helper() var err error @@ -33,7 +34,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo } err = runProviderCommand(ctx, t, func() error { - return wd.Refresh(ctx) + return wd.RefreshJSON(ctx, w) }, wd, providers) if err != nil { return err diff --git a/helper/resource/teststep_providers_test.go b/helper/resource/teststep_providers_test.go index 2aac4a56c..981635910 100644 --- a/helper/resource/teststep_providers_test.go +++ b/helper/resource/teststep_providers_test.go @@ -2301,6 +2301,116 @@ func TestTest_TestStep_ProviderFactories_ExpectWarningDetail(t *testing.T) { }) } +func TestTest_TestStep_ProviderFactories_ExpectErrorSummaryRefreshOnly(t *testing.T) { + t.Parallel() + + Test(t, TestCase{ + ProviderFactories: map[string]func() (*schema.Provider, error){ + "random": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "random_password": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics { + d.SetId("id") + return nil + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) (diags diag.Diagnostics) { + // Only generate diag when ReadContext is called for the second + // time during terraform refresh. + if d.Get("id") == "new_id" { + return append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "error diagnostic - summary", + }) + } + + // Update id when ReadContext is called following resource creation + // during terraform apply. + d.SetId("new_id") + + return + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, + }, + }, + }, nil + }, + }, + Steps: []TestStep{ + { + Config: `resource "random_password" "test" { }`, + }, + { + RefreshState: true, + ExpectError: regexp.MustCompile(`.*error diagnostic - summary`), + }, + }, + }) +} + +func TestTest_TestStep_ProviderFactories_ExpectWarningSummaryRefreshOnly(t *testing.T) { + t.Parallel() + + Test(t, TestCase{ + ProviderFactories: map[string]func() (*schema.Provider, error){ + "random": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "random_password": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics { + d.SetId("id") + return nil + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) (diags diag.Diagnostics) { + // Only generate diag when ReadContext is called for the second + // time during terraform refresh. + if d.Get("id") == "new_id" { + return append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning diagnostic - summary", + }) + } + + // Update id when ReadContext is called following resource creation + // during terraform apply. + d.SetId("new_id") + + return + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, + }, + }, + }, nil + }, + }, + Steps: []TestStep{ + { + Config: `resource "random_password" "test" { }`, + }, + { + RefreshState: true, + ExpectWarning: regexp.MustCompile(`.*warning diagnostic - summary`), + }, + }, + }) +} + func setTimeForTest(t time.Time) func() { return func() { getTimeForTest = func() time.Time { diff --git a/internal/plugintest/working_dir.go b/internal/plugintest/working_dir.go index b779135ea..0114077b7 100644 --- a/internal/plugintest/working_dir.go +++ b/internal/plugintest/working_dir.go @@ -382,6 +382,17 @@ func (wd *WorkingDir) Refresh(ctx context.Context) error { return err } +// RefreshJSON runs terraform refresh +func (wd *WorkingDir) RefreshJSON(ctx context.Context, w io.Writer) error { + logging.HelperResourceTrace(ctx, "Calling Terraform CLI refresh command") + + err := wd.tf.RefreshJSON(context.Background(), w, tfexec.Reattach(wd.reattachInfo)) + + logging.HelperResourceTrace(ctx, "Called Terraform CLI refresh command") + + return err +} + // Schemas returns an object describing the provider schemas. // // If the schemas cannot be read, Schemas returns an error. From 021c2e8923b625228a5d873d401225751a90cdb1 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Thu, 12 Jan 2023 17:47:25 +0000 Subject: [PATCH 07/32] Use framework testprovider files for setup of test using ProtoV5ProviderFactories (#16) --- helper/resource/testing_new.go | 4 +- helper/resource/testing_new_config.go | 40 +++++++++++- helper/resource/teststep_providers_test.go | 60 +++++++++++++++++- internal/testing/testprovider/provider.go | 66 +++++++++++++++++++ internal/testing/testprovider/resource.go | 74 ++++++++++++++++++++++ 5 files changed, 239 insertions(+), 5 deletions(-) create mode 100644 internal/testing/testprovider/provider.go create mode 100644 internal/testing/testprovider/resource.go diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 802d0afa4..bc68940b0 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -424,12 +424,12 @@ func diagnosticFound(wd *plugintest.WorkingDir, r *regexp.Regexp, severity tfjso txt := scanner.Text() if json.Unmarshal([]byte(txt), &outer) == nil { + jsonOutput = append(jsonOutput, txt) + if outer.Diagnostic.Severity == "" { continue } - jsonOutput = append(jsonOutput, txt) - if !r.MatchString(outer.Diagnostic.Summary) && !r.MatchString(outer.Diagnostic.Detail) { continue } diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index 9cc9001b6..440d53798 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -4,10 +4,16 @@ package resource import ( + "bufio" "context" + "encoding/json" "errors" "fmt" "io" + "log" + "os" + "path/filepath" + "strings" tfjson "github.com/hashicorp/terraform-json" "github.com/mitchellh/go-testing-interface" @@ -18,6 +24,34 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) +func getJSONOutput(wd *plugintest.WorkingDir) []string { + stdoutFile, err := os.Open(filepath.Join(wd.GetBaseDir(), "stdout.txt")) + if err != nil { + log.Fatal(err) + } + + scanner := bufio.NewScanner(stdoutFile) + var jsonOutput []string + + for scanner.Scan() { + var outer struct { + Diagnostic tfjson.Diagnostic + } + + txt := scanner.Text() + + if json.Unmarshal([]byte(txt), &outer) == nil { + jsonOutput = append(jsonOutput, txt) + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + + return jsonOutput +} + func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, w io.Writer) error { t.Helper() @@ -29,7 +63,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // require a refresh before applying // failing to do this will result in data sources not being updated err = runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, w) + return wd.Refresh(ctx) }, wd, providers) if err != nil { return fmt.Errorf("Error running pre-apply refresh: %w", err) @@ -68,6 +102,8 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint } // Apply the diff, creating real resources + // TODO: Update all Errorf outputs that should use contents of streaming json + // after calling terraform-exec with `-json`. err = runProviderCommand(ctx, t, func() error { return wd.ApplyJSON(ctx, w) }, wd, providers) @@ -75,7 +111,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if step.Destroy { return fmt.Errorf("Error running destroy: %w", err) } - return fmt.Errorf("Error running apply: %w", err) + return fmt.Errorf("Error running apply: %s", strings.Join(getJSONOutput(wd), "\n")) } // Get the new state diff --git a/helper/resource/teststep_providers_test.go b/helper/resource/teststep_providers_test.go index 981635910..6773b4f44 100644 --- a/helper/resource/teststep_providers_test.go +++ b/helper/resource/teststep_providers_test.go @@ -13,13 +13,19 @@ import ( "github.com/google/go-cmp/cmp" "github.com/hashicorp/go-cty/cty" + fwdiag "github.com/hashicorp/terraform-plugin-framework/diag" + "github.com/hashicorp/terraform-plugin-framework/provider" + "github.com/hashicorp/terraform-plugin-framework/providerserver" + "github.com/hashicorp/terraform-plugin-framework/resource" + fwresourceschema "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/types" "github.com/hashicorp/terraform-plugin-go/tfprotov5" "github.com/hashicorp/terraform-plugin-go/tfprotov6" - "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider" "github.com/hashicorp/terraform-plugin-testing/terraform" ) @@ -2411,6 +2417,58 @@ func TestTest_TestStep_ProviderFactories_ExpectWarningSummaryRefreshOnly(t *test }) } +func TestTest_TestStep_ProviderFactories_ExpectWarningSummaryPlanOnly(t *testing.T) { + t.Parallel() + + Test(t, TestCase{ + ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ + "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ + MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + resp.TypeName = "random" + }, + ResourcesMethod: func(ctx context.Context) []func() resource.Resource { + return []func() resource.Resource{ + func() resource.Resource { + return &testprovider.Resource{ + MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_password" + }, + SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = fwresourceschema.Schema{ + Attributes: map[string]fwresourceschema.Attribute{ + "id": fwresourceschema.StringAttribute{ + Computed: true, + }, + }, + } + }, + CreateMethod: func(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + var data struct { + Id types.String `tfsdk:"id"` + } + + data.Id = types.StringValue("example-id") + + diags := resp.State.Set(ctx, &data) + resp.Diagnostics.Append(diags...) + + resp.Diagnostics.Append(fwdiag.NewWarningDiagnostic("warning diagnostic - detail", "")) + }, + } + }, + } + }, + }), + }, + Steps: []TestStep{ + { + Config: `resource "random_password" "test" { }`, + ExpectWarning: regexp.MustCompile(`.*warning diagnostic - detail`), + }, + }, + }) +} + func setTimeForTest(t time.Time) func() { return func() { getTimeForTest = func() time.Time { diff --git a/internal/testing/testprovider/provider.go b/internal/testing/testprovider/provider.go new file mode 100644 index 000000000..f63fe3dbb --- /dev/null +++ b/internal/testing/testprovider/provider.go @@ -0,0 +1,66 @@ +package testprovider + +import ( + "context" + + "github.com/hashicorp/terraform-plugin-framework/datasource" + "github.com/hashicorp/terraform-plugin-framework/provider" + "github.com/hashicorp/terraform-plugin-framework/resource" +) + +var _ provider.Provider = &Provider{} + +// Provider is declarative provider.Provider for unit testing. +type Provider struct { + // Provider interface methods + MetadataMethod func(context.Context, provider.MetadataRequest, *provider.MetadataResponse) + ConfigureMethod func(context.Context, provider.ConfigureRequest, *provider.ConfigureResponse) + SchemaMethod func(context.Context, provider.SchemaRequest, *provider.SchemaResponse) + DataSourcesMethod func(context.Context) []func() datasource.DataSource + ResourcesMethod func(context.Context) []func() resource.Resource +} + +// Configure satisfies the provider.Provider interface. +func (p *Provider) Configure(ctx context.Context, req provider.ConfigureRequest, resp *provider.ConfigureResponse) { + if p == nil || p.ConfigureMethod == nil { + return + } + + p.ConfigureMethod(ctx, req, resp) +} + +// DataSources satisfies the provider.Provider interface. +func (p *Provider) DataSources(ctx context.Context) []func() datasource.DataSource { + if p == nil || p.DataSourcesMethod == nil { + return nil + } + + return p.DataSourcesMethod(ctx) +} + +// Metadata satisfies the provider.Provider interface. +func (p *Provider) Metadata(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + if p == nil || p.MetadataMethod == nil { + return + } + + p.MetadataMethod(ctx, req, resp) +} + +// Schema satisfies the provider.Provider interface. +func (p *Provider) Schema(ctx context.Context, req provider.SchemaRequest, resp *provider.SchemaResponse) { + if p == nil || p.SchemaMethod == nil { + return + } + + p.SchemaMethod(ctx, req, resp) +} + +// Resources satisfies the provider.Provider interface. +func (p *Provider) Resources(ctx context.Context) []func() resource.Resource { + if p == nil || p.ResourcesMethod == nil { + return nil + } + + return p.ResourcesMethod(ctx) +} diff --git a/internal/testing/testprovider/resource.go b/internal/testing/testprovider/resource.go new file mode 100644 index 000000000..b5bd496d9 --- /dev/null +++ b/internal/testing/testprovider/resource.go @@ -0,0 +1,74 @@ +package testprovider + +import ( + "context" + + "github.com/hashicorp/terraform-plugin-framework/resource" +) + +var _ resource.Resource = &Resource{} + +// Resource is declarative resource.Resource for unit testing. +type Resource struct { + // Resource interface methods + MetadataMethod func(context.Context, resource.MetadataRequest, *resource.MetadataResponse) + SchemaMethod func(context.Context, resource.SchemaRequest, *resource.SchemaResponse) + CreateMethod func(context.Context, resource.CreateRequest, *resource.CreateResponse) + DeleteMethod func(context.Context, resource.DeleteRequest, *resource.DeleteResponse) + ReadMethod func(context.Context, resource.ReadRequest, *resource.ReadResponse) + UpdateMethod func(context.Context, resource.UpdateRequest, *resource.UpdateResponse) +} + +// Metadata satisfies the resource.Resource interface. +func (r *Resource) Metadata(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + if r.MetadataMethod == nil { + return + } + + r.MetadataMethod(ctx, req, resp) +} + +// Schema satisfies the resource.Resource interface. +func (r *Resource) Schema(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + if r.SchemaMethod == nil { + return + } + + r.SchemaMethod(ctx, req, resp) +} + +// Create satisfies the resource.Resource interface. +func (r *Resource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + if r.CreateMethod == nil { + return + } + + r.CreateMethod(ctx, req, resp) +} + +// Delete satisfies the resource.Resource interface. +func (r *Resource) Delete(ctx context.Context, req resource.DeleteRequest, resp *resource.DeleteResponse) { + if r.DeleteMethod == nil { + return + } + + r.DeleteMethod(ctx, req, resp) +} + +// Read satisfies the resource.Resource interface. +func (r *Resource) Read(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) { + if r.ReadMethod == nil { + return + } + + r.ReadMethod(ctx, req, resp) +} + +// Update satisfies the resource.Resource interface. +func (r *Resource) Update(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse) { + if r.UpdateMethod == nil { + return + } + + r.UpdateMethod(ctx, req, resp) +} From 5e649d32ee374c0e384649ebd7ba3c7c9abab1b7 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Thu, 12 Jan 2023 17:47:44 +0000 Subject: [PATCH 08/32] Temporarily use terraform-exec sha (#16) --- go.mod | 3 ++- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0634047be..b0b2f027e 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,9 @@ require ( github.com/hashicorp/hc-install v0.4.0 github.com/hashicorp/hcl/v2 v2.15.0 github.com/hashicorp/logutils v1.0.0 - github.com/hashicorp/terraform-exec v0.17.3 + github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98 github.com/hashicorp/terraform-json v0.14.0 + github.com/hashicorp/terraform-plugin-framework v1.0.1 github.com/hashicorp/terraform-plugin-go v0.14.3 github.com/hashicorp/terraform-plugin-log v0.7.0 github.com/hashicorp/terraform-plugin-sdk/v2 v2.24.1 diff --git a/go.sum b/go.sum index e06c51538..d188998f3 100644 --- a/go.sum +++ b/go.sum @@ -93,10 +93,12 @@ github.com/hashicorp/hcl/v2 v2.15.0 h1:CPDXO6+uORPjKflkWCCwoWc9uRp+zSIPcCQ+BrxV7 github.com/hashicorp/hcl/v2 v2.15.0/go.mod h1:JRmR89jycNkrrqnMmvPDMd56n1rQJ2Q6KocSLCMCXng= github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/terraform-exec v0.17.3 h1:MX14Kvnka/oWGmIkyuyvL6POx25ZmKrjlaclkx3eErU= -github.com/hashicorp/terraform-exec v0.17.3/go.mod h1:+NELG0EqQekJzhvikkeQsOAZpsw0cv/03rbeQJqscAI= +github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98 h1:WkSxIVXfV/u9tvDGXdSw4v2eCpRMSt7mYRBseX+OaSU= +github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s= github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= +github.com/hashicorp/terraform-plugin-framework v1.0.1 h1:apX2jtaEKa15+do6H2izBJdl1dEH2w5BPVkDJ3Q3mKA= +github.com/hashicorp/terraform-plugin-framework v1.0.1/go.mod h1:FV97t2BZOARkL7NNlsc/N25c84MyeSSz72uPp7Vq1lg= github.com/hashicorp/terraform-plugin-go v0.14.3 h1:nlnJ1GXKdMwsC8g1Nh05tK2wsC3+3BL/DBBxFEki+j0= github.com/hashicorp/terraform-plugin-go v0.14.3/go.mod h1:7ees7DMZ263q8wQ6E4RdIdR6nHHJtrdt4ogX5lPkX1A= github.com/hashicorp/terraform-plugin-log v0.7.0 h1:SDxJUyT8TwN4l5b5/VkiTIaQgY6R+Y2BQ0sRZftGKQs= From 9a77e42b8f8647ef0f097778f7fd1479d8644b95 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Fri, 13 Jan 2023 08:11:30 +0000 Subject: [PATCH 09/32] Using CreatePlanJSON() for generating streaming JSON output when running terraform plan (#16) --- helper/resource/testing_new_config.go | 6 +- helper/resource/teststep_providers_test.go | 128 +++++++++++++++++++- internal/plugintest/working_dir.go | 30 +++++ internal/testing/testplanmodifier/string.go | 44 +++++++ 4 files changed, 200 insertions(+), 8 deletions(-) create mode 100644 internal/testing/testplanmodifier/string.go diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index 440d53798..f3d9c286e 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -80,7 +80,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if step.Destroy { return wd.CreateDestroyPlan(ctx) } - return wd.CreatePlan(ctx) + return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { return fmt.Errorf("Error running pre-apply plan: %w", err) @@ -152,7 +152,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if step.Destroy { return wd.CreateDestroyPlan(ctx) } - return wd.CreatePlan(ctx) + return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { return fmt.Errorf("Error running post-apply plan: %w", err) @@ -196,7 +196,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if step.Destroy { return wd.CreateDestroyPlan(ctx) } - return wd.CreatePlan(ctx) + return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { return fmt.Errorf("Error running second post-apply plan: %w", err) diff --git a/helper/resource/teststep_providers_test.go b/helper/resource/teststep_providers_test.go index 6773b4f44..728859b13 100644 --- a/helper/resource/teststep_providers_test.go +++ b/helper/resource/teststep_providers_test.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/terraform-plugin-framework/providerserver" "github.com/hashicorp/terraform-plugin-framework/resource" fwresourceschema "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" "github.com/hashicorp/terraform-plugin-framework/types" "github.com/hashicorp/terraform-plugin-go/tfprotov5" "github.com/hashicorp/terraform-plugin-go/tfprotov6" @@ -25,6 +26,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-testing/internal/testing/testplanmodifier" "github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider" "github.com/hashicorp/terraform-plugin-testing/terraform" ) @@ -2307,7 +2309,7 @@ func TestTest_TestStep_ProviderFactories_ExpectWarningDetail(t *testing.T) { }) } -func TestTest_TestStep_ProviderFactories_ExpectErrorSummaryRefreshOnly(t *testing.T) { +func TestTest_TestStep_ProviderFactories_ExpectErrorRefresh(t *testing.T) { t.Parallel() Test(t, TestCase{ @@ -2362,7 +2364,7 @@ func TestTest_TestStep_ProviderFactories_ExpectErrorSummaryRefreshOnly(t *testin }) } -func TestTest_TestStep_ProviderFactories_ExpectWarningSummaryRefreshOnly(t *testing.T) { +func TestTest_TestStep_ProviderFactories_ExpectWarningRefresh(t *testing.T) { t.Parallel() Test(t, TestCase{ @@ -2417,9 +2419,11 @@ func TestTest_TestStep_ProviderFactories_ExpectWarningSummaryRefreshOnly(t *test }) } -func TestTest_TestStep_ProviderFactories_ExpectWarningSummaryPlanOnly(t *testing.T) { +func TestTest_TestStep_ProviderFactories_ExpectErrorPlan(t *testing.T) { t.Parallel() + readCount := 0 + Test(t, TestCase{ ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ @@ -2438,6 +2442,99 @@ func TestTest_TestStep_ProviderFactories_ExpectWarningSummaryPlanOnly(t *testing Attributes: map[string]fwresourceschema.Attribute{ "id": fwresourceschema.StringAttribute{ Computed: true, + PlanModifiers: []planmodifier.String{ + testplanmodifier.String{ + PlanModifyStringMethod: func(ctx context.Context, req planmodifier.StringRequest, resp *planmodifier.StringResponse) { + if req.PlanValue.ValueString() == "new_id" { + resp.Diagnostics.Append(fwdiag.NewErrorDiagnostic("error diagnostic - summary", "")) + } + }, + }, + }, + }, + }, + } + }, + CreateMethod: func(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + var data struct { + Id types.String `tfsdk:"id"` + } + + data.Id = types.StringValue("id") + + diags := resp.State.Set(ctx, &data) + resp.Diagnostics.Append(diags...) + }, + ReadMethod: func(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) { + var data struct { + Id types.String `tfsdk:"id"` + } + + diags := req.State.Get(ctx, &data) + resp.Diagnostics.Append(diags...) + + // Update id when Read is called following resource creation + // during terraform apply. + if readCount == 1 { + data.Id = types.StringValue("new_id") + } + + readCount++ + + diags = resp.State.Set(ctx, &data) + resp.Diagnostics.Append(diags...) + }, + } + }, + } + }, + }), + }, + Steps: []TestStep{ + { + Config: `resource "random_password" "test" { }`, + }, + { + Config: `resource "random_password" "test" { }`, + PlanOnly: true, + ExpectError: regexp.MustCompile(`.*error diagnostic - summary`), + }, + }, + }) +} + +func TestTest_TestStep_ProviderFactories_ExpectWarningPlan(t *testing.T) { + t.Parallel() + + readCount := 0 + + Test(t, TestCase{ + ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ + "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ + MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + resp.TypeName = "random" + }, + ResourcesMethod: func(ctx context.Context) []func() resource.Resource { + return []func() resource.Resource{ + func() resource.Resource { + return &testprovider.Resource{ + MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_password" + }, + SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = fwresourceschema.Schema{ + Attributes: map[string]fwresourceschema.Attribute{ + "id": fwresourceschema.StringAttribute{ + Computed: true, + PlanModifiers: []planmodifier.String{ + testplanmodifier.String{ + PlanModifyStringMethod: func(ctx context.Context, req planmodifier.StringRequest, resp *planmodifier.StringResponse) { + if req.PlanValue.ValueString() == "new_id" { + resp.Diagnostics.Append(fwdiag.NewWarningDiagnostic("warning diagnostic - summary", "")) + } + }, + }, + }, }, }, } @@ -2451,8 +2548,25 @@ func TestTest_TestStep_ProviderFactories_ExpectWarningSummaryPlanOnly(t *testing diags := resp.State.Set(ctx, &data) resp.Diagnostics.Append(diags...) + }, + ReadMethod: func(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) { + var data struct { + Id types.String `tfsdk:"id"` + } + + diags := req.State.Get(ctx, &data) + resp.Diagnostics.Append(diags...) - resp.Diagnostics.Append(fwdiag.NewWarningDiagnostic("warning diagnostic - detail", "")) + // Update id when Read is called following resource creation + // during terraform apply. + if readCount == 1 { + data.Id = types.StringValue("new_id") + } + + readCount++ + + diags = resp.State.Set(ctx, &data) + resp.Diagnostics.Append(diags...) }, } }, @@ -2461,9 +2575,13 @@ func TestTest_TestStep_ProviderFactories_ExpectWarningSummaryPlanOnly(t *testing }), }, Steps: []TestStep{ + { + Config: `resource "random_password" "test" { }`, + }, { Config: `resource "random_password" "test" { }`, - ExpectWarning: regexp.MustCompile(`.*warning diagnostic - detail`), + PlanOnly: true, + ExpectWarning: regexp.MustCompile(`.*warning diagnostic - summary`), }, }, }) diff --git a/internal/plugintest/working_dir.go b/internal/plugintest/working_dir.go index 0114077b7..dd94f7673 100644 --- a/internal/plugintest/working_dir.go +++ b/internal/plugintest/working_dir.go @@ -204,6 +204,36 @@ func (wd *WorkingDir) CreatePlan(ctx context.Context) error { return nil } +// CreatePlanJSON runs "terraform plan" with `-json` to create a saved plan file, +// which if successful will then be used for the next call to Apply. +func (wd *WorkingDir) CreatePlanJSON(ctx context.Context, w io.Writer) error { + logging.HelperResourceTrace(ctx, "Calling Terraform CLI plan command") + + hasChanges, err := wd.tf.PlanJSON(context.Background(), w, tfexec.Reattach(wd.reattachInfo), tfexec.Refresh(false), tfexec.Out(PlanFileName)) + + logging.HelperResourceTrace(ctx, "Called Terraform CLI plan command") + + if err != nil { + return err + } + + if !hasChanges { + logging.HelperResourceTrace(ctx, "Created plan with no changes") + + return nil + } + + stdout, err := wd.SavedPlanRawStdout(ctx) + + if err != nil { + return fmt.Errorf("error retrieving formatted plan output: %w", err) + } + + logging.HelperResourceTrace(ctx, "Created plan with changes", map[string]any{logging.KeyTestTerraformPlan: stdout}) + + return nil +} + // CreateDestroyPlan runs "terraform plan -destroy" to create a saved plan // file, which if successful will then be used for the next call to Apply. func (wd *WorkingDir) CreateDestroyPlan(ctx context.Context) error { diff --git a/internal/testing/testplanmodifier/string.go b/internal/testing/testplanmodifier/string.go new file mode 100644 index 000000000..cf26340ce --- /dev/null +++ b/internal/testing/testplanmodifier/string.go @@ -0,0 +1,44 @@ +package testplanmodifier + +import ( + "context" + + "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" +) + +var _ planmodifier.String = &String{} + +// String is declarative planmodifier.String for unit testing. +type String struct { + // String interface methods + DescriptionMethod func(context.Context) string + MarkdownDescriptionMethod func(context.Context) string + PlanModifyStringMethod func(context.Context, planmodifier.StringRequest, *planmodifier.StringResponse) +} + +// Description satisfies the planmodifier.String interface. +func (v String) Description(ctx context.Context) string { + if v.DescriptionMethod == nil { + return "" + } + + return v.DescriptionMethod(ctx) +} + +// MarkdownDescription satisfies the planmodifier.String interface. +func (v String) MarkdownDescription(ctx context.Context) string { + if v.MarkdownDescriptionMethod == nil { + return "" + } + + return v.MarkdownDescriptionMethod(ctx) +} + +// PlanModifyString satisfies the planmodifier.String interface. +func (v String) PlanModifyString(ctx context.Context, req planmodifier.StringRequest, resp *planmodifier.StringResponse) { + if v.PlanModifyStringMethod == nil { + return + } + + v.PlanModifyStringMethod(ctx, req, resp) +} From f496bfefa4fbc34b20257191553a2e0a75c77621 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Fri, 13 Jan 2023 09:55:34 +0000 Subject: [PATCH 10/32] Refactoring stdout related functions (#16) --- helper/resource/json.go | 97 ++++++++++++++++++++ helper/resource/testing_new.go | 61 ++---------- helper/resource/testing_new_config.go | 58 +++--------- helper/resource/testing_new_refresh_state.go | 6 +- internal/plugintest/working_dir.go | 30 ++++++ 5 files changed, 147 insertions(+), 105 deletions(-) diff --git a/helper/resource/json.go b/helper/resource/json.go index 9cd6a1b98..5f342a110 100644 --- a/helper/resource/json.go +++ b/helper/resource/json.go @@ -4,12 +4,109 @@ package resource import ( + "bufio" "bytes" "encoding/json" + "io" + "log" + "os" + "path/filepath" + "regexp" + "strings" + + tfjson "github.com/hashicorp/terraform-json" + + "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) +const stdoutFile = "stdout.txt" + func unmarshalJSON(data []byte, v interface{}) error { dec := json.NewDecoder(bytes.NewReader(data)) dec.UseNumber() return dec.Decode(v) } + +func stdoutWriter(wd *plugintest.WorkingDir) (io.WriteCloser, error) { + return os.Create(filepath.Join(wd.GetBaseDir(), stdoutFile)) +} + +func stdoutReader(wd *plugintest.WorkingDir) (io.ReadCloser, error) { + return os.Open(filepath.Join(wd.GetBaseDir(), stdoutFile)) +} + +func getJSONOutput(wd *plugintest.WorkingDir) []string { + reader, err := stdoutReader(wd) + if err != nil { + log.Fatal(err) + } + + defer reader.Close() + + scanner := bufio.NewScanner(reader) + var jsonOutput []string + + for scanner.Scan() { + var outer struct{} + + txt := scanner.Text() + + if json.Unmarshal([]byte(txt), &outer) == nil { + jsonOutput = append(jsonOutput, txt) + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + + return jsonOutput +} + +func getJSONOutputStr(wd *plugintest.WorkingDir) string { + return strings.Join(getJSONOutput(wd), "\n") +} + +func diagnosticFound(wd *plugintest.WorkingDir, r *regexp.Regexp, severity tfjson.DiagnosticSeverity) (bool, []string) { + reader, err := stdoutReader(wd) + if err != nil { + log.Fatal(err) + } + + defer reader.Close() + + scanner := bufio.NewScanner(reader) + var jsonOutput []string + + for scanner.Scan() { + var outer struct { + Diagnostic tfjson.Diagnostic + } + + txt := scanner.Text() + + if json.Unmarshal([]byte(txt), &outer) == nil { + jsonOutput = append(jsonOutput, txt) + + if outer.Diagnostic.Severity == "" { + continue + } + + if !r.MatchString(outer.Diagnostic.Summary) && !r.MatchString(outer.Diagnostic.Detail) { + continue + } + + if outer.Diagnostic.Severity != severity { + continue + } + + return true, jsonOutput + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + + return false, jsonOutput +} diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index bc68940b0..370d210ec 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -4,15 +4,9 @@ package resource import ( - "bufio" "context" - "encoding/json" "fmt" - "log" - "os" - "path/filepath" "reflect" - "regexp" "strings" "github.com/google/go-cmp/cmp" @@ -126,15 +120,14 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // acts as default for import tests var appliedCfg string - // stdoutFile is used as the handle to the file that contains - // json output from running Terraform commands. - stdoutFile, err := os.Create(filepath.Join(wd.GetBaseDir(), "stdout.txt")) + // w (io.WriteCloser) is supplied to all terraform commands that are using streaming json output. + w, err := stdoutWriter(wd) if err != nil { t.Fatalf("unable to create file for writing stdout: %w", err) } defer func() { - if stdoutFile != nil { - err := stdoutFile.Close() + if w != nil { + err := w.Close() if err != nil { logging.HelperResourceError(ctx, "Error closing file containing json output from Terraform commands", @@ -272,7 +265,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.RefreshState { logging.HelperResourceTrace(ctx, "TestStep is RefreshState mode") - err := testStepNewRefreshState(ctx, t, wd, step, providers, stdoutFile) + err := testStepNewRefreshState(ctx, t, wd, step, providers, w) if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -337,7 +330,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.Config != "" { logging.HelperResourceTrace(ctx, "TestStep is Config mode") - err := testStepNewConfig(ctx, t, c, wd, step, providers, stdoutFile) + err := testStepNewConfig(ctx, t, c, wd, step, providers, w) if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -407,48 +400,6 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } } -func diagnosticFound(wd *plugintest.WorkingDir, r *regexp.Regexp, severity tfjson.DiagnosticSeverity) (bool, []string) { - stdoutFile, err := os.Open(filepath.Join(wd.GetBaseDir(), "stdout.txt")) - if err != nil { - log.Fatal(err) - } - - scanner := bufio.NewScanner(stdoutFile) - var jsonOutput []string - - for scanner.Scan() { - var outer struct { - Diagnostic tfjson.Diagnostic - } - - txt := scanner.Text() - - if json.Unmarshal([]byte(txt), &outer) == nil { - jsonOutput = append(jsonOutput, txt) - - if outer.Diagnostic.Severity == "" { - continue - } - - if !r.MatchString(outer.Diagnostic.Summary) && !r.MatchString(outer.Diagnostic.Detail) { - continue - } - - if outer.Diagnostic.Severity != severity { - continue - } - - return true, jsonOutput - } - } - - if err := scanner.Err(); err != nil { - log.Fatal(err) - } - - return false, jsonOutput -} - func getState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir) (*terraform.State, error) { t.Helper() diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index f3d9c286e..450e3d52a 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -4,16 +4,10 @@ package resource import ( - "bufio" "context" - "encoding/json" "errors" "fmt" "io" - "log" - "os" - "path/filepath" - "strings" tfjson "github.com/hashicorp/terraform-json" "github.com/mitchellh/go-testing-interface" @@ -24,34 +18,6 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func getJSONOutput(wd *plugintest.WorkingDir) []string { - stdoutFile, err := os.Open(filepath.Join(wd.GetBaseDir(), "stdout.txt")) - if err != nil { - log.Fatal(err) - } - - scanner := bufio.NewScanner(stdoutFile) - var jsonOutput []string - - for scanner.Scan() { - var outer struct { - Diagnostic tfjson.Diagnostic - } - - txt := scanner.Text() - - if json.Unmarshal([]byte(txt), &outer) == nil { - jsonOutput = append(jsonOutput, txt) - } - } - - if err := scanner.Err(); err != nil { - log.Fatal(err) - } - - return jsonOutput -} - func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, w io.Writer) error { t.Helper() @@ -63,10 +29,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // require a refresh before applying // failing to do this will result in data sources not being updated err = runProviderCommand(ctx, t, func() error { - return wd.Refresh(ctx) + return wd.RefreshJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running pre-apply refresh: %w", err) + return fmt.Errorf("Error running pre-apply refresh: %s", getJSONOutputStr(wd)) } // If this step is a PlanOnly step, skip over this first Plan and @@ -78,12 +44,12 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // Plan! err := runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlan(ctx) + return wd.CreateDestroyPlanJSON(ctx, w) } return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running pre-apply plan: %w", err) + return fmt.Errorf("Error running pre-apply plan: %s", getJSONOutputStr(wd)) } // We need to keep a copy of the state prior to destroying such @@ -102,8 +68,6 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint } // Apply the diff, creating real resources - // TODO: Update all Errorf outputs that should use contents of streaming json - // after calling terraform-exec with `-json`. err = runProviderCommand(ctx, t, func() error { return wd.ApplyJSON(ctx, w) }, wd, providers) @@ -111,7 +75,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if step.Destroy { return fmt.Errorf("Error running destroy: %w", err) } - return fmt.Errorf("Error running apply: %s", strings.Join(getJSONOutput(wd), "\n")) + return fmt.Errorf("Error running apply: %s", getJSONOutputStr(wd)) } // Get the new state @@ -150,12 +114,12 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do a plan err = runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlan(ctx) + return wd.CreateDestroyPlanJSON(ctx, w) } return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running post-apply plan: %w", err) + return fmt.Errorf("Error running post-apply plan: %s", getJSONOutputStr(wd)) } var plan *tfjson.Plan @@ -184,22 +148,22 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do a refresh if !step.Destroy || (step.Destroy && !step.PreventPostDestroyRefresh) { err := runProviderCommand(ctx, t, func() error { - return wd.Refresh(ctx) + return wd.RefreshJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running post-apply refresh: %w", err) + return fmt.Errorf("Error running post-apply refresh: %s", getJSONOutputStr(wd)) } } // do another plan err = runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlan(ctx) + return wd.CreateDestroyPlanJSON(ctx, w) } return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running second post-apply plan: %w", err) + return fmt.Errorf("Error running second post-apply plan: %s", getJSONOutputStr(wd)) } err = runProviderCommand(ctx, t, func() error { diff --git a/helper/resource/testing_new_refresh_state.go b/helper/resource/testing_new_refresh_state.go index 181544cd5..742b6b593 100644 --- a/helper/resource/testing_new_refresh_state.go +++ b/helper/resource/testing_new_refresh_state.go @@ -37,7 +37,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return wd.RefreshJSON(ctx, w) }, wd, providers) if err != nil { - return err + return fmt.Errorf("%s", getJSONOutputStr(wd)) } var refreshState *terraform.State @@ -65,10 +65,10 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo // do a plan err = runProviderCommand(ctx, t, func() error { - return wd.CreatePlan(ctx) + return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running post-apply plan: %w", err) + return fmt.Errorf("Error running post-apply plan: %s", getJSONOutputStr(wd)) } var plan *tfjson.Plan diff --git a/internal/plugintest/working_dir.go b/internal/plugintest/working_dir.go index dd94f7673..e71c1c3f7 100644 --- a/internal/plugintest/working_dir.go +++ b/internal/plugintest/working_dir.go @@ -264,6 +264,36 @@ func (wd *WorkingDir) CreateDestroyPlan(ctx context.Context) error { return nil } +// CreateDestroyPlanJSON runs "terraform plan -destroy -json" to create a saved plan +// file, which if successful will then be used for the next call to Apply. +func (wd *WorkingDir) CreateDestroyPlanJSON(ctx context.Context, w io.Writer) error { + logging.HelperResourceTrace(ctx, "Calling Terraform CLI plan -destroy command") + + hasChanges, err := wd.tf.PlanJSON(context.Background(), w, tfexec.Reattach(wd.reattachInfo), tfexec.Refresh(false), tfexec.Out(PlanFileName), tfexec.Destroy(true)) + + logging.HelperResourceTrace(ctx, "Called Terraform CLI plan -destroy command") + + if err != nil { + return err + } + + if !hasChanges { + logging.HelperResourceTrace(ctx, "Created destroy plan with no changes") + + return nil + } + + stdout, err := wd.SavedPlanRawStdout(ctx) + + if err != nil { + return fmt.Errorf("error retrieving formatted plan output: %w", err) + } + + logging.HelperResourceTrace(ctx, "Created destroy plan with changes", map[string]any{logging.KeyTestTerraformPlan: stdout}) + + return nil +} + // Apply runs "terraform apply". If CreatePlan has previously completed // successfully and the saved plan has not been cleared in the meantime then // this will apply the saved plan. Otherwise, it will implicitly create a new From da0f4e997b5669ca740b169f383e3175cdf84bce Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Mon, 16 Jan 2023 12:05:50 +0000 Subject: [PATCH 11/32] Fallback to using standard terraform-exec commands (e.g., Apply() rather than ApplyJSON()) if running commands with -json flag raises an ErrVersionMismatch error (#16) --- go.mod | 2 +- go.sum | 2 + helper/resource/testing_new_config.go | 89 ++++++++++++++++++-- helper/resource/testing_new_refresh_state.go | 26 +++++- 4 files changed, 108 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index b0b2f027e..90e22da88 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/hashicorp/hc-install v0.4.0 github.com/hashicorp/hcl/v2 v2.15.0 github.com/hashicorp/logutils v1.0.0 - github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98 + github.com/hashicorp/terraform-exec v0.17.4-0.20230116095935-bc76870e2b9b github.com/hashicorp/terraform-json v0.14.0 github.com/hashicorp/terraform-plugin-framework v1.0.1 github.com/hashicorp/terraform-plugin-go v0.14.3 diff --git a/go.sum b/go.sum index d188998f3..8dd93d606 100644 --- a/go.sum +++ b/go.sum @@ -95,6 +95,8 @@ github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98 h1:WkSxIVXfV/u9tvDGXdSw4v2eCpRMSt7mYRBseX+OaSU= github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= +github.com/hashicorp/terraform-exec v0.17.4-0.20230116095935-bc76870e2b9b h1:z4a1oo2M/yWj2ljEoXsFQRkmp2dRqvq4Y9HjQl3eBz8= +github.com/hashicorp/terraform-exec v0.17.4-0.20230116095935-bc76870e2b9b/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s= github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= github.com/hashicorp/terraform-plugin-framework v1.0.1 h1:apX2jtaEKa15+do6H2izBJdl1dEH2w5BPVkDJ3Q3mKA= diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index 450e3d52a..d16e75019 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -9,6 +9,7 @@ import ( "fmt" "io" + "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" "github.com/mitchellh/go-testing-interface" @@ -32,7 +33,17 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return wd.RefreshJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running pre-apply refresh: %s", getJSONOutputStr(wd)) + target := &tfexec.ErrVersionMismatch{} + if errors.As(err, &target) { + err = runProviderCommand(ctx, t, func() error { + return wd.Refresh(ctx) + }, wd, providers) + if err != nil { + return fmt.Errorf("Error running pre-apply refresh: %w", err) + } + } else { + return fmt.Errorf("Error running pre-apply refresh: %s", getJSONOutputStr(wd)) + } } // If this step is a PlanOnly step, skip over this first Plan and @@ -49,7 +60,20 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running pre-apply plan: %s", getJSONOutputStr(wd)) + target := &tfexec.ErrVersionMismatch{} + if errors.As(err, &target) { + err = runProviderCommand(ctx, t, func() error { + if step.Destroy { + return wd.CreateDestroyPlan(ctx) + } + return wd.CreatePlan(ctx) + }, wd, providers) + if err != nil { + return fmt.Errorf("Error running pre-apply plan: %w", err) + } + } else { + return fmt.Errorf("Error running pre-apply plan: %s", getJSONOutputStr(wd)) + } } // We need to keep a copy of the state prior to destroying such @@ -72,10 +96,23 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return wd.ApplyJSON(ctx, w) }, wd, providers) if err != nil { - if step.Destroy { - return fmt.Errorf("Error running destroy: %w", err) + target := &tfexec.ErrVersionMismatch{} + if errors.As(err, &target) { + err = runProviderCommand(ctx, t, func() error { + return wd.Apply(ctx) + }, wd, providers) + if err != nil { + if step.Destroy { + return fmt.Errorf("Error running destroy: %w", err) + } + return fmt.Errorf("Error running apply: %w", err) + } + } else { + if step.Destroy { + return fmt.Errorf("Error running destroy: %s", getJSONOutputStr(wd)) + } + return fmt.Errorf("Error running apply: %s", getJSONOutputStr(wd)) } - return fmt.Errorf("Error running apply: %s", getJSONOutputStr(wd)) } // Get the new state @@ -119,7 +156,20 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running post-apply plan: %s", getJSONOutputStr(wd)) + target := &tfexec.ErrVersionMismatch{} + if errors.As(err, &target) { + err = runProviderCommand(ctx, t, func() error { + if step.Destroy { + return wd.CreateDestroyPlan(ctx) + } + return wd.CreatePlan(ctx) + }, wd, providers) + if err != nil { + return fmt.Errorf("Error running post-apply plan: %w", err) + } + } else { + return fmt.Errorf("Error running post-apply plan: %s", getJSONOutputStr(wd)) + } } var plan *tfjson.Plan @@ -151,7 +201,17 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return wd.RefreshJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running post-apply refresh: %s", getJSONOutputStr(wd)) + target := &tfexec.ErrVersionMismatch{} + if errors.As(err, &target) { + err = runProviderCommand(ctx, t, func() error { + return wd.Refresh(ctx) + }, wd, providers) + if err != nil { + return fmt.Errorf("Error running post-apply refresh: %w", err) + } + } else { + return fmt.Errorf("Error running post-apply refresh: %s", getJSONOutputStr(wd)) + } } } @@ -163,7 +223,20 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running second post-apply plan: %s", getJSONOutputStr(wd)) + target := &tfexec.ErrVersionMismatch{} + if errors.As(err, &target) { + err = runProviderCommand(ctx, t, func() error { + if step.Destroy { + return wd.CreateDestroyPlan(ctx) + } + return wd.CreatePlan(ctx) + }, wd, providers) + if err != nil { + return fmt.Errorf("Error running second post-apply plan: %w", err) + } + } else { + return fmt.Errorf("Error running second post-apply plan: %s", getJSONOutputStr(wd)) + } } err = runProviderCommand(ctx, t, func() error { diff --git a/helper/resource/testing_new_refresh_state.go b/helper/resource/testing_new_refresh_state.go index 742b6b593..871f3d90b 100644 --- a/helper/resource/testing_new_refresh_state.go +++ b/helper/resource/testing_new_refresh_state.go @@ -5,9 +5,11 @@ package resource import ( "context" + "errors" "fmt" "io" + "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" "github.com/mitchellh/go-testing-interface" @@ -37,7 +39,17 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return wd.RefreshJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("%s", getJSONOutputStr(wd)) + target := &tfexec.ErrVersionMismatch{} + if errors.As(err, &target) { + err = runProviderCommand(ctx, t, func() error { + return wd.Refresh(ctx) + }, wd, providers) + if err != nil { + return fmt.Errorf("Error running refresh: %w", err) + } + } else { + return fmt.Errorf("Error running refresh: %s", getJSONOutputStr(wd)) + } } var refreshState *terraform.State @@ -68,7 +80,17 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return wd.CreatePlanJSON(ctx, w) }, wd, providers) if err != nil { - return fmt.Errorf("Error running post-apply plan: %s", getJSONOutputStr(wd)) + target := &tfexec.ErrVersionMismatch{} + if errors.As(err, &target) { + err = runProviderCommand(ctx, t, func() error { + return wd.CreatePlan(ctx) + }, wd, providers) + if err != nil { + return fmt.Errorf("Error running post-apply plan: %w", err) + } + } else { + return fmt.Errorf("Error running post-apply plan: %s", getJSONOutputStr(wd)) + } } var plan *tfjson.Plan From 554cbd943196e0ccec0ad73a68bfb7f70bc42d33 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Mon, 16 Jan 2023 14:44:39 +0000 Subject: [PATCH 12/32] Updating test coverage for apply, plan, refresh and destroy (#16) --- helper/resource/teststep_providers_test.go | 494 ++++++++++++------ .../testprovider/resourcewithmodifyplan.go | 27 + 2 files changed, 372 insertions(+), 149 deletions(-) create mode 100644 internal/testing/testprovider/resourcewithmodifyplan.go diff --git a/helper/resource/teststep_providers_test.go b/helper/resource/teststep_providers_test.go index 728859b13..d371e47a9 100644 --- a/helper/resource/teststep_providers_test.go +++ b/helper/resource/teststep_providers_test.go @@ -18,7 +18,6 @@ import ( "github.com/hashicorp/terraform-plugin-framework/providerserver" "github.com/hashicorp/terraform-plugin-framework/resource" fwresourceschema "github.com/hashicorp/terraform-plugin-framework/resource/schema" - "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" "github.com/hashicorp/terraform-plugin-framework/types" "github.com/hashicorp/terraform-plugin-go/tfprotov5" "github.com/hashicorp/terraform-plugin-go/tfprotov6" @@ -26,7 +25,6 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" - "github.com/hashicorp/terraform-plugin-testing/internal/testing/testplanmodifier" "github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider" "github.com/hashicorp/terraform-plugin-testing/terraform" ) @@ -2313,50 +2311,66 @@ func TestTest_TestStep_ProviderFactories_ExpectErrorRefresh(t *testing.T) { t.Parallel() Test(t, TestCase{ - ProviderFactories: map[string]func() (*schema.Provider, error){ - "random": func() (*schema.Provider, error) { //nolint:unparam // required signature - return &schema.Provider{ - ResourcesMap: map[string]*schema.Resource{ - "random_password": { - CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics { - d.SetId("id") - return nil - }, - DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { - return nil - }, - ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) (diags diag.Diagnostics) { - // Only generate diag when ReadContext is called for the second - // time during terraform refresh. - if d.Get("id") == "new_id" { - return append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: "error diagnostic - summary", - }) - } - - // Update id when ReadContext is called following resource creation - // during terraform apply. - d.SetId("new_id") - - return - }, - Schema: map[string]*schema.Schema{ - "id": { - Computed: true, - Type: schema.TypeString, + Steps: []TestStep{ + { + ProviderFactories: map[string]func() (*schema.Provider, error){ + "random": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "random_password": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics { + d.SetId("id") + return nil + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) (diags diag.Diagnostics) { + return nil + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, }, }, - }, + }, nil }, - }, nil - }, - }, - Steps: []TestStep{ - { + }, Config: `resource "random_password" "test" { }`, }, { + ProviderFactories: map[string]func() (*schema.Provider, error){ + "random": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "random_password": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics { + d.SetId("id") + return nil + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) (diags diag.Diagnostics) { + return append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "error diagnostic - summary", + }) + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, + }, + }, + }, nil + }, + }, RefreshState: true, ExpectError: regexp.MustCompile(`.*error diagnostic - summary`), }, @@ -2422,79 +2436,83 @@ func TestTest_TestStep_ProviderFactories_ExpectWarningRefresh(t *testing.T) { func TestTest_TestStep_ProviderFactories_ExpectErrorPlan(t *testing.T) { t.Parallel() - readCount := 0 - Test(t, TestCase{ - ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ - "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ - MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { - resp.TypeName = "random" - }, - ResourcesMethod: func(ctx context.Context) []func() resource.Resource { - return []func() resource.Resource{ - func() resource.Resource { - return &testprovider.Resource{ - MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { - resp.TypeName = req.ProviderTypeName + "_password" - }, - SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { - resp.Schema = fwresourceschema.Schema{ - Attributes: map[string]fwresourceschema.Attribute{ - "id": fwresourceschema.StringAttribute{ - Computed: true, - PlanModifiers: []planmodifier.String{ - testplanmodifier.String{ - PlanModifyStringMethod: func(ctx context.Context, req planmodifier.StringRequest, resp *planmodifier.StringResponse) { - if req.PlanValue.ValueString() == "new_id" { - resp.Diagnostics.Append(fwdiag.NewErrorDiagnostic("error diagnostic - summary", "")) - } - }, + Steps: []TestStep{ + { + ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ + "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ + MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + resp.TypeName = "random" + }, + ResourcesMethod: func(ctx context.Context) []func() resource.Resource { + return []func() resource.Resource{ + func() resource.Resource { + return &testprovider.Resource{ + MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_password" + }, + SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = fwresourceschema.Schema{ + Attributes: map[string]fwresourceschema.Attribute{ + "id": fwresourceschema.StringAttribute{ + Computed: true, }, }, - }, + } }, - } - }, - CreateMethod: func(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { - var data struct { - Id types.String `tfsdk:"id"` - } - - data.Id = types.StringValue("id") - - diags := resp.State.Set(ctx, &data) - resp.Diagnostics.Append(diags...) - }, - ReadMethod: func(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) { - var data struct { - Id types.String `tfsdk:"id"` - } + CreateMethod: func(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + var data struct { + Id types.String `tfsdk:"id"` + } - diags := req.State.Get(ctx, &data) - resp.Diagnostics.Append(diags...) + data.Id = types.StringValue("id") - // Update id when Read is called following resource creation - // during terraform apply. - if readCount == 1 { - data.Id = types.StringValue("new_id") + diags := resp.State.Set(ctx, &data) + resp.Diagnostics.Append(diags...) + }, } - - readCount++ - - diags = resp.State.Set(ctx, &data) - resp.Diagnostics.Append(diags...) }, } }, - } + }), }, - }), - }, - Steps: []TestStep{ - { Config: `resource "random_password" "test" { }`, }, { + ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ + "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ + MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + resp.TypeName = "random" + }, + ResourcesMethod: func(ctx context.Context) []func() resource.Resource { + return []func() resource.Resource{ + func() resource.Resource { + return &testprovider.ResourceWithModifyPlan{ + Resource: &testprovider.Resource{ + MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_password" + }, + SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = fwresourceschema.Schema{ + Attributes: map[string]fwresourceschema.Attribute{ + "id": fwresourceschema.StringAttribute{ + Computed: true, + }, + }, + } + }, + }, + ModifyPlanMethod: func(ctx context.Context, req resource.ModifyPlanRequest, resp *resource.ModifyPlanResponse) { + if !req.Plan.Raw.IsNull() { + resp.Diagnostics.Append(fwdiag.NewErrorDiagnostic("error diagnostic - summary", "")) + } + }, + } + }, + } + }, + }), + }, Config: `resource "random_password" "test" { }`, PlanOnly: true, ExpectError: regexp.MustCompile(`.*error diagnostic - summary`), @@ -2506,81 +2524,259 @@ func TestTest_TestStep_ProviderFactories_ExpectErrorPlan(t *testing.T) { func TestTest_TestStep_ProviderFactories_ExpectWarningPlan(t *testing.T) { t.Parallel() - readCount := 0 - Test(t, TestCase{ - ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ - "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ - MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { - resp.TypeName = "random" - }, - ResourcesMethod: func(ctx context.Context) []func() resource.Resource { - return []func() resource.Resource{ - func() resource.Resource { - return &testprovider.Resource{ - MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { - resp.TypeName = req.ProviderTypeName + "_password" + Steps: []TestStep{ + { + ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ + "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ + MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + resp.TypeName = "random" + }, + ResourcesMethod: func(ctx context.Context) []func() resource.Resource { + return []func() resource.Resource{ + func() resource.Resource { + return &testprovider.Resource{ + MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_password" + }, + SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = fwresourceschema.Schema{ + Attributes: map[string]fwresourceschema.Attribute{ + "id": fwresourceschema.StringAttribute{ + Computed: true, + }, + }, + } + }, + CreateMethod: func(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + var data struct { + Id types.String `tfsdk:"id"` + } + + data.Id = types.StringValue("example-id") + + diags := resp.State.Set(ctx, &data) + resp.Diagnostics.Append(diags...) + }, + } }, - SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { - resp.Schema = fwresourceschema.Schema{ - Attributes: map[string]fwresourceschema.Attribute{ - "id": fwresourceschema.StringAttribute{ - Computed: true, - PlanModifiers: []planmodifier.String{ - testplanmodifier.String{ - PlanModifyStringMethod: func(ctx context.Context, req planmodifier.StringRequest, resp *planmodifier.StringResponse) { - if req.PlanValue.ValueString() == "new_id" { - resp.Diagnostics.Append(fwdiag.NewWarningDiagnostic("warning diagnostic - summary", "")) - } + } + }, + }), + }, + Config: `resource "random_password" "test" { }`, + }, + { + ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ + "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ + MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + resp.TypeName = "random" + }, + ResourcesMethod: func(ctx context.Context) []func() resource.Resource { + return []func() resource.Resource{ + func() resource.Resource { + return &testprovider.ResourceWithModifyPlan{ + Resource: &testprovider.Resource{ + MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_password" + }, + SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = fwresourceschema.Schema{ + Attributes: map[string]fwresourceschema.Attribute{ + "id": fwresourceschema.StringAttribute{ + Computed: true, }, }, - }, + } }, }, + ModifyPlanMethod: func(ctx context.Context, req resource.ModifyPlanRequest, resp *resource.ModifyPlanResponse) { + resp.Diagnostics.Append(fwdiag.NewWarningDiagnostic("warning diagnostic - summary", "")) + }, } }, - CreateMethod: func(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { - var data struct { - Id types.String `tfsdk:"id"` - } + } + }, + }), + }, + Config: `resource "random_password" "test" { }`, + PlanOnly: true, + ExpectWarning: regexp.MustCompile(`.*warning diagnostic - summary`), + }, + }, + }) +} - data.Id = types.StringValue("example-id") +func TestTest_TestStep_ProviderFactories_ExpectErrorDestroy(t *testing.T) { + t.Parallel() - diags := resp.State.Set(ctx, &data) - resp.Diagnostics.Append(diags...) - }, - ReadMethod: func(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) { - var data struct { - Id types.String `tfsdk:"id"` - } + deleteCount := 0 - diags := req.State.Get(ctx, &data) - resp.Diagnostics.Append(diags...) + Test(t, TestCase{ + Steps: []TestStep{ + { + ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ + "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ + MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + resp.TypeName = "random" + }, + ResourcesMethod: func(ctx context.Context) []func() resource.Resource { + return []func() resource.Resource{ + func() resource.Resource { + return &testprovider.Resource{ + MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_password" + }, + SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = fwresourceschema.Schema{ + Attributes: map[string]fwresourceschema.Attribute{ + "id": fwresourceschema.StringAttribute{ + Computed: true, + }, + }, + } + }, + CreateMethod: func(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + var data struct { + Id types.String `tfsdk:"id"` + } - // Update id when Read is called following resource creation - // during terraform apply. - if readCount == 1 { - data.Id = types.StringValue("new_id") - } + data.Id = types.StringValue("example-id") - readCount++ + diags := resp.State.Set(ctx, &data) + resp.Diagnostics.Append(diags...) + }, + } + }, + } + }, + }), + }, + Config: `resource "random_password" "test" { }`, + }, + { + ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ + "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ + MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + resp.TypeName = "random" + }, + ResourcesMethod: func(ctx context.Context) []func() resource.Resource { + return []func() resource.Resource{ + func() resource.Resource { + return &testprovider.Resource{ + MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_password" + }, + SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = fwresourceschema.Schema{ + Attributes: map[string]fwresourceschema.Attribute{ + "id": fwresourceschema.StringAttribute{ + Computed: true, + }, + }, + } + }, + DeleteMethod: func(ctx context.Context, req resource.DeleteRequest, resp *resource.DeleteResponse) { + // TODO: deleteCount is used so that when Delete is called during apply, diagnostic is added but + // when Delete is called during runPostTestDestroy it does not add diagnostic. + if deleteCount < 1 { + resp.Diagnostics.Append(fwdiag.NewErrorDiagnostic("error diagnostic - summary", "")) + } - diags = resp.State.Set(ctx, &data) - resp.Diagnostics.Append(diags...) + deleteCount++ + }, + } }, } }, - } + }), }, - }), + Config: `resource "random_password" "test" { }`, + Destroy: true, + ExpectError: regexp.MustCompile(`.*error diagnostic - summary`), + }, }, + }) +} + +func TestTest_TestStep_ProviderFactories_ExpectWarningDestroy(t *testing.T) { + t.Parallel() + + Test(t, TestCase{ Steps: []TestStep{ { + ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ + "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ + MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + resp.TypeName = "random" + }, + ResourcesMethod: func(ctx context.Context) []func() resource.Resource { + return []func() resource.Resource{ + func() resource.Resource { + return &testprovider.Resource{ + MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_password" + }, + SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = fwresourceschema.Schema{ + Attributes: map[string]fwresourceschema.Attribute{ + "id": fwresourceschema.StringAttribute{ + Computed: true, + }, + }, + } + }, + CreateMethod: func(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + var data struct { + Id types.String `tfsdk:"id"` + } + + data.Id = types.StringValue("example-id") + + diags := resp.State.Set(ctx, &data) + resp.Diagnostics.Append(diags...) + }, + } + }, + } + }, + }), + }, Config: `resource "random_password" "test" { }`, }, { + ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){ + "random": providerserver.NewProtocol5WithError(&testprovider.Provider{ + MetadataMethod: func(ctx context.Context, req provider.MetadataRequest, resp *provider.MetadataResponse) { + resp.TypeName = "random" + }, + ResourcesMethod: func(ctx context.Context) []func() resource.Resource { + return []func() resource.Resource{ + func() resource.Resource { + return &testprovider.Resource{ + MetadataMethod: func(ctx context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_password" + }, + SchemaMethod: func(ctx context.Context, req resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = fwresourceschema.Schema{ + Attributes: map[string]fwresourceschema.Attribute{ + "id": fwresourceschema.StringAttribute{ + Computed: true, + }, + }, + } + }, + DeleteMethod: func(ctx context.Context, req resource.DeleteRequest, resp *resource.DeleteResponse) { + resp.Diagnostics.Append(fwdiag.NewWarningDiagnostic("warning diagnostic - summary", "")) + }, + } + }, + } + }, + }), + }, Config: `resource "random_password" "test" { }`, - PlanOnly: true, + Destroy: true, ExpectWarning: regexp.MustCompile(`.*warning diagnostic - summary`), }, }, diff --git a/internal/testing/testprovider/resourcewithmodifyplan.go b/internal/testing/testprovider/resourcewithmodifyplan.go new file mode 100644 index 000000000..0162a5694 --- /dev/null +++ b/internal/testing/testprovider/resourcewithmodifyplan.go @@ -0,0 +1,27 @@ +package testprovider + +import ( + "context" + + "github.com/hashicorp/terraform-plugin-framework/resource" +) + +var _ resource.Resource = &ResourceWithModifyPlan{} +var _ resource.ResourceWithModifyPlan = &ResourceWithModifyPlan{} + +// ResourceWithModifyPlan is a declarative resource.ResourceWithModifyPlan for unit testing. +type ResourceWithModifyPlan struct { + *Resource + + // ResourceWithModifyPlan interface methods + ModifyPlanMethod func(context.Context, resource.ModifyPlanRequest, *resource.ModifyPlanResponse) +} + +// ModifyPlan satisfies the resource.ResourceWithModifyPlan interface. +func (p *ResourceWithModifyPlan) ModifyPlan(ctx context.Context, req resource.ModifyPlanRequest, resp *resource.ModifyPlanResponse) { + if p.ModifyPlanMethod == nil { + return + } + + p.ModifyPlanMethod(ctx, req, resp) +} From 89be0c3f388c92bf2b7a21fd5a5326f94347ef3c Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Mon, 16 Jan 2023 17:43:59 +0000 Subject: [PATCH 13/32] Switching to using bytes.Buffer for terraform-exec streaming json output (#16) --- helper/resource/json.go | 97 ---------------- helper/resource/stdout.go | 112 +++++++++++++++++++ helper/resource/stdout_test.go | 85 ++++++++++++++ helper/resource/testing_new.go | 32 ++---- helper/resource/testing_new_config.go | 35 +++--- helper/resource/testing_new_refresh_state.go | 11 +- 6 files changed, 227 insertions(+), 145 deletions(-) create mode 100644 helper/resource/stdout.go create mode 100644 helper/resource/stdout_test.go diff --git a/helper/resource/json.go b/helper/resource/json.go index 5f342a110..9cd6a1b98 100644 --- a/helper/resource/json.go +++ b/helper/resource/json.go @@ -4,109 +4,12 @@ package resource import ( - "bufio" "bytes" "encoding/json" - "io" - "log" - "os" - "path/filepath" - "regexp" - "strings" - - tfjson "github.com/hashicorp/terraform-json" - - "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -const stdoutFile = "stdout.txt" - func unmarshalJSON(data []byte, v interface{}) error { dec := json.NewDecoder(bytes.NewReader(data)) dec.UseNumber() return dec.Decode(v) } - -func stdoutWriter(wd *plugintest.WorkingDir) (io.WriteCloser, error) { - return os.Create(filepath.Join(wd.GetBaseDir(), stdoutFile)) -} - -func stdoutReader(wd *plugintest.WorkingDir) (io.ReadCloser, error) { - return os.Open(filepath.Join(wd.GetBaseDir(), stdoutFile)) -} - -func getJSONOutput(wd *plugintest.WorkingDir) []string { - reader, err := stdoutReader(wd) - if err != nil { - log.Fatal(err) - } - - defer reader.Close() - - scanner := bufio.NewScanner(reader) - var jsonOutput []string - - for scanner.Scan() { - var outer struct{} - - txt := scanner.Text() - - if json.Unmarshal([]byte(txt), &outer) == nil { - jsonOutput = append(jsonOutput, txt) - } - } - - if err := scanner.Err(); err != nil { - log.Fatal(err) - } - - return jsonOutput -} - -func getJSONOutputStr(wd *plugintest.WorkingDir) string { - return strings.Join(getJSONOutput(wd), "\n") -} - -func diagnosticFound(wd *plugintest.WorkingDir, r *regexp.Regexp, severity tfjson.DiagnosticSeverity) (bool, []string) { - reader, err := stdoutReader(wd) - if err != nil { - log.Fatal(err) - } - - defer reader.Close() - - scanner := bufio.NewScanner(reader) - var jsonOutput []string - - for scanner.Scan() { - var outer struct { - Diagnostic tfjson.Diagnostic - } - - txt := scanner.Text() - - if json.Unmarshal([]byte(txt), &outer) == nil { - jsonOutput = append(jsonOutput, txt) - - if outer.Diagnostic.Severity == "" { - continue - } - - if !r.MatchString(outer.Diagnostic.Summary) && !r.MatchString(outer.Diagnostic.Detail) { - continue - } - - if outer.Diagnostic.Severity != severity { - continue - } - - return true, jsonOutput - } - } - - if err := scanner.Err(); err != nil { - log.Fatal(err) - } - - return false, jsonOutput -} diff --git a/helper/resource/stdout.go b/helper/resource/stdout.go new file mode 100644 index 000000000..b22d8ae9d --- /dev/null +++ b/helper/resource/stdout.go @@ -0,0 +1,112 @@ +package resource + +import ( + "bufio" + "bytes" + "encoding/json" + "io" + "log" + "regexp" + "strings" + + tfjson "github.com/hashicorp/terraform-json" +) + +var _ io.Writer = &Stdout{} +var _ io.Reader = &Stdout{} + +type Stdout struct { + buf *bytes.Buffer +} + +func NewStdout() *Stdout { + return &Stdout{ + buf: new(bytes.Buffer), + } +} + +func (s *Stdout) Write(p []byte) (n int, err error) { + if s.buf == nil { + log.Fatal("call NewStdout to initialise buffer") + } + + return s.buf.Write(p) +} + +func (s *Stdout) Read(p []byte) (n int, err error) { + if s.buf == nil { + log.Fatal("call NewStdout to initialise buffer") + } + + return s.buf.Read(p) +} + +func (s *Stdout) GetJSONOutput() []string { + if s.buf == nil { + log.Fatal("call NewStdout to initialise buffer") + } + + scanner := bufio.NewScanner(s.buf) + var jsonOutput []string + + for scanner.Scan() { + var outer struct{} + + txt := scanner.Text() + + if json.Unmarshal([]byte(txt), &outer) == nil { + jsonOutput = append(jsonOutput, txt) + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + + return jsonOutput +} + +func (s *Stdout) GetJSONOutputStr() string { + return strings.Join(s.GetJSONOutput(), "\n") +} + +func (s *Stdout) DiagnosticFound(r *regexp.Regexp, severity tfjson.DiagnosticSeverity) (bool, []string) { + if s.buf == nil { + log.Fatal("call NewStdout to initialise buffer") + } + + scanner := bufio.NewScanner(s.buf) + var jsonOutput []string + + for scanner.Scan() { + var outer struct { + Diagnostic tfjson.Diagnostic + } + + txt := scanner.Text() + + if json.Unmarshal([]byte(txt), &outer) == nil { + jsonOutput = append(jsonOutput, txt) + + if outer.Diagnostic.Severity == "" { + continue + } + + if !r.MatchString(outer.Diagnostic.Summary) && !r.MatchString(outer.Diagnostic.Detail) { + continue + } + + if outer.Diagnostic.Severity != severity { + continue + } + + return true, jsonOutput + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + + return false, jsonOutput +} diff --git a/helper/resource/stdout_test.go b/helper/resource/stdout_test.go new file mode 100644 index 000000000..256380af4 --- /dev/null +++ b/helper/resource/stdout_test.go @@ -0,0 +1,85 @@ +package resource + +import ( + "regexp" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + tfjson "github.com/hashicorp/terraform-json" +) + +var stdoutJSON = []string{ + `{"@level":"info","@message":"Terraform 1.3.2","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.232751Z","terraform":"1.3.2","type":"version","ui":"1.0"}`, + `{"@level":"warn","@message":"Warning: Empty or non-existent state","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.250725Z","diagnostic":{"severity":"warning","summary":"Empty or non-existent state","detail":"There are currently no remote objects tracked in the state, so there is nothing to refresh."},"type":"diagnostic"}`, + `{"@level":"info","@message":"Outputs: 0","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.251120Z","outputs":{},"type":"outputs"}`, + `{"@level":"info","@message":"random_password.test: Plan to create","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.282021Z","change":{"resource":{"addr":"random_password.test","module":"","resource":"random_password.test","implied_provider":"random","resource_type":"random_password","resource_name":"test","resource_key":null},"action":"create"},"type":"planned_change"}`, + `{"@level":"info","@message":"Plan: 1 to add, 0 to change, 0 to destroy.","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.282054Z","changes":{"add":1,"change":0,"remove":0,"operation":"plan"},"type":"change_summary"}`, + `{"format_version":"1.0"}`, + `{"@level":"error","@message":"Error: error diagnostic - summary","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.626572Z","diagnostic":{"severity":"error","summary":"error diagnostic - summary","detail":""},"type":"diagnostic"}`, +} + +func TestStdout_GetJSONOutputStr(t *testing.T) { + t.Parallel() + + stdout := NewStdout() + + for _, v := range stdoutJSON { + stdout.Write([]byte(v + "\n")) + } + + jsonOutputStr := stdout.GetJSONOutputStr() + + if diff := cmp.Diff(jsonOutputStr, strings.Join(stdoutJSON, "\n")); diff != "" { + t.Errorf("unexpected difference: %s", diff) + } +} + +func TestStdout_DiagnosticFound(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + regex *regexp.Regexp + severity tfjson.DiagnosticSeverity + expected bool + expectedJSON []string + }{ + "found": { + regex: regexp.MustCompile(`.*error diagnostic - summary`), + severity: tfjson.DiagnosticSeverityError, + expected: true, + expectedJSON: stdoutJSON, + }, + "not-found": { + regex: regexp.MustCompile(`.*warning diagnostic - summary`), + severity: tfjson.DiagnosticSeverityError, + expected: false, + expectedJSON: stdoutJSON, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + stdout := NewStdout() + + for _, v := range stdoutJSON { + _, err := stdout.Write([]byte(v + "\n")) + if err != nil { + t.Errorf("error writing to stdout: %s", err) + } + } + + isFound, output := stdout.DiagnosticFound(testCase.regex, testCase.severity) + + if diff := cmp.Diff(isFound, testCase.expected); diff != "" { + t.Errorf("unexpected difference: %s", diff) + } + if diff := cmp.Diff(output, testCase.expectedJSON); diff != "" { + t.Errorf("unexpected difference: %s", diff) + } + }) + } + +} diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 370d210ec..0ad2f0beb 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -120,24 +120,8 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // acts as default for import tests var appliedCfg string - // w (io.WriteCloser) is supplied to all terraform commands that are using streaming json output. - w, err := stdoutWriter(wd) - if err != nil { - t.Fatalf("unable to create file for writing stdout: %w", err) - } - defer func() { - if w != nil { - err := w.Close() - if err != nil { - logging.HelperResourceError(ctx, - "Error closing file containing json output from Terraform commands", - map[string]interface{}{logging.KeyError: err}, - ) - t.Fatalf("Error closing file containing json from Terraform commands: %s", err.Error()) - return - } - } - }() + // stdout (io.Writer) is supplied to all terraform commands that are using streaming json output. + stdout := NewStdout() for stepIndex, step := range c.Steps { stepNumber := stepIndex + 1 // 1-based indexing for humans @@ -265,7 +249,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.RefreshState { logging.HelperResourceTrace(ctx, "TestStep is RefreshState mode") - err := testStepNewRefreshState(ctx, t, wd, step, providers, w) + err := testStepNewRefreshState(ctx, t, wd, step, providers, stdout) if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -278,7 +262,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } errorFound := step.ExpectError.MatchString(err.Error()) - jsonErrorFound, jsonDiags := diagnosticFound(wd, step.ExpectError, tfjson.DiagnosticSeverityError) + jsonErrorFound, jsonDiags := stdout.DiagnosticFound(step.ExpectError, tfjson.DiagnosticSeverityError) errorOutput := []string{err.Error()} @@ -311,7 +295,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - warningFound, jsonDiags := diagnosticFound(wd, step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + warningFound, jsonDiags := stdout.DiagnosticFound(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { logging.HelperResourceError(ctx, @@ -330,7 +314,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.Config != "" { logging.HelperResourceTrace(ctx, "TestStep is Config mode") - err := testStepNewConfig(ctx, t, c, wd, step, providers, w) + err := testStepNewConfig(ctx, t, c, wd, step, providers, stdout) if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -343,7 +327,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } errorFound := step.ExpectError.MatchString(err.Error()) - jsonErrorFound, jsonDiags := diagnosticFound(wd, step.ExpectError, tfjson.DiagnosticSeverityError) + jsonErrorFound, jsonDiags := stdout.DiagnosticFound(step.ExpectError, tfjson.DiagnosticSeverityError) errorOutput := []string{err.Error()} @@ -378,7 +362,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - warningFound, jsonDiags := diagnosticFound(wd, step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + warningFound, jsonDiags := stdout.DiagnosticFound(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { logging.HelperResourceError(ctx, diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index d16e75019..21058dcfd 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -7,7 +7,6 @@ import ( "context" "errors" "fmt" - "io" "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" @@ -19,7 +18,7 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, w io.Writer) error { +func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, stdout *Stdout) error { t.Helper() err := wd.SetConfig(ctx, step.mergedConfig(ctx, c)) @@ -30,7 +29,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // require a refresh before applying // failing to do this will result in data sources not being updated err = runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, w) + return wd.RefreshJSON(ctx, stdout) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -42,7 +41,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running pre-apply refresh: %w", err) } } else { - return fmt.Errorf("Error running pre-apply refresh: %s", getJSONOutputStr(wd)) + return fmt.Errorf("Error running pre-apply refresh: %s", stdout.GetJSONOutputStr()) } } @@ -55,9 +54,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // Plan! err := runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, w) + return wd.CreateDestroyPlanJSON(ctx, stdout) } - return wd.CreatePlanJSON(ctx, w) + return wd.CreatePlanJSON(ctx, stdout) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -72,7 +71,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running pre-apply plan: %w", err) } } else { - return fmt.Errorf("Error running pre-apply plan: %s", getJSONOutputStr(wd)) + return fmt.Errorf("Error running pre-apply plan: %s", stdout.GetJSONOutputStr()) } } @@ -93,7 +92,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // Apply the diff, creating real resources err = runProviderCommand(ctx, t, func() error { - return wd.ApplyJSON(ctx, w) + return wd.ApplyJSON(ctx, stdout) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -109,9 +108,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint } } else { if step.Destroy { - return fmt.Errorf("Error running destroy: %s", getJSONOutputStr(wd)) + return fmt.Errorf("Error running destroy: %s", stdout.GetJSONOutputStr()) } - return fmt.Errorf("Error running apply: %s", getJSONOutputStr(wd)) + return fmt.Errorf("Error running apply: %s", stdout.GetJSONOutputStr()) } } @@ -151,9 +150,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do a plan err = runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, w) + return wd.CreateDestroyPlanJSON(ctx, stdout) } - return wd.CreatePlanJSON(ctx, w) + return wd.CreatePlanJSON(ctx, stdout) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -168,7 +167,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running post-apply plan: %w", err) } } else { - return fmt.Errorf("Error running post-apply plan: %s", getJSONOutputStr(wd)) + return fmt.Errorf("Error running post-apply plan: %s", stdout.GetJSONOutputStr()) } } @@ -198,7 +197,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do a refresh if !step.Destroy || (step.Destroy && !step.PreventPostDestroyRefresh) { err := runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, w) + return wd.RefreshJSON(ctx, stdout) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -210,7 +209,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running post-apply refresh: %w", err) } } else { - return fmt.Errorf("Error running post-apply refresh: %s", getJSONOutputStr(wd)) + return fmt.Errorf("Error running post-apply refresh: %s", stdout.GetJSONOutputStr()) } } } @@ -218,9 +217,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do another plan err = runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, w) + return wd.CreateDestroyPlanJSON(ctx, stdout) } - return wd.CreatePlanJSON(ctx, w) + return wd.CreatePlanJSON(ctx, stdout) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -235,7 +234,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running second post-apply plan: %w", err) } } else { - return fmt.Errorf("Error running second post-apply plan: %s", getJSONOutputStr(wd)) + return fmt.Errorf("Error running second post-apply plan: %s", stdout.GetJSONOutputStr()) } } diff --git a/helper/resource/testing_new_refresh_state.go b/helper/resource/testing_new_refresh_state.go index 871f3d90b..19399612c 100644 --- a/helper/resource/testing_new_refresh_state.go +++ b/helper/resource/testing_new_refresh_state.go @@ -7,7 +7,6 @@ import ( "context" "errors" "fmt" - "io" "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" @@ -19,7 +18,7 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, w io.Writer) error { +func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, stdout *Stdout) error { t.Helper() var err error @@ -36,7 +35,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo } err = runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, w) + return wd.RefreshJSON(ctx, stdout) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -48,7 +47,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return fmt.Errorf("Error running refresh: %w", err) } } else { - return fmt.Errorf("Error running refresh: %s", getJSONOutputStr(wd)) + return fmt.Errorf("Error running refresh: %s", stdout.GetJSONOutputStr()) } } @@ -77,7 +76,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo // do a plan err = runProviderCommand(ctx, t, func() error { - return wd.CreatePlanJSON(ctx, w) + return wd.CreatePlanJSON(ctx, stdout) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -89,7 +88,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return fmt.Errorf("Error running post-apply plan: %w", err) } } else { - return fmt.Errorf("Error running post-apply plan: %s", getJSONOutputStr(wd)) + return fmt.Errorf("Error running post-apply plan: %s", stdout.GetJSONOutputStr()) } } From ad2edebb9afd493711bf7f14c904beaa849aa0e1 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Wed, 18 Jan 2023 08:21:59 +0000 Subject: [PATCH 14/32] Adding changelog entry and updating documentation (#16) --- .changes/unreleased/FEATURES-20230118-083204.yaml | 8 ++++++++ .changie.yaml | 2 +- helper/resource/testing.go | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/FEATURES-20230118-083204.yaml diff --git a/.changes/unreleased/FEATURES-20230118-083204.yaml b/.changes/unreleased/FEATURES-20230118-083204.yaml new file mode 100644 index 000000000..3203f5565 --- /dev/null +++ b/.changes/unreleased/FEATURES-20230118-083204.yaml @@ -0,0 +1,8 @@ +kind: FEATURES +body: 'helper/resource: Added `TestStep` type `ExpectWarning` field, which enables + verifying that a warning has been emitted during plan, apply, refresh and destroy. + Requires that Terraform 0.15.3 or later is used as it makes use of the Terraform + [machine-readable UI](https://developer.hashicorp.com/terraform/internals/machine-readable-ui)' +time: 2023-01-18T08:32:04.688764Z +custom: + Issue: "16" diff --git a/.changie.yaml b/.changie.yaml index 08921f33d..c5eac1ecf 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -7,7 +7,7 @@ kindFormat: '{{.Kind}}:' changeFormat: '* {{.Body}} ([#{{.Custom.Issue}}](https://github.com/hashicorp/terraform-plugin-testing/issues/{{.Custom.Issue}}))' custom: - key: Issue - label: Issue/PR Number + label: Issue Number type: int minInt: 1 kinds: diff --git a/helper/resource/testing.go b/helper/resource/testing.go index 311224c94..571bd22df 100644 --- a/helper/resource/testing.go +++ b/helper/resource/testing.go @@ -501,6 +501,9 @@ type TestStep struct { // test to pass. ExpectError *regexp.Regexp + // ExpectWarning requires the use of Terraform 0.15.3 or later as it makes + // use of the Terraform [machine-readable UI](https://developer.hashicorp.com/terraform/internals/machine-readable-ui). + // // ExpectWarning allows the construction of test cases that we expect to pass // with a warning. The specified regexp must match against stdout for the // test to pass. From 28c66762d6bf06641c6998a7cd310f68f27ba808 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Wed, 18 Jan 2023 09:29:08 +0000 Subject: [PATCH 15/32] Adding ExpectNonEmptyPlan to TestTest_TestStep_ProviderFactories_ExpectWarningRefresh (#16) --- helper/resource/teststep_providers_test.go | 100 ++++++++++++--------- 1 file changed, 60 insertions(+), 40 deletions(-) diff --git a/helper/resource/teststep_providers_test.go b/helper/resource/teststep_providers_test.go index d371e47a9..9d122615e 100644 --- a/helper/resource/teststep_providers_test.go +++ b/helper/resource/teststep_providers_test.go @@ -2382,52 +2382,72 @@ func TestTest_TestStep_ProviderFactories_ExpectWarningRefresh(t *testing.T) { t.Parallel() Test(t, TestCase{ - ProviderFactories: map[string]func() (*schema.Provider, error){ - "random": func() (*schema.Provider, error) { //nolint:unparam // required signature - return &schema.Provider{ - ResourcesMap: map[string]*schema.Resource{ - "random_password": { - CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics { - d.SetId("id") - return nil - }, - DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { - return nil - }, - ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) (diags diag.Diagnostics) { - // Only generate diag when ReadContext is called for the second - // time during terraform refresh. - if d.Get("id") == "new_id" { - return append(diags, diag.Diagnostic{ - Severity: diag.Warning, - Summary: "warning diagnostic - summary", - }) - } - - // Update id when ReadContext is called following resource creation - // during terraform apply. - d.SetId("new_id") - - return - }, - Schema: map[string]*schema.Schema{ - "id": { - Computed: true, - Type: schema.TypeString, + Steps: []TestStep{ + { + ProviderFactories: map[string]func() (*schema.Provider, error){ + "random": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "random_password": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics { + d.SetId("id") + return nil + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) (diags diag.Diagnostics) { + return nil + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, }, }, - }, + }, nil }, - }, nil - }, - }, - Steps: []TestStep{ - { + }, Config: `resource "random_password" "test" { }`, }, { - RefreshState: true, - ExpectWarning: regexp.MustCompile(`.*warning diagnostic - summary`), + ProviderFactories: map[string]func() (*schema.Provider, error){ + "random": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "random_password": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics { + d.SetId("id") + return nil + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) (diags diag.Diagnostics) { + return append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning diagnostic - summary", + }) + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, + }, + }, + }, nil + }, + }, + RefreshState: true, + // ExpectNonEmptyPlan is set to true otherwise following error is generated: + // # random_password.test will be destroyed + // # (because random_password.test is not in configuration) + ExpectNonEmptyPlan: true, + ExpectWarning: regexp.MustCompile(`.*warning diagnostic - summary`), }, }, }) From 3ccd60667756f3a80dc4d68a6091ecce14869279 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Wed, 18 Jan 2023 09:32:39 +0000 Subject: [PATCH 16/32] Removing unused method (#16) --- internal/plugintest/working_dir.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/plugintest/working_dir.go b/internal/plugintest/working_dir.go index e71c1c3f7..32178940e 100644 --- a/internal/plugintest/working_dir.go +++ b/internal/plugintest/working_dir.go @@ -48,10 +48,6 @@ type WorkingDir struct { reattachInfo tfexec.ReattachInfo } -func (wd *WorkingDir) GetBaseDir() string { - return wd.baseDir -} - // Close deletes the directories and files created to represent the receiving // working directory. After this method is called, the working directory object // is invalid and may no longer be used. From af3a9dd51ebe580b02a62ad19348bb139cb6882e Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Mon, 23 Jan 2023 10:56:59 +0000 Subject: [PATCH 17/32] Renaming struct from Stdout to TerraformJSONBuffer (#16) --- .changie.yaml | 2 +- go.mod | 2 +- go.sum | 2 + helper/resource/stdout.go | 112 ------------------ helper/resource/testing_new.go | 16 +-- helper/resource/testing_new_config.go | 34 +++--- helper/resource/testing_new_refresh_state.go | 2 +- internal/plugintest/terraform_json_buffer.go | 112 ++++++++++++++++++ .../plugintest/terraform_json_buffer_test.go | 28 ++--- 9 files changed, 156 insertions(+), 154 deletions(-) delete mode 100644 helper/resource/stdout.go create mode 100644 internal/plugintest/terraform_json_buffer.go rename helper/resource/stdout_test.go => internal/plugintest/terraform_json_buffer_test.go (79%) diff --git a/.changie.yaml b/.changie.yaml index c5eac1ecf..08921f33d 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -7,7 +7,7 @@ kindFormat: '{{.Kind}}:' changeFormat: '* {{.Body}} ([#{{.Custom.Issue}}](https://github.com/hashicorp/terraform-plugin-testing/issues/{{.Custom.Issue}}))' custom: - key: Issue - label: Issue Number + label: Issue/PR Number type: int minInt: 1 kinds: diff --git a/go.mod b/go.mod index 90e22da88..31dc514c9 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/hashicorp/hc-install v0.4.0 github.com/hashicorp/hcl/v2 v2.15.0 github.com/hashicorp/logutils v1.0.0 - github.com/hashicorp/terraform-exec v0.17.4-0.20230116095935-bc76870e2b9b + github.com/hashicorp/terraform-exec v0.17.4-0.20230120174504-9acd44a7e5da github.com/hashicorp/terraform-json v0.14.0 github.com/hashicorp/terraform-plugin-framework v1.0.1 github.com/hashicorp/terraform-plugin-go v0.14.3 diff --git a/go.sum b/go.sum index 8dd93d606..6df28a46d 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98 h1:WkS github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= github.com/hashicorp/terraform-exec v0.17.4-0.20230116095935-bc76870e2b9b h1:z4a1oo2M/yWj2ljEoXsFQRkmp2dRqvq4Y9HjQl3eBz8= github.com/hashicorp/terraform-exec v0.17.4-0.20230116095935-bc76870e2b9b/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= +github.com/hashicorp/terraform-exec v0.17.4-0.20230120174504-9acd44a7e5da h1:St2EWMkwfc56l9PqejkEV/r4S55Em9iXYvW2qdtyH+4= +github.com/hashicorp/terraform-exec v0.17.4-0.20230120174504-9acd44a7e5da/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s= github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= github.com/hashicorp/terraform-plugin-framework v1.0.1 h1:apX2jtaEKa15+do6H2izBJdl1dEH2w5BPVkDJ3Q3mKA= diff --git a/helper/resource/stdout.go b/helper/resource/stdout.go deleted file mode 100644 index b22d8ae9d..000000000 --- a/helper/resource/stdout.go +++ /dev/null @@ -1,112 +0,0 @@ -package resource - -import ( - "bufio" - "bytes" - "encoding/json" - "io" - "log" - "regexp" - "strings" - - tfjson "github.com/hashicorp/terraform-json" -) - -var _ io.Writer = &Stdout{} -var _ io.Reader = &Stdout{} - -type Stdout struct { - buf *bytes.Buffer -} - -func NewStdout() *Stdout { - return &Stdout{ - buf: new(bytes.Buffer), - } -} - -func (s *Stdout) Write(p []byte) (n int, err error) { - if s.buf == nil { - log.Fatal("call NewStdout to initialise buffer") - } - - return s.buf.Write(p) -} - -func (s *Stdout) Read(p []byte) (n int, err error) { - if s.buf == nil { - log.Fatal("call NewStdout to initialise buffer") - } - - return s.buf.Read(p) -} - -func (s *Stdout) GetJSONOutput() []string { - if s.buf == nil { - log.Fatal("call NewStdout to initialise buffer") - } - - scanner := bufio.NewScanner(s.buf) - var jsonOutput []string - - for scanner.Scan() { - var outer struct{} - - txt := scanner.Text() - - if json.Unmarshal([]byte(txt), &outer) == nil { - jsonOutput = append(jsonOutput, txt) - } - } - - if err := scanner.Err(); err != nil { - log.Fatal(err) - } - - return jsonOutput -} - -func (s *Stdout) GetJSONOutputStr() string { - return strings.Join(s.GetJSONOutput(), "\n") -} - -func (s *Stdout) DiagnosticFound(r *regexp.Regexp, severity tfjson.DiagnosticSeverity) (bool, []string) { - if s.buf == nil { - log.Fatal("call NewStdout to initialise buffer") - } - - scanner := bufio.NewScanner(s.buf) - var jsonOutput []string - - for scanner.Scan() { - var outer struct { - Diagnostic tfjson.Diagnostic - } - - txt := scanner.Text() - - if json.Unmarshal([]byte(txt), &outer) == nil { - jsonOutput = append(jsonOutput, txt) - - if outer.Diagnostic.Severity == "" { - continue - } - - if !r.MatchString(outer.Diagnostic.Summary) && !r.MatchString(outer.Diagnostic.Detail) { - continue - } - - if outer.Diagnostic.Severity != severity { - continue - } - - return true, jsonOutput - } - } - - if err := scanner.Err(); err != nil { - log.Fatal(err) - } - - return false, jsonOutput -} diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 0ad2f0beb..6cd5ed078 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -120,8 +120,8 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // acts as default for import tests var appliedCfg string - // stdout (io.Writer) is supplied to all terraform commands that are using streaming json output. - stdout := NewStdout() + // terraformJSONBuffer (io.Writer) is supplied to all terraform commands that are using streaming json output. + terraformJSONBuffer := plugintest.NewTerraformJSONBuffer() for stepIndex, step := range c.Steps { stepNumber := stepIndex + 1 // 1-based indexing for humans @@ -249,7 +249,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.RefreshState { logging.HelperResourceTrace(ctx, "TestStep is RefreshState mode") - err := testStepNewRefreshState(ctx, t, wd, step, providers, stdout) + err := testStepNewRefreshState(ctx, t, wd, step, providers, terraformJSONBuffer) if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -262,7 +262,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } errorFound := step.ExpectError.MatchString(err.Error()) - jsonErrorFound, jsonDiags := stdout.DiagnosticFound(step.ExpectError, tfjson.DiagnosticSeverityError) + jsonErrorFound, jsonDiags := terraformJSONBuffer.DiagnosticFound(step.ExpectError, tfjson.DiagnosticSeverityError) errorOutput := []string{err.Error()} @@ -295,7 +295,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - warningFound, jsonDiags := stdout.DiagnosticFound(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + warningFound, jsonDiags := terraformJSONBuffer.DiagnosticFound(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { logging.HelperResourceError(ctx, @@ -314,7 +314,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.Config != "" { logging.HelperResourceTrace(ctx, "TestStep is Config mode") - err := testStepNewConfig(ctx, t, c, wd, step, providers, stdout) + err := testStepNewConfig(ctx, t, c, wd, step, providers, terraformJSONBuffer) if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -327,7 +327,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } errorFound := step.ExpectError.MatchString(err.Error()) - jsonErrorFound, jsonDiags := stdout.DiagnosticFound(step.ExpectError, tfjson.DiagnosticSeverityError) + jsonErrorFound, jsonDiags := terraformJSONBuffer.DiagnosticFound(step.ExpectError, tfjson.DiagnosticSeverityError) errorOutput := []string{err.Error()} @@ -362,7 +362,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - warningFound, jsonDiags := stdout.DiagnosticFound(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + warningFound, jsonDiags := terraformJSONBuffer.DiagnosticFound(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { logging.HelperResourceError(ctx, diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index 21058dcfd..1f501d4c9 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -18,7 +18,7 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, stdout *Stdout) error { +func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, terraformJSONBuffer *plugintest.TerraformJSONBuffer) error { t.Helper() err := wd.SetConfig(ctx, step.mergedConfig(ctx, c)) @@ -29,7 +29,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // require a refresh before applying // failing to do this will result in data sources not being updated err = runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, stdout) + return wd.RefreshJSON(ctx, terraformJSONBuffer) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -41,7 +41,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running pre-apply refresh: %w", err) } } else { - return fmt.Errorf("Error running pre-apply refresh: %s", stdout.GetJSONOutputStr()) + return fmt.Errorf("Error running pre-apply refresh: %s", terraformJSONBuffer.GetJSONOutputStr()) } } @@ -54,9 +54,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // Plan! err := runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, stdout) + return wd.CreateDestroyPlanJSON(ctx, terraformJSONBuffer) } - return wd.CreatePlanJSON(ctx, stdout) + return wd.CreatePlanJSON(ctx, terraformJSONBuffer) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -71,7 +71,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running pre-apply plan: %w", err) } } else { - return fmt.Errorf("Error running pre-apply plan: %s", stdout.GetJSONOutputStr()) + return fmt.Errorf("Error running pre-apply plan: %s", terraformJSONBuffer.GetJSONOutputStr()) } } @@ -92,7 +92,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // Apply the diff, creating real resources err = runProviderCommand(ctx, t, func() error { - return wd.ApplyJSON(ctx, stdout) + return wd.ApplyJSON(ctx, terraformJSONBuffer) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -108,9 +108,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint } } else { if step.Destroy { - return fmt.Errorf("Error running destroy: %s", stdout.GetJSONOutputStr()) + return fmt.Errorf("Error running destroy: %s", terraformJSONBuffer.GetJSONOutputStr()) } - return fmt.Errorf("Error running apply: %s", stdout.GetJSONOutputStr()) + return fmt.Errorf("Error running apply: %s", terraformJSONBuffer.GetJSONOutputStr()) } } @@ -150,9 +150,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do a plan err = runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, stdout) + return wd.CreateDestroyPlanJSON(ctx, terraformJSONBuffer) } - return wd.CreatePlanJSON(ctx, stdout) + return wd.CreatePlanJSON(ctx, terraformJSONBuffer) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -167,7 +167,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running post-apply plan: %w", err) } } else { - return fmt.Errorf("Error running post-apply plan: %s", stdout.GetJSONOutputStr()) + return fmt.Errorf("Error running post-apply plan: %s", terraformJSONBuffer.GetJSONOutputStr()) } } @@ -197,7 +197,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do a refresh if !step.Destroy || (step.Destroy && !step.PreventPostDestroyRefresh) { err := runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, stdout) + return wd.RefreshJSON(ctx, terraformJSONBuffer) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -209,7 +209,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running post-apply refresh: %w", err) } } else { - return fmt.Errorf("Error running post-apply refresh: %s", stdout.GetJSONOutputStr()) + return fmt.Errorf("Error running post-apply refresh: %s", terraformJSONBuffer.GetJSONOutputStr()) } } } @@ -217,9 +217,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do another plan err = runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, stdout) + return wd.CreateDestroyPlanJSON(ctx, terraformJSONBuffer) } - return wd.CreatePlanJSON(ctx, stdout) + return wd.CreatePlanJSON(ctx, terraformJSONBuffer) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -234,7 +234,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running second post-apply plan: %w", err) } } else { - return fmt.Errorf("Error running second post-apply plan: %s", stdout.GetJSONOutputStr()) + return fmt.Errorf("Error running second post-apply plan: %s", terraformJSONBuffer.GetJSONOutputStr()) } } diff --git a/helper/resource/testing_new_refresh_state.go b/helper/resource/testing_new_refresh_state.go index 19399612c..0bf341f2e 100644 --- a/helper/resource/testing_new_refresh_state.go +++ b/helper/resource/testing_new_refresh_state.go @@ -18,7 +18,7 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, stdout *Stdout) error { +func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, stdout *plugintest.TerraformJSONBuffer) error { t.Helper() var err error diff --git a/internal/plugintest/terraform_json_buffer.go b/internal/plugintest/terraform_json_buffer.go new file mode 100644 index 000000000..0a8b7b585 --- /dev/null +++ b/internal/plugintest/terraform_json_buffer.go @@ -0,0 +1,112 @@ +package plugintest + +import ( + "bufio" + "bytes" + "encoding/json" + "io" + "log" + "regexp" + "strings" + + tfjson "github.com/hashicorp/terraform-json" +) + +var _ io.Writer = &TerraformJSONBuffer{} +var _ io.Reader = &TerraformJSONBuffer{} + +type TerraformJSONBuffer struct { + buf *bytes.Buffer +} + +func NewTerraformJSONBuffer() *TerraformJSONBuffer { + return &TerraformJSONBuffer{ + buf: new(bytes.Buffer), + } +} + +func (b *TerraformJSONBuffer) Write(p []byte) (n int, err error) { + if b.buf == nil { + log.Fatal("call NewTerraformJSONBuffer to initialise buffer") + } + + return b.buf.Write(p) +} + +func (b *TerraformJSONBuffer) Read(p []byte) (n int, err error) { + if b.buf == nil { + log.Fatal("call NewTerraformJSONBuffer to initialise buffer") + } + + return b.buf.Read(p) +} + +func (b *TerraformJSONBuffer) GetJSONOutput() []string { + if b.buf == nil { + log.Fatal("call NewTerraformJSONBuffer to initialise buffer") + } + + scanner := bufio.NewScanner(b.buf) + var jsonOutput []string + + for scanner.Scan() { + var outer struct{} + + txt := scanner.Text() + + if json.Unmarshal([]byte(txt), &outer) == nil { + jsonOutput = append(jsonOutput, txt) + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + + return jsonOutput +} + +func (b *TerraformJSONBuffer) GetJSONOutputStr() string { + return strings.Join(b.GetJSONOutput(), "\n") +} + +func (b *TerraformJSONBuffer) DiagnosticFound(r *regexp.Regexp, severity tfjson.DiagnosticSeverity) (bool, []string) { + if b.buf == nil { + log.Fatal("call NewTerraformJSONBuffer to initialise buffer") + } + + scanner := bufio.NewScanner(b.buf) + var jsonOutput []string + + for scanner.Scan() { + var outer struct { + Diagnostic tfjson.Diagnostic + } + + txt := scanner.Text() + + if json.Unmarshal([]byte(txt), &outer) == nil { + jsonOutput = append(jsonOutput, txt) + + if outer.Diagnostic.Severity == "" { + continue + } + + if !r.MatchString(outer.Diagnostic.Summary) && !r.MatchString(outer.Diagnostic.Detail) { + continue + } + + if outer.Diagnostic.Severity != severity { + continue + } + + return true, jsonOutput + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + + return false, jsonOutput +} diff --git a/helper/resource/stdout_test.go b/internal/plugintest/terraform_json_buffer_test.go similarity index 79% rename from helper/resource/stdout_test.go rename to internal/plugintest/terraform_json_buffer_test.go index 256380af4..53750bfed 100644 --- a/helper/resource/stdout_test.go +++ b/internal/plugintest/terraform_json_buffer_test.go @@ -1,4 +1,4 @@ -package resource +package plugintest import ( "regexp" @@ -9,7 +9,7 @@ import ( tfjson "github.com/hashicorp/terraform-json" ) -var stdoutJSON = []string{ +var terraformJSONOutput = []string{ `{"@level":"info","@message":"Terraform 1.3.2","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.232751Z","terraform":"1.3.2","type":"version","ui":"1.0"}`, `{"@level":"warn","@message":"Warning: Empty or non-existent state","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.250725Z","diagnostic":{"severity":"warning","summary":"Empty or non-existent state","detail":"There are currently no remote objects tracked in the state, so there is nothing to refresh."},"type":"diagnostic"}`, `{"@level":"info","@message":"Outputs: 0","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.251120Z","outputs":{},"type":"outputs"}`, @@ -22,15 +22,15 @@ var stdoutJSON = []string{ func TestStdout_GetJSONOutputStr(t *testing.T) { t.Parallel() - stdout := NewStdout() + terraformJSONBuffer := NewTerraformJSONBuffer() - for _, v := range stdoutJSON { - stdout.Write([]byte(v + "\n")) + for _, v := range terraformJSONOutput { + terraformJSONBuffer.Write([]byte(v + "\n")) } - jsonOutputStr := stdout.GetJSONOutputStr() + jsonOutputStr := terraformJSONBuffer.GetJSONOutputStr() - if diff := cmp.Diff(jsonOutputStr, strings.Join(stdoutJSON, "\n")); diff != "" { + if diff := cmp.Diff(jsonOutputStr, strings.Join(terraformJSONOutput, "\n")); diff != "" { t.Errorf("unexpected difference: %s", diff) } } @@ -48,13 +48,13 @@ func TestStdout_DiagnosticFound(t *testing.T) { regex: regexp.MustCompile(`.*error diagnostic - summary`), severity: tfjson.DiagnosticSeverityError, expected: true, - expectedJSON: stdoutJSON, + expectedJSON: terraformJSONOutput, }, "not-found": { regex: regexp.MustCompile(`.*warning diagnostic - summary`), severity: tfjson.DiagnosticSeverityError, expected: false, - expectedJSON: stdoutJSON, + expectedJSON: terraformJSONOutput, }, } @@ -62,16 +62,16 @@ func TestStdout_DiagnosticFound(t *testing.T) { name, testCase := name, testCase t.Run(name, func(t *testing.T) { - stdout := NewStdout() + terraformJSONBuffer := NewTerraformJSONBuffer() - for _, v := range stdoutJSON { - _, err := stdout.Write([]byte(v + "\n")) + for _, v := range terraformJSONOutput { + _, err := terraformJSONBuffer.Write([]byte(v + "\n")) if err != nil { - t.Errorf("error writing to stdout: %s", err) + t.Errorf("error writing to terraformJSONBuffer: %s", err) } } - isFound, output := stdout.DiagnosticFound(testCase.regex, testCase.severity) + isFound, output := terraformJSONBuffer.DiagnosticFound(testCase.regex, testCase.severity) if diff := cmp.Diff(isFound, testCase.expected); diff != "" { t.Errorf("unexpected difference: %s", diff) From 74ea0f6145707e22832b216c6ce444a493247dd6 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Mon, 23 Jan 2023 14:26:00 +0000 Subject: [PATCH 18/32] Adding Parse() function to TerraformJSONBuffer and separating out tfjson.Diagnostic into a separate TerraformJSONDiagnostics type (#16) --- helper/resource/testing_new.go | 26 +-- helper/resource/testing_new_config.go | 35 ++-- helper/resource/testing_new_refresh_state.go | 11 +- internal/plugintest/terraform_json_buffer.go | 89 +++++---- .../plugintest/terraform_json_buffer_test.go | 178 +++++++++++++----- 5 files changed, 216 insertions(+), 123 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 6cd5ed078..3b17a3652 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -120,8 +120,8 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // acts as default for import tests var appliedCfg string - // terraformJSONBuffer (io.Writer) is supplied to all terraform commands that are using streaming json output. - terraformJSONBuffer := plugintest.NewTerraformJSONBuffer() + // tfJSON (io.Writer) is supplied to all terraform commands that are using streaming json output. + tfJSON := plugintest.NewTerraformJSONBuffer() for stepIndex, step := range c.Steps { stepNumber := stepIndex + 1 // 1-based indexing for humans @@ -249,7 +249,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.RefreshState { logging.HelperResourceTrace(ctx, "TestStep is RefreshState mode") - err := testStepNewRefreshState(ctx, t, wd, step, providers, terraformJSONBuffer) + err := testStepNewRefreshState(ctx, t, wd, step, providers, tfJSON) if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -262,12 +262,12 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } errorFound := step.ExpectError.MatchString(err.Error()) - jsonErrorFound, jsonDiags := terraformJSONBuffer.DiagnosticFound(step.ExpectError, tfjson.DiagnosticSeverityError) + jsonErrorFound := tfJSON.Diagnostics().Contains(step.ExpectError, tfjson.DiagnosticSeverityError) errorOutput := []string{err.Error()} if jsonErrorFound { - errorOutput = jsonDiags + errorOutput = tfJSON.JsonOutput() } if !errorFound && !jsonErrorFound { @@ -295,14 +295,14 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - warningFound, jsonDiags := terraformJSONBuffer.DiagnosticFound(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + warningFound := tfJSON.Diagnostics().Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { logging.HelperResourceError(ctx, fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), map[string]interface{}{logging.KeyError: err}, ) - t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(jsonDiags, "\n")) + t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(tfJSON.JsonOutput(), "\n")) } } @@ -314,7 +314,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.Config != "" { logging.HelperResourceTrace(ctx, "TestStep is Config mode") - err := testStepNewConfig(ctx, t, c, wd, step, providers, terraformJSONBuffer) + err := testStepNewConfig(ctx, t, c, wd, step, providers, tfJSON) if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -327,12 +327,12 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } errorFound := step.ExpectError.MatchString(err.Error()) - jsonErrorFound, jsonDiags := terraformJSONBuffer.DiagnosticFound(step.ExpectError, tfjson.DiagnosticSeverityError) + jsonErrorFound := tfJSON.Diagnostics().Contains(step.ExpectError, tfjson.DiagnosticSeverityError) errorOutput := []string{err.Error()} if jsonErrorFound { - errorOutput = jsonDiags + errorOutput = tfJSON.JsonOutput() } if !errorFound && !jsonErrorFound { @@ -340,7 +340,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest fmt.Sprintf("Expected an error with pattern (%s)", step.ExpectError.String()), map[string]interface{}{logging.KeyError: strings.Join(errorOutput, "")}, ) - t.Fatalf("Step %d/%d, expected an error matching pattern, no match on: %s", stepNumber, len(c.Steps), jsonDiags) + t.Fatalf("Step %d/%d, expected an error matching pattern, no match on: %s", stepNumber, len(c.Steps), errorOutput) } } else { if err != nil && c.ErrorCheck != nil { @@ -362,14 +362,14 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - warningFound, jsonDiags := terraformJSONBuffer.DiagnosticFound(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + warningFound := tfJSON.Diagnostics().Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { logging.HelperResourceError(ctx, fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), map[string]interface{}{logging.KeyError: err}, ) - t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(jsonDiags, "\n")) + t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(tfJSON.JsonOutput(), "\n")) } } diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index 1f501d4c9..2f0460707 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "strings" "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" @@ -18,7 +19,7 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, terraformJSONBuffer *plugintest.TerraformJSONBuffer) error { +func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, tfJSON *plugintest.TerraformJSONBuffer) error { t.Helper() err := wd.SetConfig(ctx, step.mergedConfig(ctx, c)) @@ -29,7 +30,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // require a refresh before applying // failing to do this will result in data sources not being updated err = runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, terraformJSONBuffer) + return wd.RefreshJSON(ctx, tfJSON) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -41,7 +42,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running pre-apply refresh: %w", err) } } else { - return fmt.Errorf("Error running pre-apply refresh: %s", terraformJSONBuffer.GetJSONOutputStr()) + return fmt.Errorf("Error running pre-apply refresh: %s", strings.Join(tfJSON.JsonOutput(), "\n")) } } @@ -54,9 +55,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // Plan! err := runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, terraformJSONBuffer) + return wd.CreateDestroyPlanJSON(ctx, tfJSON) } - return wd.CreatePlanJSON(ctx, terraformJSONBuffer) + return wd.CreatePlanJSON(ctx, tfJSON) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -71,7 +72,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running pre-apply plan: %w", err) } } else { - return fmt.Errorf("Error running pre-apply plan: %s", terraformJSONBuffer.GetJSONOutputStr()) + return fmt.Errorf("Error running pre-apply plan: %s", strings.Join(tfJSON.JsonOutput(), "\n")) } } @@ -92,7 +93,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // Apply the diff, creating real resources err = runProviderCommand(ctx, t, func() error { - return wd.ApplyJSON(ctx, terraformJSONBuffer) + return wd.ApplyJSON(ctx, tfJSON) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -108,9 +109,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint } } else { if step.Destroy { - return fmt.Errorf("Error running destroy: %s", terraformJSONBuffer.GetJSONOutputStr()) + return fmt.Errorf("Error running destroy: %s", strings.Join(tfJSON.JsonOutput(), "\n")) } - return fmt.Errorf("Error running apply: %s", terraformJSONBuffer.GetJSONOutputStr()) + return fmt.Errorf("Error running apply: %s", strings.Join(tfJSON.JsonOutput(), "\n")) } } @@ -150,9 +151,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do a plan err = runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, terraformJSONBuffer) + return wd.CreateDestroyPlanJSON(ctx, tfJSON) } - return wd.CreatePlanJSON(ctx, terraformJSONBuffer) + return wd.CreatePlanJSON(ctx, tfJSON) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -167,7 +168,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running post-apply plan: %w", err) } } else { - return fmt.Errorf("Error running post-apply plan: %s", terraformJSONBuffer.GetJSONOutputStr()) + return fmt.Errorf("Error running post-apply plan: %s", strings.Join(tfJSON.JsonOutput(), "\n")) } } @@ -197,7 +198,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do a refresh if !step.Destroy || (step.Destroy && !step.PreventPostDestroyRefresh) { err := runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, terraformJSONBuffer) + return wd.RefreshJSON(ctx, tfJSON) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -209,7 +210,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running post-apply refresh: %w", err) } } else { - return fmt.Errorf("Error running post-apply refresh: %s", terraformJSONBuffer.GetJSONOutputStr()) + return fmt.Errorf("Error running post-apply refresh: %s", strings.Join(tfJSON.JsonOutput(), "\n")) } } } @@ -217,9 +218,9 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do another plan err = runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, terraformJSONBuffer) + return wd.CreateDestroyPlanJSON(ctx, tfJSON) } - return wd.CreatePlanJSON(ctx, terraformJSONBuffer) + return wd.CreatePlanJSON(ctx, tfJSON) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -234,7 +235,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return fmt.Errorf("Error running second post-apply plan: %w", err) } } else { - return fmt.Errorf("Error running second post-apply plan: %s", terraformJSONBuffer.GetJSONOutputStr()) + return fmt.Errorf("Error running second post-apply plan: %s", strings.Join(tfJSON.JsonOutput(), "\n")) } } diff --git a/helper/resource/testing_new_refresh_state.go b/helper/resource/testing_new_refresh_state.go index 0bf341f2e..9dc61410f 100644 --- a/helper/resource/testing_new_refresh_state.go +++ b/helper/resource/testing_new_refresh_state.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "strings" "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" @@ -18,7 +19,7 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, stdout *plugintest.TerraformJSONBuffer) error { +func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, tfJSON *plugintest.TerraformJSONBuffer) error { t.Helper() var err error @@ -35,7 +36,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo } err = runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, stdout) + return wd.RefreshJSON(ctx, tfJSON) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -47,7 +48,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return fmt.Errorf("Error running refresh: %w", err) } } else { - return fmt.Errorf("Error running refresh: %s", stdout.GetJSONOutputStr()) + return fmt.Errorf("Error running refresh: %s", strings.Join(tfJSON.JsonOutput(), "\n")) } } @@ -76,7 +77,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo // do a plan err = runProviderCommand(ctx, t, func() error { - return wd.CreatePlanJSON(ctx, stdout) + return wd.CreatePlanJSON(ctx, tfJSON) }, wd, providers) if err != nil { target := &tfexec.ErrVersionMismatch{} @@ -88,7 +89,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return fmt.Errorf("Error running post-apply plan: %w", err) } } else { - return fmt.Errorf("Error running post-apply plan: %s", stdout.GetJSONOutputStr()) + return fmt.Errorf("Error running post-apply plan: %s", strings.Join(tfJSON.JsonOutput(), "\n")) } } diff --git a/internal/plugintest/terraform_json_buffer.go b/internal/plugintest/terraform_json_buffer.go index 0a8b7b585..cb978feb7 100644 --- a/internal/plugintest/terraform_json_buffer.go +++ b/internal/plugintest/terraform_json_buffer.go @@ -7,7 +7,6 @@ import ( "io" "log" "regexp" - "strings" tfjson "github.com/hashicorp/terraform-json" ) @@ -15,8 +14,29 @@ import ( var _ io.Writer = &TerraformJSONBuffer{} var _ io.Reader = &TerraformJSONBuffer{} +type TerraformJSONDiagnostics []tfjson.Diagnostic + +func (d TerraformJSONDiagnostics) Contains(r *regexp.Regexp, severity tfjson.DiagnosticSeverity) bool { + for _, v := range d { + if v.Severity != severity { + continue + } + + if !r.MatchString(v.Summary) && !r.MatchString(v.Detail) { + continue + } + + return true + } + + return false +} + type TerraformJSONBuffer struct { - buf *bytes.Buffer + buf *bytes.Buffer + diagnostics TerraformJSONDiagnostics + jsonOutput []string + parsed bool } func NewTerraformJSONBuffer() *TerraformJSONBuffer { @@ -41,21 +61,29 @@ func (b *TerraformJSONBuffer) Read(p []byte) (n int, err error) { return b.buf.Read(p) } -func (b *TerraformJSONBuffer) GetJSONOutput() []string { +func (b *TerraformJSONBuffer) Parse() { if b.buf == nil { log.Fatal("call NewTerraformJSONBuffer to initialise buffer") } scanner := bufio.NewScanner(b.buf) - var jsonOutput []string for scanner.Scan() { - var outer struct{} + var outer struct { + Diagnostic tfjson.Diagnostic + } txt := scanner.Text() + // This will only capture buffer entries that can be unmarshalled + // as JSON. If there are entries in the buffer that are not JSON + // they will be discarded. if json.Unmarshal([]byte(txt), &outer) == nil { - jsonOutput = append(jsonOutput, txt) + b.jsonOutput = append(b.jsonOutput, txt) + + if outer.Diagnostic.Severity != "" { + b.diagnostics = append(b.diagnostics, outer.Diagnostic) + } } } @@ -63,50 +91,21 @@ func (b *TerraformJSONBuffer) GetJSONOutput() []string { log.Fatal(err) } - return jsonOutput -} - -func (b *TerraformJSONBuffer) GetJSONOutputStr() string { - return strings.Join(b.GetJSONOutput(), "\n") + b.parsed = true } -func (b *TerraformJSONBuffer) DiagnosticFound(r *regexp.Regexp, severity tfjson.DiagnosticSeverity) (bool, []string) { - if b.buf == nil { - log.Fatal("call NewTerraformJSONBuffer to initialise buffer") +func (b *TerraformJSONBuffer) Diagnostics() TerraformJSONDiagnostics { + if !b.parsed { + b.Parse() } - scanner := bufio.NewScanner(b.buf) - var jsonOutput []string - - for scanner.Scan() { - var outer struct { - Diagnostic tfjson.Diagnostic - } - - txt := scanner.Text() - - if json.Unmarshal([]byte(txt), &outer) == nil { - jsonOutput = append(jsonOutput, txt) - - if outer.Diagnostic.Severity == "" { - continue - } - - if !r.MatchString(outer.Diagnostic.Summary) && !r.MatchString(outer.Diagnostic.Detail) { - continue - } - - if outer.Diagnostic.Severity != severity { - continue - } - - return true, jsonOutput - } - } + return b.diagnostics +} - if err := scanner.Err(); err != nil { - log.Fatal(err) +func (b *TerraformJSONBuffer) JsonOutput() []string { + if !b.parsed { + b.Parse() } - return false, jsonOutput + return b.jsonOutput } diff --git a/internal/plugintest/terraform_json_buffer_test.go b/internal/plugintest/terraform_json_buffer_test.go index 53750bfed..134a623c5 100644 --- a/internal/plugintest/terraform_json_buffer_test.go +++ b/internal/plugintest/terraform_json_buffer_test.go @@ -2,7 +2,6 @@ package plugintest import ( "regexp" - "strings" "testing" "github.com/google/go-cmp/cmp" @@ -15,71 +14,164 @@ var terraformJSONOutput = []string{ `{"@level":"info","@message":"Outputs: 0","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.251120Z","outputs":{},"type":"outputs"}`, `{"@level":"info","@message":"random_password.test: Plan to create","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.282021Z","change":{"resource":{"addr":"random_password.test","module":"","resource":"random_password.test","implied_provider":"random","resource_type":"random_password","resource_name":"test","resource_key":null},"action":"create"},"type":"planned_change"}`, `{"@level":"info","@message":"Plan: 1 to add, 0 to change, 0 to destroy.","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.282054Z","changes":{"add":1,"change":0,"remove":0,"operation":"plan"},"type":"change_summary"}`, - `{"format_version":"1.0"}`, `{"@level":"error","@message":"Error: error diagnostic - summary","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.626572Z","diagnostic":{"severity":"error","summary":"error diagnostic - summary","detail":""},"type":"diagnostic"}`, } -func TestStdout_GetJSONOutputStr(t *testing.T) { +func TestTerraformJSONDiagnostics_Contains(t *testing.T) { t.Parallel() - terraformJSONBuffer := NewTerraformJSONBuffer() + testCases := map[string]struct { + diags []tfjson.Diagnostic + regex *regexp.Regexp + severity tfjson.DiagnosticSeverity + expected bool + }{ + "severity-not-found": { + diags: []tfjson.Diagnostic{ + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error summary", + Detail: "error detail", + }, + }, + regex: regexp.MustCompile("error summary"), + severity: tfjson.DiagnosticSeverityWarning, + expected: false, + }, + "summary-not-found": { + diags: []tfjson.Diagnostic{ + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "warning summary", + Detail: "warning detail", + }, + }, + regex: regexp.MustCompile("error detail"), + severity: tfjson.DiagnosticSeverityError, + expected: false, + }, + "summary-found": { + diags: []tfjson.Diagnostic{ + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error summary", + Detail: "error detail", + }, + }, + regex: regexp.MustCompile("error summary"), + severity: tfjson.DiagnosticSeverityError, + expected: true, + }, + "detail-found": { + diags: []tfjson.Diagnostic{ + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error summary", + Detail: "error detail", + }, + }, + regex: regexp.MustCompile("error detail"), + severity: tfjson.DiagnosticSeverityError, + expected: true, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + var tfJSONDiagnostics TerraformJSONDiagnostics = testCase.diags + + isFound := tfJSONDiagnostics.Contains(testCase.regex, testCase.severity) + + if diff := cmp.Diff(isFound, testCase.expected); diff != "" { + t.Errorf("unexpected difference: %s", diff) + } + }) + } +} + +func TestTerraformJSONBuffer_Parse(t *testing.T) { + t.Parallel() + + tfJSON := NewTerraformJSONBuffer() for _, v := range terraformJSONOutput { - terraformJSONBuffer.Write([]byte(v + "\n")) + _, err := tfJSON.Write([]byte(v + "\n")) + if err != nil { + t.Fatalf("cannot write to tfJSON: %s", err) + } } - jsonOutputStr := terraformJSONBuffer.GetJSONOutputStr() + tfJSON.Parse() + + if diff := cmp.Diff(tfJSON.jsonOutput, terraformJSONOutput); diff != "" { + t.Errorf("unexpected difference: %s", diff) + } + + var tfJSONDiagnostics TerraformJSONDiagnostics = []tfjson.Diagnostic{ + { + Severity: tfjson.DiagnosticSeverityWarning, + Summary: "Empty or non-existent state", + Detail: "There are currently no remote objects tracked in the state, so there is nothing to refresh.", + }, + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error diagnostic - summary", + }, + } - if diff := cmp.Diff(jsonOutputStr, strings.Join(terraformJSONOutput, "\n")); diff != "" { + if diff := cmp.Diff(tfJSON.diagnostics, tfJSONDiagnostics); diff != "" { t.Errorf("unexpected difference: %s", diff) } } -func TestStdout_DiagnosticFound(t *testing.T) { +func TestTerraformJSONBuffer_Diagnostics(t *testing.T) { t.Parallel() - testCases := map[string]struct { - regex *regexp.Regexp - severity tfjson.DiagnosticSeverity - expected bool - expectedJSON []string - }{ - "found": { - regex: regexp.MustCompile(`.*error diagnostic - summary`), - severity: tfjson.DiagnosticSeverityError, - expected: true, - expectedJSON: terraformJSONOutput, + tfJSON := NewTerraformJSONBuffer() + + for _, v := range terraformJSONOutput { + _, err := tfJSON.Write([]byte(v + "\n")) + if err != nil { + t.Fatalf("cannot write to tfJSON: %s", err) + } + } + + tfJSON.Parse() + + var tfJSONDiagnostics TerraformJSONDiagnostics = []tfjson.Diagnostic{ + { + Severity: tfjson.DiagnosticSeverityWarning, + Summary: "Empty or non-existent state", + Detail: "There are currently no remote objects tracked in the state, so there is nothing to refresh.", }, - "not-found": { - regex: regexp.MustCompile(`.*warning diagnostic - summary`), - severity: tfjson.DiagnosticSeverityError, - expected: false, - expectedJSON: terraformJSONOutput, + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error diagnostic - summary", }, } - for name, testCase := range testCases { - name, testCase := name, testCase - - t.Run(name, func(t *testing.T) { - terraformJSONBuffer := NewTerraformJSONBuffer() + if diff := cmp.Diff(tfJSON.Diagnostics(), tfJSONDiagnostics); diff != "" { + t.Errorf("unexpected difference: %s", diff) + } +} - for _, v := range terraformJSONOutput { - _, err := terraformJSONBuffer.Write([]byte(v + "\n")) - if err != nil { - t.Errorf("error writing to terraformJSONBuffer: %s", err) - } - } +func TestTerraformJSONBuffer_JsonOutput(t *testing.T) { + t.Parallel() - isFound, output := terraformJSONBuffer.DiagnosticFound(testCase.regex, testCase.severity) + tfJSON := NewTerraformJSONBuffer() - if diff := cmp.Diff(isFound, testCase.expected); diff != "" { - t.Errorf("unexpected difference: %s", diff) - } - if diff := cmp.Diff(output, testCase.expectedJSON); diff != "" { - t.Errorf("unexpected difference: %s", diff) - } - }) + for _, v := range terraformJSONOutput { + _, err := tfJSON.Write([]byte(v + "\n")) + if err != nil { + t.Fatalf("cannot write to tfJSON: %s", err) + } } + tfJSON.Parse() + + if diff := cmp.Diff(tfJSON.JsonOutput(), terraformJSONOutput); diff != "" { + t.Errorf("unexpected difference: %s", diff) + } } From e93c329abf1dfb65f9fe22fd887745193d25d438 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Mon, 23 Jan 2023 16:20:11 +0000 Subject: [PATCH 19/32] Using file for test fixture (#16) --- .../plugintest/terraform_json_buffer_test.go | 80 ++++++++++++++----- internal/testdata/terraform-json-output.txt | 6 ++ 2 files changed, 66 insertions(+), 20 deletions(-) create mode 100644 internal/testdata/terraform-json-output.txt diff --git a/internal/plugintest/terraform_json_buffer_test.go b/internal/plugintest/terraform_json_buffer_test.go index 134a623c5..1cd6fab1f 100644 --- a/internal/plugintest/terraform_json_buffer_test.go +++ b/internal/plugintest/terraform_json_buffer_test.go @@ -1,6 +1,8 @@ package plugintest import ( + "bufio" + "os" "regexp" "testing" @@ -8,15 +10,6 @@ import ( tfjson "github.com/hashicorp/terraform-json" ) -var terraformJSONOutput = []string{ - `{"@level":"info","@message":"Terraform 1.3.2","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.232751Z","terraform":"1.3.2","type":"version","ui":"1.0"}`, - `{"@level":"warn","@message":"Warning: Empty or non-existent state","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.250725Z","diagnostic":{"severity":"warning","summary":"Empty or non-existent state","detail":"There are currently no remote objects tracked in the state, so there is nothing to refresh."},"type":"diagnostic"}`, - `{"@level":"info","@message":"Outputs: 0","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.251120Z","outputs":{},"type":"outputs"}`, - `{"@level":"info","@message":"random_password.test: Plan to create","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.282021Z","change":{"resource":{"addr":"random_password.test","module":"","resource":"random_password.test","implied_provider":"random","resource_type":"random_password","resource_name":"test","resource_key":null},"action":"create"},"type":"planned_change"}`, - `{"@level":"info","@message":"Plan: 1 to add, 0 to change, 0 to destroy.","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.282054Z","changes":{"add":1,"change":0,"remove":0,"operation":"plan"},"type":"change_summary"}`, - `{"@level":"error","@message":"Error: error diagnostic - summary","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.626572Z","diagnostic":{"severity":"error","summary":"error diagnostic - summary","detail":""},"type":"diagnostic"}`, -} - func TestTerraformJSONDiagnostics_Contains(t *testing.T) { t.Parallel() @@ -96,16 +89,33 @@ func TestTerraformJSONBuffer_Parse(t *testing.T) { tfJSON := NewTerraformJSONBuffer() - for _, v := range terraformJSONOutput { - _, err := tfJSON.Write([]byte(v + "\n")) + file, err := os.Open("../testdata/terraform-json-output.txt") + if err != nil { + t.Errorf("cannot read file: %s", err) + } + defer file.Close() + + var fileEntries []string + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + txt := scanner.Text() + + _, err := tfJSON.Write([]byte(txt + "\n")) if err != nil { - t.Fatalf("cannot write to tfJSON: %s", err) + t.Errorf("cannot write to tfJSON: %s", err) } + + fileEntries = append(fileEntries, txt) + } + + if err := scanner.Err(); err != nil { + t.Errorf("scanner error: %s", err) } tfJSON.Parse() - if diff := cmp.Diff(tfJSON.jsonOutput, terraformJSONOutput); diff != "" { + if diff := cmp.Diff(tfJSON.jsonOutput, fileEntries); diff != "" { t.Errorf("unexpected difference: %s", diff) } @@ -131,13 +141,26 @@ func TestTerraformJSONBuffer_Diagnostics(t *testing.T) { tfJSON := NewTerraformJSONBuffer() - for _, v := range terraformJSONOutput { - _, err := tfJSON.Write([]byte(v + "\n")) + file, err := os.Open("../testdata/terraform-json-output.txt") + if err != nil { + t.Errorf("cannot read file: %s", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + txt := scanner.Text() + + _, err := tfJSON.Write([]byte(txt + "\n")) if err != nil { - t.Fatalf("cannot write to tfJSON: %s", err) + t.Errorf("cannot write to tfJSON: %s", err) } } + if err := scanner.Err(); err != nil { + t.Errorf("scanner error: %s", err) + } + tfJSON.Parse() var tfJSONDiagnostics TerraformJSONDiagnostics = []tfjson.Diagnostic{ @@ -162,16 +185,33 @@ func TestTerraformJSONBuffer_JsonOutput(t *testing.T) { tfJSON := NewTerraformJSONBuffer() - for _, v := range terraformJSONOutput { - _, err := tfJSON.Write([]byte(v + "\n")) + file, err := os.Open("../testdata/terraform-json-output.txt") + if err != nil { + t.Errorf("cannot read file: %s", err) + } + defer file.Close() + + var fileEntries []string + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + txt := scanner.Text() + + _, err := tfJSON.Write([]byte(txt + "\n")) if err != nil { - t.Fatalf("cannot write to tfJSON: %s", err) + t.Errorf("cannot write to tfJSON: %s", err) } + + fileEntries = append(fileEntries, txt) + } + + if err := scanner.Err(); err != nil { + t.Errorf("scanner error: %s", err) } tfJSON.Parse() - if diff := cmp.Diff(tfJSON.JsonOutput(), terraformJSONOutput); diff != "" { + if diff := cmp.Diff(tfJSON.JsonOutput(), fileEntries); diff != "" { t.Errorf("unexpected difference: %s", diff) } } diff --git a/internal/testdata/terraform-json-output.txt b/internal/testdata/terraform-json-output.txt new file mode 100644 index 000000000..b8dd96bda --- /dev/null +++ b/internal/testdata/terraform-json-output.txt @@ -0,0 +1,6 @@ +{"@level":"info","@message":"Terraform 1.3.2","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.232751Z","terraform":"1.3.2","type":"version","ui":"1.0"} +{"@level":"warn","@message":"Warning: Empty or non-existent state","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.250725Z","diagnostic":{"severity":"warning","summary":"Empty or non-existent state","detail":"There are currently no remote objects tracked in the state, so there is nothing to refresh."},"type":"diagnostic"} +{"@level":"info","@message":"Outputs: 0","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.251120Z","outputs":{},"type":"outputs"} +{"@level":"info","@message":"random_password.test: Plan to create","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.282021Z","change":{"resource":{"addr":"random_password.test","module":"","resource":"random_password.test","implied_provider":"random","resource_type":"random_password","resource_name":"test","resource_key":null},"action":"create"},"type":"planned_change"} +{"@level":"info","@message":"Plan: 1 to add, 0 to change, 0 to destroy.","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.282054Z","changes":{"add":1,"change":0,"remove":0,"operation":"plan"},"type":"change_summary"} +{"@level":"error","@message":"Error: error diagnostic - summary","@module":"terraform.ui","@timestamp":"2023-01-16T17:02:14.626572Z","diagnostic":{"severity":"error","summary":"error diagnostic - summary","detail":""},"type":"diagnostic"} \ No newline at end of file From 32cb52770bda2b3552dfc72880fab8a551ae52ab Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Tue, 24 Jan 2023 11:40:47 +0000 Subject: [PATCH 20/32] Refactored to use command-specific responses and to instantiate instances of NewTerraformJSONBuffer within each command (#16) --- helper/resource/testing_new.go | 47 ++--- helper/resource/testing_new_config.go | 201 ++++++++----------- helper/resource/testing_new_refresh_state.go | 54 ++--- internal/plugintest/terraform_json_buffer.go | 24 ++- internal/plugintest/working_dir.go | 137 ++++++++++--- 5 files changed, 260 insertions(+), 203 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 3b17a3652..0ceaa4a1c 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -120,9 +120,6 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // acts as default for import tests var appliedCfg string - // tfJSON (io.Writer) is supplied to all terraform commands that are using streaming json output. - tfJSON := plugintest.NewTerraformJSONBuffer() - for stepIndex, step := range c.Steps { stepNumber := stepIndex + 1 // 1-based indexing for humans ctx = logging.TestStepNumberContext(ctx, stepNumber) @@ -249,7 +246,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.RefreshState { logging.HelperResourceTrace(ctx, "TestStep is RefreshState mode") - err := testStepNewRefreshState(ctx, t, wd, step, providers, tfJSON) + tfJSONDiags, stdout, err := testStepNewRefreshState(ctx, t, wd, step, providers) if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -262,22 +259,20 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } errorFound := step.ExpectError.MatchString(err.Error()) - jsonErrorFound := tfJSON.Diagnostics().Contains(step.ExpectError, tfjson.DiagnosticSeverityError) - - errorOutput := []string{err.Error()} + jsonErrorFound := tfJSONDiags.Contains(step.ExpectError, tfjson.DiagnosticSeverityError) - if jsonErrorFound { - errorOutput = tfJSON.JsonOutput() - } + errorOutput := []string{err.Error(), stdout} if !errorFound && !jsonErrorFound { logging.HelperResourceError(ctx, fmt.Sprintf("Error running refresh: expected an error with pattern (%s)", step.ExpectError.String()), map[string]interface{}{logging.KeyError: strings.Join(errorOutput, "")}, ) - t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), err) + t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), strings.Join(errorOutput, "\n")) } } else { + // TODO: ErrorCheck will be broken if errors that are being checked that would have + // previously been present in err are now in stdout. if err != nil && c.ErrorCheck != nil { logging.HelperResourceDebug(ctx, "Calling TestCase ErrorCheck") err = c.ErrorCheck(err) @@ -295,14 +290,16 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - warningFound := tfJSON.Diagnostics().Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + warningFound := tfJSONDiags.Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { logging.HelperResourceError(ctx, fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), map[string]interface{}{logging.KeyError: err}, ) - t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(tfJSON.JsonOutput(), "\n")) + // TODO: Using stdout will show all entries in the Terraform stdout whereas the warning will only + // be found if it's in the JSON output. + t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), stdout) } } @@ -314,7 +311,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.Config != "" { logging.HelperResourceTrace(ctx, "TestStep is Config mode") - err := testStepNewConfig(ctx, t, c, wd, step, providers, tfJSON) + tfJSONDiags, stdout, err := testStepNewConfig(ctx, t, c, wd, step, providers) if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -327,13 +324,9 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } errorFound := step.ExpectError.MatchString(err.Error()) - jsonErrorFound := tfJSON.Diagnostics().Contains(step.ExpectError, tfjson.DiagnosticSeverityError) + jsonErrorFound := tfJSONDiags.Contains(step.ExpectError, tfjson.DiagnosticSeverityError) - errorOutput := []string{err.Error()} - - if jsonErrorFound { - errorOutput = tfJSON.JsonOutput() - } + errorOutput := []string{err.Error(), stdout} if !errorFound && !jsonErrorFound { logging.HelperResourceError(ctx, @@ -343,6 +336,8 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest t.Fatalf("Step %d/%d, expected an error matching pattern, no match on: %s", stepNumber, len(c.Steps), errorOutput) } } else { + // TODO: ErrorCheck will be broken if errors that are being checked that would have + // previously been present in err are now in stdout. if err != nil && c.ErrorCheck != nil { logging.HelperResourceDebug(ctx, "Calling TestCase ErrorCheck") @@ -362,14 +357,16 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - warningFound := tfJSON.Diagnostics().Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + warningFound := tfJSONDiags.Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { logging.HelperResourceError(ctx, fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), map[string]interface{}{logging.KeyError: err}, ) - t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(tfJSON.JsonOutput(), "\n")) + // TODO: Using stdout will show all entries in the Terraform stdout whereas the warning will only + // be found if it's in the JSON output. + t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), stdout) } } @@ -437,7 +434,11 @@ func testIDRefresh(ctx context.Context, t testing.T, c TestCase, wd *plugintest. // Refresh! err = runProviderCommand(ctx, t, func() error { - err = wd.Refresh(ctx) + // TODO: Need to handle the possibility that the error no longer requires all + // of the required info for debugging in the case that terraform refresh with + // -json has been run and the information that was in the error is now in + // RefreshResponse.Stdout + _, err := wd.Refresh(ctx) if err != nil { t.Fatalf("Error running terraform refresh: %s", err) } diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index 2f0460707..2c8d73d5b 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -7,9 +7,7 @@ import ( "context" "errors" "fmt" - "strings" - "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" "github.com/mitchellh/go-testing-interface" @@ -19,31 +17,29 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, tfJSON *plugintest.TerraformJSONBuffer) error { +func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) (plugintest.TerraformJSONDiagnostics, string, error) { t.Helper() + var tfJSONDiags plugintest.TerraformJSONDiagnostics + var stdout string + err := wd.SetConfig(ctx, step.mergedConfig(ctx, c)) if err != nil { - return fmt.Errorf("Error setting config: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Error setting config: %w", err) } // require a refresh before applying // failing to do this will result in data sources not being updated err = runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, tfJSON) + refreshResponse, err := wd.Refresh(ctx) + + tfJSONDiags = append(tfJSONDiags, refreshResponse.Diagnostics...) + stdout += refreshResponse.Stdout + + return err }, wd, providers) if err != nil { - target := &tfexec.ErrVersionMismatch{} - if errors.As(err, &target) { - err = runProviderCommand(ctx, t, func() error { - return wd.Refresh(ctx) - }, wd, providers) - if err != nil { - return fmt.Errorf("Error running pre-apply refresh: %w", err) - } - } else { - return fmt.Errorf("Error running pre-apply refresh: %s", strings.Join(tfJSON.JsonOutput(), "\n")) - } + return tfJSONDiags, stdout, fmt.Errorf("Error running pre-apply refresh: %w", err) } // If this step is a PlanOnly step, skip over this first Plan and @@ -55,25 +51,22 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // Plan! err := runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, tfJSON) + createDestroyPlanResponse, err := wd.CreateDestroyPlan(ctx) + + tfJSONDiags = append(tfJSONDiags, createDestroyPlanResponse.Diagnostics...) + stdout += createDestroyPlanResponse.Stdout + + return err } - return wd.CreatePlanJSON(ctx, tfJSON) + createPlanResponse, err := wd.CreatePlan(ctx) + + tfJSONDiags = append(tfJSONDiags, createPlanResponse.Diagnostics...) + stdout += createPlanResponse.Stdout + + return err }, wd, providers) if err != nil { - target := &tfexec.ErrVersionMismatch{} - if errors.As(err, &target) { - err = runProviderCommand(ctx, t, func() error { - if step.Destroy { - return wd.CreateDestroyPlan(ctx) - } - return wd.CreatePlan(ctx) - }, wd, providers) - if err != nil { - return fmt.Errorf("Error running pre-apply plan: %w", err) - } - } else { - return fmt.Errorf("Error running pre-apply plan: %s", strings.Join(tfJSON.JsonOutput(), "\n")) - } + return tfJSONDiags, stdout, fmt.Errorf("Error running pre-apply plan: %w", err) } // We need to keep a copy of the state prior to destroying such @@ -88,31 +81,24 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return nil }, wd, providers) if err != nil { - return fmt.Errorf("Error retrieving pre-apply state: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Error retrieving pre-apply state: %w", err) } // Apply the diff, creating real resources err = runProviderCommand(ctx, t, func() error { - return wd.ApplyJSON(ctx, tfJSON) + applyResponse, err := wd.Apply(ctx) + + tfJSONDiags = append(tfJSONDiags, applyResponse.Diagnostics...) + stdout += applyResponse.Stdout + + return err }, wd, providers) if err != nil { - target := &tfexec.ErrVersionMismatch{} - if errors.As(err, &target) { - err = runProviderCommand(ctx, t, func() error { - return wd.Apply(ctx) - }, wd, providers) - if err != nil { - if step.Destroy { - return fmt.Errorf("Error running destroy: %w", err) - } - return fmt.Errorf("Error running apply: %w", err) - } - } else { - if step.Destroy { - return fmt.Errorf("Error running destroy: %s", strings.Join(tfJSON.JsonOutput(), "\n")) - } - return fmt.Errorf("Error running apply: %s", strings.Join(tfJSON.JsonOutput(), "\n")) + if step.Destroy { + return tfJSONDiags, stdout, fmt.Errorf("Error running destroy: %w", err) } + + return tfJSONDiags, stdout, fmt.Errorf("Error running apply: %w", err) } // Get the new state @@ -125,7 +111,7 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return nil }, wd, providers) if err != nil { - return fmt.Errorf("Error retrieving state after apply: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Error retrieving state after apply: %w", err) } // Run any configured checks @@ -135,11 +121,11 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint state.IsBinaryDrivenTest = true if step.Destroy { if err := step.Check(stateBeforeApplication); err != nil { - return fmt.Errorf("Check failed: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Check failed: %w", err) } } else { if err := step.Check(state); err != nil { - return fmt.Errorf("Check failed: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Check failed: %w", err) } } } @@ -151,25 +137,22 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // do a plan err = runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, tfJSON) + createDestroyPlanResponse, err := wd.CreateDestroyPlan(ctx) + + tfJSONDiags = append(tfJSONDiags, createDestroyPlanResponse.Diagnostics...) + stdout += createDestroyPlanResponse.Stdout + + return err } - return wd.CreatePlanJSON(ctx, tfJSON) + createPlanResponse, err := wd.CreatePlan(ctx) + + tfJSONDiags = append(tfJSONDiags, createPlanResponse.Diagnostics...) + stdout += createPlanResponse.Stdout + + return err }, wd, providers) if err != nil { - target := &tfexec.ErrVersionMismatch{} - if errors.As(err, &target) { - err = runProviderCommand(ctx, t, func() error { - if step.Destroy { - return wd.CreateDestroyPlan(ctx) - } - return wd.CreatePlan(ctx) - }, wd, providers) - if err != nil { - return fmt.Errorf("Error running post-apply plan: %w", err) - } - } else { - return fmt.Errorf("Error running post-apply plan: %s", strings.Join(tfJSON.JsonOutput(), "\n")) - } + return tfJSONDiags, stdout, fmt.Errorf("Error running post-apply plan: %w", err) } var plan *tfjson.Plan @@ -179,64 +162,56 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return fmt.Errorf("Error retrieving post-apply plan: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Error retrieving post-apply plan: %w", err) } if !planIsEmpty(plan) && !step.ExpectNonEmptyPlan { - var stdout string + var savedPlanRawStdout string err = runProviderCommand(ctx, t, func() error { var err error - stdout, err = wd.SavedPlanRawStdout(ctx) + savedPlanRawStdout, err = wd.SavedPlanRawStdout(ctx) return err }, wd, providers) if err != nil { - return fmt.Errorf("Error retrieving formatted plan output: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Error retrieving formatted plan output: %w", err) } - return fmt.Errorf("After applying this test step, the plan was not empty.\nstdout:\n\n%s", stdout) + return tfJSONDiags, stdout, fmt.Errorf("After applying this test step, the plan was not empty.\nstdout:\n\n%s", savedPlanRawStdout) } // do a refresh if !step.Destroy || (step.Destroy && !step.PreventPostDestroyRefresh) { - err := runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, tfJSON) + err = runProviderCommand(ctx, t, func() error { + refreshResponse, err := wd.Refresh(ctx) + + tfJSONDiags = append(tfJSONDiags, refreshResponse.Diagnostics...) + stdout += refreshResponse.Stdout + + return err }, wd, providers) if err != nil { - target := &tfexec.ErrVersionMismatch{} - if errors.As(err, &target) { - err = runProviderCommand(ctx, t, func() error { - return wd.Refresh(ctx) - }, wd, providers) - if err != nil { - return fmt.Errorf("Error running post-apply refresh: %w", err) - } - } else { - return fmt.Errorf("Error running post-apply refresh: %s", strings.Join(tfJSON.JsonOutput(), "\n")) - } + return tfJSONDiags, stdout, fmt.Errorf("Error running post-apply refresh: %w", err) } } // do another plan err = runProviderCommand(ctx, t, func() error { if step.Destroy { - return wd.CreateDestroyPlanJSON(ctx, tfJSON) + createDestroyPlanResponse, err := wd.CreateDestroyPlan(ctx) + + tfJSONDiags = append(tfJSONDiags, createDestroyPlanResponse.Diagnostics...) + stdout += createDestroyPlanResponse.Stdout + + return err } - return wd.CreatePlanJSON(ctx, tfJSON) + createPlanResponse, err := wd.CreatePlan(ctx) + + tfJSONDiags = append(tfJSONDiags, createPlanResponse.Diagnostics...) + stdout += createPlanResponse.Stdout + + return err }, wd, providers) if err != nil { - target := &tfexec.ErrVersionMismatch{} - if errors.As(err, &target) { - err = runProviderCommand(ctx, t, func() error { - if step.Destroy { - return wd.CreateDestroyPlan(ctx) - } - return wd.CreatePlan(ctx) - }, wd, providers) - if err != nil { - return fmt.Errorf("Error running second post-apply plan: %w", err) - } - } else { - return fmt.Errorf("Error running second post-apply plan: %s", strings.Join(tfJSON.JsonOutput(), "\n")) - } + return tfJSONDiags, stdout, fmt.Errorf("Error running second post-apply plan: %w", err) } err = runProviderCommand(ctx, t, func() error { @@ -245,23 +220,23 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return fmt.Errorf("Error retrieving second post-apply plan: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Error retrieving second post-apply plan: %w", err) } // check if plan is empty if !planIsEmpty(plan) && !step.ExpectNonEmptyPlan { - var stdout string + var savedPlanRawStdout string err = runProviderCommand(ctx, t, func() error { var err error - stdout, err = wd.SavedPlanRawStdout(ctx) + savedPlanRawStdout, err = wd.SavedPlanRawStdout(ctx) return err }, wd, providers) if err != nil { - return fmt.Errorf("Error retrieving formatted second plan output: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Error retrieving formatted second plan output: %w", err) } - return fmt.Errorf("After applying this test step and performing a `terraform refresh`, the plan was not empty.\nstdout\n\n%s", stdout) + return tfJSONDiags, stdout, fmt.Errorf("After applying this test step and performing a `terraform refresh`, the plan was not empty.\nstdout\n\n%s", savedPlanRawStdout) } else if step.ExpectNonEmptyPlan && planIsEmpty(plan) { - return errors.New("Expected a non-empty plan, but got an empty plan") + return tfJSONDiags, stdout, errors.New("Expected a non-empty plan, but got an empty plan") } // ID-ONLY REFRESH @@ -281,11 +256,11 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint }, wd, providers) if err != nil { - return err + return tfJSONDiags, stdout, err } if state.Empty() { - return nil + return tfJSONDiags, stdout, nil } var idRefreshCheck *terraform.ResourceState @@ -309,11 +284,11 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // caught a different bug. if idRefreshCheck != nil { if err := testIDRefresh(ctx, t, c, wd, step, idRefreshCheck, providers); err != nil { - return fmt.Errorf( + return tfJSONDiags, stdout, fmt.Errorf( "[ERROR] Test: ID-only test failed: %s", err) } } } - return nil + return tfJSONDiags, stdout, nil } diff --git a/helper/resource/testing_new_refresh_state.go b/helper/resource/testing_new_refresh_state.go index 9dc61410f..a42e1d46c 100644 --- a/helper/resource/testing_new_refresh_state.go +++ b/helper/resource/testing_new_refresh_state.go @@ -5,11 +5,8 @@ package resource import ( "context" - "errors" "fmt" - "strings" - "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" "github.com/mitchellh/go-testing-interface" @@ -19,9 +16,12 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories, tfJSON *plugintest.TerraformJSONBuffer) error { +func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) (plugintest.TerraformJSONDiagnostics, string, error) { t.Helper() + var tfJSONDiags plugintest.TerraformJSONDiagnostics + var stdout string + var err error // Explicitly ensure prior state exists before refresh. err = runProviderCommand(ctx, t, func() error { @@ -36,20 +36,15 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo } err = runProviderCommand(ctx, t, func() error { - return wd.RefreshJSON(ctx, tfJSON) + refreshResponse, err := wd.Refresh(ctx) + + tfJSONDiags = append(tfJSONDiags, refreshResponse.Diagnostics...) + stdout += refreshResponse.Stdout + + return err }, wd, providers) if err != nil { - target := &tfexec.ErrVersionMismatch{} - if errors.As(err, &target) { - err = runProviderCommand(ctx, t, func() error { - return wd.Refresh(ctx) - }, wd, providers) - if err != nil { - return fmt.Errorf("Error running refresh: %w", err) - } - } else { - return fmt.Errorf("Error running refresh: %s", strings.Join(tfJSON.JsonOutput(), "\n")) - } + return tfJSONDiags, stdout, fmt.Errorf("Error running refresh: %w", err) } var refreshState *terraform.State @@ -77,20 +72,15 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo // do a plan err = runProviderCommand(ctx, t, func() error { - return wd.CreatePlanJSON(ctx, tfJSON) + createPlanResponse, err := wd.CreatePlan(ctx) + + tfJSONDiags = append(tfJSONDiags, createPlanResponse.Diagnostics...) + stdout += createPlanResponse.Stdout + + return err }, wd, providers) if err != nil { - target := &tfexec.ErrVersionMismatch{} - if errors.As(err, &target) { - err = runProviderCommand(ctx, t, func() error { - return wd.CreatePlan(ctx) - }, wd, providers) - if err != nil { - return fmt.Errorf("Error running post-apply plan: %w", err) - } - } else { - return fmt.Errorf("Error running post-apply plan: %s", strings.Join(tfJSON.JsonOutput(), "\n")) - } + return tfJSONDiags, stdout, fmt.Errorf("Error running post-apply plan: %w", err) } var plan *tfjson.Plan @@ -100,7 +90,7 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return err }, wd, providers) if err != nil { - return fmt.Errorf("Error retrieving post-apply plan: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Error retrieving post-apply plan: %w", err) } if !planIsEmpty(plan) && !step.ExpectNonEmptyPlan { @@ -111,10 +101,10 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return err }, wd, providers) if err != nil { - return fmt.Errorf("Error retrieving formatted plan output: %w", err) + return tfJSONDiags, stdout, fmt.Errorf("Error retrieving formatted plan output: %w", err) } - return fmt.Errorf("After refreshing state during this test step, a followup plan was not empty.\nstdout:\n\n%s", stdout) + return tfJSONDiags, stdout, fmt.Errorf("After refreshing state during this test step, a followup plan was not empty.\nstdout:\n\n%s", stdout) } - return nil + return tfJSONDiags, stdout, nil } diff --git a/internal/plugintest/terraform_json_buffer.go b/internal/plugintest/terraform_json_buffer.go index cb978feb7..6b72954fc 100644 --- a/internal/plugintest/terraform_json_buffer.go +++ b/internal/plugintest/terraform_json_buffer.go @@ -11,9 +11,6 @@ import ( tfjson "github.com/hashicorp/terraform-json" ) -var _ io.Writer = &TerraformJSONBuffer{} -var _ io.Reader = &TerraformJSONBuffer{} - type TerraformJSONDiagnostics []tfjson.Diagnostic func (d TerraformJSONDiagnostics) Contains(r *regexp.Regexp, severity tfjson.DiagnosticSeverity) bool { @@ -32,10 +29,17 @@ func (d TerraformJSONDiagnostics) Contains(r *regexp.Regexp, severity tfjson.Dia return false } +var _ io.Writer = &TerraformJSONBuffer{} +var _ io.Reader = &TerraformJSONBuffer{} + +// TerraformJSONBuffer is used for storing and processing streaming +// JSON output generated when running terraform commands with the +// `-json` flag. type TerraformJSONBuffer struct { buf *bytes.Buffer diagnostics TerraformJSONDiagnostics jsonOutput []string + rawOutput string parsed bool } @@ -69,12 +73,14 @@ func (b *TerraformJSONBuffer) Parse() { scanner := bufio.NewScanner(b.buf) for scanner.Scan() { + txt := scanner.Text() + + b.rawOutput += "\n" + txt + var outer struct { Diagnostic tfjson.Diagnostic } - txt := scanner.Text() - // This will only capture buffer entries that can be unmarshalled // as JSON. If there are entries in the buffer that are not JSON // they will be discarded. @@ -109,3 +115,11 @@ func (b *TerraformJSONBuffer) JsonOutput() []string { return b.jsonOutput } + +func (b *TerraformJSONBuffer) RawOutput() string { + if !b.parsed { + b.Parse() + } + + return b.rawOutput +} diff --git a/internal/plugintest/working_dir.go b/internal/plugintest/working_dir.go index 32178940e..c86662b81 100644 --- a/internal/plugintest/working_dir.go +++ b/internal/plugintest/working_dir.go @@ -6,6 +6,7 @@ package plugintest import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -170,9 +171,28 @@ func (wd *WorkingDir) planFilename() string { return filepath.Join(wd.baseDir, PlanFileName) } -// CreatePlan runs "terraform plan" to create a saved plan file, which if successful -// will then be used for the next call to Apply. -func (wd *WorkingDir) CreatePlan(ctx context.Context) error { +type CreatePlanResponse struct { + Diagnostics []tfjson.Diagnostic + Stdout string +} + +// CreatePlan runs CreatePlanJSON first and will fall back to running terraform plan if an error +// is returned from CreatePlanJSON indicating that the version of Terraform that is being used +// does not support the `-json` flag with "terraform plan". CreatePlan will create a saved plan +// file, which if successful will then be used for the next call to Apply. +func (wd *WorkingDir) CreatePlan(ctx context.Context) (CreatePlanResponse, error) { + tfJSON := NewTerraformJSONBuffer() + createPlanResponse := CreatePlanResponse{} + + err := wd.CreatePlanJSON(ctx, tfJSON) + target := &tfexec.ErrVersionMismatch{} + if !errors.As(err, &target) { + createPlanResponse.Diagnostics = tfJSON.Diagnostics() + createPlanResponse.Stdout = tfJSON.RawOutput() + + return createPlanResponse, err + } + logging.HelperResourceTrace(ctx, "Calling Terraform CLI plan command") hasChanges, err := wd.tf.Plan(context.Background(), tfexec.Reattach(wd.reattachInfo), tfexec.Refresh(false), tfexec.Out(PlanFileName)) @@ -180,34 +200,34 @@ func (wd *WorkingDir) CreatePlan(ctx context.Context) error { logging.HelperResourceTrace(ctx, "Called Terraform CLI plan command") if err != nil { - return err + return createPlanResponse, err } if !hasChanges { logging.HelperResourceTrace(ctx, "Created plan with no changes") - return nil + return createPlanResponse, nil } stdout, err := wd.SavedPlanRawStdout(ctx) if err != nil { - return fmt.Errorf("error retrieving formatted plan output: %w", err) + return createPlanResponse, fmt.Errorf("error retrieving formatted plan output: %w", err) } logging.HelperResourceTrace(ctx, "Created plan with changes", map[string]any{logging.KeyTestTerraformPlan: stdout}) - return nil + return createPlanResponse, nil } // CreatePlanJSON runs "terraform plan" with `-json` to create a saved plan file, // which if successful will then be used for the next call to Apply. func (wd *WorkingDir) CreatePlanJSON(ctx context.Context, w io.Writer) error { - logging.HelperResourceTrace(ctx, "Calling Terraform CLI plan command") + logging.HelperResourceTrace(ctx, "Calling Terraform CLI plan -json command") hasChanges, err := wd.tf.PlanJSON(context.Background(), w, tfexec.Reattach(wd.reattachInfo), tfexec.Refresh(false), tfexec.Out(PlanFileName)) - logging.HelperResourceTrace(ctx, "Called Terraform CLI plan command") + logging.HelperResourceTrace(ctx, "Called Terraform CLI plan -json command") if err != nil { return err @@ -230,9 +250,28 @@ func (wd *WorkingDir) CreatePlanJSON(ctx context.Context, w io.Writer) error { return nil } -// CreateDestroyPlan runs "terraform plan -destroy" to create a saved plan +type CreateDestroyPlanResponse struct { + Diagnostics []tfjson.Diagnostic + Stdout string +} + +// CreateDestroyPlan runs CreateDestroyPlanJSON first and will fall back to running terraform plan if an error +// is returned from ApplyJSON indicating that the version of Terraform that is being used +// does not support the `-json` flag with "terraform plan -destroy". CreateDestroyPlan will create a saved plan // file, which if successful will then be used for the next call to Apply. -func (wd *WorkingDir) CreateDestroyPlan(ctx context.Context) error { +func (wd *WorkingDir) CreateDestroyPlan(ctx context.Context) (CreateDestroyPlanResponse, error) { + tfJSON := NewTerraformJSONBuffer() + createDestroyPlanResponse := CreateDestroyPlanResponse{} + + err := wd.CreateDestroyPlanJSON(ctx, tfJSON) + target := &tfexec.ErrVersionMismatch{} + if !errors.As(err, &target) { + createDestroyPlanResponse.Diagnostics = tfJSON.Diagnostics() + createDestroyPlanResponse.Stdout = tfJSON.RawOutput() + + return createDestroyPlanResponse, err + } + logging.HelperResourceTrace(ctx, "Calling Terraform CLI plan -destroy command") hasChanges, err := wd.tf.Plan(context.Background(), tfexec.Reattach(wd.reattachInfo), tfexec.Refresh(false), tfexec.Out(PlanFileName), tfexec.Destroy(true)) @@ -240,34 +279,34 @@ func (wd *WorkingDir) CreateDestroyPlan(ctx context.Context) error { logging.HelperResourceTrace(ctx, "Called Terraform CLI plan -destroy command") if err != nil { - return err + return createDestroyPlanResponse, err } if !hasChanges { logging.HelperResourceTrace(ctx, "Created destroy plan with no changes") - return nil + return createDestroyPlanResponse, nil } stdout, err := wd.SavedPlanRawStdout(ctx) if err != nil { - return fmt.Errorf("error retrieving formatted plan output: %w", err) + return createDestroyPlanResponse, fmt.Errorf("error retrieving formatted plan output: %w", err) } logging.HelperResourceTrace(ctx, "Created destroy plan with changes", map[string]any{logging.KeyTestTerraformPlan: stdout}) - return nil + return createDestroyPlanResponse, nil } // CreateDestroyPlanJSON runs "terraform plan -destroy -json" to create a saved plan // file, which if successful will then be used for the next call to Apply. func (wd *WorkingDir) CreateDestroyPlanJSON(ctx context.Context, w io.Writer) error { - logging.HelperResourceTrace(ctx, "Calling Terraform CLI plan -destroy command") + logging.HelperResourceTrace(ctx, "Calling Terraform CLI plan -destroy -json command") hasChanges, err := wd.tf.PlanJSON(context.Background(), w, tfexec.Reattach(wd.reattachInfo), tfexec.Refresh(false), tfexec.Out(PlanFileName), tfexec.Destroy(true)) - logging.HelperResourceTrace(ctx, "Called Terraform CLI plan -destroy command") + logging.HelperResourceTrace(ctx, "Called Terraform CLI plan -destroy -json command") if err != nil { return err @@ -290,11 +329,30 @@ func (wd *WorkingDir) CreateDestroyPlanJSON(ctx context.Context, w io.Writer) er return nil } -// Apply runs "terraform apply". If CreatePlan has previously completed +type ApplyResponse struct { + Diagnostics []tfjson.Diagnostic + Stdout string +} + +// Apply runs ApplyJSON first and will fall back to running terraform apply if an error +// is returned from ApplyJSON indicating that the version of Terraform that is being used +// does not support the `-json` flag with "terraform apply". If CreatePlan has previously completed // successfully and the saved plan has not been cleared in the meantime then // this will apply the saved plan. Otherwise, it will implicitly create a new // plan and apply it. -func (wd *WorkingDir) Apply(ctx context.Context) error { +func (wd *WorkingDir) Apply(ctx context.Context) (ApplyResponse, error) { + tfJSON := NewTerraformJSONBuffer() + applyResponse := ApplyResponse{} + + err := wd.ApplyJSON(ctx, tfJSON) + target := &tfexec.ErrVersionMismatch{} + if !errors.As(err, &target) { + applyResponse.Diagnostics = tfJSON.Diagnostics() + applyResponse.Stdout = tfJSON.RawOutput() + + return applyResponse, err + } + args := []tfexec.ApplyOption{tfexec.Reattach(wd.reattachInfo), tfexec.Refresh(false)} if wd.HasSavedPlan() { args = append(args, tfexec.DirOrPlan(PlanFileName)) @@ -302,11 +360,11 @@ func (wd *WorkingDir) Apply(ctx context.Context) error { logging.HelperResourceTrace(ctx, "Calling Terraform CLI apply command") - err := wd.tf.Apply(context.Background(), args...) + err = wd.tf.Apply(context.Background(), args...) logging.HelperResourceTrace(ctx, "Called Terraform CLI apply command") - return err + return applyResponse, err } // ApplyJSON runs "terraform apply" with the `-json` flag. The streaming JSON output @@ -320,11 +378,11 @@ func (wd *WorkingDir) ApplyJSON(ctx context.Context, w io.Writer) error { args = append(args, tfexec.DirOrPlan(PlanFileName)) } - logging.HelperResourceTrace(ctx, "Calling Terraform CLI apply command") + logging.HelperResourceTrace(ctx, "Calling Terraform CLI apply -json command") err := wd.tf.ApplyJSON(context.Background(), w, args...) - logging.HelperResourceTrace(ctx, "Called Terraform CLI apply command") + logging.HelperResourceTrace(ctx, "Called Terraform CLI apply -json command") return err } @@ -427,24 +485,43 @@ func (wd *WorkingDir) Taint(ctx context.Context, address string) error { return err } -// Refresh runs terraform refresh -func (wd *WorkingDir) Refresh(ctx context.Context) error { +type RefreshResponse struct { + Diagnostics []tfjson.Diagnostic + Stdout string +} + +// Refresh runs RefreshJSON first and will fall back to running terraform refresh if an error +// is returned from RefreshJSON indicating that the version of Terraform that is being used +// does not support the `-json` flag. +func (wd *WorkingDir) Refresh(ctx context.Context) (RefreshResponse, error) { + tfJSON := NewTerraformJSONBuffer() + refreshResponse := RefreshResponse{} + + err := wd.RefreshJSON(ctx, tfJSON) + target := &tfexec.ErrVersionMismatch{} + if !errors.As(err, &target) { + refreshResponse.Diagnostics = tfJSON.Diagnostics() + refreshResponse.Stdout = tfJSON.RawOutput() + + return refreshResponse, err + } + logging.HelperResourceTrace(ctx, "Calling Terraform CLI refresh command") - err := wd.tf.Refresh(context.Background(), tfexec.Reattach(wd.reattachInfo)) + err = wd.tf.Refresh(context.Background(), tfexec.Reattach(wd.reattachInfo)) logging.HelperResourceTrace(ctx, "Called Terraform CLI refresh command") - return err + return refreshResponse, err } -// RefreshJSON runs terraform refresh +// RefreshJSON runs terraform refresh with `-json` flag func (wd *WorkingDir) RefreshJSON(ctx context.Context, w io.Writer) error { - logging.HelperResourceTrace(ctx, "Calling Terraform CLI refresh command") + logging.HelperResourceTrace(ctx, "Calling Terraform CLI refresh -json command") err := wd.tf.RefreshJSON(context.Background(), w, tfexec.Reattach(wd.reattachInfo)) - logging.HelperResourceTrace(ctx, "Called Terraform CLI refresh command") + logging.HelperResourceTrace(ctx, "Called Terraform CLI refresh -json command") return err } From 72c60c052051279ed9efca419cd741701a92432e Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Tue, 24 Jan 2023 14:32:09 +0000 Subject: [PATCH 21/32] Joining strings for error output (#16) --- helper/resource/testing_new.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 0ceaa4a1c..f27c2e1eb 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -268,7 +268,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest fmt.Sprintf("Error running refresh: expected an error with pattern (%s)", step.ExpectError.String()), map[string]interface{}{logging.KeyError: strings.Join(errorOutput, "")}, ) - t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), strings.Join(errorOutput, "\n")) + t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), strings.Join(errorOutput, "")) } } else { // TODO: ErrorCheck will be broken if errors that are being checked that would have @@ -333,7 +333,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest fmt.Sprintf("Expected an error with pattern (%s)", step.ExpectError.String()), map[string]interface{}{logging.KeyError: strings.Join(errorOutput, "")}, ) - t.Fatalf("Step %d/%d, expected an error matching pattern, no match on: %s", stepNumber, len(c.Steps), errorOutput) + t.Fatalf("Step %d/%d, expected an error matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(errorOutput, "")) } } else { // TODO: ErrorCheck will be broken if errors that are being checked that would have From 9357173076cb6887bfcf6bb6d66b58d6c3db0626 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Tue, 24 Jan 2023 14:54:17 +0000 Subject: [PATCH 22/32] Returning response structs for testStepNewRefreshState() and testStepNewConfig() (#16) --- helper/resource/testing_new.go | 8 +- helper/resource/testing_new_config.go | 123 +++++++++++++++---- helper/resource/testing_new_refresh_state.go | 37 ++++-- 3 files changed, 134 insertions(+), 34 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index f27c2e1eb..836f672e7 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -246,7 +246,9 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.RefreshState { logging.HelperResourceTrace(ctx, "TestStep is RefreshState mode") - tfJSONDiags, stdout, err := testStepNewRefreshState(ctx, t, wd, step, providers) + testStepNewRefreshStateResponse, err := testStepNewRefreshState(ctx, t, wd, step, providers) + tfJSONDiags := testStepNewRefreshStateResponse.tfJSONDiags + stdout := testStepNewRefreshStateResponse.stdout if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -311,7 +313,9 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.Config != "" { logging.HelperResourceTrace(ctx, "TestStep is Config mode") - tfJSONDiags, stdout, err := testStepNewConfig(ctx, t, c, wd, step, providers) + testStepNewConfigResponse, err := testStepNewConfig(ctx, t, c, wd, step, providers) + tfJSONDiags := testStepNewConfigResponse.tfJSONDiags + stdout := testStepNewConfigResponse.stdout if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index 2c8d73d5b..92d2260ec 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -17,7 +17,12 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) (plugintest.TerraformJSONDiagnostics, string, error) { +type testStepNewConfigResponse struct { + tfJSONDiags plugintest.TerraformJSONDiagnostics + stdout string +} + +func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) (testStepNewConfigResponse, error) { t.Helper() var tfJSONDiags plugintest.TerraformJSONDiagnostics @@ -25,7 +30,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint err := wd.SetConfig(ctx, step.mergedConfig(ctx, c)) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error setting config: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error setting config: %w", err) } // require a refresh before applying @@ -39,7 +47,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error running pre-apply refresh: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error running pre-apply refresh: %w", err) } // If this step is a PlanOnly step, skip over this first Plan and @@ -66,7 +77,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error running pre-apply plan: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error running pre-apply plan: %w", err) } // We need to keep a copy of the state prior to destroying such @@ -81,7 +95,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return nil }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error retrieving pre-apply state: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error retrieving pre-apply state: %w", err) } // Apply the diff, creating real resources @@ -95,10 +112,16 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint }, wd, providers) if err != nil { if step.Destroy { - return tfJSONDiags, stdout, fmt.Errorf("Error running destroy: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error running destroy: %w", err) } - return tfJSONDiags, stdout, fmt.Errorf("Error running apply: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error running apply: %w", err) } // Get the new state @@ -111,7 +134,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return nil }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error retrieving state after apply: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error retrieving state after apply: %w", err) } // Run any configured checks @@ -121,11 +147,17 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint state.IsBinaryDrivenTest = true if step.Destroy { if err := step.Check(stateBeforeApplication); err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Check failed: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Check failed: %w", err) } } else { if err := step.Check(state); err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Check failed: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Check failed: %w", err) } } } @@ -152,7 +184,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error running post-apply plan: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error running post-apply plan: %w", err) } var plan *tfjson.Plan @@ -162,7 +197,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error retrieving post-apply plan: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error retrieving post-apply plan: %w", err) } if !planIsEmpty(plan) && !step.ExpectNonEmptyPlan { @@ -173,9 +211,15 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error retrieving formatted plan output: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error retrieving formatted plan output: %w", err) } - return tfJSONDiags, stdout, fmt.Errorf("After applying this test step, the plan was not empty.\nstdout:\n\n%s", savedPlanRawStdout) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("After applying this test step, the plan was not empty.\nstdout:\n\n%s", savedPlanRawStdout) } // do a refresh @@ -189,7 +233,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error running post-apply refresh: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error running post-apply refresh: %w", err) } } @@ -211,7 +258,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error running second post-apply plan: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error running second post-apply plan: %w", err) } err = runProviderCommand(ctx, t, func() error { @@ -220,7 +270,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error retrieving second post-apply plan: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error retrieving second post-apply plan: %w", err) } // check if plan is empty @@ -232,11 +285,20 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error retrieving formatted second plan output: %w", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error retrieving formatted second plan output: %w", err) } - return tfJSONDiags, stdout, fmt.Errorf("After applying this test step and performing a `terraform refresh`, the plan was not empty.\nstdout\n\n%s", savedPlanRawStdout) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("After applying this test step and performing a `terraform refresh`, the plan was not empty.\nstdout\n\n%s", savedPlanRawStdout) } else if step.ExpectNonEmptyPlan && planIsEmpty(plan) { - return tfJSONDiags, stdout, errors.New("Expected a non-empty plan, but got an empty plan") + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, errors.New("Expected a non-empty plan, but got an empty plan") } // ID-ONLY REFRESH @@ -256,11 +318,17 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint }, wd, providers) if err != nil { - return tfJSONDiags, stdout, err + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, err } if state.Empty() { - return tfJSONDiags, stdout, nil + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, nil } var idRefreshCheck *terraform.ResourceState @@ -284,11 +352,16 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint // caught a different bug. if idRefreshCheck != nil { if err := testIDRefresh(ctx, t, c, wd, step, idRefreshCheck, providers); err != nil { - return tfJSONDiags, stdout, fmt.Errorf( - "[ERROR] Test: ID-only test failed: %s", err) + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("[ERROR] Test: ID-only test failed: %s", err) } } } - return tfJSONDiags, stdout, nil + return testStepNewConfigResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, nil } diff --git a/helper/resource/testing_new_refresh_state.go b/helper/resource/testing_new_refresh_state.go index a42e1d46c..908184ee9 100644 --- a/helper/resource/testing_new_refresh_state.go +++ b/helper/resource/testing_new_refresh_state.go @@ -16,7 +16,12 @@ import ( "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" ) -func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) (plugintest.TerraformJSONDiagnostics, string, error) { +type testStepNewRefreshStateResponse struct { + tfJSONDiags plugintest.TerraformJSONDiagnostics + stdout string +} + +func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) (testStepNewRefreshStateResponse, error) { t.Helper() var tfJSONDiags plugintest.TerraformJSONDiagnostics @@ -44,7 +49,10 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error running refresh: %w", err) + return testStepNewRefreshStateResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error running refresh: %w", err) } var refreshState *terraform.State @@ -80,7 +88,10 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error running post-apply plan: %w", err) + return testStepNewRefreshStateResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error running post-apply plan: %w", err) } var plan *tfjson.Plan @@ -90,7 +101,10 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error retrieving post-apply plan: %w", err) + return testStepNewRefreshStateResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error retrieving post-apply plan: %w", err) } if !planIsEmpty(plan) && !step.ExpectNonEmptyPlan { @@ -101,10 +115,19 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return err }, wd, providers) if err != nil { - return tfJSONDiags, stdout, fmt.Errorf("Error retrieving formatted plan output: %w", err) + return testStepNewRefreshStateResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("Error retrieving formatted plan output: %w", err) } - return tfJSONDiags, stdout, fmt.Errorf("After refreshing state during this test step, a followup plan was not empty.\nstdout:\n\n%s", stdout) + return testStepNewRefreshStateResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, fmt.Errorf("After refreshing state during this test step, a followup plan was not empty.\nstdout:\n\n%s", stdout) } - return tfJSONDiags, stdout, nil + return testStepNewRefreshStateResponse{ + tfJSONDiags: tfJSONDiags, + stdout: stdout, + }, nil } From a12aaa1a623d95acbf87212c58ef90184ea70c7d Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Wed, 25 Jan 2023 09:14:01 +0000 Subject: [PATCH 23/32] Bumping to latest main for terraform-exec (#16) --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 31dc514c9..db9755e3e 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/hashicorp/hc-install v0.4.0 github.com/hashicorp/hcl/v2 v2.15.0 github.com/hashicorp/logutils v1.0.0 - github.com/hashicorp/terraform-exec v0.17.4-0.20230120174504-9acd44a7e5da + github.com/hashicorp/terraform-exec v0.17.4-0.20230124151948-5eada031c166 github.com/hashicorp/terraform-json v0.14.0 github.com/hashicorp/terraform-plugin-framework v1.0.1 github.com/hashicorp/terraform-plugin-go v0.14.3 diff --git a/go.sum b/go.sum index 6df28a46d..f8763754b 100644 --- a/go.sum +++ b/go.sum @@ -93,12 +93,16 @@ github.com/hashicorp/hcl/v2 v2.15.0 h1:CPDXO6+uORPjKflkWCCwoWc9uRp+zSIPcCQ+BrxV7 github.com/hashicorp/hcl/v2 v2.15.0/go.mod h1:JRmR89jycNkrrqnMmvPDMd56n1rQJ2Q6KocSLCMCXng= github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/terraform-exec v0.17.3 h1:MX14Kvnka/oWGmIkyuyvL6POx25ZmKrjlaclkx3eErU= +github.com/hashicorp/terraform-exec v0.17.3/go.mod h1:+NELG0EqQekJzhvikkeQsOAZpsw0cv/03rbeQJqscAI= github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98 h1:WkSxIVXfV/u9tvDGXdSw4v2eCpRMSt7mYRBseX+OaSU= github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= github.com/hashicorp/terraform-exec v0.17.4-0.20230116095935-bc76870e2b9b h1:z4a1oo2M/yWj2ljEoXsFQRkmp2dRqvq4Y9HjQl3eBz8= github.com/hashicorp/terraform-exec v0.17.4-0.20230116095935-bc76870e2b9b/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= github.com/hashicorp/terraform-exec v0.17.4-0.20230120174504-9acd44a7e5da h1:St2EWMkwfc56l9PqejkEV/r4S55Em9iXYvW2qdtyH+4= github.com/hashicorp/terraform-exec v0.17.4-0.20230120174504-9acd44a7e5da/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= +github.com/hashicorp/terraform-exec v0.17.4-0.20230124151948-5eada031c166 h1:WCvC+VknMywOyDXTIzvHHYWknjNBKI0/dmIR9EZd0bw= +github.com/hashicorp/terraform-exec v0.17.4-0.20230124151948-5eada031c166/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s= github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= github.com/hashicorp/terraform-plugin-framework v1.0.1 h1:apX2jtaEKa15+do6H2izBJdl1dEH2w5BPVkDJ3Q3mKA= From 2fb79ba249aee9cc0a58e2d9908b1d5173495008 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Wed, 25 Jan 2023 10:42:35 +0000 Subject: [PATCH 24/32] Adding contents of error diagnostics to error supplied to TestCase ErrorCheck (#16) --- helper/resource/testing_new.go | 44 ++++++++++++++++-- helper/resource/teststep_providers_test.go | 54 ++++++++++++++++++++++ 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 836f672e7..749c5d217 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -273,11 +273,30 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), strings.Join(errorOutput, "")) } } else { - // TODO: ErrorCheck will be broken if errors that are being checked that would have - // previously been present in err are now in stdout. if err != nil && c.ErrorCheck != nil { logging.HelperResourceDebug(ctx, "Calling TestCase ErrorCheck") + + // This is a "best effort" to supply ErrorCheck with the contents of the + // error diagnostics returned from running the Terraform command when + // executed with the -json flag. If ErrorCheck functions are using + // regexp to perform matches that include line breaks then these will + // likely no longer behave as expected and therefore potentially + // represent a breaking change. + var diagStrings []string + + for _, v := range tfJSONDiags { + if v.Severity != tfjson.DiagnosticSeverityError { + continue + } + + diagStrings = append(diagStrings, "Error: "+v.Summary) + diagStrings = append(diagStrings, v.Detail) + } + + err = fmt.Errorf("%s\n"+strings.Join(diagStrings, "\n"), err) + err = c.ErrorCheck(err) + logging.HelperResourceDebug(ctx, "Called TestCase ErrorCheck") } if err != nil { @@ -340,11 +359,28 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest t.Fatalf("Step %d/%d, expected an error matching pattern, no match on: %s", stepNumber, len(c.Steps), strings.Join(errorOutput, "")) } } else { - // TODO: ErrorCheck will be broken if errors that are being checked that would have - // previously been present in err are now in stdout. if err != nil && c.ErrorCheck != nil { logging.HelperResourceDebug(ctx, "Calling TestCase ErrorCheck") + // This is a "best effort" to supply ErrorCheck with the contents of the + // error diagnostics returned from running the Terraform command when + // executed with the -json flag. If ErrorCheck functions are using + // regexp to perform matches that include line breaks then these will + // likely no longer behave as expected and therefore potentially + // represent a breaking change. + var diagStrings []string + + for _, v := range tfJSONDiags { + if v.Severity != tfjson.DiagnosticSeverityError { + continue + } + + diagStrings = append(diagStrings, "Error: "+v.Summary) + diagStrings = append(diagStrings, v.Detail) + } + + err = fmt.Errorf("%s\n"+strings.Join(diagStrings, "\n"), err) + err = c.ErrorCheck(err) logging.HelperResourceDebug(ctx, "Called TestCase ErrorCheck") diff --git a/helper/resource/teststep_providers_test.go b/helper/resource/teststep_providers_test.go index 9d122615e..e433ee0d3 100644 --- a/helper/resource/teststep_providers_test.go +++ b/helper/resource/teststep_providers_test.go @@ -2803,6 +2803,60 @@ func TestTest_TestStep_ProviderFactories_ExpectWarningDestroy(t *testing.T) { }) } +func TestTest_TestStep_ProviderFactories_ErrorCheck(t *testing.T) { + t.Parallel() + + Test(t, TestCase{ + ErrorCheck: func(err error) error { + r := regexp.MustCompile("error summary") + + if r.MatchString(err.Error()) { + return nil + } + + return err + }, + ProviderFactories: map[string]func() (*schema.Provider, error){ + "example": func() (*schema.Provider, error) { //nolint:unparam // required signature + return &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "example_resource": { + CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics { + d.SetId("id") + + return diag.Diagnostics{ + diag.Diagnostic{ + Severity: 0, + Summary: "error summary", + Detail: "error detail", + }, + } + }, + DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) diag.Diagnostics { + return nil + }, + Schema: map[string]*schema.Schema{ + "id": { + Computed: true, + Type: schema.TypeString, + }, + }, + }, + }, + }, nil + }, + }, + Steps: []TestStep{ + { + Config: `resource "example_resource" "test" { }`, + }, + }, + }) +} + func setTimeForTest(t time.Time) func() { return func() { getTimeForTest = func() time.Time { From ad5ebb1713a85baf4c92bfe064f561d2ef11a6a1 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Wed, 25 Jan 2023 11:11:14 +0000 Subject: [PATCH 25/32] Supplying stdout to test output (#16) --- helper/resource/testing_new.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 749c5d217..dce69b9e7 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -5,6 +5,7 @@ package resource import ( "context" + "encoding/json" "fmt" "reflect" "strings" @@ -318,9 +319,14 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), map[string]interface{}{logging.KeyError: err}, ) - // TODO: Using stdout will show all entries in the Terraform stdout whereas the warning will only - // be found if it's in the JSON output. - t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), stdout) + + tfJSONDiagsMarshalled, err := json.Marshal(tfJSONDiags) + if err != nil { + t.Fatalf("could not marshal tfJSON: %s", err) + } + + t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), tfJSONDiagsMarshalled) + t.Fatalf("Stdout: %s", stdout) } } @@ -404,9 +410,14 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest fmt.Sprintf("Expected a warning with pattern (%s)", step.ExpectWarning.String()), map[string]interface{}{logging.KeyError: err}, ) - // TODO: Using stdout will show all entries in the Terraform stdout whereas the warning will only - // be found if it's in the JSON output. - t.Fatalf("Step %d/%d, expected a warning matching pattern, no match on: %s", stepNumber, len(c.Steps), stdout) + + tfJSONDiagsMarshalled, err := json.Marshal(tfJSONDiags) + if err != nil { + t.Fatalf("could not marshal tfJSON: %s", err) + } + + t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), tfJSONDiagsMarshalled) + t.Fatalf("Stdout: %s", stdout) } } @@ -474,13 +485,9 @@ func testIDRefresh(ctx context.Context, t testing.T, c TestCase, wd *plugintest. // Refresh! err = runProviderCommand(ctx, t, func() error { - // TODO: Need to handle the possibility that the error no longer requires all - // of the required info for debugging in the case that terraform refresh with - // -json has been run and the information that was in the error is now in - // RefreshResponse.Stdout - _, err := wd.Refresh(ctx) + refreshResponse, err := wd.Refresh(ctx) if err != nil { - t.Fatalf("Error running terraform refresh: %s", err) + t.Fatalf("Error running terraform refresh: %s", err, refreshResponse.Stdout) } state, err = getState(ctx, t, wd) if err != nil { From a30a88fe6fc6fb16ac84cb4f9df73d73559524ee Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Wed, 25 Jan 2023 13:53:33 +0000 Subject: [PATCH 26/32] Renaming vars (#16) --- helper/resource/testing_new.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index dce69b9e7..b8f957bc0 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -248,8 +248,8 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest logging.HelperResourceTrace(ctx, "TestStep is RefreshState mode") testStepNewRefreshStateResponse, err := testStepNewRefreshState(ctx, t, wd, step, providers) - tfJSONDiags := testStepNewRefreshStateResponse.tfJSONDiags - stdout := testStepNewRefreshStateResponse.stdout + refreshTfJSONDiags := testStepNewRefreshStateResponse.tfJSONDiags + refreshStdout := testStepNewRefreshStateResponse.stdout if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -262,9 +262,9 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } errorFound := step.ExpectError.MatchString(err.Error()) - jsonErrorFound := tfJSONDiags.Contains(step.ExpectError, tfjson.DiagnosticSeverityError) + jsonErrorFound := refreshTfJSONDiags.Contains(step.ExpectError, tfjson.DiagnosticSeverityError) - errorOutput := []string{err.Error(), stdout} + errorOutput := []string{err.Error(), refreshStdout} if !errorFound && !jsonErrorFound { logging.HelperResourceError(ctx, @@ -285,7 +285,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // represent a breaking change. var diagStrings []string - for _, v := range tfJSONDiags { + for _, v := range refreshTfJSONDiags { if v.Severity != tfjson.DiagnosticSeverityError { continue } @@ -312,7 +312,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - warningFound := tfJSONDiags.Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + warningFound := refreshTfJSONDiags.Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { logging.HelperResourceError(ctx, @@ -320,13 +320,13 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest map[string]interface{}{logging.KeyError: err}, ) - tfJSONDiagsMarshalled, err := json.Marshal(tfJSONDiags) + tfJSONDiagsMarshalled, err := json.Marshal(refreshTfJSONDiags) if err != nil { t.Fatalf("could not marshal tfJSON: %s", err) } t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), tfJSONDiagsMarshalled) - t.Fatalf("Stdout: %s", stdout) + t.Fatalf("Stdout: %s", refreshStdout) } } @@ -339,8 +339,8 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest logging.HelperResourceTrace(ctx, "TestStep is Config mode") testStepNewConfigResponse, err := testStepNewConfig(ctx, t, c, wd, step, providers) - tfJSONDiags := testStepNewConfigResponse.tfJSONDiags - stdout := testStepNewConfigResponse.stdout + newConfigTfJSONDiags := testStepNewConfigResponse.tfJSONDiags + newConfigStdout := testStepNewConfigResponse.stdout if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -353,9 +353,9 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } errorFound := step.ExpectError.MatchString(err.Error()) - jsonErrorFound := tfJSONDiags.Contains(step.ExpectError, tfjson.DiagnosticSeverityError) + jsonErrorFound := newConfigTfJSONDiags.Contains(step.ExpectError, tfjson.DiagnosticSeverityError) - errorOutput := []string{err.Error(), stdout} + errorOutput := []string{err.Error(), newConfigStdout} if !errorFound && !jsonErrorFound { logging.HelperResourceError(ctx, @@ -376,7 +376,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // represent a breaking change. var diagStrings []string - for _, v := range tfJSONDiags { + for _, v := range newConfigTfJSONDiags { if v.Severity != tfjson.DiagnosticSeverityError { continue } @@ -403,7 +403,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest if step.ExpectWarning != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectWarning") - warningFound := tfJSONDiags.Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) + warningFound := newConfigTfJSONDiags.Contains(step.ExpectWarning, tfjson.DiagnosticSeverityWarning) if !warningFound { logging.HelperResourceError(ctx, @@ -411,13 +411,13 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest map[string]interface{}{logging.KeyError: err}, ) - tfJSONDiagsMarshalled, err := json.Marshal(tfJSONDiags) + tfJSONDiagsMarshalled, err := json.Marshal(newConfigTfJSONDiags) if err != nil { t.Fatalf("could not marshal tfJSON: %s", err) } t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), tfJSONDiagsMarshalled) - t.Fatalf("Stdout: %s", stdout) + t.Fatalf("Stdout: %s", newConfigStdout) } } From 870e540805a6ab152ee72b4ccf943d100dd55158 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Mon, 30 Jan 2023 14:25:15 +0000 Subject: [PATCH 27/32] Return diagnostics rather than the whole of stdout (#16) --- helper/resource/testing_new.go | 64 +++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index b8f957bc0..2b143b95d 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -249,7 +249,6 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest testStepNewRefreshStateResponse, err := testStepNewRefreshState(ctx, t, wd, step, providers) refreshTfJSONDiags := testStepNewRefreshStateResponse.tfJSONDiags - refreshStdout := testStepNewRefreshStateResponse.stdout if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -264,14 +263,25 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest errorFound := step.ExpectError.MatchString(err.Error()) jsonErrorFound := refreshTfJSONDiags.Contains(step.ExpectError, tfjson.DiagnosticSeverityError) - errorOutput := []string{err.Error(), refreshStdout} + errorOutput := []string{err.Error()} + + for _, v := range refreshTfJSONDiags { + if v.Severity == tfjson.DiagnosticSeverityError { + b, err := json.Marshal(v) + if err != nil { + t.Errorf("could not marshal tfjson diagnostic: %s", err) + } + + errorOutput = append(errorOutput, string(b)) + } + } if !errorFound && !jsonErrorFound { logging.HelperResourceError(ctx, fmt.Sprintf("Error running refresh: expected an error with pattern (%s)", step.ExpectError.String()), map[string]interface{}{logging.KeyError: strings.Join(errorOutput, "")}, ) - t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), strings.Join(errorOutput, "")) + t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), strings.Join(errorOutput, "\n")) } } else { if err != nil && c.ErrorCheck != nil { @@ -305,7 +315,21 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest "Error running refresh", map[string]interface{}{logging.KeyError: err}, ) - t.Fatalf("Step %d/%d error running refresh: %s", stepNumber, len(c.Steps), err) + + errorOutput := []string{err.Error()} + + for _, v := range refreshTfJSONDiags { + if v.Severity == tfjson.DiagnosticSeverityError { + b, err := json.Marshal(v) + if err != nil { + t.Errorf("could not marshal tfjson diagnostic: %s", err) + } + + errorOutput = append(errorOutput, string(b)) + } + } + + t.Fatalf("Step %d/%d error: %s", stepNumber, len(c.Steps), strings.Join(errorOutput, "\n")) } } @@ -326,7 +350,6 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), tfJSONDiagsMarshalled) - t.Fatalf("Stdout: %s", refreshStdout) } } @@ -340,7 +363,6 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest testStepNewConfigResponse, err := testStepNewConfig(ctx, t, c, wd, step, providers) newConfigTfJSONDiags := testStepNewConfigResponse.tfJSONDiags - newConfigStdout := testStepNewConfigResponse.stdout if step.ExpectError != nil { logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError") @@ -355,7 +377,18 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest errorFound := step.ExpectError.MatchString(err.Error()) jsonErrorFound := newConfigTfJSONDiags.Contains(step.ExpectError, tfjson.DiagnosticSeverityError) - errorOutput := []string{err.Error(), newConfigStdout} + errorOutput := []string{err.Error()} + + for _, v := range newConfigTfJSONDiags { + if v.Severity == tfjson.DiagnosticSeverityError { + b, err := json.Marshal(v) + if err != nil { + t.Errorf("could not marshal tfjson Diagnostic: %s", err) + } + + errorOutput = append(errorOutput, string(b)) + } + } if !errorFound && !jsonErrorFound { logging.HelperResourceError(ctx, @@ -396,7 +429,21 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest "Unexpected error", map[string]interface{}{logging.KeyError: err}, ) - t.Fatalf("Step %d/%d error: %s", stepNumber, len(c.Steps), err) + + errorOutput := []string{err.Error()} + + for _, v := range newConfigTfJSONDiags { + if v.Severity == tfjson.DiagnosticSeverityError { + b, err := json.Marshal(v) + if err != nil { + t.Errorf("could not marshall tfjson Diagnostic: %s", err) + } + + errorOutput = append(errorOutput, string(b)) + } + } + + t.Fatalf("Step %d/%d error: %s", stepNumber, len(c.Steps), strings.Join(errorOutput, "\n")) } } @@ -417,7 +464,6 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest } t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), tfJSONDiagsMarshalled) - t.Fatalf("Stdout: %s", newConfigStdout) } } From 25b46814bc0a541d211971e5974c8baff4fbbc37 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Mon, 30 Jan 2023 17:02:08 +0000 Subject: [PATCH 28/32] Only return error diagnostics for errors and warning diagnostics for warnings (#16) --- helper/resource/testing_new.go | 94 ++++++++-------- internal/plugintest/terraform_json_buffer.go | 24 +++++ .../plugintest/terraform_json_buffer_test.go | 101 ++++++++++++++++++ 3 files changed, 170 insertions(+), 49 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index 2b143b95d..b5f813656 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -265,15 +265,13 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest errorOutput := []string{err.Error()} - for _, v := range refreshTfJSONDiags { - if v.Severity == tfjson.DiagnosticSeverityError { - b, err := json.Marshal(v) - if err != nil { - t.Errorf("could not marshal tfjson diagnostic: %s", err) - } - - errorOutput = append(errorOutput, string(b)) + for _, v := range refreshTfJSONDiags.Errors() { + b, err := json.Marshal(v) + if err != nil { + t.Errorf("could not marshal tfjson diagnostic: %s", err) } + + errorOutput = append(errorOutput, string(b)) } if !errorFound && !jsonErrorFound { @@ -295,11 +293,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // represent a breaking change. var diagStrings []string - for _, v := range refreshTfJSONDiags { - if v.Severity != tfjson.DiagnosticSeverityError { - continue - } - + for _, v := range refreshTfJSONDiags.Errors() { diagStrings = append(diagStrings, "Error: "+v.Summary) diagStrings = append(diagStrings, v.Detail) } @@ -319,14 +313,12 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest errorOutput := []string{err.Error()} for _, v := range refreshTfJSONDiags { - if v.Severity == tfjson.DiagnosticSeverityError { - b, err := json.Marshal(v) - if err != nil { - t.Errorf("could not marshal tfjson diagnostic: %s", err) - } - - errorOutput = append(errorOutput, string(b)) + b, err := json.Marshal(v) + if err != nil { + t.Errorf("could not marshal tfjson diagnostic: %s", err) } + + errorOutput = append(errorOutput, string(b)) } t.Fatalf("Step %d/%d error: %s", stepNumber, len(c.Steps), strings.Join(errorOutput, "\n")) @@ -344,12 +336,18 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest map[string]interface{}{logging.KeyError: err}, ) - tfJSONDiagsMarshalled, err := json.Marshal(refreshTfJSONDiags) - if err != nil { - t.Fatalf("could not marshal tfJSON: %s", err) + var warningDiags []string + + for _, v := range refreshTfJSONDiags.Warnings() { + b, err := json.Marshal(v) + if err != nil { + t.Errorf("could not marshal tfjson diagnostic: %s", err) + } + + warningDiags = append(warningDiags, string(b)) } - t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), tfJSONDiagsMarshalled) + t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), strings.Join(warningDiags, "\n")) } } @@ -379,15 +377,13 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest errorOutput := []string{err.Error()} - for _, v := range newConfigTfJSONDiags { - if v.Severity == tfjson.DiagnosticSeverityError { - b, err := json.Marshal(v) - if err != nil { - t.Errorf("could not marshal tfjson Diagnostic: %s", err) - } - - errorOutput = append(errorOutput, string(b)) + for _, v := range newConfigTfJSONDiags.Errors() { + b, err := json.Marshal(v) + if err != nil { + t.Errorf("could not marshal tfjson Diagnostic: %s", err) } + + errorOutput = append(errorOutput, string(b)) } if !errorFound && !jsonErrorFound { @@ -409,11 +405,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest // represent a breaking change. var diagStrings []string - for _, v := range newConfigTfJSONDiags { - if v.Severity != tfjson.DiagnosticSeverityError { - continue - } - + for _, v := range newConfigTfJSONDiags.Errors() { diagStrings = append(diagStrings, "Error: "+v.Summary) diagStrings = append(diagStrings, v.Detail) } @@ -432,15 +424,13 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest errorOutput := []string{err.Error()} - for _, v := range newConfigTfJSONDiags { - if v.Severity == tfjson.DiagnosticSeverityError { - b, err := json.Marshal(v) - if err != nil { - t.Errorf("could not marshall tfjson Diagnostic: %s", err) - } - - errorOutput = append(errorOutput, string(b)) + for _, v := range newConfigTfJSONDiags.Errors() { + b, err := json.Marshal(v) + if err != nil { + t.Errorf("could not marshall tfjson Diagnostic: %s", err) } + + errorOutput = append(errorOutput, string(b)) } t.Fatalf("Step %d/%d error: %s", stepNumber, len(c.Steps), strings.Join(errorOutput, "\n")) @@ -458,12 +448,18 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest map[string]interface{}{logging.KeyError: err}, ) - tfJSONDiagsMarshalled, err := json.Marshal(newConfigTfJSONDiags) - if err != nil { - t.Fatalf("could not marshal tfJSON: %s", err) + var warningDiags []string + + for _, v := range newConfigTfJSONDiags.Warnings() { + b, err := json.Marshal(v) + if err != nil { + t.Errorf("could not marshal tfjson diagnostic: %s", err) + } + + warningDiags = append(warningDiags, string(b)) } - t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), tfJSONDiagsMarshalled) + t.Errorf("Step %d/%d, expected a warning matching pattern: %q, no match on: %s", stepNumber, len(c.Steps), step.ExpectWarning.String(), strings.Join(warningDiags, "\n")) } } diff --git a/internal/plugintest/terraform_json_buffer.go b/internal/plugintest/terraform_json_buffer.go index 6b72954fc..49af4e8ef 100644 --- a/internal/plugintest/terraform_json_buffer.go +++ b/internal/plugintest/terraform_json_buffer.go @@ -29,6 +29,30 @@ func (d TerraformJSONDiagnostics) Contains(r *regexp.Regexp, severity tfjson.Dia return false } +func (d TerraformJSONDiagnostics) Errors() TerraformJSONDiagnostics { + var tfJSONDiagnostics TerraformJSONDiagnostics + + for _, v := range d { + if v.Severity == tfjson.DiagnosticSeverityError { + tfJSONDiagnostics = append(tfJSONDiagnostics, v) + } + } + + return tfJSONDiagnostics +} + +func (d TerraformJSONDiagnostics) Warnings() TerraformJSONDiagnostics { + var tfJSONDiagnostics TerraformJSONDiagnostics + + for _, v := range d { + if v.Severity == tfjson.DiagnosticSeverityWarning { + tfJSONDiagnostics = append(tfJSONDiagnostics, v) + } + } + + return tfJSONDiagnostics +} + var _ io.Writer = &TerraformJSONBuffer{} var _ io.Reader = &TerraformJSONBuffer{} diff --git a/internal/plugintest/terraform_json_buffer_test.go b/internal/plugintest/terraform_json_buffer_test.go index 1cd6fab1f..718538e03 100644 --- a/internal/plugintest/terraform_json_buffer_test.go +++ b/internal/plugintest/terraform_json_buffer_test.go @@ -84,6 +84,107 @@ func TestTerraformJSONDiagnostics_Contains(t *testing.T) { } } +func TestTerraformJSONDiagnostics_Errors(t *testing.T) { + testCases := map[string]struct { + diags []tfjson.Diagnostic + expected TerraformJSONDiagnostics + }{ + "errors-found": { + diags: []tfjson.Diagnostic{ + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error 1 summary", + Detail: "error 1 detail", + }, + { + Severity: tfjson.DiagnosticSeverityWarning, + Summary: "warning summary", + Detail: "warning detail", + }, + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error 2 summary", + Detail: "error 2 detail", + }, + }, + expected: []tfjson.Diagnostic{ + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error 1 summary", + Detail: "error 1 detail", + }, + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error 2 summary", + Detail: "error 2 detail", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + var tfJSONDiagnostics TerraformJSONDiagnostics = testCase.diags + + actual := tfJSONDiagnostics.Errors() + + if diff := cmp.Diff(actual, testCase.expected); diff != "" { + t.Errorf("unexpected difference: %s", diff) + } + }) + } +} + +func TestTerraformJSONDiagnostics_Warnings(t *testing.T) { + testCases := map[string]struct { + diags []tfjson.Diagnostic + expected TerraformJSONDiagnostics + }{ + "warnings-found": { + diags: []tfjson.Diagnostic{ + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error 1 summary", + Detail: "error 1 detail", + }, + { + Severity: tfjson.DiagnosticSeverityWarning, + Summary: "warning summary", + Detail: "warning detail", + }, + { + Severity: tfjson.DiagnosticSeverityError, + Summary: "error 2 summary", + Detail: "error 2 detail", + }, + }, + expected: []tfjson.Diagnostic{ + { + Severity: tfjson.DiagnosticSeverityWarning, + Summary: "warning summary", + Detail: "warning detail", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + var tfJSONDiagnostics TerraformJSONDiagnostics = testCase.diags + + actual := tfJSONDiagnostics.Warnings() + + if diff := cmp.Diff(actual, testCase.expected); diff != "" { + t.Errorf("unexpected difference: %s", diff) + } + }) + } +} + func TestTerraformJSONBuffer_Parse(t *testing.T) { t.Parallel() From d27b6c2d85c6a84322abbd2a717953c8bbf2bd42 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Mon, 30 Jan 2023 17:35:32 +0000 Subject: [PATCH 29/32] Return errors rather than log and exit. Remove usage of TFJSON stdout (#16) --- helper/resource/testing_new.go | 4 +- helper/resource/testing_new_config.go | 34 --------------- helper/resource/testing_new_refresh_state.go | 7 ---- internal/plugintest/terraform_json_buffer.go | 41 ++++++++++++------- .../plugintest/terraform_json_buffer_test.go | 21 +++++++--- internal/plugintest/working_dir.go | 28 +++++++------ 6 files changed, 59 insertions(+), 76 deletions(-) diff --git a/helper/resource/testing_new.go b/helper/resource/testing_new.go index b5f813656..f8aadeb54 100644 --- a/helper/resource/testing_new.go +++ b/helper/resource/testing_new.go @@ -527,9 +527,9 @@ func testIDRefresh(ctx context.Context, t testing.T, c TestCase, wd *plugintest. // Refresh! err = runProviderCommand(ctx, t, func() error { - refreshResponse, err := wd.Refresh(ctx) + _, err := wd.Refresh(ctx) if err != nil { - t.Fatalf("Error running terraform refresh: %s", err, refreshResponse.Stdout) + t.Fatalf("Error running terraform refresh: %s", err) } state, err = getState(ctx, t, wd) if err != nil { diff --git a/helper/resource/testing_new_config.go b/helper/resource/testing_new_config.go index 92d2260ec..41ac69360 100644 --- a/helper/resource/testing_new_config.go +++ b/helper/resource/testing_new_config.go @@ -19,20 +19,17 @@ import ( type testStepNewConfigResponse struct { tfJSONDiags plugintest.TerraformJSONDiagnostics - stdout string } func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) (testStepNewConfigResponse, error) { t.Helper() var tfJSONDiags plugintest.TerraformJSONDiagnostics - var stdout string err := wd.SetConfig(ctx, step.mergedConfig(ctx, c)) if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error setting config: %w", err) } @@ -42,14 +39,12 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint refreshResponse, err := wd.Refresh(ctx) tfJSONDiags = append(tfJSONDiags, refreshResponse.Diagnostics...) - stdout += refreshResponse.Stdout return err }, wd, providers) if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error running pre-apply refresh: %w", err) } @@ -65,21 +60,18 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint createDestroyPlanResponse, err := wd.CreateDestroyPlan(ctx) tfJSONDiags = append(tfJSONDiags, createDestroyPlanResponse.Diagnostics...) - stdout += createDestroyPlanResponse.Stdout return err } createPlanResponse, err := wd.CreatePlan(ctx) tfJSONDiags = append(tfJSONDiags, createPlanResponse.Diagnostics...) - stdout += createPlanResponse.Stdout return err }, wd, providers) if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error running pre-apply plan: %w", err) } @@ -97,7 +89,6 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error retrieving pre-apply state: %w", err) } @@ -106,7 +97,6 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint applyResponse, err := wd.Apply(ctx) tfJSONDiags = append(tfJSONDiags, applyResponse.Diagnostics...) - stdout += applyResponse.Stdout return err }, wd, providers) @@ -114,13 +104,11 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if step.Destroy { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error running destroy: %w", err) } return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error running apply: %w", err) } @@ -136,7 +124,6 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error retrieving state after apply: %w", err) } @@ -149,14 +136,12 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if err := step.Check(stateBeforeApplication); err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Check failed: %w", err) } } else { if err := step.Check(state); err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Check failed: %w", err) } } @@ -172,21 +157,18 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint createDestroyPlanResponse, err := wd.CreateDestroyPlan(ctx) tfJSONDiags = append(tfJSONDiags, createDestroyPlanResponse.Diagnostics...) - stdout += createDestroyPlanResponse.Stdout return err } createPlanResponse, err := wd.CreatePlan(ctx) tfJSONDiags = append(tfJSONDiags, createPlanResponse.Diagnostics...) - stdout += createPlanResponse.Stdout return err }, wd, providers) if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error running post-apply plan: %w", err) } @@ -199,7 +181,6 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error retrieving post-apply plan: %w", err) } @@ -213,12 +194,10 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error retrieving formatted plan output: %w", err) } return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("After applying this test step, the plan was not empty.\nstdout:\n\n%s", savedPlanRawStdout) } @@ -228,14 +207,12 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint refreshResponse, err := wd.Refresh(ctx) tfJSONDiags = append(tfJSONDiags, refreshResponse.Diagnostics...) - stdout += refreshResponse.Stdout return err }, wd, providers) if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error running post-apply refresh: %w", err) } } @@ -246,21 +223,18 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint createDestroyPlanResponse, err := wd.CreateDestroyPlan(ctx) tfJSONDiags = append(tfJSONDiags, createDestroyPlanResponse.Diagnostics...) - stdout += createDestroyPlanResponse.Stdout return err } createPlanResponse, err := wd.CreatePlan(ctx) tfJSONDiags = append(tfJSONDiags, createPlanResponse.Diagnostics...) - stdout += createPlanResponse.Stdout return err }, wd, providers) if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error running second post-apply plan: %w", err) } @@ -272,7 +246,6 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error retrieving second post-apply plan: %w", err) } @@ -287,17 +260,14 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error retrieving formatted second plan output: %w", err) } return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("After applying this test step and performing a `terraform refresh`, the plan was not empty.\nstdout\n\n%s", savedPlanRawStdout) } else if step.ExpectNonEmptyPlan && planIsEmpty(plan) { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, errors.New("Expected a non-empty plan, but got an empty plan") } @@ -320,14 +290,12 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, err } if state.Empty() { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, nil } @@ -354,7 +322,6 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint if err := testIDRefresh(ctx, t, c, wd, step, idRefreshCheck, providers); err != nil { return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("[ERROR] Test: ID-only test failed: %s", err) } } @@ -362,6 +329,5 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint return testStepNewConfigResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, nil } diff --git a/helper/resource/testing_new_refresh_state.go b/helper/resource/testing_new_refresh_state.go index 908184ee9..cc08c52e5 100644 --- a/helper/resource/testing_new_refresh_state.go +++ b/helper/resource/testing_new_refresh_state.go @@ -25,7 +25,6 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo t.Helper() var tfJSONDiags plugintest.TerraformJSONDiagnostics - var stdout string var err error // Explicitly ensure prior state exists before refresh. @@ -44,14 +43,12 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo refreshResponse, err := wd.Refresh(ctx) tfJSONDiags = append(tfJSONDiags, refreshResponse.Diagnostics...) - stdout += refreshResponse.Stdout return err }, wd, providers) if err != nil { return testStepNewRefreshStateResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error running refresh: %w", err) } @@ -83,14 +80,12 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo createPlanResponse, err := wd.CreatePlan(ctx) tfJSONDiags = append(tfJSONDiags, createPlanResponse.Diagnostics...) - stdout += createPlanResponse.Stdout return err }, wd, providers) if err != nil { return testStepNewRefreshStateResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error running post-apply plan: %w", err) } @@ -103,7 +98,6 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo if err != nil { return testStepNewRefreshStateResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, fmt.Errorf("Error retrieving post-apply plan: %w", err) } @@ -128,6 +122,5 @@ func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.Wo return testStepNewRefreshStateResponse{ tfJSONDiags: tfJSONDiags, - stdout: stdout, }, nil } diff --git a/internal/plugintest/terraform_json_buffer.go b/internal/plugintest/terraform_json_buffer.go index 49af4e8ef..fdbb23cae 100644 --- a/internal/plugintest/terraform_json_buffer.go +++ b/internal/plugintest/terraform_json_buffer.go @@ -4,8 +4,8 @@ import ( "bufio" "bytes" "encoding/json" + "fmt" "io" - "log" "regexp" tfjson "github.com/hashicorp/terraform-json" @@ -75,7 +75,7 @@ func NewTerraformJSONBuffer() *TerraformJSONBuffer { func (b *TerraformJSONBuffer) Write(p []byte) (n int, err error) { if b.buf == nil { - log.Fatal("call NewTerraformJSONBuffer to initialise buffer") + return 0, fmt.Errorf("cannot write to uninitialized buffer, use NewTerraformJSONBuffer") } return b.buf.Write(p) @@ -83,15 +83,15 @@ func (b *TerraformJSONBuffer) Write(p []byte) (n int, err error) { func (b *TerraformJSONBuffer) Read(p []byte) (n int, err error) { if b.buf == nil { - log.Fatal("call NewTerraformJSONBuffer to initialise buffer") + return 0, fmt.Errorf("cannot write to uninitialized buffer, use NewTerraformJSONBuffer") } return b.buf.Read(p) } -func (b *TerraformJSONBuffer) Parse() { +func (b *TerraformJSONBuffer) Parse() error { if b.buf == nil { - log.Fatal("call NewTerraformJSONBuffer to initialise buffer") + return fmt.Errorf("cannot write to uninitialized buffer, use NewTerraformJSONBuffer") } scanner := bufio.NewScanner(b.buf) @@ -118,32 +118,43 @@ func (b *TerraformJSONBuffer) Parse() { } if err := scanner.Err(); err != nil { - log.Fatal(err) + return fmt.Errorf("error scanning buffer: %w", err) } b.parsed = true + + return nil } -func (b *TerraformJSONBuffer) Diagnostics() TerraformJSONDiagnostics { +func (b *TerraformJSONBuffer) Diagnostics() (TerraformJSONDiagnostics, error) { if !b.parsed { - b.Parse() + err := b.Parse() + if err != nil { + return nil, err + } } - return b.diagnostics + return b.diagnostics, nil } -func (b *TerraformJSONBuffer) JsonOutput() []string { +func (b *TerraformJSONBuffer) JsonOutput() ([]string, error) { if !b.parsed { - b.Parse() + err := b.Parse() + if err != nil { + return nil, err + } } - return b.jsonOutput + return b.jsonOutput, nil } -func (b *TerraformJSONBuffer) RawOutput() string { +func (b *TerraformJSONBuffer) RawOutput() (string, error) { if !b.parsed { - b.Parse() + err := b.Parse() + if err != nil { + return "", err + } } - return b.rawOutput + return b.rawOutput, nil } diff --git a/internal/plugintest/terraform_json_buffer_test.go b/internal/plugintest/terraform_json_buffer_test.go index 718538e03..85e1350e6 100644 --- a/internal/plugintest/terraform_json_buffer_test.go +++ b/internal/plugintest/terraform_json_buffer_test.go @@ -214,7 +214,10 @@ func TestTerraformJSONBuffer_Parse(t *testing.T) { t.Errorf("scanner error: %s", err) } - tfJSON.Parse() + err = tfJSON.Parse() + if err != nil { + t.Errorf("unexpected error: %s", err) + } if diff := cmp.Diff(tfJSON.jsonOutput, fileEntries); diff != "" { t.Errorf("unexpected difference: %s", diff) @@ -262,8 +265,6 @@ func TestTerraformJSONBuffer_Diagnostics(t *testing.T) { t.Errorf("scanner error: %s", err) } - tfJSON.Parse() - var tfJSONDiagnostics TerraformJSONDiagnostics = []tfjson.Diagnostic{ { Severity: tfjson.DiagnosticSeverityWarning, @@ -276,7 +277,12 @@ func TestTerraformJSONBuffer_Diagnostics(t *testing.T) { }, } - if diff := cmp.Diff(tfJSON.Diagnostics(), tfJSONDiagnostics); diff != "" { + diags, err := tfJSON.Diagnostics() + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if diff := cmp.Diff(diags, tfJSONDiagnostics); diff != "" { t.Errorf("unexpected difference: %s", diff) } } @@ -310,9 +316,12 @@ func TestTerraformJSONBuffer_JsonOutput(t *testing.T) { t.Errorf("scanner error: %s", err) } - tfJSON.Parse() + jsonOutput, err := tfJSON.JsonOutput() + if err != nil { + t.Errorf("unexpected error: %s", err) + } - if diff := cmp.Diff(tfJSON.JsonOutput(), fileEntries); diff != "" { + if diff := cmp.Diff(jsonOutput, fileEntries); diff != "" { t.Errorf("unexpected difference: %s", diff) } } diff --git a/internal/plugintest/working_dir.go b/internal/plugintest/working_dir.go index c86662b81..4ebd79ac8 100644 --- a/internal/plugintest/working_dir.go +++ b/internal/plugintest/working_dir.go @@ -173,7 +173,6 @@ func (wd *WorkingDir) planFilename() string { type CreatePlanResponse struct { Diagnostics []tfjson.Diagnostic - Stdout string } // CreatePlan runs CreatePlanJSON first and will fall back to running terraform plan if an error @@ -187,8 +186,10 @@ func (wd *WorkingDir) CreatePlan(ctx context.Context) (CreatePlanResponse, error err := wd.CreatePlanJSON(ctx, tfJSON) target := &tfexec.ErrVersionMismatch{} if !errors.As(err, &target) { - createPlanResponse.Diagnostics = tfJSON.Diagnostics() - createPlanResponse.Stdout = tfJSON.RawOutput() + createPlanResponse.Diagnostics, err = tfJSON.Diagnostics() + if err != nil { + return createPlanResponse, err + } return createPlanResponse, err } @@ -252,7 +253,6 @@ func (wd *WorkingDir) CreatePlanJSON(ctx context.Context, w io.Writer) error { type CreateDestroyPlanResponse struct { Diagnostics []tfjson.Diagnostic - Stdout string } // CreateDestroyPlan runs CreateDestroyPlanJSON first and will fall back to running terraform plan if an error @@ -266,8 +266,10 @@ func (wd *WorkingDir) CreateDestroyPlan(ctx context.Context) (CreateDestroyPlanR err := wd.CreateDestroyPlanJSON(ctx, tfJSON) target := &tfexec.ErrVersionMismatch{} if !errors.As(err, &target) { - createDestroyPlanResponse.Diagnostics = tfJSON.Diagnostics() - createDestroyPlanResponse.Stdout = tfJSON.RawOutput() + createDestroyPlanResponse.Diagnostics, err = tfJSON.Diagnostics() + if err != nil { + return createDestroyPlanResponse, err + } return createDestroyPlanResponse, err } @@ -331,7 +333,6 @@ func (wd *WorkingDir) CreateDestroyPlanJSON(ctx context.Context, w io.Writer) er type ApplyResponse struct { Diagnostics []tfjson.Diagnostic - Stdout string } // Apply runs ApplyJSON first and will fall back to running terraform apply if an error @@ -347,8 +348,10 @@ func (wd *WorkingDir) Apply(ctx context.Context) (ApplyResponse, error) { err := wd.ApplyJSON(ctx, tfJSON) target := &tfexec.ErrVersionMismatch{} if !errors.As(err, &target) { - applyResponse.Diagnostics = tfJSON.Diagnostics() - applyResponse.Stdout = tfJSON.RawOutput() + applyResponse.Diagnostics, err = tfJSON.Diagnostics() + if err != nil { + return applyResponse, err + } return applyResponse, err } @@ -487,7 +490,6 @@ func (wd *WorkingDir) Taint(ctx context.Context, address string) error { type RefreshResponse struct { Diagnostics []tfjson.Diagnostic - Stdout string } // Refresh runs RefreshJSON first and will fall back to running terraform refresh if an error @@ -500,8 +502,10 @@ func (wd *WorkingDir) Refresh(ctx context.Context) (RefreshResponse, error) { err := wd.RefreshJSON(ctx, tfJSON) target := &tfexec.ErrVersionMismatch{} if !errors.As(err, &target) { - refreshResponse.Diagnostics = tfJSON.Diagnostics() - refreshResponse.Stdout = tfJSON.RawOutput() + refreshResponse.Diagnostics, err = tfJSON.Diagnostics() + if err != nil { + return refreshResponse, err + } return refreshResponse, err } From 1369356e9a2c915698fe99588037d59173f99805 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Tue, 31 Jan 2023 07:56:37 +0000 Subject: [PATCH 30/32] Switching to using bufio.NewReader in order to be able to handle lines exceeding 64kb (#16) --- internal/plugintest/terraform_json_buffer.go | 34 ++++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/internal/plugintest/terraform_json_buffer.go b/internal/plugintest/terraform_json_buffer.go index fdbb23cae..f769ded6f 100644 --- a/internal/plugintest/terraform_json_buffer.go +++ b/internal/plugintest/terraform_json_buffer.go @@ -89,15 +89,39 @@ func (b *TerraformJSONBuffer) Read(p []byte) (n int, err error) { return b.buf.Read(p) } +func read(r *bufio.Reader) ([]byte, error) { + var ( + isPrefix = true + err error + line, ln []byte + ) + + for isPrefix && err == nil { + line, isPrefix, err = r.ReadLine() + ln = append(ln, line...) + } + + return ln, err +} + func (b *TerraformJSONBuffer) Parse() error { if b.buf == nil { return fmt.Errorf("cannot write to uninitialized buffer, use NewTerraformJSONBuffer") } - scanner := bufio.NewScanner(b.buf) + reader := bufio.NewReader(b.buf) - for scanner.Scan() { - txt := scanner.Text() + for { + line, err := read(reader) + if err != nil { + if err == io.EOF { + break + } + + return fmt.Errorf("cannot read line: %s", err) + } + + txt := string(line) b.rawOutput += "\n" + txt @@ -117,10 +141,6 @@ func (b *TerraformJSONBuffer) Parse() error { } } - if err := scanner.Err(); err != nil { - return fmt.Errorf("error scanning buffer: %w", err) - } - b.parsed = true return nil From bba27119527adcf0c2faee455be4c5af8f59e899 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Tue, 31 Jan 2023 09:19:40 +0000 Subject: [PATCH 31/32] Adding NewTerraformJSONBufferFromFile and ReadFile funcs (#16) --- internal/plugintest/terraform_json_buffer.go | 57 ++++++++-- .../plugintest/terraform_json_buffer_test.go | 105 +++++++----------- 2 files changed, 89 insertions(+), 73 deletions(-) diff --git a/internal/plugintest/terraform_json_buffer.go b/internal/plugintest/terraform_json_buffer.go index f769ded6f..005d949c5 100644 --- a/internal/plugintest/terraform_json_buffer.go +++ b/internal/plugintest/terraform_json_buffer.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "os" "regexp" tfjson "github.com/hashicorp/terraform-json" @@ -73,6 +74,24 @@ func NewTerraformJSONBuffer() *TerraformJSONBuffer { } } +func NewTerraformJSONBufferFromFile(path string) (*TerraformJSONBuffer, error) { + tfJSON := NewTerraformJSONBuffer() + + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("cannot read file: %w", err) + } + + defer file.Close() + + _, err = io.Copy(tfJSON, file) + if err != nil { + return nil, fmt.Errorf("cannot copy file contents to buffer: %w", err) + } + + return tfJSON, nil +} + func (b *TerraformJSONBuffer) Write(p []byte) (n int, err error) { if b.buf == nil { return 0, fmt.Errorf("cannot write to uninitialized buffer, use NewTerraformJSONBuffer") @@ -89,19 +108,20 @@ func (b *TerraformJSONBuffer) Read(p []byte) (n int, err error) { return b.buf.Read(p) } -func read(r *bufio.Reader) ([]byte, error) { - var ( - isPrefix = true - err error - line, ln []byte - ) +func (b *TerraformJSONBuffer) ReadFile(path string) error { + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("cannot read file: %w", err) + } - for isPrefix && err == nil { - line, isPrefix, err = r.ReadLine() - ln = append(ln, line...) + defer file.Close() + + _, err = io.Copy(b, file) + if err != nil { + return fmt.Errorf("cannot copy file contents to buffer: %w", err) } - return ln, err + return nil } func (b *TerraformJSONBuffer) Parse() error { @@ -118,7 +138,7 @@ func (b *TerraformJSONBuffer) Parse() error { break } - return fmt.Errorf("cannot read line: %s", err) + return fmt.Errorf("cannot read line during Parse: %s", err) } txt := string(line) @@ -178,3 +198,18 @@ func (b *TerraformJSONBuffer) RawOutput() (string, error) { return b.rawOutput, nil } + +func read(r *bufio.Reader) ([]byte, error) { + var ( + isPrefix = true + err error + line, ln []byte + ) + + for isPrefix && err == nil { + line, isPrefix, err = r.ReadLine() + ln = append(ln, line...) + } + + return ln, err +} diff --git a/internal/plugintest/terraform_json_buffer_test.go b/internal/plugintest/terraform_json_buffer_test.go index 85e1350e6..dafa34b34 100644 --- a/internal/plugintest/terraform_json_buffer_test.go +++ b/internal/plugintest/terraform_json_buffer_test.go @@ -2,8 +2,10 @@ package plugintest import ( "bufio" + "fmt" "os" "regexp" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -189,37 +191,26 @@ func TestTerraformJSONBuffer_Parse(t *testing.T) { t.Parallel() tfJSON := NewTerraformJSONBuffer() - - file, err := os.Open("../testdata/terraform-json-output.txt") + err := tfJSON.ReadFile("../testdata/terraform-json-output.txt") if err != nil { - t.Errorf("cannot read file: %s", err) - } - defer file.Close() - - var fileEntries []string - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - txt := scanner.Text() - - _, err := tfJSON.Write([]byte(txt + "\n")) - if err != nil { - t.Errorf("cannot write to tfJSON: %s", err) - } - - fileEntries = append(fileEntries, txt) + t.Errorf("ReadFile err: %s", err) } - if err := scanner.Err(); err != nil { - t.Errorf("scanner error: %s", err) + entries, err := fileEntries("../testdata/terraform-json-output.txt") + if err != nil { + t.Errorf("fileEntries error: %s", err) } err = tfJSON.Parse() if err != nil { - t.Errorf("unexpected error: %s", err) + t.Errorf("parse error: %s", err) } - if diff := cmp.Diff(tfJSON.jsonOutput, fileEntries); diff != "" { + if diff := cmp.Diff(strings.Split(strings.Trim(tfJSON.rawOutput, "\n"), "\n"), entries); diff != "" { + t.Errorf("unexpected difference: %s", diff) + } + + if diff := cmp.Diff(tfJSON.jsonOutput, entries); diff != "" { t.Errorf("unexpected difference: %s", diff) } @@ -243,26 +234,9 @@ func TestTerraformJSONBuffer_Parse(t *testing.T) { func TestTerraformJSONBuffer_Diagnostics(t *testing.T) { t.Parallel() - tfJSON := NewTerraformJSONBuffer() - - file, err := os.Open("../testdata/terraform-json-output.txt") + tfJSON, err := NewTerraformJSONBufferFromFile("../testdata/terraform-json-output.txt") if err != nil { - t.Errorf("cannot read file: %s", err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - txt := scanner.Text() - - _, err := tfJSON.Write([]byte(txt + "\n")) - if err != nil { - t.Errorf("cannot write to tfJSON: %s", err) - } - } - - if err := scanner.Err(); err != nil { - t.Errorf("scanner error: %s", err) + t.Errorf("NewTerraformJSONBufferFromFile err: %s", err) } var tfJSONDiagnostics TerraformJSONDiagnostics = []tfjson.Diagnostic{ @@ -279,7 +253,7 @@ func TestTerraformJSONBuffer_Diagnostics(t *testing.T) { diags, err := tfJSON.Diagnostics() if err != nil { - t.Errorf("unexpected error: %s", err) + t.Errorf("Diagnostics error: %s", err) } if diff := cmp.Diff(diags, tfJSONDiagnostics); diff != "" { @@ -292,28 +266,14 @@ func TestTerraformJSONBuffer_JsonOutput(t *testing.T) { tfJSON := NewTerraformJSONBuffer() - file, err := os.Open("../testdata/terraform-json-output.txt") + err := tfJSON.ReadFile("../testdata/terraform-json-output.txt") if err != nil { - t.Errorf("cannot read file: %s", err) - } - defer file.Close() - - var fileEntries []string - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - txt := scanner.Text() - - _, err := tfJSON.Write([]byte(txt + "\n")) - if err != nil { - t.Errorf("cannot write to tfJSON: %s", err) - } - - fileEntries = append(fileEntries, txt) + t.Errorf("ReadFile err: %s", err) } - if err := scanner.Err(); err != nil { - t.Errorf("scanner error: %s", err) + entries, err := fileEntries("../testdata/terraform-json-output.txt") + if err != nil { + t.Errorf("fileEntries error: %s", err) } jsonOutput, err := tfJSON.JsonOutput() @@ -321,7 +281,28 @@ func TestTerraformJSONBuffer_JsonOutput(t *testing.T) { t.Errorf("unexpected error: %s", err) } - if diff := cmp.Diff(jsonOutput, fileEntries); diff != "" { + if diff := cmp.Diff(jsonOutput, entries); diff != "" { t.Errorf("unexpected difference: %s", diff) } } + +func fileEntries(filePath string) ([]string, error) { + var fileEntries []string + + file, err := os.Open(filePath) + if err != nil { + return fileEntries, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + fileEntries = append(fileEntries, scanner.Text()) + } + + if err := scanner.Err(); err != nil { + return fileEntries, fmt.Errorf("scanner error: %s", err) + } + + return fileEntries, nil +} From bc8bc6de9b4bd1719f31b8b49bf1df6b6b476ab2 Mon Sep 17 00:00:00 2001 From: Benjamin Bennett Date: Tue, 31 Jan 2023 10:40:26 +0000 Subject: [PATCH 32/32] Linting (#16) --- go.sum | 8 -------- helper/resource/teststep_providers_test.go | 4 ++-- helper/resource/wait_test.go | 4 ++-- internal/plugintest/terraform_json_buffer_test.go | 10 ++++++++++ 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/go.sum b/go.sum index f8763754b..e7a873fab 100644 --- a/go.sum +++ b/go.sum @@ -93,14 +93,6 @@ github.com/hashicorp/hcl/v2 v2.15.0 h1:CPDXO6+uORPjKflkWCCwoWc9uRp+zSIPcCQ+BrxV7 github.com/hashicorp/hcl/v2 v2.15.0/go.mod h1:JRmR89jycNkrrqnMmvPDMd56n1rQJ2Q6KocSLCMCXng= github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/terraform-exec v0.17.3 h1:MX14Kvnka/oWGmIkyuyvL6POx25ZmKrjlaclkx3eErU= -github.com/hashicorp/terraform-exec v0.17.3/go.mod h1:+NELG0EqQekJzhvikkeQsOAZpsw0cv/03rbeQJqscAI= -github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98 h1:WkSxIVXfV/u9tvDGXdSw4v2eCpRMSt7mYRBseX+OaSU= -github.com/hashicorp/terraform-exec v0.17.4-0.20230110163400-c387e4f7bf98/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= -github.com/hashicorp/terraform-exec v0.17.4-0.20230116095935-bc76870e2b9b h1:z4a1oo2M/yWj2ljEoXsFQRkmp2dRqvq4Y9HjQl3eBz8= -github.com/hashicorp/terraform-exec v0.17.4-0.20230116095935-bc76870e2b9b/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= -github.com/hashicorp/terraform-exec v0.17.4-0.20230120174504-9acd44a7e5da h1:St2EWMkwfc56l9PqejkEV/r4S55Em9iXYvW2qdtyH+4= -github.com/hashicorp/terraform-exec v0.17.4-0.20230120174504-9acd44a7e5da/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= github.com/hashicorp/terraform-exec v0.17.4-0.20230124151948-5eada031c166 h1:WCvC+VknMywOyDXTIzvHHYWknjNBKI0/dmIR9EZd0bw= github.com/hashicorp/terraform-exec v0.17.4-0.20230124151948-5eada031c166/go.mod h1:5M9hP3RX39/2AWDpAEWf0whTJFibOUN7DfH1E1lCZN8= github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s= diff --git a/helper/resource/teststep_providers_test.go b/helper/resource/teststep_providers_test.go index 53823d9fe..affa876d3 100644 --- a/helper/resource/teststep_providers_test.go +++ b/helper/resource/teststep_providers_test.go @@ -8,7 +8,7 @@ import ( "fmt" "os" "path/filepath" - "regexp" + "regexp" "strconv" "strings" "testing" @@ -28,8 +28,8 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" - "github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider" "github.com/hashicorp/terraform-plugin-testing/internal/plugintest" + "github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider" "github.com/hashicorp/terraform-plugin-testing/terraform" ) diff --git a/helper/resource/wait_test.go b/helper/resource/wait_test.go index 07f523ad4..fbb66cf72 100644 --- a/helper/resource/wait_test.go +++ b/helper/resource/wait_test.go @@ -30,9 +30,9 @@ func TestRetry(t *testing.T) { } // make sure a slow StateRefreshFunc is allowed to complete after timeout +// +//nolint:paralleltest // Test fails when running using t.Parallel() func TestRetry_grace(t *testing.T) { - t.Parallel() - f := func() *RetryError { time.Sleep(1 * time.Second) return nil diff --git a/internal/plugintest/terraform_json_buffer_test.go b/internal/plugintest/terraform_json_buffer_test.go index dafa34b34..a1c8a95a0 100644 --- a/internal/plugintest/terraform_json_buffer_test.go +++ b/internal/plugintest/terraform_json_buffer_test.go @@ -75,6 +75,8 @@ func TestTerraformJSONDiagnostics_Contains(t *testing.T) { name, testCase := name, testCase t.Run(name, func(t *testing.T) { + t.Parallel() + var tfJSONDiagnostics TerraformJSONDiagnostics = testCase.diags isFound := tfJSONDiagnostics.Contains(testCase.regex, testCase.severity) @@ -87,6 +89,8 @@ func TestTerraformJSONDiagnostics_Contains(t *testing.T) { } func TestTerraformJSONDiagnostics_Errors(t *testing.T) { + t.Parallel() + testCases := map[string]struct { diags []tfjson.Diagnostic expected TerraformJSONDiagnostics @@ -128,6 +132,8 @@ func TestTerraformJSONDiagnostics_Errors(t *testing.T) { name, testCase := name, testCase t.Run(name, func(t *testing.T) { + t.Parallel() + var tfJSONDiagnostics TerraformJSONDiagnostics = testCase.diags actual := tfJSONDiagnostics.Errors() @@ -140,6 +146,8 @@ func TestTerraformJSONDiagnostics_Errors(t *testing.T) { } func TestTerraformJSONDiagnostics_Warnings(t *testing.T) { + t.Parallel() + testCases := map[string]struct { diags []tfjson.Diagnostic expected TerraformJSONDiagnostics @@ -176,6 +184,8 @@ func TestTerraformJSONDiagnostics_Warnings(t *testing.T) { name, testCase := name, testCase t.Run(name, func(t *testing.T) { + t.Parallel() + var tfJSONDiagnostics TerraformJSONDiagnostics = testCase.diags actual := tfJSONDiagnostics.Warnings()