Skip to content

Commit

Permalink
Use crosstestimpl.T instead of testing.T in all PF crosstests (#2584)
Browse files Browse the repository at this point in the history 
Redoing #2583 on
top of the existing PR stack in the area.
The `crosstestimpl.T` interface is compatible with both `testing.T` and
`rapid.T`, allowing us to use cross-tests in rapid property-based tests.
  • Loading branch information
@VenelinMartinov
VenelinMartinov authored Nov 5, 2024
1 parent b278128 commit a7af6e8
Show file tree
Hide file tree
Showing 8 changed files with 1,552 additions and 116 deletions.
6 changes: 6 additions & 0 deletions dynamic/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,16 @@ require (

require (
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/hashicorp/logutils v1.0.0 // indirect
github.com/hashicorp/terraform-plugin-sdk/v2 v2.33.0 // indirect
github.com/nxadm/tail v1.4.11 // indirect
github.com/pulumi/terraform-diff-reader v0.0.2 // indirect
github.com/teekennedy/goldmark-markdown v0.3.0 // indirect
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
github.com/yuin/goldmark v1.7.4 // indirect
google.golang.org/appengine v1.6.8 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gotest.tools v2.2.0+incompatible // indirect
)

require (
Expand Down
1,465 changes: 1,465 additions & 0 deletions dynamic/go.sum
View file

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion pkg/internal/tests/cross-tests/impl/diff.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ func GetPulumiDiffResponse(t T, entries []grpclog.GrpcLogEntry) PulumiDiffResp {
diffResponse := PulumiDiffResp{}
found := false
for _, entry := range entries {
t.Logf("entry.Method: %s", entry.Method)
if entry.Method == "/pulumirpc.ResourceProvider/Diff" {
require.False(t, found, "expected to find only one Diff entry in the gRPC log")
err := json.Unmarshal(entry.Response, &diffResponse)
Expand Down
4 changes: 3 additions & 1 deletion pkg/pf/tests/diff_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ func TestSimpleNoDiff(t *testing.T) {
}

res := crosstests.Diff(t, sch,
map[string]cty.Value{"key": cty.StringVal("value")}, map[string]cty.Value{"key": cty.StringVal("value1")})
map[string]cty.Value{"key": cty.StringVal("value")},
map[string]cty.Value{"key": cty.StringVal("value1")},
)

autogold.Expect(`
Terraform used the selected providers to generate the following execution
Expand Down
37 changes: 14 additions & 23 deletions pkg/pf/tests/internal/cross-tests/configure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,20 +82,17 @@ func MakeConfigure(schema schema.Schema, tfConfig map[string]cty.Value, options
// +--------------------+ +---------------------+
//
// Configure should be safe to run in parallel.
func Configure(t *testing.T, schema schema.Schema, tfConfig map[string]cty.Value, options ...ConfigureOption) {
func Configure(t T, schema schema.Schema, tfConfig map[string]cty.Value, options ...ConfigureOption) {
skipUnlessLinux(t)

var opts configureOpts
for _, f := range options {
f(&opts)
}

// By default, logs only show when they are on a failed test. By logging to
// topLevelT, we can log items to be shown if downstream tests fail.
topLevelT := t
const providerName = "test"

prov := func(config *tfsdk.Config) *pb.Provider {
provbuilder := func(config *tfsdk.Config) *pb.Provider {
return pb.NewProvider(pb.NewProviderArgs{
TypeName: providerName,
ProviderSchema: schema,
Expand All @@ -109,8 +106,8 @@ func Configure(t *testing.T, schema schema.Schema, tfConfig map[string]cty.Value
}

var tfOutput, puOutput tfsdk.Config
t.Run("tf", func(t *testing.T) {
defer propagateSkip(topLevelT, t)
// Run the TF part
{
var hcl bytes.Buffer
err := writeProvider(&hcl, schema, providerName, tfConfig)
require.NoError(t, err)
Expand All @@ -121,26 +118,26 @@ func Configure(t *testing.T, schema schema.Schema, tfConfig map[string]cty.Value
resource "` + providerName + `_res" "res" {}
`)

prov := prov(&tfOutput)
prov := provbuilder(&tfOutput)
driver := tfcheck.NewTfDriver(t, t.TempDir(), prov.TypeName, prov)

driver.Write(t, hcl.String())
plan, err := driver.Plan(t)
require.NoError(t, err)
err = driver.Apply(t, plan)
require.NoError(t, err)
})
}

t.Run("bridged", func(t *testing.T) {
defer propagateSkip(topLevelT, t)
// Run the Pulumi part
{
dir := t.TempDir()

var puConfig resource.PropertyMap
if opts.puConfig != nil {
puConfig = *opts.puConfig
} else {
puConfig = crosstestsimpl.InferPulumiValue(t,
tfbridge.ShimProvider(prov(nil)).Schema(),
tfbridge.ShimProvider(provbuilder(nil)).Schema(),
opts.resourceInfo,
cty.ObjectVal(tfConfig),
)
Expand All @@ -162,16 +159,16 @@ resource "` + providerName + `_res" "res" {}

bytes, err := yaml.Marshal(pulumiYaml)
require.NoError(t, err)
topLevelT.Logf("Pulumi.yaml:\n%s", string(bytes))
err = os.WriteFile(filepath.Join(dir, "Pulumi.yaml"), bytes, 0600)
t.Logf("Pulumi.yaml:\n%s", string(bytes))
err = os.WriteFile(filepath.Join(dir, "Pulumi.yaml"), bytes, 0o600)
require.NoError(t, err)

makeProvider := func(providers.PulumiTest) (pulumirpc.ResourceProviderServer, error) {
ctx, sink := context.Background(), testLogSink{t}

p := info.Provider{
Name: providerName,
P: tfbridge.ShimProvider(prov(&puOutput)),
P: tfbridge.ShimProvider(provbuilder(&puOutput)),
Version: "0.1.0-dev",
UpstreamRepoPath: ".",
Config: opts.resourceInfo,
Expand Down Expand Up @@ -204,15 +201,9 @@ resource "` + providerName + `_res" "res" {}
)
contract.Ignore(test.Preview(t)) // Assert that the preview succeeded, but not the result.
contract.Ignore(test.Up(t)) // Assert that the update succeeded, but not the result.
})
}

skipCompare := t.Failed() || t.Skipped()
t.Run("compare", func(t *testing.T) {
if skipCompare {
t.Skipf("skipping since earlier steps did not complete")
}
assert.Equal(t, tfOutput, puOutput)
})
assert.Equal(t, tfOutput, puOutput)
}

type configureOpts struct {
Expand Down
124 changes: 49 additions & 75 deletions pkg/pf/tests/internal/cross-tests/diff.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package crosstests

import (
"bytes"
"testing"

rschema "github.com/hashicorp/terraform-plugin-framework/resource/schema"
crosstests "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/tests/cross-tests"
Expand All @@ -26,14 +25,13 @@ import (
"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/tfbridge"
"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tests/tfcheck"
"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info"
"github.com/pulumi/pulumi/sdk/v3/go/auto"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
"github.com/stretchr/testify/require"
"github.com/zclconf/go-cty/cty"
"gopkg.in/yaml.v3"
)

func yamlResource(t *testing.T, properties resource.PropertyMap) map[string]any {
func yamlResource(t T, properties resource.PropertyMap) map[string]any {
return map[string]any{
"name": "project",
"runtime": "yaml",
Expand All @@ -50,18 +48,14 @@ func yamlResource(t *testing.T, properties resource.PropertyMap) map[string]any
// when computed by Terraform and Pulumi.
//
// Diff should be safe to run in parallel.
func Diff(t *testing.T, schema rschema.Schema, tfConfig1, tfConfig2 map[string]cty.Value, options ...DiffOption) crosstestsimpl.DiffResult {
func Diff(t T, schema rschema.Schema, tfConfig1, tfConfig2 map[string]cty.Value, options ...DiffOption) crosstestsimpl.DiffResult {
skipUnlessLinux(t)

var opts diffOpts
for _, f := range options {
f(&opts)
}

// By default, logs only show when they are on a failed test. By logging to
// topLevelT, we can log items to be shown if downstream tests fail.
topLevelT := t

prov := pb.NewProvider(pb.NewProviderArgs{
AllResources: []pb.Resource{{
Name: "test",
Expand All @@ -71,88 +65,68 @@ func Diff(t *testing.T, schema rschema.Schema, tfConfig1, tfConfig2 map[string]c

shimProvider := tfbridge.ShimProvider(prov)

var tfOut string
var pulumiOut string
var tfChanges tfcheck.TFChange
var pulumiRes auto.UpResult
var diffResponse crosstestsimpl.PulumiDiffResp
t.Run("tf", func(t *testing.T) {
defer propagateSkip(topLevelT, t)
var hcl1 bytes.Buffer

err := writeResource(&hcl1, schema, "testprovider_test", "res", tfConfig1)
require.NoError(t, err)

driver := tfcheck.NewTfDriver(t, t.TempDir(), prov.TypeName, prov)

driver.Write(t, hcl1.String())
plan, err := driver.Plan(t)
require.NoError(t, err)
err = driver.Apply(t, plan)
require.NoError(t, err)

var hcl2 bytes.Buffer
err = writeResource(&hcl2, schema, "testprovider_test", "res", tfConfig2)
require.NoError(t, err)
driver.Write(t, hcl2.String())
plan, err = driver.Plan(t)
require.NoError(t, err)
tfChanges = driver.ParseChangesFromTFPlan(plan)
tfOut = plan.StdOut
})
// Run the TF part
var hcl1 bytes.Buffer

t.Run("bridged", func(t *testing.T) {
defer propagateSkip(topLevelT, t)
err := writeResource(&hcl1, schema, "testprovider_test", "res", tfConfig1)
require.NoError(t, err)

puConfig1 := crosstestsimpl.InferPulumiValue(t,
shimProvider.ResourcesMap().Get("testprovider_test").Schema(),
opts.resourceInfo,
cty.ObjectVal(tfConfig1),
)
pulumiYaml1 := yamlResource(t, puConfig1)
driver := tfcheck.NewTfDriver(t, t.TempDir(), prov.TypeName, prov)

puConfig2 := crosstestsimpl.InferPulumiValue(t,
shimProvider.ResourcesMap().Get("testprovider_test").Schema(),
opts.resourceInfo,
cty.ObjectVal(tfConfig2),
)
pulumiYaml2 := yamlResource(t, puConfig2)
driver.Write(t, hcl1.String())
plan, err := driver.Plan(t)
require.NoError(t, err)
err = driver.Apply(t, plan)
require.NoError(t, err)

bytes, err := yaml.Marshal(pulumiYaml1)
require.NoError(t, err)
topLevelT.Logf("Pulumi.yaml:\n%s", string(bytes))
var hcl2 bytes.Buffer
err = writeResource(&hcl2, schema, "testprovider_test", "res", tfConfig2)
require.NoError(t, err)
driver.Write(t, hcl2.String())
plan, err = driver.Plan(t)
require.NoError(t, err)
tfChanges := driver.ParseChangesFromTFPlan(plan)

pt, err := pulcheck.PulCheck(t, bridgedProvider(prov), string(bytes))
require.NoError(t, err)
pt.Up(t)
// Run the Pulumi part

bytes, err = yaml.Marshal(pulumiYaml2)
require.NoError(t, err)
topLevelT.Logf("Pulumi.yaml:\n%s", string(bytes))
pt.WritePulumiYaml(t, string(bytes))
puConfig1 := crosstestsimpl.InferPulumiValue(t,
shimProvider.ResourcesMap().Get("testprovider_test").Schema(),
opts.resourceInfo,
cty.ObjectVal(tfConfig1),
)
pulumiYaml1 := yamlResource(t, puConfig1)

previewRes := pt.Preview(t)
pulumiOut = previewRes.StdOut
puConfig2 := crosstestsimpl.InferPulumiValue(t,
shimProvider.ResourcesMap().Get("testprovider_test").Schema(),
opts.resourceInfo,
cty.ObjectVal(tfConfig2),
)
pulumiYaml2 := yamlResource(t, puConfig2)

pulumiRes = pt.Up(t)
bytes, err := yaml.Marshal(pulumiYaml1)
require.NoError(t, err)
t.Logf("Pulumi.yaml:\n%s", string(bytes))

diffResponse = crosstestsimpl.GetPulumiDiffResponse(t, pt.GrpcLog(t).Entries)
})
pt, err := pulcheck.PulCheck(t, bridgedProvider(prov), string(bytes))
require.NoError(t, err)
pt.Up(t)

skipCompare := t.Failed() || t.Skipped()
t.Run("compare", func(t *testing.T) {
if skipCompare {
t.Skipf("skipping since earlier steps did not complete")
}
bytes, err = yaml.Marshal(pulumiYaml2)
require.NoError(t, err)
t.Logf("Pulumi.yaml:\n%s", string(bytes))
pt.WritePulumiYaml(t, string(bytes))

crosstestsimpl.VerifyBasicDiffAgreement(t, tfChanges.Actions, pulumiRes.Summary, diffResponse)
})
previewRes := pt.Preview(t)
pulumiRes := pt.Up(t)
diffResponse := crosstestsimpl.GetPulumiDiffResponse(t, pt.GrpcLog(t).Entries)

crosstestsimpl.VerifyBasicDiffAgreement(t, tfChanges.Actions, pulumiRes.Summary, diffResponse)

return crosstestsimpl.DiffResult{
TFDiff: tfChanges,
PulumiDiff: diffResponse,
TFOut: tfOut,
PulumiOut: pulumiOut,
TFOut: plan.StdOut,
PulumiOut: previewRes.StdOut,
}
}

Expand Down
15 changes: 6 additions & 9 deletions pkg/pf/tests/internal/cross-tests/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ import (
"os"
"runtime"
"strings"
"testing"

crosstestsimpl "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/tests/cross-tests/impl"
"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/tests/internal/providerbuilder"
"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/tfbridge"
tfbridge0 "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge"
Expand All @@ -30,17 +30,14 @@ import (
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
)

func propagateSkip(parent, child *testing.T) {
if child.Skipped() {
parent.Skipf("skipping due to skipped child test")
}
}
type T = crosstestsimpl.T

type testLogSink struct{ t *testing.T }
type testLogSink struct{ t T }

func (s testLogSink) Log(_ context.Context, sev diag.Severity, urn resource.URN, msg string) error {
return s.log("LOG", sev, urn, msg)
}

func (s testLogSink) LogStatus(_ context.Context, sev diag.Severity, urn resource.URN, msg string) error {
return s.log("STATUS", sev, urn, msg)
}
Expand All @@ -54,7 +51,7 @@ func (s testLogSink) log(kind string, sev diag.Severity, urn resource.URN, msg s
return nil
}

func skipUnlessLinux(t *testing.T) {
func skipUnlessLinux(t T) {
if ci, ok := os.LookupEnv("CI"); ok && ci == "true" && !strings.Contains(strings.ToLower(runtime.GOOS), "linux") {
t.Skip("Skipping on non-Linux platforms as our CI does not yet install Terraform CLI required for these tests")
}
Expand All @@ -73,4 +70,4 @@ func bridgedProvider(prov *providerbuilder.Provider) info.Provider {
provider.MustComputeTokens(tokens.SingleModule(prov.TypeName, "index", tokens.MakeStandard(prov.TypeName)))

return provider
}
}
Loading

0 comments on commit a7af6e8

Please sign in to comment.