From 54bb19ffd79865436a82eced45664faaaf0e9775 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 10:31:04 -0400 Subject: [PATCH 01/29] Start the unrec package --- pkg/tfgen/internal/unrec/package.go | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 pkg/tfgen/internal/unrec/package.go diff --git a/pkg/tfgen/internal/unrec/package.go b/pkg/tfgen/internal/unrec/package.go new file mode 100644 index 000000000..df46f8a91 --- /dev/null +++ b/pkg/tfgen/internal/unrec/package.go @@ -0,0 +1,11 @@ +// Package unrec implements recursion detection over [Pulumi Package Schema]. +// +// The implementation is generic to all providers but the intent is tailored for bridged providers specifically. TF +// cannot represent recursive types but uses unrolling up to level N, for example for Statement in waf2 web_acl in AWS +// it uses unrolling to level 3. This creates too many types in the Pulumi projection. +// +// This package is aimed at detecting the unrolled recursion and tying it back into recursive types. +// +// [Pulumi Package Schema]: https://www.pulumi.com/docs/guides/pulumi-packages/schema/ +package unrec + From bb0f5d9908b8746abdaeac422e23ee86e09be06b Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 10:34:17 -0400 Subject: [PATCH 02/29] Add equality_classes utility --- pkg/tfgen/internal/unrec/equality_classes.go | 68 +++++++++++++++++++ .../internal/unrec/equality_classes_test.go | 55 +++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 pkg/tfgen/internal/unrec/equality_classes.go create mode 100644 pkg/tfgen/internal/unrec/equality_classes_test.go diff --git a/pkg/tfgen/internal/unrec/equality_classes.go b/pkg/tfgen/internal/unrec/equality_classes.go new file mode 100644 index 000000000..91274220e --- /dev/null +++ b/pkg/tfgen/internal/unrec/equality_classes.go @@ -0,0 +1,68 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +func equalityClasses[T any](eq func(T, T) bool, slice []T) [][]T { + switch len(slice) { + case 0: + return nil + case 1: + return [][]T{slice} + case 2: + a, b := slice[0], slice[1] + if eq(a, b) { + return [][]T{slice} + } + return [][]T{{a}, {b}} + default: + n := len(slice) + left := slice[0 : n/2] + right := slice[n/2:] + part1 := equalityClasses(eq, left) + part2 := equalityClasses(eq, right) + return mergeEqualityClasses(eq, part1, part2) + } +} + +func sameEqualityClasses[T any](eq func(T, T) bool, xs []T, ys []T) bool { + if len(xs) == 0 || len(ys) == 0 { + return len(xs) == len(ys) + } + return eq(xs[0], ys[0]) +} + +func mergeEqualityClasses[T any](eq func(T, T) bool, xs [][]T, ys [][]T) (result [][]T) { + acc := [][]T{} + for _, xc := range xs { + if len(xc) == 0 { + continue + } + acc = append(acc, xc) + } + for _, yc := range ys { + matched := false + for i, ac := range acc { + if sameEqualityClasses(eq, ac, yc) { + acc[i] = append(append([]T{}, ac...), yc...) + matched = true + break + } + } + if !matched { + acc = append(acc, yc) + } + } + return acc +} diff --git a/pkg/tfgen/internal/unrec/equality_classes_test.go b/pkg/tfgen/internal/unrec/equality_classes_test.go new file mode 100644 index 000000000..08196e80f --- /dev/null +++ b/pkg/tfgen/internal/unrec/equality_classes_test.go @@ -0,0 +1,55 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "testing" + + "pgregory.net/rapid" +) + +func TestEqualityClasses(t *testing.T) { + eq := func(a int, b int) bool { + return a%3 == b%3 + } + + rapid.Check(t, func(t *rapid.T) { + s := rapid.SliceOf(rapid.Int()).Draw(t, "s") + actual := equalityClasses(eq, s) + t.Logf("actual %+v", actual) + for i, c1 := range actual { + for j, c2 := range actual { + if i == j { + continue + } + if sameEqualityClasses(eq, c1, c2) { + t.Fatalf("found same equality classes in the result") + } + } + } + for _, c := range actual { + for i, e1 := range c { + for j, e2 := range c { + if i == j { + continue + } + if !eq(e1, e2) { + t.Fatalf("found entries in the same eq class") + } + } + } + } + }) +} From f2c736c483b3349f8cb18ab2a079410ce4430880 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 10:46:58 -0400 Subject: [PATCH 03/29] Util for parsing tokens --- pkg/tfgen/internal/unrec/package_spec_util.go | 29 +++++++++++ .../internal/unrec/package_spec_util_test.go | 48 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 pkg/tfgen/internal/unrec/package_spec_util.go create mode 100644 pkg/tfgen/internal/unrec/package_spec_util_test.go diff --git a/pkg/tfgen/internal/unrec/package_spec_util.go b/pkg/tfgen/internal/unrec/package_spec_util.go new file mode 100644 index 000000000..477b731f6 --- /dev/null +++ b/pkg/tfgen/internal/unrec/package_spec_util.go @@ -0,0 +1,29 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "strings" + + "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" +) + +func parseLocalRef(rawRef string) (tokens.Type, bool) { + if !strings.HasPrefix(rawRef, "#/types/") { + return "", false + } + cleanRef := strings.TrimPrefix(rawRef, "#/types/") + return tokens.Type(cleanRef), true +} diff --git a/pkg/tfgen/internal/unrec/package_spec_util_test.go b/pkg/tfgen/internal/unrec/package_spec_util_test.go new file mode 100644 index 000000000..0d769eb15 --- /dev/null +++ b/pkg/tfgen/internal/unrec/package_spec_util_test.go @@ -0,0 +1,48 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "testing" + + "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" + "github.com/stretchr/testify/require" +) + +func TestParseLocalRef(t *testing.T) { + type testCase struct { + ref string + expect tokens.Type + } + testCases := []testCase{ + { + "#/types/aws:opsworks/RailsAppLayerCloudwatchConfiguration:RailsAppLayerCloudwatchConfiguration", + "aws:opsworks/RailsAppLayerCloudwatchConfiguration:RailsAppLayerCloudwatchConfiguration", + }, + { + "pulumi.json#/Asset", + "", + }, + } + for _, tc := range testCases { + tok, ok := parseLocalRef(tc.ref) + if tc.expect != "" { + require.True(t, ok) + require.Equal(t, tc.expect, tok) + } else { + require.False(t, ok) + } + } +} From 01ccdac7b5ae0e4aa624b1d5eb1d0dcfbd31402c Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 11:05:11 -0400 Subject: [PATCH 04/29] Add an example schema --- .../internal/unrec/example_schema_test.go | 38 +++ pkg/tfgen/internal/unrec/gen.go | 189 +++++++++++ .../internal/unrec/testdata/test-schema.json | 320 ++++++++++++++++++ 3 files changed, 547 insertions(+) create mode 100644 pkg/tfgen/internal/unrec/example_schema_test.go create mode 100644 pkg/tfgen/internal/unrec/gen.go create mode 100644 pkg/tfgen/internal/unrec/testdata/test-schema.json diff --git a/pkg/tfgen/internal/unrec/example_schema_test.go b/pkg/tfgen/internal/unrec/example_schema_test.go new file mode 100644 index 000000000..313e29c52 --- /dev/null +++ b/pkg/tfgen/internal/unrec/example_schema_test.go @@ -0,0 +1,38 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "os" + "path/filepath" + "testing" + + "encoding/json" + pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" +) + +//go:generate go run gen.go +func exampleSchema(t *testing.T) *pschema.PackageSpec { + b, err := os.ReadFile(filepath.Join("testdata", "test-schema.json")) + if err != nil { + t.Fatal(err) + } + var r pschema.PackageSpec + err = json.Unmarshal(b, &r) + if err != nil { + t.Fatal(err) + } + return &r +} diff --git a/pkg/tfgen/internal/unrec/gen.go b/pkg/tfgen/internal/unrec/gen.go new file mode 100644 index 000000000..ccc5a3d07 --- /dev/null +++ b/pkg/tfgen/internal/unrec/gen.go @@ -0,0 +1,189 @@ +//go:build ignore + +package main + +import ( + "encoding/json" + "io" + "log" + "os" + "path/filepath" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfgen" + shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/sdk-v2" + pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" + "github.com/pulumi/pulumi/sdk/v3/go/common/diag" + "github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors" +) + +func main() { + schema, err := exampleSchema() + if err != nil { + log.Fatal(err) + } + bytes, err := json.MarshalIndent(schema, "", " ") + if err != nil { + log.Fatal(err) + } + p := filepath.Join("testdata", "test-schema.json") + err = os.WriteFile(p, bytes, 0600) + if err != nil { + log.Fatal(err) + } + log.Println("Wrote", p) +} + +func exampleSchema() (pschema.PackageSpec, error) { + return tfgen.GenerateSchema(info.Provider{ + P: shim.NewProvider( + &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "web_acl": { + Schema: map[string]*schema.Schema{ + "statement": webACLRootStatementSchema(3), + }, + }, + }, + }, + ), + Name: "myprov", + Resources: map[string]*info.Resource{ + "web_acl": { + Tok: "myprov:index:WebAcl", + }, + }, + }, diag.DefaultSink(io.Discard, io.Discard, diag.FormatOptions{Color: colors.Never})) +} + +func webACLRootStatementSchema(level int) *schema.Schema { + return &schema.Schema{ + Type: schema.TypeList, + Required: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "and_statement": statementSchema(level), + "rate_based_statement": rateBasedStatementSchema(level), + }, + }, + } +} + +func rateBasedStatementSchema(level int) *schema.Schema { + return &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "aggregate_key_type": { + Type: schema.TypeString, + Optional: true, + }, + "scope_down_statement": scopeDownStatementSchema(level - 1), + }, + }, + } +} + +func scopeDownStatementSchema(level int) *schema.Schema { + return &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "and_statement": statementSchema(level), + "xss_match_statement": xssMatchStatementSchema(), + }, + }, + } +} + +func statementSchema(level int) *schema.Schema { + if level > 1 { + return &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "statement": { + Type: schema.TypeList, + Required: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "and_statement": statementSchema(level - 1), + "xss_match_statement": xssMatchStatementSchema(), + }, + }, + }, + }, + }, + } + } + + return &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "statement": { + Type: schema.TypeList, + Required: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "xss_match_statement": xssMatchStatementSchema(), + }, + }, + }, + }, + }, + } +} + +func xssMatchStatementSchema() *schema.Schema { + return &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "field_to_match": fieldToMatchSchema(), + }, + }, + } +} + +func fieldToMatchSchema() *schema.Schema { + return &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: fieldToMatchBaseSchema(), + } +} + +var listOfEmptyObjectSchema *schema.Schema = &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{}, + }, +} + +func emptySchema() *schema.Schema { + return listOfEmptyObjectSchema +} + +func fieldToMatchBaseSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "method": emptySchema(), + }, + } +} diff --git a/pkg/tfgen/internal/unrec/testdata/test-schema.json b/pkg/tfgen/internal/unrec/testdata/test-schema.json new file mode 100644 index 000000000..96dbe7cef --- /dev/null +++ b/pkg/tfgen/internal/unrec/testdata/test-schema.json @@ -0,0 +1,320 @@ +{ + "name": "myprov", + "attribution": "This Pulumi package is based on the [`myprov` Terraform Provider](https://github.com/terraform-providers/terraform-provider-myprov).", + "meta": { + "moduleFormat": "(.*)(?:/[^/]*)" + }, + "language": { + "nodejs": { + "readme": "\u003e This provider is a derived work of the [Terraform Provider](https://github.com/terraform-providers/terraform-provider-myprov)\n\u003e distributed under [MPL 2.0](https://www.mozilla.org/en-US/MPL/2.0/). If you encounter a bug or missing feature,\n\u003e first check the [`pulumi-myprov` repo](/issues); however, if that doesn't turn up anything,\n\u003e please consult the source [`terraform-provider-myprov` repo](https://github.com/terraform-providers/terraform-provider-myprov/issues).", + "compatibility": "tfbridge20", + "disableUnionOutputTypes": true + }, + "python": { + "readme": "\u003e This provider is a derived work of the [Terraform Provider](https://github.com/terraform-providers/terraform-provider-myprov)\n\u003e distributed under [MPL 2.0](https://www.mozilla.org/en-US/MPL/2.0/). If you encounter a bug or missing feature,\n\u003e first check the [`pulumi-myprov` repo](/issues); however, if that doesn't turn up anything,\n\u003e please consult the source [`terraform-provider-myprov` repo](https://github.com/terraform-providers/terraform-provider-myprov/issues).", + "compatibility": "tfbridge20", + "pyproject": {} + } + }, + "config": {}, + "types": { + "myprov:index/WebAclStatement:WebAclStatement": { + "properties": { + "andStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement" + }, + "rateBasedStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement": { + "properties": { + "statements": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement" + } + } + }, + "type": "object", + "required": [ + "statements" + ] + }, + "myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement": { + "properties": { + "andStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement" + }, + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement": { + "properties": { + "statements": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement" + } + } + }, + "type": "object", + "required": [ + "statements" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement": { + "properties": { + "andStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement" + }, + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement": { + "properties": { + "statements": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement" + } + } + }, + "type": "object", + "required": [ + "statements" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement": { + "properties": { + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement": { + "properties": { + "aggregateKeyType": { + "type": "string" + }, + "scopeDownStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement": { + "properties": { + "andStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement" + }, + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement": { + "properties": { + "statements": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement" + } + } + }, + "type": "object", + "required": [ + "statements" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement": { + "properties": { + "andStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement" + }, + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement": { + "properties": { + "statements": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement" + } + } + }, + "type": "object", + "required": [ + "statements" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement": { + "properties": { + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + } + }, + "provider": { + "description": "The provider type for the myprov package. By default, resources use package-wide configuration\nsettings, however an explicit `Provider` instance may be created and passed during resource\nconstruction to achieve fine-grained programmatic control over provider settings. See the\n[documentation](https://www.pulumi.com/docs/reference/programming-model/#providers) for more information.\n" + }, + "resources": { + "myprov:index:WebAcl": { + "properties": { + "statement": { + "$ref": "#/types/myprov:index/WebAclStatement:WebAclStatement" + } + }, + "required": [ + "statement" + ], + "inputProperties": { + "statement": { + "$ref": "#/types/myprov:index/WebAclStatement:WebAclStatement" + } + }, + "requiredInputs": [ + "statement" + ], + "stateInputs": { + "description": "Input properties used for looking up and filtering WebAcl resources.\n", + "properties": { + "statement": { + "$ref": "#/types/myprov:index/WebAclStatement:WebAclStatement" + } + }, + "type": "object" + } + } + } +} \ No newline at end of file From 940d1219e52b443cbca6399bda6a37a803e82f6c Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 11:12:49 -0400 Subject: [PATCH 05/29] Add a type visitor --- pkg/tfgen/internal/unrec/type_visitor.go | 106 ++++++++++++++++++ pkg/tfgen/internal/unrec/type_visitor_test.go | 57 ++++++++++ 2 files changed, 163 insertions(+) create mode 100644 pkg/tfgen/internal/unrec/type_visitor.go create mode 100644 pkg/tfgen/internal/unrec/type_visitor_test.go diff --git a/pkg/tfgen/internal/unrec/type_visitor.go b/pkg/tfgen/internal/unrec/type_visitor.go new file mode 100644 index 000000000..fedbc091c --- /dev/null +++ b/pkg/tfgen/internal/unrec/type_visitor.go @@ -0,0 +1,106 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "slices" + + pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" + "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" +) + +type typeVisitor struct { + Schema *pschema.PackageSpec + Visit func(ancestors []tokens.Type, current tokens.Type) + + parent *typeVisitor // internal + visiting tokens.Type // internal + seen map[tokens.Type]struct{} // internal +} + +func (tv *typeVisitor) VisitRoots(roots []tokens.Type) { + for _, ty := range roots { + tv.visitLocalType(ty) + } +} + +func (tv *typeVisitor) push(ty tokens.Type) *typeVisitor { + copy := *tv + copy.parent = tv + copy.visiting = ty + return © +} + +func (tv *typeVisitor) ancestors() []tokens.Type { + var result []tokens.Type + x := tv + for x != nil { + if x.visiting != "" { + result = append(result, x.visiting) + } + x = x.parent + } + slices.Reverse(result) + return result +} + +func (tv *typeVisitor) visitLocalType(ty tokens.Type) { + cts, found := tv.Schema.Types[string(ty)] + if !found { + return + } + if tv.seen == nil { + tv.seen = map[tokens.Type]struct{}{} + } + if _, seen := tv.seen[ty]; seen { + return + } + tv.seen[ty] = struct{}{} + tv.Visit(tv.ancestors(), ty) + tv.push(ty).visitComplexTypeSpec(cts) +} + +func (tv *typeVisitor) visitComplexTypeSpec(cts pschema.ComplexTypeSpec) { + if cts.Enum != nil { + return + } + tv.visitObjectTypeSpec(cts.ObjectTypeSpec) +} + +func (tv *typeVisitor) visitObjectTypeSpec(ots pschema.ObjectTypeSpec) { + for _, prop := range ots.Properties { + tv.visitPropertySpec(prop) + } +} + +func (tv *typeVisitor) visitPropertySpec(ots pschema.PropertySpec) { + tv.visitTypeSpec(ots.TypeSpec) +} + +func (tv *typeVisitor) visitTypeSpec(ots pschema.TypeSpec) { + if localType, ok := parseLocalRef(ots.Ref); ok { + tv.visitLocalType(localType) + return + } + if ots.Items != nil { + tv.visitTypeSpec(*ots.Items) + } + if ots.AdditionalProperties != nil { + tv.visitTypeSpec(*ots.AdditionalProperties) + } + for _, ty := range ots.OneOf { + tv.visitTypeSpec(ty) + } +} diff --git a/pkg/tfgen/internal/unrec/type_visitor_test.go b/pkg/tfgen/internal/unrec/type_visitor_test.go new file mode 100644 index 000000000..e0bc829ff --- /dev/null +++ b/pkg/tfgen/internal/unrec/type_visitor_test.go @@ -0,0 +1,57 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "testing" + + "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" + "github.com/stretchr/testify/require" +) + +func TestTypeVisitor(t *testing.T) { + s := exampleSchema(t) + + roots := []tokens.Type{} + for _, r := range s.Resources { + for _, p := range r.Properties { + if ref, ok := parseLocalRef(p.TypeSpec.Ref); ok { + roots = append(roots, ref) + } + } + } + + count := 0 + + visited := map[tokens.Type][]tokens.Type{} + + vis := &typeVisitor{Schema: s, Visit: func(ancestors []tokens.Type, current tokens.Type) { + visited[current] = ancestors + count++ + }} + vis.VisitRoots(roots) + + require.Equal(t, 31, count) + + tok := "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement" + require.Equal(t, []tokens.Type{ + "myprov:index/WebAclStatement:WebAclStatement", + "myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement", + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement", + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement", + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement", + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement", + }, visited[tokens.Type(tok)]) +} From f5b6f438b88e6395fdb51c6be9b8ff7839aaecfb Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 11:16:56 -0400 Subject: [PATCH 06/29] Add type comparer --- pkg/tfgen/internal/unrec/comparer.go | 366 ++++++++++++++++++ pkg/tfgen/internal/unrec/package_spec_util.go | 29 ++ 2 files changed, 395 insertions(+) create mode 100644 pkg/tfgen/internal/unrec/comparer.go diff --git a/pkg/tfgen/internal/unrec/comparer.go b/pkg/tfgen/internal/unrec/comparer.go new file mode 100644 index 000000000..62d85b483 --- /dev/null +++ b/pkg/tfgen/internal/unrec/comparer.go @@ -0,0 +1,366 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "bytes" + "reflect" + + pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" + "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" +) + +type comparer struct { + // Local type-ref comparisons are scoped to a package schema. + schema *pschema.PackageSpec +} + +func (cmp *comparer) LessThanTypeRefs(a, b tokens.Type) bool { + return cmp.LessThanOrEqualTypeRefs(a, b) && !cmp.EqualTypeRefs(a, b) +} + +// A type will be considered "less than" another type if both are locally defined object types and A defines a subset of +// B's properties. This is useful to deal with property dropout during recursive type expansions. +func (cmp *comparer) LessThanOrEqualTypeRefs(a, b tokens.Type) bool { + if a == b { + return true + } + aT, gotA := cmp.schema.Types[string(a)] + bT, gotB := cmp.schema.Types[string(b)] + if !gotA || !gotB { + return false + } + if aT.Enum != nil || bT.Enum != nil { + return false + } + + // Check that everything other than the properties is the same. + spec1 := aT.ObjectTypeSpec + spec2 := bT.ObjectTypeSpec + spec1.Properties = nil + spec2.Properties = nil + spec1.Plain = nil + spec2.Plain = nil + spec1.Required = nil + spec2.Required = nil + if !cmp.EqualObjectTypeSpecs(spec1, spec2) { + return false + } + + // Finally check that properties satisfy the less-than relation. + aProps := newXPropertyMap(aT.ObjectTypeSpec) + bProps := newXPropertyMap(bT.ObjectTypeSpec) + if !cmp.LessThanOrEqualXPropertyMaps(aProps, bProps) { + return false + } + + return true +} + +func (cmp *comparer) LessThanOrEqualXPropertyMaps(a, b xPropertyMap) bool { + // Do not define the relation for the degenerate case of empty maps. + if len(a) == 0 { + return false + } + for aK, aP := range a { + bP, ok := b[aK] + // Every key in A should also be a key in B. + if !ok { + return false + } + if !cmp.LessThanOrEqualXProperties(aP, bP) { + return false + } + } + return true +} + +func (cmp *comparer) LessThanOrEqualXProperties(a, b xProperty) bool { + if !cmp.LessThanOrEqualPropertySpecs(a.PropertySpec, b.PropertySpec) { + return false + } + // Everything else should be equal. + if a.IsRequired != b.IsRequired { + return false + } + if a.IsPlain != b.IsPlain { + return false + } + return true +} + +func (cmp *comparer) LessThanOrEqualPropertySpecs(a, b pschema.PropertySpec) bool { + if !cmp.LessThanOrEqualTypeSpecs(a.TypeSpec, b.TypeSpec) { + return false + } + // Everything else should be equal. + a.TypeSpec = pschema.TypeSpec{} + b.TypeSpec = pschema.TypeSpec{} + return cmp.EqualPropertySpecs(&a, &b) +} + +func (cmp *comparer) LessThanOrEqualTypeSpecs(a, b pschema.TypeSpec) bool { + aT, aOk := parseLocalRef(a.Ref) + bT, bOk := parseLocalRef(a.Ref) + if aOk && bOk && !cmp.LessThanOrEqualTypeRefs(aT, bT) { + return false + } + // Everything else should be equal. + a.Ref = "" + b.Ref = "" + return cmp.EqualTypeSpecs(&a, &b) +} + +func (cmp *comparer) EqualTypeRefs(a, b tokens.Type) bool { + if a == b { + return true + } + aT, gotA := cmp.schema.Types[string(a)] + bT, gotB := cmp.schema.Types[string(b)] + if gotA && gotB { + return cmp.EqualComplexTypeSpecs(&aT, &bT) + } + return false +} + +func (cmp *comparer) EqualRawRefs(a, b string) bool { + if a == b { + return true + } + aT, ok1 := parseLocalRef(a) + bT, ok2 := parseLocalRef(b) + if ok1 && ok2 { + return cmp.EqualTypeRefs(aT, bT) + } + return false +} + +func (cmp *comparer) EqualXPropertyMaps(a, b xPropertyMap) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + if len(a) != len(b) { + return false + } + for k, av := range a { + bv, ok := b[k] + if !ok { + return false + } + if !cmp.EqualXProperties(av, bv) { + return false + } + } + return true +} + +func (cmp *comparer) EqualXProperties(a, b xProperty) bool { + if a.IsRequired != b.IsRequired { + return false + } + if a.IsPlain != b.IsPlain { + return false + } + if !cmp.EqualPropertySpecs(&a.PropertySpec, &b.PropertySpec) { + return false + } + return true +} + +func (cmp *comparer) EqualObjectTypeSpecs(a, b pschema.ObjectTypeSpec) bool { + if a.Type != b.Type { + return false + } + if a.IsOverlay != b.IsOverlay { + return false + } + if !cmp.EqualLanguageMaps(a.Language, b.Language) { + return false + } + if a.Description != b.Description { + return false + } + if !cmp.EqualXPropertyMaps(newXPropertyMap(a), newXPropertyMap(b)) { + return false + } + return true +} + +func (cmp *comparer) EqualComplexTypeSpecs(a, b *pschema.ComplexTypeSpec) bool { + // Do not identify enum equality yet. + if a.Enum != nil || b.Enum != nil { + return false + } + if !cmp.EqualObjectTypeSpecs(a.ObjectTypeSpec, b.ObjectTypeSpec) { + return false + } + return true +} + +func (cmp *comparer) EqualLanguageMaps(a, b map[string]pschema.RawMessage) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + if len(a) != len(b) { + return false + } + for k, av := range a { + bv, ok := b[k] + if !ok { + return false + } + if !bytes.Equal(av, bv) { + return false + } + } + return true +} + +func (cmp *comparer) EqualTypeSpecLists(a, b []pschema.TypeSpec) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + if len(a) != len(b) { + return false + } + for i := range a { + if !cmp.EqualTypeSpecs(&a[i], &b[i]) { + return false + } + } + return true +} + +func (cmp *comparer) EqualStringSlices(a, b []string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func (cmp *comparer) EqualStringMaps(a, b map[string]string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + if len(a) != len(b) { + return false + } + for k, av := range a { + bv, ok := b[k] + if !ok { + return false + } + if av != bv { + return false + } + } + return true +} + +func (cmp *comparer) EqualDiscriminatorSpecs(a, b *pschema.DiscriminatorSpec) bool { + if a == nil || b == nil { + return a == b + } + if a.PropertyName != b.PropertyName { + return false + } + if !cmp.EqualStringMaps(a.Mapping, b.Mapping) { + return false + } + return true +} + +func (cmp *comparer) EqualTypeSpecs(a, b *pschema.TypeSpec) bool { + if a == nil || b == nil { + return a == b + } + if a.Type != b.Type { + return false + } + if !cmp.EqualRawRefs(a.Ref, b.Ref) { + return false + } + if !cmp.EqualTypeSpecs(a.AdditionalProperties, b.AdditionalProperties) { + return false + } + if !cmp.EqualTypeSpecs(a.Items, b.Items) { + return false + } + if !cmp.EqualTypeSpecLists(a.OneOf, b.OneOf) { + return false + } + if !cmp.EqualDiscriminatorSpecs(a.Discriminator, b.Discriminator) { + return false + } + if a.Plain != b.Plain { + return false + } + return true +} + +func (cmp *comparer) EqualPropertySpecs(a, b *pschema.PropertySpec) bool { + if !cmp.EqualTypeSpecs(&a.TypeSpec, &b.TypeSpec) { + return false + } + if a.Description != b.Description { + return false + } + if !reflect.DeepEqual(a.Const, b.Const) { + return false + } + if !reflect.DeepEqual(a.Default, b.Default) { + return false + } + if !cmp.EqualDefaultSpecs(a.DefaultInfo, b.DefaultInfo) { + return false + } + if a.DeprecationMessage != b.DeprecationMessage { + return false + } + if !cmp.EqualLanguageMaps(a.Language, b.Language) { + return false + } + if a.Secret != b.Secret { + return false + } + if a.ReplaceOnChanges != b.ReplaceOnChanges { + return false + } + if a.WillReplaceOnChanges != b.WillReplaceOnChanges { + return false + } + return true +} + +func (cmp *comparer) EqualDefaultSpecs(a, b *pschema.DefaultSpec) bool { + if a == nil || b == nil { + return a == b + } + if !cmp.EqualLanguageMaps(a.Language, b.Language) { + return false + } + if !cmp.EqualStringSlices(a.Environment, b.Environment) { + return false + } + return true +} diff --git a/pkg/tfgen/internal/unrec/package_spec_util.go b/pkg/tfgen/internal/unrec/package_spec_util.go index 477b731f6..de4947b80 100644 --- a/pkg/tfgen/internal/unrec/package_spec_util.go +++ b/pkg/tfgen/internal/unrec/package_spec_util.go @@ -17,6 +17,7 @@ package unrec import ( "strings" + pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" ) @@ -27,3 +28,31 @@ func parseLocalRef(rawRef string) (tokens.Type, bool) { cleanRef := strings.TrimPrefix(rawRef, "#/types/") return tokens.Type(cleanRef), true } + +type xProperty struct { + pschema.PropertySpec + IsRequired bool + IsPlain bool +} + +type xPropertyMap map[string]xProperty + +func newXPropertyMap(s pschema.ObjectTypeSpec) xPropertyMap { + m := make(xPropertyMap) + for _, prop := range s.Plain { + p := m[prop] + p.IsPlain = true + m[prop] = p + } + for _, prop := range s.Required { + p := m[prop] + p.IsRequired = true + m[prop] = p + } + for prop, spec := range s.Properties { + p := m[prop] + p.PropertySpec = spec + m[prop] = p + } + return m +} From a0d44d8c4da29e1a512bdfc99924b77e01e838e4 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 11:21:54 -0400 Subject: [PATCH 07/29] Add recursive type detector algo --- .../internal/unrec/recursion_detector.go | 97 +++++++++++++++++++ .../internal/unrec/recursion_detector_test.go | 43 ++++++++ 2 files changed, 140 insertions(+) create mode 100644 pkg/tfgen/internal/unrec/recursion_detector.go create mode 100644 pkg/tfgen/internal/unrec/recursion_detector_test.go diff --git a/pkg/tfgen/internal/unrec/recursion_detector.go b/pkg/tfgen/internal/unrec/recursion_detector.go new file mode 100644 index 000000000..6d9e0ff48 --- /dev/null +++ b/pkg/tfgen/internal/unrec/recursion_detector.go @@ -0,0 +1,97 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "sort" + + pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" + "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" +) + +// Detects unrolled recursion. +// +// TF does not support recursive type definitions but Pulumi does. The idea here is to take a projection of TF recursion +// unrolling trick and fold it back. +// +// Suppose ancestor(T1, T2) if T2 describes a sub-property of T1. +// +// Suppose T1 <= T2 if T1 has the subset of properties of T2 with types that are themselves Tx <= Ty. +// +// Suppose props(T) defines the set of property names for an object type. +// +// The algo looks for this pattern to detect recursion: +// +// ancestor(T1, T2), ancestor(T2, T3), T3 <= T2 <= T1, props(T1)==props(T2) +// +// This needs to use an approximate and not strict equality because the leaf node of a recursively unrolled type will +// drop recursive properties and therefore not strictly match the ancestor. +type recursionDetector struct { + schema *pschema.PackageSpec + detectedRecursiveTypes map[tokens.Type]struct{} + cmp comparer +} + +func newRecursionDetector(schema *pschema.PackageSpec) *recursionDetector { + return &recursionDetector{ + schema: schema, + detectedRecursiveTypes: map[tokens.Type]struct{}{}, + cmp: comparer{schema: schema}, + } +} + +func (rd *recursionDetector) Detect(roots []tokens.Type) []tokens.Type { + vis := &typeVisitor{Schema: rd.schema, Visit: rd.visit} + vis.VisitRoots(roots) + result := []tokens.Type{} + for t := range rd.detectedRecursiveTypes { + result = append(result, t) + } + sort.Slice(result, func(i, j int) bool { + return result[i] < result[j] + }) + return result +} + +func (rd *recursionDetector) visit(ancestors []tokens.Type, current tokens.Type) { + for i, ai := range ancestors { + if _, visited := rd.detectedRecursiveTypes[ai]; visited { + continue + } + for _, aj := range ancestors[i+1:] { + if rd.detect(ai, aj, current) { + if rd.detectedRecursiveTypes == nil { + rd.detectedRecursiveTypes = map[tokens.Type]struct{}{} + } + rd.detectedRecursiveTypes[ai] = struct{}{} + } + } + } +} + +func (rd *recursionDetector) detect(t1, t2, t3 tokens.Type) bool { + return rd.cmp.LessThanOrEqualTypeRefs(t3, t2) && + rd.cmp.LessThanOrEqualTypeRefs(t2, t1) && + rd.hasSameProps(t1, t2) +} + +func (rd *recursionDetector) hasSameProps(t1, t2 tokens.Type) bool { + cts1, ok1 := rd.schema.Types[string(t1)] + cts2, ok2 := rd.schema.Types[string(t2)] + if !ok1 || !ok2 { + return false + } + return len(cts1.Properties) == len(cts2.Properties) +} diff --git a/pkg/tfgen/internal/unrec/recursion_detector_test.go b/pkg/tfgen/internal/unrec/recursion_detector_test.go new file mode 100644 index 000000000..89468c544 --- /dev/null +++ b/pkg/tfgen/internal/unrec/recursion_detector_test.go @@ -0,0 +1,43 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "testing" + + "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" + "github.com/stretchr/testify/require" +) + +func TestRecursionDetector(t *testing.T) { + s := exampleSchema(t) + rd := newRecursionDetector(s) + + roots := []tokens.Type{} + for _, r := range s.Resources { + for _, p := range r.Properties { + if ref, ok := parseLocalRef(p.TypeSpec.Ref); ok { + roots = append(roots, ref) + } + } + } + + expect := []tokens.Type{ + "myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement", + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement", + } + + require.Equal(t, expect, rd.Detect(roots)) +} From bd290edd92efe975bac5cd4bb9be0e779ff0f766 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 12:34:17 -0400 Subject: [PATCH 08/29] Schema simplification - first pass --- pkg/tfgen/internal/unrec/package_spec_util.go | 159 +++++++++ pkg/tfgen/internal/unrec/simplify.go | 132 ++++++++ pkg/tfgen/internal/unrec/simplify_test.go | 35 ++ .../unrec/testdata/TestSimplify.golden | 320 ++++++++++++++++++ 4 files changed, 646 insertions(+) create mode 100644 pkg/tfgen/internal/unrec/simplify.go create mode 100644 pkg/tfgen/internal/unrec/simplify_test.go create mode 100644 pkg/tfgen/internal/unrec/testdata/TestSimplify.golden diff --git a/pkg/tfgen/internal/unrec/package_spec_util.go b/pkg/tfgen/internal/unrec/package_spec_util.go index de4947b80..40d8886e2 100644 --- a/pkg/tfgen/internal/unrec/package_spec_util.go +++ b/pkg/tfgen/internal/unrec/package_spec_util.go @@ -15,10 +15,13 @@ package unrec import ( + "encoding/json" + "fmt" "strings" pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" ) func parseLocalRef(rawRef string) (tokens.Type, bool) { @@ -56,3 +59,159 @@ func newXPropertyMap(s pschema.ObjectTypeSpec) xPropertyMap { } return m } + +type typeRefs map[tokens.Type]struct{} + +func newTypeRefs(tok ...tokens.Type) typeRefs { + tr := make(typeRefs) + tr.Add(tok...) + return tr +} + +func (refs typeRefs) Add(tok ...tokens.Type) { + for _, t := range tok { + refs[t] = struct{}{} + } +} + +func (refs typeRefs) Slice() []tokens.Type { + x := []tokens.Type{} + for k := range refs { + x = append(x, k) + } + return x +} + +func (refs typeRefs) Best() tokens.Type { + var best tokens.Type + for k := range refs { + if best == "" || len(k) < len(best) { + best = k + } + } + contract.Assertf(best != "", "expected to find at least one element") + return best +} + +func findGenericTypeReferences(x any) (typeRefs, error) { + bytes, err := json.Marshal(x) + if err != nil { + return nil, err + } + + var s any + if err := json.Unmarshal(bytes, &s); err != nil { + return nil, err + } + tr := make(typeRefs, 0) + transformMaps(func(m map[string]any) { + ref, ok := detectRef(m) + if ok { + tr[ref] = struct{}{} + } + }, s) + return tr, nil +} + +func detectRef(m map[string]any) (tokens.Type, bool) { + ref, gotRef := m["$ref"] + if !gotRef { + return "", false + } + refs, ok := ref.(string) + if !ok { + return "", false + } + return parseLocalRef(refs) +} + +func transformMaps(transform func(map[string]any), value any) any { + switch value := value.(type) { + case map[string]any: + transform(value) + for k := range value { + value[k] = transformMaps(transform, value[k]) + } + return value + case []any: + for i := range value { + value[i] = transformMaps(transform, value[i]) + } + return value + default: + return value + } +} + +func rewriteTypeRefs(rewrites map[tokens.Type]tokens.Type, schema *pschema.PackageSpec) error { + bytes, err := json.Marshal(schema) + if err != nil { + return err + } + + var s any + if err := json.Unmarshal(bytes, &s); err != nil { + return err + } + + rewrite := func(m map[string]any) { + cleanRef, ok := detectRef(m) + if !ok { + return + } + if modifiedRef, ok := rewrites[cleanRef]; ok { + m["$ref"] = fmt.Sprintf("#/types/%s", modifiedRef) + } + } + + modifiedS := transformMaps(rewrite, s) + + modifiedBytes, err := json.Marshal(modifiedS) + if err != nil { + return err + } + + var modifiedSchema pschema.PackageSpec + if err := json.Unmarshal( + modifiedBytes, &modifiedSchema); err != nil { + return err + } + *schema = modifiedSchema + + return nil +} + +func findResourceTypeReferences(r pschema.ResourceSpec) (typeRefs, error) { + return findGenericTypeReferences(r) +} + +func findFunctionTypeReferences(f pschema.FunctionSpec) (typeRefs, error) { + return findGenericTypeReferences(f) +} + +func findTypeReferenceTransitiveClosure(spec *pschema.PackageSpec, refs typeRefs) (typeRefs, error) { + queue := []tokens.Type{} + for r := range refs { + queue = append(queue, r) + } + seen := typeRefs{} + for len(queue) > 0 { + r := queue[0] + queue = queue[1:] + if _, ok := seen[r]; ok { + continue + } + t, ok := spec.Types[string(r)] + if ok { + moreRefs, err := findGenericTypeReferences(t) + if err != nil { + return nil, err + } + for m := range moreRefs { + queue = append(queue, m) + } + } + seen[r] = struct{}{} + } + return seen, nil +} diff --git a/pkg/tfgen/internal/unrec/simplify.go b/pkg/tfgen/internal/unrec/simplify.go new file mode 100644 index 000000000..5b2310de2 --- /dev/null +++ b/pkg/tfgen/internal/unrec/simplify.go @@ -0,0 +1,132 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "context" + "fmt" + + pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" + "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" +) + +func SimplifyRecursiveTypes(ctx context.Context, schema *pschema.PackageSpec) error { + rewriteRules, err := computeAllRewriteRules(schema) + if err != nil { + return err + } + return rewriteTypeRefs(rewriteRules, schema) +} + +func computeAllRewriteRules(schema *pschema.PackageSpec) (map[tokens.Type]tokens.Type, error) { + cmp := &comparer{schema: schema} + + rewriteRules := make(map[tokens.Type]tokens.Type) + + for _, r := range schema.Resources { + typeRefs, err := findResourceTypeReferences(r) + if err != nil { + return nil, err + } + if err := computeRewriteRules(cmp, schema, typeRefs, rewriteRules); err != nil { + return nil, err + } + } + + for _, f := range schema.Functions { + typeRefs, err := findFunctionTypeReferences(f) + if err != nil { + return nil, err + } + if err := computeRewriteRules(cmp, schema, typeRefs, rewriteRules); err != nil { + return nil, err + } + } + + fmt.Println("REWRITE RULES", len(rewriteRules)) + for k, v := range rewriteRules { + fmt.Println("REWRITE", k, v) + + } + + return rewriteRules, nil +} + +// Starting from roots, detect recursive type roots. Simplify all types reachable from the recursive type roots. +// Simplification rewrites type A with type B if A<=B according to [comparer.LessThanTypeRefs]. +// +// Although scoped to the progeny of recursive types, this may still end up over-eagerly rewriting logically distinct +// but structurally identical types and may require some refinement. +func computeRewriteRules( + cmp *comparer, + schema *pschema.PackageSpec, + roots typeRefs, + rewriteRules map[tokens.Type]tokens.Type, +) error { + rd := newRecursionDetector(schema) + recursionRoots := rd.Detect(roots.Slice()) + + allRefs, err := findTypeReferenceTransitiveClosure(schema, newTypeRefs(recursionRoots...)) + if err != nil { + return err + } + eqcs := typeEqualityClasses(cmp, allRefs) + + for i, eqc := range eqcs { + best := eqc.Best() + + // Check if there are any other classes that have bigger elements. + isDominant := true + for j, otherc := range eqcs { + if i != j && cmp.LessThanTypeRefs(best, otherc.Best()) { + isDominant = false + break + } + } + + // Only consider classes that are not dominated by another class. + if isDominant { + // Add rewrite rules for all types in the class itself. + for _, typ := range eqc.Slice() { + if typ != best { + rewriteRules[typ] = best + } + } + // Add rewrite rules for all classes it dominates. + for j, otherc := range eqcs { + if i != j && cmp.LessThanTypeRefs(otherc.Best(), best) { + for _, typ := range otherc.Slice() { + rewriteRules[typ] = best + } + } + } + } + } + return nil +} + +func typeEqualityClasses(cmp *comparer, types typeRefs) []typeRefs { + eq := func(a, b tokens.Type) bool { + return cmp.EqualTypeRefs(a, b) + } + eClasses := equalityClasses(eq, types.Slice()) + var result []typeRefs + for _, c := range eClasses { + tr := make(typeRefs) + tr.Add(c...) + result = append(result, tr) + } + return result +} diff --git a/pkg/tfgen/internal/unrec/simplify_test.go b/pkg/tfgen/internal/unrec/simplify_test.go new file mode 100644 index 000000000..02fd81f04 --- /dev/null +++ b/pkg/tfgen/internal/unrec/simplify_test.go @@ -0,0 +1,35 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "context" + "testing" + + "encoding/json" + "github.com/hexops/autogold/v2" + "github.com/stretchr/testify/require" +) + +func TestSimplify(t *testing.T) { + s := exampleSchema(t) + err := SimplifyRecursiveTypes(context.Background(), s) + require.NoError(t, err) + + bytes, err := json.MarshalIndent(s, "", " ") + require.NoError(t, err) + + autogold.ExpectFile(t, autogold.Raw(string(bytes))) +} diff --git a/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden b/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden new file mode 100644 index 000000000..11a420fcf --- /dev/null +++ b/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden @@ -0,0 +1,320 @@ +{ + "name": "myprov", + "attribution": "This Pulumi package is based on the [`myprov` Terraform Provider](https://github.com/terraform-providers/terraform-provider-myprov).", + "meta": { + "moduleFormat": "(.*)(?:/[^/]*)" + }, + "language": { + "nodejs": { + "compatibility": "tfbridge20", + "disableUnionOutputTypes": true, + "readme": "\u003e This provider is a derived work of the [Terraform Provider](https://github.com/terraform-providers/terraform-provider-myprov)\n\u003e distributed under [MPL 2.0](https://www.mozilla.org/en-US/MPL/2.0/). If you encounter a bug or missing feature,\n\u003e first check the [`pulumi-myprov` repo](/issues); however, if that doesn't turn up anything,\n\u003e please consult the source [`terraform-provider-myprov` repo](https://github.com/terraform-providers/terraform-provider-myprov/issues)." + }, + "python": { + "compatibility": "tfbridge20", + "pyproject": {}, + "readme": "\u003e This provider is a derived work of the [Terraform Provider](https://github.com/terraform-providers/terraform-provider-myprov)\n\u003e distributed under [MPL 2.0](https://www.mozilla.org/en-US/MPL/2.0/). If you encounter a bug or missing feature,\n\u003e first check the [`pulumi-myprov` repo](/issues); however, if that doesn't turn up anything,\n\u003e please consult the source [`terraform-provider-myprov` repo](https://github.com/terraform-providers/terraform-provider-myprov/issues)." + } + }, + "config": {}, + "types": { + "myprov:index/WebAclStatement:WebAclStatement": { + "properties": { + "andStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement" + }, + "rateBasedStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement": { + "properties": { + "statements": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement" + } + } + }, + "type": "object", + "required": [ + "statements" + ] + }, + "myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement": { + "properties": { + "andStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement" + }, + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement": { + "properties": { + "statements": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement" + } + } + }, + "type": "object", + "required": [ + "statements" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement": { + "properties": { + "andStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement" + }, + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement": { + "properties": { + "statements": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement" + } + } + }, + "type": "object", + "required": [ + "statements" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement": { + "properties": { + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement": { + "properties": { + "aggregateKeyType": { + "type": "string" + }, + "scopeDownStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement": { + "properties": { + "andStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement" + }, + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement": { + "properties": { + "statements": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement" + } + } + }, + "type": "object", + "required": [ + "statements" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement": { + "properties": { + "andStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement" + }, + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement": { + "properties": { + "statements": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement" + } + } + }, + "type": "object", + "required": [ + "statements" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement": { + "properties": { + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + } + }, + "provider": { + "description": "The provider type for the myprov package. By default, resources use package-wide configuration\nsettings, however an explicit `Provider` instance may be created and passed during resource\nconstruction to achieve fine-grained programmatic control over provider settings. See the\n[documentation](https://www.pulumi.com/docs/reference/programming-model/#providers) for more information.\n" + }, + "resources": { + "myprov:index:WebAcl": { + "properties": { + "statement": { + "$ref": "#/types/myprov:index/WebAclStatement:WebAclStatement" + } + }, + "required": [ + "statement" + ], + "inputProperties": { + "statement": { + "$ref": "#/types/myprov:index/WebAclStatement:WebAclStatement" + } + }, + "requiredInputs": [ + "statement" + ], + "stateInputs": { + "description": "Input properties used for looking up and filtering WebAcl resources.\n", + "properties": { + "statement": { + "$ref": "#/types/myprov:index/WebAclStatement:WebAclStatement" + } + }, + "type": "object" + } + } + } +} \ No newline at end of file From dc32a76e57eb6c2f80fe62d31738163fc5d0ed6e Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 16:07:20 -0400 Subject: [PATCH 09/29] Debug and test comparer --- pkg/tfgen/internal/unrec/comparer.go | 6 +- pkg/tfgen/internal/unrec/comparer_test.go | 92 +++++++++++++++++++++++ 2 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 pkg/tfgen/internal/unrec/comparer_test.go diff --git a/pkg/tfgen/internal/unrec/comparer.go b/pkg/tfgen/internal/unrec/comparer.go index 62d85b483..c2f414928 100644 --- a/pkg/tfgen/internal/unrec/comparer.go +++ b/pkg/tfgen/internal/unrec/comparer.go @@ -70,10 +70,6 @@ func (cmp *comparer) LessThanOrEqualTypeRefs(a, b tokens.Type) bool { } func (cmp *comparer) LessThanOrEqualXPropertyMaps(a, b xPropertyMap) bool { - // Do not define the relation for the degenerate case of empty maps. - if len(a) == 0 { - return false - } for aK, aP := range a { bP, ok := b[aK] // Every key in A should also be a key in B. @@ -113,7 +109,7 @@ func (cmp *comparer) LessThanOrEqualPropertySpecs(a, b pschema.PropertySpec) boo func (cmp *comparer) LessThanOrEqualTypeSpecs(a, b pschema.TypeSpec) bool { aT, aOk := parseLocalRef(a.Ref) - bT, bOk := parseLocalRef(a.Ref) + bT, bOk := parseLocalRef(b.Ref) if aOk && bOk && !cmp.LessThanOrEqualTypeRefs(aT, bT) { return false } diff --git a/pkg/tfgen/internal/unrec/comparer_test.go b/pkg/tfgen/internal/unrec/comparer_test.go new file mode 100644 index 000000000..d71b1e018 --- /dev/null +++ b/pkg/tfgen/internal/unrec/comparer_test.go @@ -0,0 +1,92 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unrec + +import ( + "testing" + + "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" + "github.com/stretchr/testify/require" +) + +func TestComparer(t *testing.T) { + s := exampleSchema(t) + cmp := &comparer{s} + + allTypes := []tokens.Type{} + for t := range s.Types { + allTypes = append(allTypes, tokens.Type(t)) + } + + t.Run("A=B is reflexive", func(t *testing.T) { + checkReflexive(t, allTypes, cmp.EqualTypeRefs) + }) + + t.Run("A=B is symmetric", func(t *testing.T) { + checkSymmetric(t, allTypes, cmp.EqualTypeRefs) + }) + + t.Run("A=B is transitive", func(t *testing.T) { + checkTransitive(t, allTypes, cmp.EqualTypeRefs) + }) + + t.Run("A <= B is reflexive", func(t *testing.T) { + checkReflexive(t, allTypes, cmp.LessThanOrEqualTypeRefs) + }) + + t.Run("A <= B is antisymmetric", func(t *testing.T) { + checkAntisymmetric(t, allTypes, cmp.EqualTypeRefs, cmp.LessThanOrEqualTypeRefs) + }) + + t.Run("A <= B is transitive", func(t *testing.T) { + checkTransitive(t, allTypes, cmp.LessThanOrEqualTypeRefs) + }) +} + +func checkReflexive[T any](t *testing.T, universe []T, r func(a, b T) bool) { + for _, a := range universe { + require.Truef(t, r(a, a), "reflexivity does not hold: a=%v", a) + } +} + +func checkSymmetric[T any](t *testing.T, universe []T, r func(a, b T) bool) { + for _, a := range universe { + for _, b := range universe { + require.Equalf(t, r(a, b), r(b, a), "symmetry does not hold: a=%v b=%v", a, b) + } + } +} + +func checkAntisymmetric[T any](t *testing.T, universe []T, eq, leq func(a, b T) bool) { + for _, a := range universe { + for _, b := range universe { + if leq(a, b) && leq(b, a) { + require.True(t, eq(a, b), "anti symmetry does not hold: a=%v b=%v", a, b) + } + } + } +} + +func checkTransitive[T any](t *testing.T, universe []T, r func(a, b T) bool) { + for _, a := range universe { + for _, b := range universe { + for _, c := range universe { + if r(a, b) && r(a, c) { + require.Truef(t, r(a, c), "transitivity does not hold: a=%v b=%v c=%v", a, b, c) + } + } + } + } +} From 33b9d8db39dac48f87e03dfeb856d676cfd290f7 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 16:35:39 -0400 Subject: [PATCH 10/29] Simlify recursion detector --- pkg/tfgen/internal/unrec/recursion_detector.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/pkg/tfgen/internal/unrec/recursion_detector.go b/pkg/tfgen/internal/unrec/recursion_detector.go index 6d9e0ff48..3cdb15d89 100644 --- a/pkg/tfgen/internal/unrec/recursion_detector.go +++ b/pkg/tfgen/internal/unrec/recursion_detector.go @@ -30,11 +30,9 @@ import ( // // Suppose T1 <= T2 if T1 has the subset of properties of T2 with types that are themselves Tx <= Ty. // -// Suppose props(T) defines the set of property names for an object type. -// // The algo looks for this pattern to detect recursion: // -// ancestor(T1, T2), ancestor(T2, T3), T3 <= T2 <= T1, props(T1)==props(T2) +// ancestor(T1, T2), ancestor(T2, T3), T3 <= T2 <= T1 // // This needs to use an approximate and not strict equality because the leaf node of a recursively unrolled type will // drop recursive properties and therefore not strictly match the ancestor. @@ -76,22 +74,12 @@ func (rd *recursionDetector) visit(ancestors []tokens.Type, current tokens.Type) rd.detectedRecursiveTypes = map[tokens.Type]struct{}{} } rd.detectedRecursiveTypes[ai] = struct{}{} + return } } } } func (rd *recursionDetector) detect(t1, t2, t3 tokens.Type) bool { - return rd.cmp.LessThanOrEqualTypeRefs(t3, t2) && - rd.cmp.LessThanOrEqualTypeRefs(t2, t1) && - rd.hasSameProps(t1, t2) -} - -func (rd *recursionDetector) hasSameProps(t1, t2 tokens.Type) bool { - cts1, ok1 := rd.schema.Types[string(t1)] - cts2, ok2 := rd.schema.Types[string(t2)] - if !ok1 || !ok2 { - return false - } - return len(cts1.Properties) == len(cts2.Properties) + return rd.cmp.LessThanOrEqualTypeRefs(t3, t2) && rd.cmp.LessThanOrEqualTypeRefs(t2, t1) } From 5953d4de4780de0f2bc25cef97a3c0b4d1be2dae Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 17:24:51 -0400 Subject: [PATCH 11/29] Fix comparer recursion for A<=B --- pkg/tfgen/internal/unrec/comparer.go | 138 +++++++--------------- pkg/tfgen/internal/unrec/comparer_test.go | 8 ++ 2 files changed, 50 insertions(+), 96 deletions(-) diff --git a/pkg/tfgen/internal/unrec/comparer.go b/pkg/tfgen/internal/unrec/comparer.go index c2f414928..65ef7fa23 100644 --- a/pkg/tfgen/internal/unrec/comparer.go +++ b/pkg/tfgen/internal/unrec/comparer.go @@ -27,99 +27,64 @@ type comparer struct { schema *pschema.PackageSpec } +func (cmp *comparer) EqualTypeRefs(a, b tokens.Type) bool { + g := &generalizedComparer{schema: cmp.schema} + g.EqualXPropertyMaps = g.strictlyEqualXPropertyMaps + return g.EqualTypeRefs(a, b) +} + func (cmp *comparer) LessThanTypeRefs(a, b tokens.Type) bool { return cmp.LessThanOrEqualTypeRefs(a, b) && !cmp.EqualTypeRefs(a, b) } // A type will be considered "less than" another type if both are locally defined object types and A defines a subset of // B's properties. This is useful to deal with property dropout during recursive type expansions. -func (cmp *comparer) LessThanOrEqualTypeRefs(a, b tokens.Type) bool { - if a == b { - return true - } - aT, gotA := cmp.schema.Types[string(a)] - bT, gotB := cmp.schema.Types[string(b)] - if !gotA || !gotB { - return false - } - if aT.Enum != nil || bT.Enum != nil { - return false - } - - // Check that everything other than the properties is the same. - spec1 := aT.ObjectTypeSpec - spec2 := bT.ObjectTypeSpec - spec1.Properties = nil - spec2.Properties = nil - spec1.Plain = nil - spec2.Plain = nil - spec1.Required = nil - spec2.Required = nil - if !cmp.EqualObjectTypeSpecs(spec1, spec2) { - return false - } - - // Finally check that properties satisfy the less-than relation. - aProps := newXPropertyMap(aT.ObjectTypeSpec) - bProps := newXPropertyMap(bT.ObjectTypeSpec) - if !cmp.LessThanOrEqualXPropertyMaps(aProps, bProps) { - return false - } +func (cmp *comparer) LessThanOrEqualTypeRefs(a, b tokens.Type) (eq bool) { + g := &generalizedComparer{schema: cmp.schema} + g.EqualXPropertyMaps = g.lessThanOrEqualXPropertyMaps + return g.EqualTypeRefs(a, b) +} - return true +// Generalizing structural comparisons to specialize for A=B and A<=B separately. +type generalizedComparer struct { + schema *pschema.PackageSpec + EqualXPropertyMaps func(xPropertyMap, xPropertyMap) bool } -func (cmp *comparer) LessThanOrEqualXPropertyMaps(a, b xPropertyMap) bool { +func (cmp *generalizedComparer) lessThanOrEqualXPropertyMaps(a, b xPropertyMap) bool { for aK, aP := range a { bP, ok := b[aK] // Every key in A should also be a key in B. if !ok { return false } - if !cmp.LessThanOrEqualXProperties(aP, bP) { + if !cmp.EqualXProperties(aP, bP) { return false } } return true } -func (cmp *comparer) LessThanOrEqualXProperties(a, b xProperty) bool { - if !cmp.LessThanOrEqualPropertySpecs(a.PropertySpec, b.PropertySpec) { - return false +func (cmp *generalizedComparer) strictlyEqualXPropertyMaps(a, b xPropertyMap) bool { + if a == nil || b == nil { + return a == nil && b == nil } - // Everything else should be equal. - if a.IsRequired != b.IsRequired { + if len(a) != len(b) { return false } - if a.IsPlain != b.IsPlain { - return false + for k, av := range a { + bv, ok := b[k] + if !ok { + return false + } + if !cmp.EqualXProperties(av, bv) { + return false + } } return true } -func (cmp *comparer) LessThanOrEqualPropertySpecs(a, b pschema.PropertySpec) bool { - if !cmp.LessThanOrEqualTypeSpecs(a.TypeSpec, b.TypeSpec) { - return false - } - // Everything else should be equal. - a.TypeSpec = pschema.TypeSpec{} - b.TypeSpec = pschema.TypeSpec{} - return cmp.EqualPropertySpecs(&a, &b) -} - -func (cmp *comparer) LessThanOrEqualTypeSpecs(a, b pschema.TypeSpec) bool { - aT, aOk := parseLocalRef(a.Ref) - bT, bOk := parseLocalRef(b.Ref) - if aOk && bOk && !cmp.LessThanOrEqualTypeRefs(aT, bT) { - return false - } - // Everything else should be equal. - a.Ref = "" - b.Ref = "" - return cmp.EqualTypeSpecs(&a, &b) -} - -func (cmp *comparer) EqualTypeRefs(a, b tokens.Type) bool { +func (cmp *generalizedComparer) EqualTypeRefs(a, b tokens.Type) bool { if a == b { return true } @@ -131,7 +96,7 @@ func (cmp *comparer) EqualTypeRefs(a, b tokens.Type) bool { return false } -func (cmp *comparer) EqualRawRefs(a, b string) bool { +func (cmp *generalizedComparer) EqualRawRefs(a, b string) bool { if a == b { return true } @@ -143,26 +108,7 @@ func (cmp *comparer) EqualRawRefs(a, b string) bool { return false } -func (cmp *comparer) EqualXPropertyMaps(a, b xPropertyMap) bool { - if a == nil || b == nil { - return a == nil && b == nil - } - if len(a) != len(b) { - return false - } - for k, av := range a { - bv, ok := b[k] - if !ok { - return false - } - if !cmp.EqualXProperties(av, bv) { - return false - } - } - return true -} - -func (cmp *comparer) EqualXProperties(a, b xProperty) bool { +func (cmp *generalizedComparer) EqualXProperties(a, b xProperty) bool { if a.IsRequired != b.IsRequired { return false } @@ -175,7 +121,7 @@ func (cmp *comparer) EqualXProperties(a, b xProperty) bool { return true } -func (cmp *comparer) EqualObjectTypeSpecs(a, b pschema.ObjectTypeSpec) bool { +func (cmp *generalizedComparer) EqualObjectTypeSpecs(a, b pschema.ObjectTypeSpec) bool { if a.Type != b.Type { return false } @@ -194,7 +140,7 @@ func (cmp *comparer) EqualObjectTypeSpecs(a, b pschema.ObjectTypeSpec) bool { return true } -func (cmp *comparer) EqualComplexTypeSpecs(a, b *pschema.ComplexTypeSpec) bool { +func (cmp *generalizedComparer) EqualComplexTypeSpecs(a, b *pschema.ComplexTypeSpec) bool { // Do not identify enum equality yet. if a.Enum != nil || b.Enum != nil { return false @@ -205,7 +151,7 @@ func (cmp *comparer) EqualComplexTypeSpecs(a, b *pschema.ComplexTypeSpec) bool { return true } -func (cmp *comparer) EqualLanguageMaps(a, b map[string]pschema.RawMessage) bool { +func (cmp *generalizedComparer) EqualLanguageMaps(a, b map[string]pschema.RawMessage) bool { if a == nil || b == nil { return a == nil && b == nil } @@ -224,7 +170,7 @@ func (cmp *comparer) EqualLanguageMaps(a, b map[string]pschema.RawMessage) bool return true } -func (cmp *comparer) EqualTypeSpecLists(a, b []pschema.TypeSpec) bool { +func (cmp *generalizedComparer) EqualTypeSpecLists(a, b []pschema.TypeSpec) bool { if a == nil || b == nil { return a == nil && b == nil } @@ -239,7 +185,7 @@ func (cmp *comparer) EqualTypeSpecLists(a, b []pschema.TypeSpec) bool { return true } -func (cmp *comparer) EqualStringSlices(a, b []string) bool { +func (cmp *generalizedComparer) EqualStringSlices(a, b []string) bool { if a == nil || b == nil { return a == nil && b == nil } @@ -254,7 +200,7 @@ func (cmp *comparer) EqualStringSlices(a, b []string) bool { return true } -func (cmp *comparer) EqualStringMaps(a, b map[string]string) bool { +func (cmp *generalizedComparer) EqualStringMaps(a, b map[string]string) bool { if a == nil || b == nil { return a == nil && b == nil } @@ -273,7 +219,7 @@ func (cmp *comparer) EqualStringMaps(a, b map[string]string) bool { return true } -func (cmp *comparer) EqualDiscriminatorSpecs(a, b *pschema.DiscriminatorSpec) bool { +func (cmp *generalizedComparer) EqualDiscriminatorSpecs(a, b *pschema.DiscriminatorSpec) bool { if a == nil || b == nil { return a == b } @@ -286,7 +232,7 @@ func (cmp *comparer) EqualDiscriminatorSpecs(a, b *pschema.DiscriminatorSpec) bo return true } -func (cmp *comparer) EqualTypeSpecs(a, b *pschema.TypeSpec) bool { +func (cmp *generalizedComparer) EqualTypeSpecs(a, b *pschema.TypeSpec) bool { if a == nil || b == nil { return a == b } @@ -314,7 +260,7 @@ func (cmp *comparer) EqualTypeSpecs(a, b *pschema.TypeSpec) bool { return true } -func (cmp *comparer) EqualPropertySpecs(a, b *pschema.PropertySpec) bool { +func (cmp *generalizedComparer) EqualPropertySpecs(a, b *pschema.PropertySpec) bool { if !cmp.EqualTypeSpecs(&a.TypeSpec, &b.TypeSpec) { return false } @@ -348,7 +294,7 @@ func (cmp *comparer) EqualPropertySpecs(a, b *pschema.PropertySpec) bool { return true } -func (cmp *comparer) EqualDefaultSpecs(a, b *pschema.DefaultSpec) bool { +func (cmp *generalizedComparer) EqualDefaultSpecs(a, b *pschema.DefaultSpec) bool { if a == nil || b == nil { return a == b } diff --git a/pkg/tfgen/internal/unrec/comparer_test.go b/pkg/tfgen/internal/unrec/comparer_test.go index d71b1e018..d57acb955 100644 --- a/pkg/tfgen/internal/unrec/comparer_test.go +++ b/pkg/tfgen/internal/unrec/comparer_test.go @@ -21,6 +21,14 @@ import ( "github.com/stretchr/testify/require" ) +func TestComparerAndStatement(t *testing.T) { + t1 := "myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement" + t2 := "myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement" + s := exampleSchema(t) + cmp := &comparer{s} + require.Truef(t, cmp.LessThanTypeRefs(tokens.Type(t1), tokens.Type(t2)), "A Date: Wed, 26 Jun 2024 17:44:03 -0400 Subject: [PATCH 12/29] Fixes --- pkg/tfgen/internal/unrec/package_spec_util.go | 8 +- .../internal/unrec/recursion_detector.go | 4 +- .../internal/unrec/recursion_detector_test.go | 5 + pkg/tfgen/internal/unrec/simplify.go | 7 - .../unrec/testdata/TestSimplify.golden | 209 +----------------- 5 files changed, 16 insertions(+), 217 deletions(-) diff --git a/pkg/tfgen/internal/unrec/package_spec_util.go b/pkg/tfgen/internal/unrec/package_spec_util.go index 40d8886e2..ba9c5176b 100644 --- a/pkg/tfgen/internal/unrec/package_spec_util.go +++ b/pkg/tfgen/internal/unrec/package_spec_util.go @@ -172,10 +172,14 @@ func rewriteTypeRefs(rewrites map[tokens.Type]tokens.Type, schema *pschema.Packa } var modifiedSchema pschema.PackageSpec - if err := json.Unmarshal( - modifiedBytes, &modifiedSchema); err != nil { + if err := json.Unmarshal(modifiedBytes, &modifiedSchema); err != nil { return err } + + for deletedType, _ := range rewrites { + delete(modifiedSchema.Types, string(deletedType)) + } + *schema = modifiedSchema return nil diff --git a/pkg/tfgen/internal/unrec/recursion_detector.go b/pkg/tfgen/internal/unrec/recursion_detector.go index 3cdb15d89..96372711a 100644 --- a/pkg/tfgen/internal/unrec/recursion_detector.go +++ b/pkg/tfgen/internal/unrec/recursion_detector.go @@ -39,14 +39,14 @@ import ( type recursionDetector struct { schema *pschema.PackageSpec detectedRecursiveTypes map[tokens.Type]struct{} - cmp comparer + cmp *comparer } func newRecursionDetector(schema *pschema.PackageSpec) *recursionDetector { return &recursionDetector{ schema: schema, detectedRecursiveTypes: map[tokens.Type]struct{}{}, - cmp: comparer{schema: schema}, + cmp: &comparer{schema: schema}, } } diff --git a/pkg/tfgen/internal/unrec/recursion_detector_test.go b/pkg/tfgen/internal/unrec/recursion_detector_test.go index 89468c544..de2570916 100644 --- a/pkg/tfgen/internal/unrec/recursion_detector_test.go +++ b/pkg/tfgen/internal/unrec/recursion_detector_test.go @@ -34,9 +34,14 @@ func TestRecursionDetector(t *testing.T) { } } + // This currently over-detects a bit, instead of just detecting roots it detects sub-roots as well. This can be + // rectified but does not seem to affect the algorithm at the moment. expect := []tokens.Type{ + "myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement", "myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement", + "myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement", "myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement", + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement", } require.Equal(t, expect, rd.Detect(roots)) diff --git a/pkg/tfgen/internal/unrec/simplify.go b/pkg/tfgen/internal/unrec/simplify.go index 5b2310de2..e270c6dfa 100644 --- a/pkg/tfgen/internal/unrec/simplify.go +++ b/pkg/tfgen/internal/unrec/simplify.go @@ -16,7 +16,6 @@ package unrec import ( "context" - "fmt" pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" @@ -55,12 +54,6 @@ func computeAllRewriteRules(schema *pschema.PackageSpec) (map[tokens.Type]tokens } } - fmt.Println("REWRITE RULES", len(rewriteRules)) - for k, v := range rewriteRules { - fmt.Println("REWRITE", k, v) - - } - return rewriteRules, nil } diff --git a/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden b/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden index 11a420fcf..5d2043aee 100644 --- a/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden +++ b/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden @@ -46,32 +46,7 @@ "myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement": { "properties": { "andStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement" - }, - "xssMatchStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement": { - "properties": { - "statements": { - "type": "array", - "items": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement" - } - } - }, - "type": "object", - "required": [ - "statements" - ] - }, - "myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement": { - "properties": { - "andStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement" + "$ref": "#/types/myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement" }, "xssMatchStatement": { "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" @@ -79,66 +54,6 @@ }, "type": "object" }, - "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement": { - "properties": { - "statements": { - "type": "array", - "items": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement" - } - } - }, - "type": "object", - "required": [ - "statements" - ] - }, - "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement": { - "properties": { - "xssMatchStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement": { - "properties": { - "fieldToMatch": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch": { - "properties": { - "method": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { - "type": "object" - }, - "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement": { - "properties": { - "fieldToMatch": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch": { - "properties": { - "method": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { - "type": "object" - }, "myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement": { "properties": { "fieldToMatch": { @@ -150,139 +65,21 @@ "myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch": { "properties": { "method": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + "$ref": "#/types/myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement" } }, "type": "object" }, - "myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { - "type": "object" - }, "myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement": { "properties": { "aggregateKeyType": { "type": "string" }, "scopeDownStatement": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement": { - "properties": { - "andStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement" - }, - "xssMatchStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement": { - "properties": { - "statements": { - "type": "array", - "items": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement" - } - } - }, - "type": "object", - "required": [ - "statements" - ] - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement": { - "properties": { - "andStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement" - }, - "xssMatchStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement": { - "properties": { - "statements": { - "type": "array", - "items": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement" - } - } - }, - "type": "object", - "required": [ - "statements" - ] - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement": { - "properties": { - "xssMatchStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement": { - "properties": { - "fieldToMatch": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatch": { - "properties": { - "method": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement": { - "properties": { - "fieldToMatch": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatch": { - "properties": { - "method": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement" } }, "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement": { - "properties": { - "fieldToMatch": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch": { - "properties": { - "method": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod": { - "type": "object" } }, "provider": { From f80c74489f1384f98923ef1aec780fabcd824b4f Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 26 Jun 2024 18:34:16 -0400 Subject: [PATCH 13/29] Treat empty objects specially --- pkg/tfgen/internal/unrec/comparer.go | 4 ++++ .../internal/unrec/recursion_detector_test.go | 16 ++++++---------- .../internal/unrec/testdata/TestSimplify.golden | 5 ++++- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/tfgen/internal/unrec/comparer.go b/pkg/tfgen/internal/unrec/comparer.go index 65ef7fa23..069b28791 100644 --- a/pkg/tfgen/internal/unrec/comparer.go +++ b/pkg/tfgen/internal/unrec/comparer.go @@ -52,6 +52,10 @@ type generalizedComparer struct { } func (cmp *generalizedComparer) lessThanOrEqualXPropertyMaps(a, b xPropertyMap) bool { + // Empty objects are treated specially and are never {}<=X. + if len(a) == 0 || len(b) == 0 { + return len(a) == len(b) + } for aK, aP := range a { bP, ok := b[aK] // Every key in A should also be a key in B. diff --git a/pkg/tfgen/internal/unrec/recursion_detector_test.go b/pkg/tfgen/internal/unrec/recursion_detector_test.go index de2570916..5936d8712 100644 --- a/pkg/tfgen/internal/unrec/recursion_detector_test.go +++ b/pkg/tfgen/internal/unrec/recursion_detector_test.go @@ -17,8 +17,8 @@ package unrec import ( "testing" + "github.com/hexops/autogold/v2" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" - "github.com/stretchr/testify/require" ) func TestRecursionDetector(t *testing.T) { @@ -36,13 +36,9 @@ func TestRecursionDetector(t *testing.T) { // This currently over-detects a bit, instead of just detecting roots it detects sub-roots as well. This can be // rectified but does not seem to affect the algorithm at the moment. - expect := []tokens.Type{ - "myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement", - "myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement", - "myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement", - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement", - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement", - } - - require.Equal(t, expect, rd.Detect(roots)) + autogold.Expect([]tokens.Type{ + tokens.Type("myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement"), + tokens.Type("myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement"), + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"), + }).Equal(t, rd.Detect(roots)) } diff --git a/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden b/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden index 5d2043aee..dcbe0bcbd 100644 --- a/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden +++ b/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden @@ -65,11 +65,14 @@ "myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch": { "properties": { "method": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement" + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod" } }, "type": "object" }, + "myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" + }, "myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement": { "properties": { "aggregateKeyType": { From 242339028d5a6f1af2c0b14ffe96b734bbe9aec9 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Thu, 27 Jun 2024 12:45:23 -0400 Subject: [PATCH 14/29] Expose --- pkg/tfgen/{internal => }/unrec/comparer.go | 0 pkg/tfgen/{internal => }/unrec/comparer_test.go | 0 pkg/tfgen/{internal => }/unrec/equality_classes.go | 0 pkg/tfgen/{internal => }/unrec/equality_classes_test.go | 0 pkg/tfgen/{internal => }/unrec/example_schema_test.go | 0 pkg/tfgen/{internal => }/unrec/gen.go | 0 pkg/tfgen/{internal => }/unrec/package.go | 0 pkg/tfgen/{internal => }/unrec/package_spec_util.go | 0 pkg/tfgen/{internal => }/unrec/package_spec_util_test.go | 0 pkg/tfgen/{internal => }/unrec/recursion_detector.go | 0 pkg/tfgen/{internal => }/unrec/recursion_detector_test.go | 0 pkg/tfgen/{internal => }/unrec/simplify.go | 0 pkg/tfgen/{internal => }/unrec/simplify_test.go | 0 pkg/tfgen/{internal => }/unrec/testdata/TestSimplify.golden | 0 pkg/tfgen/{internal => }/unrec/testdata/test-schema.json | 0 pkg/tfgen/{internal => }/unrec/type_visitor.go | 0 pkg/tfgen/{internal => }/unrec/type_visitor_test.go | 0 17 files changed, 0 insertions(+), 0 deletions(-) rename pkg/tfgen/{internal => }/unrec/comparer.go (100%) rename pkg/tfgen/{internal => }/unrec/comparer_test.go (100%) rename pkg/tfgen/{internal => }/unrec/equality_classes.go (100%) rename pkg/tfgen/{internal => }/unrec/equality_classes_test.go (100%) rename pkg/tfgen/{internal => }/unrec/example_schema_test.go (100%) rename pkg/tfgen/{internal => }/unrec/gen.go (100%) rename pkg/tfgen/{internal => }/unrec/package.go (100%) rename pkg/tfgen/{internal => }/unrec/package_spec_util.go (100%) rename pkg/tfgen/{internal => }/unrec/package_spec_util_test.go (100%) rename pkg/tfgen/{internal => }/unrec/recursion_detector.go (100%) rename pkg/tfgen/{internal => }/unrec/recursion_detector_test.go (100%) rename pkg/tfgen/{internal => }/unrec/simplify.go (100%) rename pkg/tfgen/{internal => }/unrec/simplify_test.go (100%) rename pkg/tfgen/{internal => }/unrec/testdata/TestSimplify.golden (100%) rename pkg/tfgen/{internal => }/unrec/testdata/test-schema.json (100%) rename pkg/tfgen/{internal => }/unrec/type_visitor.go (100%) rename pkg/tfgen/{internal => }/unrec/type_visitor_test.go (100%) diff --git a/pkg/tfgen/internal/unrec/comparer.go b/pkg/tfgen/unrec/comparer.go similarity index 100% rename from pkg/tfgen/internal/unrec/comparer.go rename to pkg/tfgen/unrec/comparer.go diff --git a/pkg/tfgen/internal/unrec/comparer_test.go b/pkg/tfgen/unrec/comparer_test.go similarity index 100% rename from pkg/tfgen/internal/unrec/comparer_test.go rename to pkg/tfgen/unrec/comparer_test.go diff --git a/pkg/tfgen/internal/unrec/equality_classes.go b/pkg/tfgen/unrec/equality_classes.go similarity index 100% rename from pkg/tfgen/internal/unrec/equality_classes.go rename to pkg/tfgen/unrec/equality_classes.go diff --git a/pkg/tfgen/internal/unrec/equality_classes_test.go b/pkg/tfgen/unrec/equality_classes_test.go similarity index 100% rename from pkg/tfgen/internal/unrec/equality_classes_test.go rename to pkg/tfgen/unrec/equality_classes_test.go diff --git a/pkg/tfgen/internal/unrec/example_schema_test.go b/pkg/tfgen/unrec/example_schema_test.go similarity index 100% rename from pkg/tfgen/internal/unrec/example_schema_test.go rename to pkg/tfgen/unrec/example_schema_test.go diff --git a/pkg/tfgen/internal/unrec/gen.go b/pkg/tfgen/unrec/gen.go similarity index 100% rename from pkg/tfgen/internal/unrec/gen.go rename to pkg/tfgen/unrec/gen.go diff --git a/pkg/tfgen/internal/unrec/package.go b/pkg/tfgen/unrec/package.go similarity index 100% rename from pkg/tfgen/internal/unrec/package.go rename to pkg/tfgen/unrec/package.go diff --git a/pkg/tfgen/internal/unrec/package_spec_util.go b/pkg/tfgen/unrec/package_spec_util.go similarity index 100% rename from pkg/tfgen/internal/unrec/package_spec_util.go rename to pkg/tfgen/unrec/package_spec_util.go diff --git a/pkg/tfgen/internal/unrec/package_spec_util_test.go b/pkg/tfgen/unrec/package_spec_util_test.go similarity index 100% rename from pkg/tfgen/internal/unrec/package_spec_util_test.go rename to pkg/tfgen/unrec/package_spec_util_test.go diff --git a/pkg/tfgen/internal/unrec/recursion_detector.go b/pkg/tfgen/unrec/recursion_detector.go similarity index 100% rename from pkg/tfgen/internal/unrec/recursion_detector.go rename to pkg/tfgen/unrec/recursion_detector.go diff --git a/pkg/tfgen/internal/unrec/recursion_detector_test.go b/pkg/tfgen/unrec/recursion_detector_test.go similarity index 100% rename from pkg/tfgen/internal/unrec/recursion_detector_test.go rename to pkg/tfgen/unrec/recursion_detector_test.go diff --git a/pkg/tfgen/internal/unrec/simplify.go b/pkg/tfgen/unrec/simplify.go similarity index 100% rename from pkg/tfgen/internal/unrec/simplify.go rename to pkg/tfgen/unrec/simplify.go diff --git a/pkg/tfgen/internal/unrec/simplify_test.go b/pkg/tfgen/unrec/simplify_test.go similarity index 100% rename from pkg/tfgen/internal/unrec/simplify_test.go rename to pkg/tfgen/unrec/simplify_test.go diff --git a/pkg/tfgen/internal/unrec/testdata/TestSimplify.golden b/pkg/tfgen/unrec/testdata/TestSimplify.golden similarity index 100% rename from pkg/tfgen/internal/unrec/testdata/TestSimplify.golden rename to pkg/tfgen/unrec/testdata/TestSimplify.golden diff --git a/pkg/tfgen/internal/unrec/testdata/test-schema.json b/pkg/tfgen/unrec/testdata/test-schema.json similarity index 100% rename from pkg/tfgen/internal/unrec/testdata/test-schema.json rename to pkg/tfgen/unrec/testdata/test-schema.json diff --git a/pkg/tfgen/internal/unrec/type_visitor.go b/pkg/tfgen/unrec/type_visitor.go similarity index 100% rename from pkg/tfgen/internal/unrec/type_visitor.go rename to pkg/tfgen/unrec/type_visitor.go diff --git a/pkg/tfgen/internal/unrec/type_visitor_test.go b/pkg/tfgen/unrec/type_visitor_test.go similarity index 100% rename from pkg/tfgen/internal/unrec/type_visitor_test.go rename to pkg/tfgen/unrec/type_visitor_test.go From 6891d3be995ca152dc709897f4495665f34f9b56 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 15:28:02 -0400 Subject: [PATCH 15/29] Add more variety from AWS WAFv2 demonstrating some challenges --- pkg/tfgen/unrec/gen.go | 79 ++- pkg/tfgen/unrec/testdata/TestSimplify.golden | 136 +++- pkg/tfgen/unrec/testdata/test-schema.json | 672 +++++++++++++++++++ pkg/tfgen/unrec/type_visitor_test.go | 2 +- 4 files changed, 878 insertions(+), 11 deletions(-) diff --git a/pkg/tfgen/unrec/gen.go b/pkg/tfgen/unrec/gen.go index ccc5a3d07..b9445b7d6 100644 --- a/pkg/tfgen/unrec/gen.go +++ b/pkg/tfgen/unrec/gen.go @@ -64,8 +64,10 @@ func webACLRootStatementSchema(level int) *schema.Schema { MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "and_statement": statementSchema(level), - "rate_based_statement": rateBasedStatementSchema(level), + "and_statement": statementSchema(level), + "rate_based_statement": rateBasedStatementSchema(level), + "sqli_match_statement": sqliMatchStatementSchema(), + "regex_match_statement": regexMatchStatementSchema(), }, }, } @@ -88,6 +90,24 @@ func rateBasedStatementSchema(level int) *schema.Schema { } } +func regexMatchStatementSchema() *schema.Schema { + return &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "regex_string": { + Type: schema.TypeString, + Required: true, + }, + "field_to_match": fieldToMatchSchema(), + "text_transformation": textTransformationSchema(), + }, + }, + } +} + func scopeDownStatementSchema(level int) *schema.Schema { return &schema.Schema{ Type: schema.TypeList, @@ -95,8 +115,10 @@ func scopeDownStatementSchema(level int) *schema.Schema { MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "and_statement": statementSchema(level), - "xss_match_statement": xssMatchStatementSchema(), + "and_statement": statementSchema(level), + "xss_match_statement": xssMatchStatementSchema(), + "sqli_match_statement": sqliMatchStatementSchema(), + "regex_match_statement": regexMatchStatementSchema(), }, }, } @@ -115,8 +137,10 @@ func statementSchema(level int) *schema.Schema { Required: true, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "and_statement": statementSchema(level - 1), - "xss_match_statement": xssMatchStatementSchema(), + "and_statement": statementSchema(level - 1), + "xss_match_statement": xssMatchStatementSchema(), + "sqli_match_statement": sqliMatchStatementSchema(), + "regex_match_statement": regexMatchStatementSchema(), }, }, }, @@ -136,7 +160,9 @@ func statementSchema(level int) *schema.Schema { Required: true, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "xss_match_statement": xssMatchStatementSchema(), + "xss_match_statement": xssMatchStatementSchema(), + "sqli_match_statement": sqliMatchStatementSchema(), + "regex_match_statement": regexMatchStatementSchema(), }, }, }, @@ -167,6 +193,45 @@ func fieldToMatchSchema() *schema.Schema { } } +func sqliMatchStatementSchema() *schema.Schema { + return &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "field_to_match": fieldToMatchSchema(), + "text_transformation": textTransformationSchema(), + }, + }, + } +} + +const ( + namesAttrPriority = "priority" + namesAttrType = "type" +) + +func textTransformationSchema() *schema.Schema { + return &schema.Schema{ + Type: schema.TypeSet, + Required: true, + MinItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + namesAttrPriority: { + Type: schema.TypeInt, + Required: true, + }, + namesAttrType: { + Type: schema.TypeString, + Required: true, + }, + }, + }, + } +} + var listOfEmptyObjectSchema *schema.Schema = &schema.Schema{ Type: schema.TypeList, Optional: true, diff --git a/pkg/tfgen/unrec/testdata/TestSimplify.golden b/pkg/tfgen/unrec/testdata/TestSimplify.golden index dcbe0bcbd..d9bba62f4 100644 --- a/pkg/tfgen/unrec/testdata/TestSimplify.golden +++ b/pkg/tfgen/unrec/testdata/TestSimplify.golden @@ -25,6 +25,12 @@ }, "rateBasedStatement": { "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement" + }, + "regexMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRegexMatchStatement:WebAclStatementRegexMatchStatement" + }, + "sqliMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementSqliMatchStatement:WebAclStatementSqliMatchStatement" } }, "type": "object" @@ -48,19 +54,53 @@ "andStatement": { "$ref": "#/types/myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement" }, + "regexMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementRegexMatchStatement:WebAclStatementAndStatementStatementRegexMatchStatement" + }, + "sqliMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementRegexMatchStatement:WebAclStatementAndStatementStatementRegexMatchStatement" + }, "xssMatchStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementRegexMatchStatement:WebAclStatementAndStatementStatementRegexMatchStatement" } }, "type": "object" }, - "myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement": { + "myprov:index/WebAclStatementAndStatementStatementRegexMatchStatement:WebAclStatementAndStatementStatementRegexMatchStatement": { "properties": { "fieldToMatch": { "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch" + }, + "regexString": { + "type": "string" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementAndStatementStatementSqliMatchStatementTextTransformation" + } } }, - "type": "object" + "type": "object", + "required": [ + "regexString", + "textTransformations" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementAndStatementStatementSqliMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] }, "myprov:index/WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch:WebAclStatementAndStatementStatementXssMatchStatementFieldToMatch": { "properties": { @@ -83,6 +123,96 @@ } }, "type": "object" + }, + "myprov:index/WebAclStatementRegexMatchStatement:WebAclStatementRegexMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRegexMatchStatementFieldToMatch:WebAclStatementRegexMatchStatementFieldToMatch" + }, + "regexString": { + "type": "string" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRegexMatchStatementTextTransformation:WebAclStatementRegexMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "regexString", + "textTransformations" + ] + }, + "myprov:index/WebAclStatementRegexMatchStatementFieldToMatch:WebAclStatementRegexMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRegexMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRegexMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRegexMatchStatementTextTransformation:WebAclStatementRegexMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, + "myprov:index/WebAclStatementSqliMatchStatement:WebAclStatementSqliMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementSqliMatchStatementFieldToMatch:WebAclStatementSqliMatchStatementFieldToMatch" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementSqliMatchStatementTextTransformation:WebAclStatementSqliMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "textTransformations" + ] + }, + "myprov:index/WebAclStatementSqliMatchStatementFieldToMatch:WebAclStatementSqliMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementSqliMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementSqliMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementSqliMatchStatementTextTransformation:WebAclStatementSqliMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] } }, "provider": { diff --git a/pkg/tfgen/unrec/testdata/test-schema.json b/pkg/tfgen/unrec/testdata/test-schema.json index 96dbe7cef..c38a15aa1 100644 --- a/pkg/tfgen/unrec/testdata/test-schema.json +++ b/pkg/tfgen/unrec/testdata/test-schema.json @@ -25,6 +25,12 @@ }, "rateBasedStatement": { "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement" + }, + "regexMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRegexMatchStatement:WebAclStatementRegexMatchStatement" + }, + "sqliMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementSqliMatchStatement:WebAclStatementSqliMatchStatement" } }, "type": "object" @@ -48,6 +54,12 @@ "andStatement": { "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement" }, + "regexMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementRegexMatchStatement:WebAclStatementAndStatementStatementRegexMatchStatement" + }, + "sqliMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementSqliMatchStatement:WebAclStatementAndStatementStatementSqliMatchStatement" + }, "xssMatchStatement": { "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement" } @@ -73,6 +85,12 @@ "andStatement": { "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement" }, + "regexMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatement" + }, + "sqliMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatement" + }, "xssMatchStatement": { "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement" } @@ -95,12 +113,108 @@ }, "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement": { "properties": { + "regexMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatement" + }, + "sqliMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatement" + }, "xssMatchStatement": { "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement" } }, "type": "object" }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch" + }, + "regexString": { + "type": "string" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "regexString", + "textTransformations" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "textTransformations" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatement": { "properties": { "fieldToMatch": { @@ -120,6 +234,96 @@ "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { "type": "object" }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch" + }, + "regexString": { + "type": "string" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation:WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "regexString", + "textTransformations" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation:WebAclStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "textTransformations" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch:WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatement": { "properties": { "fieldToMatch": { @@ -139,6 +343,96 @@ "myprov:index/WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { "type": "object" }, + "myprov:index/WebAclStatementAndStatementStatementRegexMatchStatement:WebAclStatementAndStatementStatementRegexMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementRegexMatchStatementFieldToMatch:WebAclStatementAndStatementStatementRegexMatchStatementFieldToMatch" + }, + "regexString": { + "type": "string" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementRegexMatchStatementTextTransformation:WebAclStatementAndStatementStatementRegexMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "regexString", + "textTransformations" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementRegexMatchStatementFieldToMatch:WebAclStatementAndStatementStatementRegexMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementRegexMatchStatementTextTransformation:WebAclStatementAndStatementStatementRegexMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementSqliMatchStatement:WebAclStatementAndStatementStatementSqliMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementSqliMatchStatementFieldToMatch:WebAclStatementAndStatementStatementSqliMatchStatementFieldToMatch" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementAndStatementStatementSqliMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "textTransformations" + ] + }, + "myprov:index/WebAclStatementAndStatementStatementSqliMatchStatementFieldToMatch:WebAclStatementAndStatementStatementSqliMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementAndStatementStatementSqliMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, "myprov:index/WebAclStatementAndStatementStatementXssMatchStatement:WebAclStatementAndStatementStatementXssMatchStatement": { "properties": { "fieldToMatch": { @@ -174,6 +468,12 @@ "andStatement": { "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement" }, + "regexMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatement" + }, + "sqliMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatement" + }, "xssMatchStatement": { "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement" } @@ -199,6 +499,12 @@ "andStatement": { "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement" }, + "regexMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatement" + }, + "sqliMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatement" + }, "xssMatchStatement": { "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement" } @@ -221,12 +527,108 @@ }, "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement": { "properties": { + "regexMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatement" + }, + "sqliMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatement" + }, "xssMatchStatement": { "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement" } }, "type": "object" }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch" + }, + "regexString": { + "type": "string" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "regexString", + "textTransformations" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementRegexMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "textTransformations" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementSqliMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatement": { "properties": { "fieldToMatch": { @@ -246,6 +648,96 @@ "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { "type": "object" }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementFieldToMatch" + }, + "regexString": { + "type": "string" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "regexString", + "textTransformations" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementRegexMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementFieldToMatch" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "textTransformations" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementSqliMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatement": { "properties": { "fieldToMatch": { @@ -265,6 +757,96 @@ "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementXssMatchStatementFieldToMatchMethod": { "type": "object" }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatch" + }, + "regexString": { + "type": "string" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "regexString", + "textTransformations" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatch" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "textTransformations" + ] + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement": { "properties": { "fieldToMatch": { @@ -283,6 +865,96 @@ }, "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod": { "type": "object" + }, + "myprov:index/WebAclStatementRegexMatchStatement:WebAclStatementRegexMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementRegexMatchStatementFieldToMatch:WebAclStatementRegexMatchStatementFieldToMatch" + }, + "regexString": { + "type": "string" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementRegexMatchStatementTextTransformation:WebAclStatementRegexMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "regexString", + "textTransformations" + ] + }, + "myprov:index/WebAclStatementRegexMatchStatementFieldToMatch:WebAclStatementRegexMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRegexMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRegexMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementRegexMatchStatementTextTransformation:WebAclStatementRegexMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] + }, + "myprov:index/WebAclStatementSqliMatchStatement:WebAclStatementSqliMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementSqliMatchStatementFieldToMatch:WebAclStatementSqliMatchStatementFieldToMatch" + }, + "textTransformations": { + "type": "array", + "items": { + "$ref": "#/types/myprov:index/WebAclStatementSqliMatchStatementTextTransformation:WebAclStatementSqliMatchStatementTextTransformation" + } + } + }, + "type": "object", + "required": [ + "textTransformations" + ] + }, + "myprov:index/WebAclStatementSqliMatchStatementFieldToMatch:WebAclStatementSqliMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementSqliMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementSqliMatchStatementFieldToMatchMethod": { + "type": "object" + }, + "myprov:index/WebAclStatementSqliMatchStatementTextTransformation:WebAclStatementSqliMatchStatementTextTransformation": { + "properties": { + "priority": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "priority", + "type" + ] } }, "provider": { diff --git a/pkg/tfgen/unrec/type_visitor_test.go b/pkg/tfgen/unrec/type_visitor_test.go index e0bc829ff..e0a143c9d 100644 --- a/pkg/tfgen/unrec/type_visitor_test.go +++ b/pkg/tfgen/unrec/type_visitor_test.go @@ -43,7 +43,7 @@ func TestTypeVisitor(t *testing.T) { }} vis.VisitRoots(roots) - require.Equal(t, 31, count) + require.Equal(t, 87, count) tok := "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement" require.Equal(t, []tokens.Type{ From 6f09ad4aef4188c3679f4fb4dd778ecfb20d2e25 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 15:49:15 -0400 Subject: [PATCH 16/29] Visitor termination --- pkg/tfgen/unrec/recursion_detector.go | 5 +++-- pkg/tfgen/unrec/type_visitor.go | 7 ++++--- pkg/tfgen/unrec/type_visitor_test.go | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/tfgen/unrec/recursion_detector.go b/pkg/tfgen/unrec/recursion_detector.go index 96372711a..cdd783511 100644 --- a/pkg/tfgen/unrec/recursion_detector.go +++ b/pkg/tfgen/unrec/recursion_detector.go @@ -63,7 +63,7 @@ func (rd *recursionDetector) Detect(roots []tokens.Type) []tokens.Type { return result } -func (rd *recursionDetector) visit(ancestors []tokens.Type, current tokens.Type) { +func (rd *recursionDetector) visit(ancestors []tokens.Type, current tokens.Type) bool { for i, ai := range ancestors { if _, visited := rd.detectedRecursiveTypes[ai]; visited { continue @@ -74,10 +74,11 @@ func (rd *recursionDetector) visit(ancestors []tokens.Type, current tokens.Type) rd.detectedRecursiveTypes = map[tokens.Type]struct{}{} } rd.detectedRecursiveTypes[ai] = struct{}{} - return + return true } } } + return true } func (rd *recursionDetector) detect(t1, t2, t3 tokens.Type) bool { diff --git a/pkg/tfgen/unrec/type_visitor.go b/pkg/tfgen/unrec/type_visitor.go index fedbc091c..041c1b3b4 100644 --- a/pkg/tfgen/unrec/type_visitor.go +++ b/pkg/tfgen/unrec/type_visitor.go @@ -23,7 +23,7 @@ import ( type typeVisitor struct { Schema *pschema.PackageSpec - Visit func(ancestors []tokens.Type, current tokens.Type) + Visit func(ancestors []tokens.Type, current tokens.Type) bool parent *typeVisitor // internal visiting tokens.Type // internal @@ -68,8 +68,9 @@ func (tv *typeVisitor) visitLocalType(ty tokens.Type) { return } tv.seen[ty] = struct{}{} - tv.Visit(tv.ancestors(), ty) - tv.push(ty).visitComplexTypeSpec(cts) + if tv.Visit(tv.ancestors(), ty) { + tv.push(ty).visitComplexTypeSpec(cts) + } } func (tv *typeVisitor) visitComplexTypeSpec(cts pschema.ComplexTypeSpec) { diff --git a/pkg/tfgen/unrec/type_visitor_test.go b/pkg/tfgen/unrec/type_visitor_test.go index e0a143c9d..94b916d63 100644 --- a/pkg/tfgen/unrec/type_visitor_test.go +++ b/pkg/tfgen/unrec/type_visitor_test.go @@ -37,9 +37,10 @@ func TestTypeVisitor(t *testing.T) { visited := map[tokens.Type][]tokens.Type{} - vis := &typeVisitor{Schema: s, Visit: func(ancestors []tokens.Type, current tokens.Type) { + vis := &typeVisitor{Schema: s, Visit: func(ancestors []tokens.Type, current tokens.Type) bool { visited[current] = ancestors count++ + return true }} vis.VisitRoots(roots) From 9166ec3a4ccc952b886960f7d62ef56f244955ef Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 15:52:50 -0400 Subject: [PATCH 17/29] Disambiguate roots --- pkg/tfgen/unrec/recursion_detector.go | 4 ++-- pkg/tfgen/unrec/recursion_detector_test.go | 6 +++--- pkg/tfgen/unrec/simplify.go | 6 +++--- pkg/tfgen/unrec/type_visitor.go | 4 ++-- pkg/tfgen/unrec/type_visitor_test.go | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/tfgen/unrec/recursion_detector.go b/pkg/tfgen/unrec/recursion_detector.go index cdd783511..ef69012c8 100644 --- a/pkg/tfgen/unrec/recursion_detector.go +++ b/pkg/tfgen/unrec/recursion_detector.go @@ -50,9 +50,9 @@ func newRecursionDetector(schema *pschema.PackageSpec) *recursionDetector { } } -func (rd *recursionDetector) Detect(roots []tokens.Type) []tokens.Type { +func (rd *recursionDetector) Detect(types []tokens.Type) []tokens.Type { vis := &typeVisitor{Schema: rd.schema, Visit: rd.visit} - vis.VisitRoots(roots) + vis.VisitTypes(types...) result := []tokens.Type{} for t := range rd.detectedRecursiveTypes { result = append(result, t) diff --git a/pkg/tfgen/unrec/recursion_detector_test.go b/pkg/tfgen/unrec/recursion_detector_test.go index 5936d8712..f86b9dcae 100644 --- a/pkg/tfgen/unrec/recursion_detector_test.go +++ b/pkg/tfgen/unrec/recursion_detector_test.go @@ -25,11 +25,11 @@ func TestRecursionDetector(t *testing.T) { s := exampleSchema(t) rd := newRecursionDetector(s) - roots := []tokens.Type{} + starterTypes := []tokens.Type{} for _, r := range s.Resources { for _, p := range r.Properties { if ref, ok := parseLocalRef(p.TypeSpec.Ref); ok { - roots = append(roots, ref) + starterTypes = append(starterTypes, ref) } } } @@ -40,5 +40,5 @@ func TestRecursionDetector(t *testing.T) { tokens.Type("myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement"), tokens.Type("myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement"), tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"), - }).Equal(t, rd.Detect(roots)) + }).Equal(t, rd.Detect(starterTypes)) } diff --git a/pkg/tfgen/unrec/simplify.go b/pkg/tfgen/unrec/simplify.go index e270c6dfa..6ee6dbeaf 100644 --- a/pkg/tfgen/unrec/simplify.go +++ b/pkg/tfgen/unrec/simplify.go @@ -57,7 +57,7 @@ func computeAllRewriteRules(schema *pschema.PackageSpec) (map[tokens.Type]tokens return rewriteRules, nil } -// Starting from roots, detect recursive type roots. Simplify all types reachable from the recursive type roots. +// Starting from starterTypes, detect recursive type roots. Simplify all types reachable from the recursive type roots. // Simplification rewrites type A with type B if A<=B according to [comparer.LessThanTypeRefs]. // // Although scoped to the progeny of recursive types, this may still end up over-eagerly rewriting logically distinct @@ -65,11 +65,11 @@ func computeAllRewriteRules(schema *pschema.PackageSpec) (map[tokens.Type]tokens func computeRewriteRules( cmp *comparer, schema *pschema.PackageSpec, - roots typeRefs, + starterTypes typeRefs, rewriteRules map[tokens.Type]tokens.Type, ) error { rd := newRecursionDetector(schema) - recursionRoots := rd.Detect(roots.Slice()) + recursionRoots := rd.Detect(starterTypes.Slice()) allRefs, err := findTypeReferenceTransitiveClosure(schema, newTypeRefs(recursionRoots...)) if err != nil { diff --git a/pkg/tfgen/unrec/type_visitor.go b/pkg/tfgen/unrec/type_visitor.go index 041c1b3b4..e0ed952bf 100644 --- a/pkg/tfgen/unrec/type_visitor.go +++ b/pkg/tfgen/unrec/type_visitor.go @@ -30,8 +30,8 @@ type typeVisitor struct { seen map[tokens.Type]struct{} // internal } -func (tv *typeVisitor) VisitRoots(roots []tokens.Type) { - for _, ty := range roots { +func (tv *typeVisitor) VisitTypes(types ...tokens.Type) { + for _, ty := range types { tv.visitLocalType(ty) } } diff --git a/pkg/tfgen/unrec/type_visitor_test.go b/pkg/tfgen/unrec/type_visitor_test.go index 94b916d63..5fae0c97d 100644 --- a/pkg/tfgen/unrec/type_visitor_test.go +++ b/pkg/tfgen/unrec/type_visitor_test.go @@ -24,11 +24,11 @@ import ( func TestTypeVisitor(t *testing.T) { s := exampleSchema(t) - roots := []tokens.Type{} + starterTypes := []tokens.Type{} for _, r := range s.Resources { for _, p := range r.Properties { if ref, ok := parseLocalRef(p.TypeSpec.Ref); ok { - roots = append(roots, ref) + starterTypes = append(starterTypes, ref) } } } @@ -42,7 +42,7 @@ func TestTypeVisitor(t *testing.T) { count++ return true }} - vis.VisitRoots(roots) + vis.VisitTypes(starterTypes...) require.Equal(t, 87, count) From 7631376821e53b77b6e0fb98dc595c1c053f412a Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 16:28:41 -0400 Subject: [PATCH 18/29] Flesh out recursion detector more --- pkg/tfgen/unrec/recursion_detector.go | 53 ++++++++++++++++------ pkg/tfgen/unrec/recursion_detector_test.go | 20 +++++--- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/pkg/tfgen/unrec/recursion_detector.go b/pkg/tfgen/unrec/recursion_detector.go index ef69012c8..7a84686b6 100644 --- a/pkg/tfgen/unrec/recursion_detector.go +++ b/pkg/tfgen/unrec/recursion_detector.go @@ -15,8 +15,6 @@ package unrec import ( - "sort" - pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" ) @@ -50,20 +48,49 @@ func newRecursionDetector(schema *pschema.PackageSpec) *recursionDetector { } } -func (rd *recursionDetector) Detect(types []tokens.Type) []tokens.Type { - vis := &typeVisitor{Schema: rd.schema, Visit: rd.visit} - vis.VisitTypes(types...) - result := []tokens.Type{} - for t := range rd.detectedRecursiveTypes { - result = append(result, t) +// Starting from the set of starterTypes, detects recursion and reports it. The keys of the resulting map are the +// recursion roots, and the values are sets of recursive instances for each root. +func (rd *recursionDetector) Detect(starterTypes []tokens.Type) map[tokens.Type]map[tokens.Type]struct{} { + // First pass: detect recursion roots. + vis := &typeVisitor{Schema: rd.schema, Visit: rd.detectRootsVisitor} + vis.VisitTypes(starterTypes...) + + detected := map[tokens.Type]map[tokens.Type]struct{}{} + + roots := []tokens.Type{} + + seenRoot := func(t tokens.Type) bool { + for _, r := range roots { + if rd.cmp.EqualTypeRefs(r, t) { + return true + } + } + return false } - sort.Slice(result, func(i, j int) bool { - return result[i] < result[j] - }) - return result + + for recursionRoot := range rd.detectedRecursiveTypes { + if !seenRoot(recursionRoot) { + roots = append(roots, recursionRoot) + detected[recursionRoot] = map[tokens.Type]struct{}{} + } + } + + // Second pass: detect instances. + vis2 := &typeVisitor{Schema: rd.schema, Visit: func(_ []tokens.Type, current tokens.Type) bool { + for _, root := range roots { + if rd.cmp.LessThanOrEqualTypeRefs(current, root) && current != root { + detected[root][current] = struct{}{} + return true + } + } + return true + }} + + vis2.VisitTypes(starterTypes...) + return detected } -func (rd *recursionDetector) visit(ancestors []tokens.Type, current tokens.Type) bool { +func (rd *recursionDetector) detectRootsVisitor(ancestors []tokens.Type, current tokens.Type) bool { for i, ai := range ancestors { if _, visited := rd.detectedRecursiveTypes[ai]; visited { continue diff --git a/pkg/tfgen/unrec/recursion_detector_test.go b/pkg/tfgen/unrec/recursion_detector_test.go index f86b9dcae..714f13c3e 100644 --- a/pkg/tfgen/unrec/recursion_detector_test.go +++ b/pkg/tfgen/unrec/recursion_detector_test.go @@ -34,11 +34,19 @@ func TestRecursionDetector(t *testing.T) { } } - // This currently over-detects a bit, instead of just detecting roots it detects sub-roots as well. This can be - // rectified but does not seem to affect the algorithm at the moment. - autogold.Expect([]tokens.Type{ - tokens.Type("myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement"), - tokens.Type("myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement"), - tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"), + autogold.Expect(map[tokens.Type]map[tokens.Type]struct{}{ + tokens.Type("myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement"): { + tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement"): {}, + tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement"): {}, + }, + tokens.Type("myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement"): { + tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement"): {}, + tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement"): {}, + }, }).Equal(t, rd.Detect(starterTypes)) } From f710fa16fbe8325a181badba38a5d301b55c1bb2 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 16:53:57 -0400 Subject: [PATCH 19/29] Stricter recursion detector --- pkg/tfgen/unrec/recursion_detector.go | 13 +++++++++++-- pkg/tfgen/unrec/recursion_detector_test.go | 3 +-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/tfgen/unrec/recursion_detector.go b/pkg/tfgen/unrec/recursion_detector.go index 7a84686b6..d0c8c21b8 100644 --- a/pkg/tfgen/unrec/recursion_detector.go +++ b/pkg/tfgen/unrec/recursion_detector.go @@ -15,6 +15,8 @@ package unrec import ( + "sort" + pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" ) @@ -59,6 +61,13 @@ func (rd *recursionDetector) Detect(starterTypes []tokens.Type) map[tokens.Type] roots := []tokens.Type{} + sort.Slice(roots, func(i, j int) bool { + if len(roots[i]) < len(roots[j]) { + return true + } + return roots[i] < roots[j] + }) + seenRoot := func(t tokens.Type) bool { for _, r := range roots { if rd.cmp.EqualTypeRefs(r, t) { @@ -78,7 +87,7 @@ func (rd *recursionDetector) Detect(starterTypes []tokens.Type) map[tokens.Type] // Second pass: detect instances. vis2 := &typeVisitor{Schema: rd.schema, Visit: func(_ []tokens.Type, current tokens.Type) bool { for _, root := range roots { - if rd.cmp.LessThanOrEqualTypeRefs(current, root) && current != root { + if rd.cmp.LessThanTypeRefs(current, root) && current != root { detected[root][current] = struct{}{} return true } @@ -109,5 +118,5 @@ func (rd *recursionDetector) detectRootsVisitor(ancestors []tokens.Type, current } func (rd *recursionDetector) detect(t1, t2, t3 tokens.Type) bool { - return rd.cmp.LessThanOrEqualTypeRefs(t3, t2) && rd.cmp.LessThanOrEqualTypeRefs(t2, t1) + return rd.cmp.LessThanTypeRefs(t3, t2) && rd.cmp.LessThanTypeRefs(t2, t1) } diff --git a/pkg/tfgen/unrec/recursion_detector_test.go b/pkg/tfgen/unrec/recursion_detector_test.go index 714f13c3e..d436e29e0 100644 --- a/pkg/tfgen/unrec/recursion_detector_test.go +++ b/pkg/tfgen/unrec/recursion_detector_test.go @@ -41,10 +41,9 @@ func TestRecursionDetector(t *testing.T) { tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement"): {}, tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement"): {}, }, - tokens.Type("myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement"): { + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"): { tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement"): {}, tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement"): {}, - tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"): {}, tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement"): {}, tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement"): {}, }, From 194b36f9e40669c6c79f36314f8ce79047abefba Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 16:55:57 -0400 Subject: [PATCH 20/29] Fix recursion detector assert --- pkg/tfgen/unrec/recursion_detector_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tfgen/unrec/recursion_detector_test.go b/pkg/tfgen/unrec/recursion_detector_test.go index d436e29e0..668c2b26b 100644 --- a/pkg/tfgen/unrec/recursion_detector_test.go +++ b/pkg/tfgen/unrec/recursion_detector_test.go @@ -41,7 +41,7 @@ func TestRecursionDetector(t *testing.T) { tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement"): {}, tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement"): {}, }, - tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"): { + tokens.Type("myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement"): { tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement"): {}, tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement"): {}, tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement"): {}, From 983d5c78954fc80abffbd2a86a36e91a698273fb Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 17:33:12 -0400 Subject: [PATCH 21/29] WIP --- pkg/tfgen/unrec/comparer.go | 26 ++- pkg/tfgen/unrec/comparer_test.go | 4 +- pkg/tfgen/unrec/package_spec_util.go | 5 + pkg/tfgen/unrec/simplify.go | 53 +++---- pkg/tfgen/unrec/testdata/TestSimplify.golden | 157 ++++++++++++++++++- 5 files changed, 209 insertions(+), 36 deletions(-) diff --git a/pkg/tfgen/unrec/comparer.go b/pkg/tfgen/unrec/comparer.go index 069b28791..87862b59f 100644 --- a/pkg/tfgen/unrec/comparer.go +++ b/pkg/tfgen/unrec/comparer.go @@ -25,10 +25,20 @@ import ( type comparer struct { // Local type-ref comparisons are scoped to a package schema. schema *pschema.PackageSpec + + // Set of implicit type rewrites to consider when comparing. + rewrites map[tokens.Type]tokens.Type +} + +func (cmp *comparer) WithRewrites(rewrites map[tokens.Type]tokens.Type) *comparer { + return &comparer{ + schema: cmp.schema, + rewrites: rewrites, + } } func (cmp *comparer) EqualTypeRefs(a, b tokens.Type) bool { - g := &generalizedComparer{schema: cmp.schema} + g := &generalizedComparer{schema: cmp.schema, rewrites: cmp.rewrites} g.EqualXPropertyMaps = g.strictlyEqualXPropertyMaps return g.EqualTypeRefs(a, b) } @@ -40,7 +50,7 @@ func (cmp *comparer) LessThanTypeRefs(a, b tokens.Type) bool { // A type will be considered "less than" another type if both are locally defined object types and A defines a subset of // B's properties. This is useful to deal with property dropout during recursive type expansions. func (cmp *comparer) LessThanOrEqualTypeRefs(a, b tokens.Type) (eq bool) { - g := &generalizedComparer{schema: cmp.schema} + g := &generalizedComparer{schema: cmp.schema, rewrites: cmp.rewrites} g.EqualXPropertyMaps = g.lessThanOrEqualXPropertyMaps return g.EqualTypeRefs(a, b) } @@ -48,6 +58,7 @@ func (cmp *comparer) LessThanOrEqualTypeRefs(a, b tokens.Type) (eq bool) { // Generalizing structural comparisons to specialize for A=B and A<=B separately. type generalizedComparer struct { schema *pschema.PackageSpec + rewrites map[tokens.Type]tokens.Type EqualXPropertyMaps func(xPropertyMap, xPropertyMap) bool } @@ -88,7 +99,18 @@ func (cmp *generalizedComparer) strictlyEqualXPropertyMaps(a, b xPropertyMap) bo return true } +func (cmp *generalizedComparer) rewrite(a tokens.Type) tokens.Type { + if cmp.rewrites == nil { + return a + } + if x, ok := cmp.rewrites[a]; ok { + return x + } + return a +} + func (cmp *generalizedComparer) EqualTypeRefs(a, b tokens.Type) bool { + a, b = cmp.rewrite(a), cmp.rewrite(b) if a == b { return true } diff --git a/pkg/tfgen/unrec/comparer_test.go b/pkg/tfgen/unrec/comparer_test.go index d57acb955..875f688f1 100644 --- a/pkg/tfgen/unrec/comparer_test.go +++ b/pkg/tfgen/unrec/comparer_test.go @@ -25,13 +25,13 @@ func TestComparerAndStatement(t *testing.T) { t1 := "myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement" t2 := "myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement" s := exampleSchema(t) - cmp := &comparer{s} + cmp := &comparer{s, nil} require.Truef(t, cmp.LessThanTypeRefs(tokens.Type(t1), tokens.Type(t2)), "A Date: Fri, 28 Jun 2024 17:52:43 -0400 Subject: [PATCH 22/29] Fix recursion detector one more time --- pkg/tfgen/unrec/recursion_detector.go | 67 ++++++++++++++-------- pkg/tfgen/unrec/recursion_detector_test.go | 1 + 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/pkg/tfgen/unrec/recursion_detector.go b/pkg/tfgen/unrec/recursion_detector.go index d0c8c21b8..13b601db2 100644 --- a/pkg/tfgen/unrec/recursion_detector.go +++ b/pkg/tfgen/unrec/recursion_detector.go @@ -15,6 +15,7 @@ package unrec import ( + "slices" "sort" pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" @@ -57,37 +58,17 @@ func (rd *recursionDetector) Detect(starterTypes []tokens.Type) map[tokens.Type] vis := &typeVisitor{Schema: rd.schema, Visit: rd.detectRootsVisitor} vis.VisitTypes(starterTypes...) - detected := map[tokens.Type]map[tokens.Type]struct{}{} - - roots := []tokens.Type{} - - sort.Slice(roots, func(i, j int) bool { - if len(roots[i]) < len(roots[j]) { - return true - } - return roots[i] < roots[j] - }) - - seenRoot := func(t tokens.Type) bool { - for _, r := range roots { - if rd.cmp.EqualTypeRefs(r, t) { - return true - } - } - return false - } + roots := rd.roots() - for recursionRoot := range rd.detectedRecursiveTypes { - if !seenRoot(recursionRoot) { - roots = append(roots, recursionRoot) - detected[recursionRoot] = map[tokens.Type]struct{}{} - } + detected := map[tokens.Type]map[tokens.Type]struct{}{} + for _, r := range roots { + detected[r] = map[tokens.Type]struct{}{} } // Second pass: detect instances. vis2 := &typeVisitor{Schema: rd.schema, Visit: func(_ []tokens.Type, current tokens.Type) bool { for _, root := range roots { - if rd.cmp.LessThanTypeRefs(current, root) && current != root { + if rd.cmp.LessThanOrEqualTypeRefs(current, root) && current != root { detected[root][current] = struct{}{} return true } @@ -99,6 +80,42 @@ func (rd *recursionDetector) Detect(starterTypes []tokens.Type) map[tokens.Type] return detected } +func (rd *recursionDetector) sorted(types []tokens.Type) []tokens.Type { + tokens := slices.Clone(types) + sort.Slice(tokens, func(i, j int) bool { + if len(tokens[i]) < len(tokens[j]) { + return true + } + return tokens[i] < tokens[j] + }) + return tokens +} + +func (rd *recursionDetector) unique(types []tokens.Type) []tokens.Type { + result := []tokens.Type{} + for _, t := range types { + seen := false + for _, s := range result { + if rd.cmp.EqualTypeRefs(s, t) { + seen = true + break + } + } + if !seen { + result = append(result, t) + } + } + return result +} + +func (rd *recursionDetector) roots() []tokens.Type { + tt := []tokens.Type{} + for t := range rd.detectedRecursiveTypes { + tt = append(tt, t) + } + return rd.unique(rd.sorted(tt)) +} + func (rd *recursionDetector) detectRootsVisitor(ancestors []tokens.Type, current tokens.Type) bool { for i, ai := range ancestors { if _, visited := rd.detectedRecursiveTypes[ai]; visited { diff --git a/pkg/tfgen/unrec/recursion_detector_test.go b/pkg/tfgen/unrec/recursion_detector_test.go index 668c2b26b..714f13c3e 100644 --- a/pkg/tfgen/unrec/recursion_detector_test.go +++ b/pkg/tfgen/unrec/recursion_detector_test.go @@ -44,6 +44,7 @@ func TestRecursionDetector(t *testing.T) { tokens.Type("myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement"): { tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement"): {}, tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"): {}, tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement"): {}, tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement"): {}, }, From a18fd4f486efed4903ac51b0c719d403aaf862e9 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 17:54:06 -0400 Subject: [PATCH 23/29] TestSimplify updated --- pkg/tfgen/unrec/testdata/TestSimplify.golden | 128 +------------------ 1 file changed, 1 insertion(+), 127 deletions(-) diff --git a/pkg/tfgen/unrec/testdata/TestSimplify.golden b/pkg/tfgen/unrec/testdata/TestSimplify.golden index 64abca18c..c0727ba4c 100644 --- a/pkg/tfgen/unrec/testdata/TestSimplify.golden +++ b/pkg/tfgen/unrec/testdata/TestSimplify.golden @@ -144,137 +144,11 @@ "type": "string" }, "scopeDownStatement": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement" + "$ref": "#/types/myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement" } }, "type": "object" }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement": { - "properties": { - "andStatement": { - "$ref": "#/types/myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement" - }, - "regexMatchStatement": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatement" - }, - "sqliMatchStatement": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatement" - }, - "xssMatchStatement": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatement": { - "properties": { - "fieldToMatch": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatch" - }, - "regexString": { - "type": "string" - }, - "textTransformations": { - "type": "array", - "items": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementTextTransformation" - } - } - }, - "type": "object", - "required": [ - "regexString", - "textTransformations" - ] - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatch": { - "properties": { - "method": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatchMethod" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementFieldToMatchMethod": { - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementRegexMatchStatementTextTransformation": { - "properties": { - "priority": { - "type": "integer" - }, - "type": { - "type": "string" - } - }, - "type": "object", - "required": [ - "priority", - "type" - ] - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatement": { - "properties": { - "fieldToMatch": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatch" - }, - "textTransformations": { - "type": "array", - "items": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementTextTransformation" - } - } - }, - "type": "object", - "required": [ - "textTransformations" - ] - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatch": { - "properties": { - "method": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatchMethod" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementFieldToMatchMethod": { - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementTextTransformation:WebAclStatementRateBasedStatementScopeDownStatementSqliMatchStatementTextTransformation": { - "properties": { - "priority": { - "type": "integer" - }, - "type": { - "type": "string" - } - }, - "type": "object", - "required": [ - "priority", - "type" - ] - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatement": { - "properties": { - "fieldToMatch": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatch": { - "properties": { - "method": { - "$ref": "#/types/myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod" - } - }, - "type": "object" - }, - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod:WebAclStatementRateBasedStatementScopeDownStatementXssMatchStatementFieldToMatchMethod": { - "type": "object" - }, "myprov:index/WebAclStatementRegexMatchStatement:WebAclStatementRegexMatchStatement": { "properties": { "fieldToMatch": { From 0f3b7423de44153e802abc9907353c5c19dcd1e9 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 18:28:55 -0400 Subject: [PATCH 24/29] More realism in the schema --- pkg/tfgen/unrec/gen.go | 1 + pkg/tfgen/unrec/recursion_detector.go | 19 ++++++++++++++++- pkg/tfgen/unrec/testdata/TestSimplify.golden | 22 ++++++++++++++++++++ pkg/tfgen/unrec/testdata/test-schema.json | 22 ++++++++++++++++++++ pkg/tfgen/unrec/type_visitor_test.go | 2 +- 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/pkg/tfgen/unrec/gen.go b/pkg/tfgen/unrec/gen.go index b9445b7d6..2bca76bdb 100644 --- a/pkg/tfgen/unrec/gen.go +++ b/pkg/tfgen/unrec/gen.go @@ -68,6 +68,7 @@ func webACLRootStatementSchema(level int) *schema.Schema { "rate_based_statement": rateBasedStatementSchema(level), "sqli_match_statement": sqliMatchStatementSchema(), "regex_match_statement": regexMatchStatementSchema(), + "xss_match_statement": xssMatchStatementSchema(), }, }, } diff --git a/pkg/tfgen/unrec/recursion_detector.go b/pkg/tfgen/unrec/recursion_detector.go index 13b601db2..a502fb965 100644 --- a/pkg/tfgen/unrec/recursion_detector.go +++ b/pkg/tfgen/unrec/recursion_detector.go @@ -135,5 +135,22 @@ func (rd *recursionDetector) detectRootsVisitor(ancestors []tokens.Type, current } func (rd *recursionDetector) detect(t1, t2, t3 tokens.Type) bool { - return rd.cmp.LessThanTypeRefs(t3, t2) && rd.cmp.LessThanTypeRefs(t2, t1) + return rd.cmp.LessThanTypeRefs(t3, t2) && rd.cmp.LessThanTypeRefs(t2, t1) && rd.sameProps(t1, t2) +} + +func (rd *recursionDetector) sameProps(t1, t2 tokens.Type) bool { + t1d, ok1 := rd.schema.Types[string(t1)] + t2d, ok2 := rd.schema.Types[string(t2)] + if !ok1 || !ok2 { + return false + } + if len(t1d.Properties) != len(t2d.Properties) { + return false + } + for k := range t1d.Properties { + if _, ok := t2d.Properties[k]; !ok { + return false + } + } + return true } diff --git a/pkg/tfgen/unrec/testdata/TestSimplify.golden b/pkg/tfgen/unrec/testdata/TestSimplify.golden index c0727ba4c..648b1f970 100644 --- a/pkg/tfgen/unrec/testdata/TestSimplify.golden +++ b/pkg/tfgen/unrec/testdata/TestSimplify.golden @@ -31,6 +31,9 @@ }, "sqliMatchStatement": { "$ref": "#/types/myprov:index/WebAclStatementSqliMatchStatement:WebAclStatementSqliMatchStatement" + }, + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementXssMatchStatement:WebAclStatementXssMatchStatement" } }, "type": "object" @@ -238,6 +241,25 @@ "priority", "type" ] + }, + "myprov:index/WebAclStatementXssMatchStatement:WebAclStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementXssMatchStatementFieldToMatch:WebAclStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementXssMatchStatementFieldToMatch:WebAclStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementXssMatchStatementFieldToMatchMethod:WebAclStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementXssMatchStatementFieldToMatchMethod:WebAclStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" } }, "provider": { diff --git a/pkg/tfgen/unrec/testdata/test-schema.json b/pkg/tfgen/unrec/testdata/test-schema.json index c38a15aa1..87a10f78f 100644 --- a/pkg/tfgen/unrec/testdata/test-schema.json +++ b/pkg/tfgen/unrec/testdata/test-schema.json @@ -31,6 +31,9 @@ }, "sqliMatchStatement": { "$ref": "#/types/myprov:index/WebAclStatementSqliMatchStatement:WebAclStatementSqliMatchStatement" + }, + "xssMatchStatement": { + "$ref": "#/types/myprov:index/WebAclStatementXssMatchStatement:WebAclStatementXssMatchStatement" } }, "type": "object" @@ -955,6 +958,25 @@ "priority", "type" ] + }, + "myprov:index/WebAclStatementXssMatchStatement:WebAclStatementXssMatchStatement": { + "properties": { + "fieldToMatch": { + "$ref": "#/types/myprov:index/WebAclStatementXssMatchStatementFieldToMatch:WebAclStatementXssMatchStatementFieldToMatch" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementXssMatchStatementFieldToMatch:WebAclStatementXssMatchStatementFieldToMatch": { + "properties": { + "method": { + "$ref": "#/types/myprov:index/WebAclStatementXssMatchStatementFieldToMatchMethod:WebAclStatementXssMatchStatementFieldToMatchMethod" + } + }, + "type": "object" + }, + "myprov:index/WebAclStatementXssMatchStatementFieldToMatchMethod:WebAclStatementXssMatchStatementFieldToMatchMethod": { + "type": "object" } }, "provider": { diff --git a/pkg/tfgen/unrec/type_visitor_test.go b/pkg/tfgen/unrec/type_visitor_test.go index 5fae0c97d..51a539114 100644 --- a/pkg/tfgen/unrec/type_visitor_test.go +++ b/pkg/tfgen/unrec/type_visitor_test.go @@ -44,7 +44,7 @@ func TestTypeVisitor(t *testing.T) { }} vis.VisitTypes(starterTypes...) - require.Equal(t, 87, count) + require.Equal(t, 90, count) tok := "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement" require.Equal(t, []tokens.Type{ From 54ba222de394ff139e8c277daf4260d23005dbcc Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 19:19:17 -0400 Subject: [PATCH 25/29] PR feedback --- pkg/tfgen/unrec/comparer.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/pkg/tfgen/unrec/comparer.go b/pkg/tfgen/unrec/comparer.go index 87862b59f..a260ef0bd 100644 --- a/pkg/tfgen/unrec/comparer.go +++ b/pkg/tfgen/unrec/comparer.go @@ -81,22 +81,7 @@ func (cmp *generalizedComparer) lessThanOrEqualXPropertyMaps(a, b xPropertyMap) } func (cmp *generalizedComparer) strictlyEqualXPropertyMaps(a, b xPropertyMap) bool { - if a == nil || b == nil { - return a == nil && b == nil - } - if len(a) != len(b) { - return false - } - for k, av := range a { - bv, ok := b[k] - if !ok { - return false - } - if !cmp.EqualXProperties(av, bv) { - return false - } - } - return true + return len(a) == len(b) && cmp.lessThanOrEqualXPropertyMaps(a, b) } func (cmp *generalizedComparer) rewrite(a tokens.Type) tokens.Type { From f52a2945043e0fceb9815a3cf8f6942fb8333155 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 19:20:34 -0400 Subject: [PATCH 26/29] More PR feedback: labels --- pkg/tfgen/unrec/equality_classes.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/tfgen/unrec/equality_classes.go b/pkg/tfgen/unrec/equality_classes.go index 91274220e..5226e7fca 100644 --- a/pkg/tfgen/unrec/equality_classes.go +++ b/pkg/tfgen/unrec/equality_classes.go @@ -51,18 +51,15 @@ func mergeEqualityClasses[T any](eq func(T, T) bool, xs [][]T, ys [][]T) (result } acc = append(acc, xc) } +outer: for _, yc := range ys { - matched := false for i, ac := range acc { if sameEqualityClasses(eq, ac, yc) { acc[i] = append(append([]T{}, ac...), yc...) - matched = true - break + continue outer } } - if !matched { - acc = append(acc, yc) - } + acc = append(acc, yc) } return acc } From fb0ca558a42ba6fb91774f240ae756941f026377 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 19:22:15 -0400 Subject: [PATCH 27/29] Comment on go generate --- pkg/tfgen/unrec/example_schema_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/tfgen/unrec/example_schema_test.go b/pkg/tfgen/unrec/example_schema_test.go index 313e29c52..85f10c989 100644 --- a/pkg/tfgen/unrec/example_schema_test.go +++ b/pkg/tfgen/unrec/example_schema_test.go @@ -23,6 +23,9 @@ import ( pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" ) +// Consults a generated test schema (see gen.go), run go generate to rebuild. This indirection allows the package not to +// build-depend on the tfgen module in case it ever needs to be exported from tfgen to avoid build cycles. +// //go:generate go run gen.go func exampleSchema(t *testing.T) *pschema.PackageSpec { b, err := os.ReadFile(filepath.Join("testdata", "test-schema.json")) From 78f60408cf7a04982d425018a1285c2e583f8631 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 19:27:21 -0400 Subject: [PATCH 28/29] Clarify the algo --- pkg/tfgen/unrec/recursion_detector.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/tfgen/unrec/recursion_detector.go b/pkg/tfgen/unrec/recursion_detector.go index a502fb965..796379329 100644 --- a/pkg/tfgen/unrec/recursion_detector.go +++ b/pkg/tfgen/unrec/recursion_detector.go @@ -35,8 +35,13 @@ import ( // // ancestor(T1, T2), ancestor(T2, T3), T3 <= T2 <= T1 // +// Such that: +// +// set(props(T1)) = set(props(T2)) +// // This needs to use an approximate and not strict equality because the leaf node of a recursively unrolled type will -// drop recursive properties and therefore not strictly match the ancestor. +// drop recursive properties and therefore not strictly match the ancestor. The prop-set condition prevents accidentally +// identifying non-recursive subset instances as recursive instances. type recursionDetector struct { schema *pschema.PackageSpec detectedRecursiveTypes map[tokens.Type]struct{} From f16155371317df515b28fc9201ff04afe5f23124 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 28 Jun 2024 19:35:48 -0400 Subject: [PATCH 29/29] Lint --- pkg/tfgen/unrec/comparer_test.go | 3 ++- pkg/tfgen/unrec/package.go | 1 - pkg/tfgen/unrec/package_spec_util.go | 2 +- pkg/tfgen/unrec/recursion_detector_test.go | 16 +--------------- .../unrec/testdata/TestRecursionDetector.golden | 15 +++++++++++++++ pkg/tfgen/unrec/testdata/TestTypeVisitor.golden | 8 ++++++++ pkg/tfgen/unrec/type_visitor_test.go | 11 +++-------- 7 files changed, 30 insertions(+), 26 deletions(-) create mode 100644 pkg/tfgen/unrec/testdata/TestRecursionDetector.golden create mode 100644 pkg/tfgen/unrec/testdata/TestTypeVisitor.golden diff --git a/pkg/tfgen/unrec/comparer_test.go b/pkg/tfgen/unrec/comparer_test.go index 875f688f1..e2e9503d4 100644 --- a/pkg/tfgen/unrec/comparer_test.go +++ b/pkg/tfgen/unrec/comparer_test.go @@ -21,7 +21,8 @@ import ( "github.com/stretchr/testify/require" ) -func TestComparerAndStatement(t *testing.T) { +func TesComparerAndStatement(t *testing.T) { + //nolint:lll t1 := "myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement" t2 := "myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement" s := exampleSchema(t) diff --git a/pkg/tfgen/unrec/package.go b/pkg/tfgen/unrec/package.go index df46f8a91..ad1756a9e 100644 --- a/pkg/tfgen/unrec/package.go +++ b/pkg/tfgen/unrec/package.go @@ -8,4 +8,3 @@ // // [Pulumi Package Schema]: https://www.pulumi.com/docs/guides/pulumi-packages/schema/ package unrec - diff --git a/pkg/tfgen/unrec/package_spec_util.go b/pkg/tfgen/unrec/package_spec_util.go index 844ae7561..c57ce0d1a 100644 --- a/pkg/tfgen/unrec/package_spec_util.go +++ b/pkg/tfgen/unrec/package_spec_util.go @@ -181,7 +181,7 @@ func rewriteTypeRefs(rewrites map[tokens.Type]tokens.Type, schema *pschema.Packa return err } - for deletedType, _ := range rewrites { + for deletedType := range rewrites { delete(modifiedSchema.Types, string(deletedType)) } diff --git a/pkg/tfgen/unrec/recursion_detector_test.go b/pkg/tfgen/unrec/recursion_detector_test.go index 714f13c3e..0cc15f4e1 100644 --- a/pkg/tfgen/unrec/recursion_detector_test.go +++ b/pkg/tfgen/unrec/recursion_detector_test.go @@ -34,19 +34,5 @@ func TestRecursionDetector(t *testing.T) { } } - autogold.Expect(map[tokens.Type]map[tokens.Type]struct{}{ - tokens.Type("myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement"): { - tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement"): {}, - tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement"): {}, - tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement"): {}, - tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement"): {}, - }, - tokens.Type("myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement"): { - tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement"): {}, - tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement"): {}, - tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"): {}, - tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement"): {}, - tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement"): {}, - }, - }).Equal(t, rd.Detect(starterTypes)) + autogold.ExpectFile(t, rd.Detect(starterTypes)) } diff --git a/pkg/tfgen/unrec/testdata/TestRecursionDetector.golden b/pkg/tfgen/unrec/testdata/TestRecursionDetector.golden new file mode 100644 index 000000000..518dda7a2 --- /dev/null +++ b/pkg/tfgen/unrec/testdata/TestRecursionDetector.golden @@ -0,0 +1,15 @@ +map[tokens.Type]map[tokens.Type]struct{}{ + tokens.Type("myprov:index/WebAclStatementAndStatement:WebAclStatementAndStatement"): { + tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatement"): {}, + tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement"): {}, + }, + tokens.Type("myprov:index/WebAclStatementAndStatementStatement:WebAclStatementAndStatementStatement"): { + tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatement"): {}, + tokens.Type("myprov:index/WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement:WebAclStatementAndStatementStatementAndStatementStatementAndStatementStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement"): {}, + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement"): {}, + }, +} diff --git a/pkg/tfgen/unrec/testdata/TestTypeVisitor.golden b/pkg/tfgen/unrec/testdata/TestTypeVisitor.golden new file mode 100644 index 000000000..9ddfe4ae3 --- /dev/null +++ b/pkg/tfgen/unrec/testdata/TestTypeVisitor.golden @@ -0,0 +1,8 @@ +[]tokens.Type{ + tokens.Type("myprov:index/WebAclStatement:WebAclStatement"), + tokens.Type("myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement"), + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement"), + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement"), + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement"), + tokens.Type("myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement"), +} diff --git a/pkg/tfgen/unrec/type_visitor_test.go b/pkg/tfgen/unrec/type_visitor_test.go index 51a539114..beac41256 100644 --- a/pkg/tfgen/unrec/type_visitor_test.go +++ b/pkg/tfgen/unrec/type_visitor_test.go @@ -17,6 +17,7 @@ package unrec import ( "testing" + "github.com/hexops/autogold/v2" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" "github.com/stretchr/testify/require" ) @@ -46,13 +47,7 @@ func TestTypeVisitor(t *testing.T) { require.Equal(t, 90, count) + //nolint:lll tok := "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatementStatement" - require.Equal(t, []tokens.Type{ - "myprov:index/WebAclStatement:WebAclStatement", - "myprov:index/WebAclStatementRateBasedStatement:WebAclStatementRateBasedStatement", - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatement:WebAclStatementRateBasedStatementScopeDownStatement", - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatement", - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatement", - "myprov:index/WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement:WebAclStatementRateBasedStatementScopeDownStatementAndStatementStatementAndStatement", - }, visited[tokens.Type(tok)]) + autogold.ExpectFile(t, visited[tokens.Type(tok)]) }