From 27e570811139561e3aa28e80c9d5c29c3b005e48 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 00:06:36 +0100 Subject: [PATCH 01/48] test: add initial test/API for exported tf --- generate/exportedtf/exportedtf.go | 40 ++++++++++ generate/exportedtf/exportedtf_test.go | 104 +++++++++++++++++++++++++ test/sandbox/sandbox.go | 5 +- 3 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 generate/exportedtf/exportedtf.go create mode 100644 generate/exportedtf/exportedtf_test.go diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go new file mode 100644 index 000000000..421a6d42f --- /dev/null +++ b/generate/exportedtf/exportedtf.go @@ -0,0 +1,40 @@ +package exportedtf + +import ( + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/mineiros-io/terramate" + "github.com/mineiros-io/terramate/stack" +) + +// Configs represents a map of configurations for exported +// Terraform code. These configurations are HCL with +// evaluated values on them. +type Configs map[string]Config + +// Config represents a configuration for exported Terraform code. +// Is contains HCL parsed code with evaluated values on it. +type Config hclsyntax.Body + +// String returns a string representation of the configuration +// that is guaranteed to be valid HCL or an empty string if the config +// itself is empty. +func (c Config) String() string { + return "" +} + +// Load loads from the file system all export_as_terraform for +// a given stack. It will navigate the file system from the stack dir until +// it reaches rootdir, loading export_as_terraform and merging them appropriately. +// +// More specific definitions (closer or at the stack) have precedence over +// less specific ones (closer or at the root dir). +// +// Metadata and globals for the stack are used on the evaluation of the +// export_as_terramate blocks. +// +// The returned result only contains evaluated values. +// +// The rootdir MUST be an absolute path. +func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (Configs, error) { + return nil, nil +} diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go new file mode 100644 index 000000000..825f766a2 --- /dev/null +++ b/generate/exportedtf/exportedtf_test.go @@ -0,0 +1,104 @@ +package exportedtf_test + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/madlambda/spells/assert" + "github.com/mineiros-io/terramate/config" + "github.com/mineiros-io/terramate/generate/exportedtf" + "github.com/mineiros-io/terramate/test" + "github.com/mineiros-io/terramate/test/hclwrite" + "github.com/mineiros-io/terramate/test/sandbox" +) + +func TestLoadExportedTerraform(t *testing.T) { + type ( + hclconfig struct { + path string + add fmt.Stringer + } + testcase struct { + name string + stack string + configs []hclconfig + want map[string]fmt.Stringer + wantErr error + } + ) + + exportAsTerraform := func(label string, builders ...hclwrite.BlockBuilder) *hclwrite.Block { + b := hclwrite.BuildBlock("export_as_terraform", builders...) + b.AddLabel(label) + return b + } + block := func(name string, builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock(name, builders...) + } + globals := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("globals", builders...) + } + expr := hclwrite.Expression + str := hclwrite.String + number := hclwrite.NumberInt + boolean := hclwrite.Boolean + + tcases := []testcase{ + { + name: "no exported terraform", + stack: "/stack", + }, + { + name: "exported terraform on stack with single block", + stack: "/stack", + configs: []hclconfig{ + { + path: "/stack", + add: globals( + str("some_string", "string"), + number("some_number", 777), + boolean("some_bool", true), + ), + }, + { + path: "/stack", + add: exportAsTerraform("test", + block("testblock", + expr("string", "global.some_string"), + expr("number", "global.some_number"), + expr("bool", "global.some_bool"), + ), + ), + }, + }, + want: map[string]fmt.Stringer{ + "test": block("testblock", + str("string_local", "string"), + number("number_local", 777), + boolean("bool_local", true), + ), + }, + }, + } + + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + s := sandbox.New(t) + stackEntry := s.CreateStack(tcase.stack) + stack := stackEntry.Load() + + for _, cfg := range tcase.configs { + path := filepath.Join(s.RootDir(), cfg.path) + test.AppendFile(t, path, config.Filename, cfg.add.String()) + } + + meta := stack.Meta() + globals := s.LoadStackGlobals(meta) + _, err := exportedtf.Load(s.RootDir(), meta, globals) + assert.IsError(t, err, tcase.wantErr) + + // TODO(katcipis): check exported terraform + }) + } +} diff --git a/test/sandbox/sandbox.go b/test/sandbox/sandbox.go index 560904670..271dbaf7b 100644 --- a/test/sandbox/sandbox.go +++ b/test/sandbox/sandbox.go @@ -250,12 +250,15 @@ func (s S) CreateModule(relpath string) DirEntry { // CreateStack will create a stack dir with the given relative path and // initializes the stack, returning a stack entry that can be used // to create files inside the stack dir. +// +// If the path is absolute, it will be considered in relation to the sandbox +// root dir. func (s S) CreateStack(relpath string) StackEntry { t := s.t t.Helper() if filepath.IsAbs(relpath) { - t.Fatalf("CreateStack() needs a relative path but given %q", relpath) + relpath = relpath[1:] } stack := newStackEntry(t, s.RootDir(), relpath) From cc159e6cd6065a3e3702dc0a8f444af342a82007 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 00:18:39 +0100 Subject: [PATCH 02/48] test: improve initial test --- generate/exportedtf/exportedtf_test.go | 39 ++++++++++++++++++++------ 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index 825f766a2..3c4526eb2 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -19,11 +19,15 @@ func TestLoadExportedTerraform(t *testing.T) { path string add fmt.Stringer } + result struct { + name string + hcl fmt.Stringer + } testcase struct { name string stack string configs []hclconfig - want map[string]fmt.Stringer + want []result wantErr error } ) @@ -39,6 +43,10 @@ func TestLoadExportedTerraform(t *testing.T) { globals := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { return hclwrite.BuildBlock("globals", builders...) } + attr := func(name, expr string) hclwrite.BlockBuilder { + t.Helper() + return hclwrite.AttributeValue(t, name, expr) + } expr := hclwrite.Expression str := hclwrite.String number := hclwrite.NumberInt @@ -65,19 +73,32 @@ func TestLoadExportedTerraform(t *testing.T) { path: "/stack", add: exportAsTerraform("test", block("testblock", - expr("string", "global.some_string"), - expr("number", "global.some_number"), expr("bool", "global.some_bool"), + expr("number", "global.some_number"), + expr("string", "global.some_string"), + expr("obj", `{ + string = global.some_string + number = global.some_number + bool = global.bool + }`), ), ), }, }, - want: map[string]fmt.Stringer{ - "test": block("testblock", - str("string_local", "string"), - number("number_local", 777), - boolean("bool_local", true), - ), + want: []result{ + { + name: "test", + hcl: block("testblock", + boolean("bool", true), + number("number", 777), + str("string", "string"), + attr("obj", `{ + bool = true + number = 777 + string = "string" + }`), + ), + }, }, }, } From cd3574c022331e7831a8ec0a31eb372d3e45df4f Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 00:24:18 +0100 Subject: [PATCH 03/48] refactor: improve type names and docs --- generate/exportedtf/exportedtf.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index 421a6d42f..7a0044550 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -6,19 +6,17 @@ import ( "github.com/mineiros-io/terramate/stack" ) -// Configs represents a map of configurations for exported -// Terraform code. These configurations are HCL with -// evaluated values on them. -type Configs map[string]Config +// TfCode represents all exported terraform code for a stack, +// mapping the exported code name to the actual Terraform code. +type TfCode map[string]Body -// Config represents a configuration for exported Terraform code. -// Is contains HCL parsed code with evaluated values on it. -type Config hclsyntax.Body +// Body represents exported Terraform code from a single block. +// Is contains parsed and evaluated code on it. +type Body hclsyntax.Body -// String returns a string representation of the configuration -// that is guaranteed to be valid HCL or an empty string if the config -// itself is empty. -func (c Config) String() string { +// String returns a string representation of the Terraform code +// or an empty string if the config itself is empty. +func (b Body) String() string { return "" } @@ -35,6 +33,6 @@ func (c Config) String() string { // The returned result only contains evaluated values. // // The rootdir MUST be an absolute path. -func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (Configs, error) { +func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (TfCode, error) { return nil, nil } From 8e483f009c72dbf88ab11ac5dccd35eb23a29ae2 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 15:00:22 +0100 Subject: [PATCH 04/48] feat: first working/incomplete version --- generate/exportedtf/exportedtf.go | 113 ++++++++++++++++++++++++- generate/exportedtf/exportedtf_test.go | 64 ++++++++++---- generate/generate.go | 2 + hcl/hcl.go | 69 +++++++++++++++ 4 files changed, 229 insertions(+), 19 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index 7a0044550..74902bf10 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -1,9 +1,17 @@ package exportedtf import ( + "fmt" + "path/filepath" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/hcl/v2/hclwrite" "github.com/mineiros-io/terramate" + "github.com/mineiros-io/terramate/config" + "github.com/mineiros-io/terramate/hcl" + "github.com/mineiros-io/terramate/hcl/eval" "github.com/mineiros-io/terramate/stack" + "github.com/rs/zerolog/log" ) // TfCode represents all exported terraform code for a stack, @@ -12,12 +20,14 @@ type TfCode map[string]Body // Body represents exported Terraform code from a single block. // Is contains parsed and evaluated code on it. -type Body hclsyntax.Body +type Body struct { + body []byte +} // String returns a string representation of the Terraform code // or an empty string if the config itself is empty. func (b Body) String() string { - return "" + return string(b.body) } // Load loads from the file system all export_as_terraform for @@ -34,5 +44,102 @@ func (b Body) String() string { // // The rootdir MUST be an absolute path. func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (TfCode, error) { - return nil, nil + stackpath := filepath.Join(rootdir, sm.Path, config.Filename) + logger := log.With(). + Str("action", "exportedtf.Load()"). + Str("path", stackpath). + Logger() + + logger.Trace().Msg("loading export_as_terraform blocks.") + + exportBlocks, err := loadExportBlocks(rootdir, stackpath) + if err != nil { + return nil, fmt.Errorf("loading exported terraform code: %v", err) + } + + evalctx, err := newEvalCtx(stackpath, sm, globals) + if err != nil { + return nil, fmt.Errorf("preparing to eval exported terraform code: %v", err) + } + + logger.Trace().Msg("generating exported terraform code.") + + res := map[string]Body{} + + for name, block := range exportBlocks { + logger := logger.With(). + Str("block", name). + Logger() + + logger.Trace().Msg("evaluating block.") + + gen := hclwrite.NewEmptyFile() + + // TODO(katcipis): test if block.Body is nil + if err := hcl.CopyBody(gen.Body(), block.Body, evalctx); err != nil { + return nil, fmt.Errorf( + "generating terraform code for stack %q block %q: %v", + stackpath, + name, + err, + ) + } + + res[name] = Body{body: gen.Bytes()} + } + + return res, nil +} + +func newEvalCtx(stackpath string, sm stack.Metadata, globals *terramate.Globals) (*eval.Context, error) { + logger := log.With(). + Str("action", "exportedtf.newEvalCtx()"). + Str("path", stackpath). + Logger() + + evalctx := eval.NewContext(stackpath) + + logger.Trace().Msg("Add stack metadata evaluation namespace.") + + err := evalctx.SetNamespace("terramate", sm.ToCtyMap()) + if err != nil { + return nil, fmt.Errorf("setting terramate namespace on eval context for stack %q: %v", + stackpath, err) + } + + logger.Trace().Msg("Add global evaluation namespace.") + + if err := evalctx.SetNamespace("global", globals.Attributes()); err != nil { + return nil, fmt.Errorf("setting global namespace on eval context for stack %q: %v", + stackpath, err) + } + + return evalctx, nil +} + +// loadExportBlocks will load all export_as_terraform blocks applying overriding +// as it goes, the returned map maps the name of the block (its label) to the original block +func loadExportBlocks(rootdir string, cfgpath string) (map[string]*hclsyntax.Block, error) { + logger := log.With(). + Str("action", "exportedtf.loadExportBlocks()"). + Str("configFile", cfgpath). + Logger() + + logger.Trace().Msg("Parsing export_as_terraform blocks.") + + blocks, err := hcl.ParseExportAsTerraformBlocks(cfgpath) + if err != nil { + return nil, fmt.Errorf("loading exported terraform code: %v", err) + } + + res := map[string]*hclsyntax.Block{} + + for _, block := range blocks { + // TODO(katcipis): properly test wrong amount of labels + // TODO(katcipis): properly test two blocks with same label on same config file + name := block.Labels[0] + res[name] = block + } + + return res, nil } diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index 3c4526eb2..ea4996f9a 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -3,14 +3,17 @@ package exportedtf_test import ( "fmt" "path/filepath" + "strings" "testing" + "github.com/google/go-cmp/cmp" "github.com/madlambda/spells/assert" "github.com/mineiros-io/terramate/config" "github.com/mineiros-io/terramate/generate/exportedtf" "github.com/mineiros-io/terramate/test" "github.com/mineiros-io/terramate/test/hclwrite" "github.com/mineiros-io/terramate/test/sandbox" + "github.com/rs/zerolog" ) func TestLoadExportedTerraform(t *testing.T) { @@ -43,10 +46,10 @@ func TestLoadExportedTerraform(t *testing.T) { globals := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { return hclwrite.BuildBlock("globals", builders...) } - attr := func(name, expr string) hclwrite.BlockBuilder { - t.Helper() - return hclwrite.AttributeValue(t, name, expr) - } + //attr := func(name, expr string) hclwrite.BlockBuilder { + //t.Helper() + //return hclwrite.AttributeValue(t, name, expr) + //} expr := hclwrite.Expression str := hclwrite.String number := hclwrite.NumberInt @@ -76,11 +79,11 @@ func TestLoadExportedTerraform(t *testing.T) { expr("bool", "global.some_bool"), expr("number", "global.some_number"), expr("string", "global.some_string"), - expr("obj", `{ - string = global.some_string - number = global.some_number - bool = global.bool - }`), + //expr("obj", `{ + //string = global.some_string + //number = global.some_number + //bool = global.bool + //}`), ), ), }, @@ -92,11 +95,11 @@ func TestLoadExportedTerraform(t *testing.T) { boolean("bool", true), number("number", 777), str("string", "string"), - attr("obj", `{ - bool = true - number = 777 - string = "string" - }`), + //attr("obj", `{ + //bool = true + //number = 777 + //string = "string" + //}`), ), }, }, @@ -116,10 +119,39 @@ func TestLoadExportedTerraform(t *testing.T) { meta := stack.Meta() globals := s.LoadStackGlobals(meta) - _, err := exportedtf.Load(s.RootDir(), meta, globals) + got, err := exportedtf.Load(s.RootDir(), meta, globals) assert.IsError(t, err, tcase.wantErr) - // TODO(katcipis): check exported terraform + for _, res := range tcase.want { + gothcl, ok := got[res.name] + if !ok { + t.Fatalf("want hcl code for %q, got: %v", res.name, got) + } + gotcode := gothcl.String() + wantcode := res.hcl.String() + + assertHCLEquals(t, gotcode, wantcode) + } }) } } + +func assertHCLEquals(t *testing.T, got string, want string) { + t.Helper() + + const trimmedChars = "\n " + + got = strings.Trim(got, trimmedChars) + want = strings.Trim(want, trimmedChars) + + if diff := cmp.Diff(want, got); diff != "" { + t.Error("generated code doesn't match expectation") + t.Errorf("want:\n%q", want) + t.Errorf("got:\n%q", got) + t.Fatalf("diff:\n%s", diff) + } +} + +func init() { + zerolog.SetGlobalLevel(zerolog.Disabled) +} diff --git a/generate/generate.go b/generate/generate.go index 5822232bb..e21d2f168 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -401,6 +401,7 @@ func PrependHeaderBytes(code []byte) []byte { const codeHeader = "// GENERATED BY TERRAMATE: DO NOT EDIT" +// TODO(katcipis): use extracted version from hcl package func copyBody(target *hclwrite.Body, src *hclsyntax.Body, evalctx *eval.Context) error { if src == nil || target == nil { return nil @@ -440,6 +441,7 @@ func copyBody(target *hclwrite.Body, src *hclsyntax.Body, evalctx *eval.Context) return nil } +// TODO(katcipis): use extracted version from hcl package func sortedAttributes(attrs hclsyntax.Attributes) []*hclsyntax.Attribute { names := make([]string, 0, len(attrs)) diff --git a/hcl/hcl.go b/hcl/hcl.go index 57d67665a..581e63503 100644 --- a/hcl/hcl.go +++ b/hcl/hcl.go @@ -23,7 +23,9 @@ import ( "github.com/hashicorp/hcl/v2/hclparse" "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/hcl/v2/hclwrite" "github.com/madlambda/spells/errutil" + "github.com/mineiros-io/terramate/hcl/eval" "github.com/rs/zerolog/log" "github.com/zclconf/go-cty/cty" ) @@ -465,6 +467,73 @@ func ParseExportAsLocalsBlocks(path string) ([]*hclsyntax.Block, error) { return parseBlocksOfType(path, "export_as_locals") } +// ParseExportAsTerraformBlocks parses export_as_terraform blocks, ignoring other blocks +func ParseExportAsTerraformBlocks(path string) ([]*hclsyntax.Block, error) { + return parseBlocksOfType(path, "export_as_terraform") +} + +// CopyBody will copy the src body to the given target, evaluating attributes using the +// given evaluation context. +// +// Returns an error if the evaluation fails. +func CopyBody(target *hclwrite.Body, src *hclsyntax.Body, evalctx *eval.Context) error { + logger := log.With(). + Str("action", "CopyBody()"). + Logger() + + logger.Trace().Msg("Sorting attributes.") + + // Avoid generating code with random attr order (map iteration is random) + attrs := sortedAttributes(src.Attributes) + + for _, attr := range attrs { + val, err := evalctx.Eval(attr.Expr) + if err != nil { + return fmt.Errorf("parsing attribute %q: %v", attr.Name, err) + } + + logger.Trace(). + Str("attribute", attr.Name). + Msg("Setting evaluated attribute.") + + target.SetAttributeValue(attr.Name, val) + } + + logger.Trace().Msg("Append blocks.") + + for _, block := range src.Blocks { + targetBlock := target.AppendNewBlock(block.Type, block.Labels) + if block.Body == nil { + continue + } + if err := CopyBody(targetBlock.Body(), block.Body, evalctx); err != nil { + return err + } + } + + return nil +} + +func sortedAttributes(attrs hclsyntax.Attributes) []*hclsyntax.Attribute { + names := make([]string, 0, len(attrs)) + + for name := range attrs { + names = append(names, name) + } + + log.Trace(). + Str("action", "sortedAttributes()"). + Msg("Sort attributes.") + sort.Strings(names) + + sorted := make([]*hclsyntax.Attribute, len(names)) + for i, name := range names { + sorted[i] = attrs[name] + } + + return sorted +} + func parseBlocksOfType(path string, blocktype string) ([]*hclsyntax.Block, error) { logger := log.With(). Str("action", "parseBlocksOfType()"). From c5ba62391418509e3e249b1c4a4d6ccb49bd84bc Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 15:02:54 +0100 Subject: [PATCH 05/48] test: add object back to test --- generate/exportedtf/exportedtf_test.go | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index ea4996f9a..2d8a2bca0 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -46,10 +46,10 @@ func TestLoadExportedTerraform(t *testing.T) { globals := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { return hclwrite.BuildBlock("globals", builders...) } - //attr := func(name, expr string) hclwrite.BlockBuilder { - //t.Helper() - //return hclwrite.AttributeValue(t, name, expr) - //} + attr := func(name, expr string) hclwrite.BlockBuilder { + t.Helper() + return hclwrite.AttributeValue(t, name, expr) + } expr := hclwrite.Expression str := hclwrite.String number := hclwrite.NumberInt @@ -79,11 +79,11 @@ func TestLoadExportedTerraform(t *testing.T) { expr("bool", "global.some_bool"), expr("number", "global.some_number"), expr("string", "global.some_string"), - //expr("obj", `{ - //string = global.some_string - //number = global.some_number - //bool = global.bool - //}`), + expr("obj", `{ + string = global.some_string + number = global.some_number + bool = global.some_bool + }`), ), ), }, @@ -95,11 +95,11 @@ func TestLoadExportedTerraform(t *testing.T) { boolean("bool", true), number("number", 777), str("string", "string"), - //attr("obj", `{ - //bool = true - //number = 777 - //string = "string" - //}`), + attr("obj", `{ + bool = true + number = 777 + string = "string" + }`), ), }, }, From 000672502fb668cbdd7d1bd8b5381ca14470069d Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 15:04:53 +0100 Subject: [PATCH 06/48] refactor: use new extracted func to copy bodies --- generate/generate.go | 64 +------------------------------------------- 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index e21d2f168..9f9c040ff 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -23,7 +23,6 @@ import ( "sort" "strings" - "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/madlambda/spells/errutil" "github.com/mineiros-io/terramate" @@ -386,7 +385,7 @@ func generateBackendCfgCode( backendBlock := tfBody.AppendNewBlock(parsed.Backend.Type, parsed.Backend.Labels) backendBody := backendBlock.Body() - if err := copyBody(backendBody, parsed.Backend.Body, evalctx); err != nil { + if err := hcl.CopyBody(backendBody, parsed.Backend.Body, evalctx); err != nil { return nil, err } @@ -401,67 +400,6 @@ func PrependHeaderBytes(code []byte) []byte { const codeHeader = "// GENERATED BY TERRAMATE: DO NOT EDIT" -// TODO(katcipis): use extracted version from hcl package -func copyBody(target *hclwrite.Body, src *hclsyntax.Body, evalctx *eval.Context) error { - if src == nil || target == nil { - return nil - } - - logger := log.With(). - Str("action", "copyBody()"). - Logger() - - logger.Trace(). - Msg("Get sorted attributes.") - - // Avoid generating code with random attr order (map iteration is random) - attrs := sortedAttributes(src.Attributes) - - for _, attr := range attrs { - val, err := evalctx.Eval(attr.Expr) - if err != nil { - return fmt.Errorf("parsing attribute %q: %v", attr.Name, err) - } - logger.Trace(). - Str("attribute", attr.Name). - Msg("Set attribute value.") - target.SetAttributeValue(attr.Name, val) - } - - logger.Trace(). - Msg("Append blocks.") - for _, block := range src.Blocks { - targetBlock := target.AppendNewBlock(block.Type, block.Labels) - targetBody := targetBlock.Body() - if err := copyBody(targetBody, block.Body, evalctx); err != nil { - return err - } - } - - return nil -} - -// TODO(katcipis): use extracted version from hcl package -func sortedAttributes(attrs hclsyntax.Attributes) []*hclsyntax.Attribute { - names := make([]string, 0, len(attrs)) - - for name := range attrs { - names = append(names, name) - } - - log.Trace(). - Str("action", "sortedAttributes()"). - Msg("Sort attributes.") - sort.Strings(names) - - sorted := make([]*hclsyntax.Attribute, len(names)) - for i, name := range names { - sorted[i] = attrs[name] - } - - return sorted -} - func writeGeneratedCode(target string, code []byte) error { logger := log.With(). Str("action", "writeGeneratedCode()"). From 0be1c5ed24d04c6642d987bf37df1b04265e34a6 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 15:05:43 +0100 Subject: [PATCH 07/48] chore: add license --- generate/exportedtf/exportedtf.go | 14 ++++++++++++++ generate/exportedtf/exportedtf_test.go | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index 74902bf10..77bf911cb 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -1,3 +1,17 @@ +// Copyright 2022 Mineiros GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package exportedtf import ( diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index 2d8a2bca0..a81457d19 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -1,3 +1,17 @@ +// Copyright 2022 Mineiros GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package exportedtf_test import ( From 8ab69c2b938acc932e2d048f24e756d51eb0c1c3 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 15:15:45 +0100 Subject: [PATCH 08/48] test: add test for empty block --- generate/exportedtf/exportedtf_test.go | 41 ++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index a81457d19..ca09f6f4e 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -49,6 +49,9 @@ func TestLoadExportedTerraform(t *testing.T) { } ) + hcldoc := func(blocks ...*hclwrite.Block) hclwrite.HCL { + return hclwrite.NewHCL(blocks...) + } exportAsTerraform := func(label string, builders ...hclwrite.BlockBuilder) *hclwrite.Block { b := hclwrite.BuildBlock("export_as_terraform", builders...) b.AddLabel(label) @@ -74,6 +77,40 @@ func TestLoadExportedTerraform(t *testing.T) { name: "no exported terraform", stack: "/stack", }, + { + name: "empty export_as_terraform block generates empty code", + stack: "/stack", + configs: []hclconfig{ + { + path: "/stack", + add: exportAsTerraform("empty"), + }, + }, + want: []result{ + { + name: "empty", + hcl: hcldoc(), + }, + }, + }, + { + name: "exported terraform on stack with single empty block", + stack: "/stack", + configs: []hclconfig{ + { + path: "/stack", + add: exportAsTerraform("emptytest", + block("empty"), + ), + }, + }, + want: []result{ + { + name: "emptytest", + hcl: block("empty"), + }, + }, + }, { name: "exported terraform on stack with single block", stack: "/stack", @@ -145,7 +182,11 @@ func TestLoadExportedTerraform(t *testing.T) { wantcode := res.hcl.String() assertHCLEquals(t, gotcode, wantcode) + + delete(got, res.name) } + + assert.EqualInts(t, 0, len(got), "got unexpected exported code: %v", got) }) } } From 0c1859da6feb936c5de311c6cb857d8c76d62127 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 15:16:08 +0100 Subject: [PATCH 09/48] chore: remove unnecessary TODO --- generate/exportedtf/exportedtf.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index 77bf911cb..c3f72f042 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -88,8 +88,6 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (TfCode logger.Trace().Msg("evaluating block.") gen := hclwrite.NewEmptyFile() - - // TODO(katcipis): test if block.Body is nil if err := hcl.CopyBody(gen.Body(), block.Body, evalctx); err != nil { return nil, fmt.Errorf( "generating terraform code for stack %q block %q: %v", @@ -98,7 +96,6 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (TfCode err, ) } - res[name] = Body{body: gen.Bytes()} } From ef1e58317153b909632fdb6584538ad6a4327061 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 19:32:54 +0100 Subject: [PATCH 10/48] refactor: make types immutable --- generate/exportedtf/exportedtf.go | 30 ++++++++++---- generate/exportedtf/exportedtf_test.go | 56 +++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index c3f72f042..7a14cdb5b 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -28,9 +28,11 @@ import ( "github.com/rs/zerolog/log" ) -// TfCode represents all exported terraform code for a stack, +// StackTf represents all exported terraform code for a stack, // mapping the exported code name to the actual Terraform code. -type TfCode map[string]Body +type StackTf struct { + tfcode map[string]Body +} // Body represents exported Terraform code from a single block. // Is contains parsed and evaluated code on it. @@ -38,6 +40,16 @@ type Body struct { body []byte } +// ExportedCode returns all exported code, mapping the name to its +// equivalent generated code. +func (s StackTf) ExportedCode() map[string]Body { + cp := map[string]Body{} + for k, v := range s.tfcode { + cp[k] = v + } + return cp +} + // String returns a string representation of the Terraform code // or an empty string if the config itself is empty. func (b Body) String() string { @@ -57,7 +69,7 @@ func (b Body) String() string { // The returned result only contains evaluated values. // // The rootdir MUST be an absolute path. -func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (TfCode, error) { +func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackTf, error) { stackpath := filepath.Join(rootdir, sm.Path, config.Filename) logger := log.With(). Str("action", "exportedtf.Load()"). @@ -68,17 +80,19 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (TfCode exportBlocks, err := loadExportBlocks(rootdir, stackpath) if err != nil { - return nil, fmt.Errorf("loading exported terraform code: %v", err) + return StackTf{}, fmt.Errorf("loading exported terraform code: %v", err) } evalctx, err := newEvalCtx(stackpath, sm, globals) if err != nil { - return nil, fmt.Errorf("preparing to eval exported terraform code: %v", err) + return StackTf{}, fmt.Errorf("preparing to eval exported terraform code: %v", err) } logger.Trace().Msg("generating exported terraform code.") - res := map[string]Body{} + res := StackTf{ + tfcode: map[string]Body{}, + } for name, block := range exportBlocks { logger := logger.With(). @@ -89,14 +103,14 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (TfCode gen := hclwrite.NewEmptyFile() if err := hcl.CopyBody(gen.Body(), block.Body, evalctx); err != nil { - return nil, fmt.Errorf( + return StackTf{}, fmt.Errorf( "generating terraform code for stack %q block %q: %v", stackpath, name, err, ) } - res[name] = Body{body: gen.Bytes()} + res.tfcode[name] = Body{body: gen.Bytes()} } return res, nil diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index ca09f6f4e..4f2d610f9 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -155,6 +155,58 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, }, + { + name: "exported terraform on stack with single nested block", + stack: "/stack", + configs: []hclconfig{ + { + path: "/stack", + add: globals( + str("some_string", "string"), + number("some_number", 777), + boolean("some_bool", true), + ), + }, + { + path: "/stack", + add: exportAsTerraform("nesting", + block("block1", + expr("bool", "global.some_bool"), + block("block2", + expr("number", "global.some_number"), + block("block3", + expr("string", "global.some_string"), + expr("obj", `{ + string = global.some_string + number = global.some_number + bool = global.some_bool + }`), + ), + ), + ), + ), + }, + }, + want: []result{ + { + name: "nesting", + hcl: block("block1", + boolean("bool", true), + block("block2", + number("number", 777), + block("block3", + str("string", "string"), + attr("obj", `{ + bool = true + number = 777 + string = "string" + }`), + ), + ), + ), + }, + }, + }, } for _, tcase := range tcases { @@ -170,9 +222,11 @@ func TestLoadExportedTerraform(t *testing.T) { meta := stack.Meta() globals := s.LoadStackGlobals(meta) - got, err := exportedtf.Load(s.RootDir(), meta, globals) + res, err := exportedtf.Load(s.RootDir(), meta, globals) assert.IsError(t, err, tcase.wantErr) + got := res.ExportedCode() + for _, res := range tcase.want { gothcl, ok := got[res.name] if !ok { From a1f70d5c1368ae52d1071ac17fc129006001fd2d Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 20:17:52 +0100 Subject: [PATCH 11/48] feat: add parent config loading + overriding strategy --- generate/exportedtf/exportedtf.go | 27 ++- generate/exportedtf/exportedtf_test.go | 232 +++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 3 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index 7a14cdb5b..a42350a44 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -17,6 +17,7 @@ package exportedtf import ( "fmt" "path/filepath" + "strings" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hcl/v2/hclwrite" @@ -70,7 +71,7 @@ func (b Body) String() string { // // The rootdir MUST be an absolute path. func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackTf, error) { - stackpath := filepath.Join(rootdir, sm.Path, config.Filename) + stackpath := filepath.Join(rootdir, sm.Path) logger := log.With(). Str("action", "exportedtf.Load()"). Str("path", stackpath). @@ -144,14 +145,21 @@ func newEvalCtx(stackpath string, sm stack.Metadata, globals *terramate.Globals) // loadExportBlocks will load all export_as_terraform blocks applying overriding // as it goes, the returned map maps the name of the block (its label) to the original block -func loadExportBlocks(rootdir string, cfgpath string) (map[string]*hclsyntax.Block, error) { +func loadExportBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Block, error) { logger := log.With(). Str("action", "exportedtf.loadExportBlocks()"). - Str("configFile", cfgpath). + Str("root", rootdir). + Str("configDir", cfgdir). Logger() logger.Trace().Msg("Parsing export_as_terraform blocks.") + if !strings.HasPrefix(cfgdir, rootdir) { + logger.Trace().Msg("config dir outside root, nothing to do") + return nil, nil + } + + cfgpath := filepath.Join(cfgdir, config.Filename) blocks, err := hcl.ParseExportAsTerraformBlocks(cfgpath) if err != nil { return nil, fmt.Errorf("loading exported terraform code: %v", err) @@ -166,5 +174,18 @@ func loadExportBlocks(rootdir string, cfgpath string) (map[string]*hclsyntax.Blo res[name] = block } + // TODO(katcipis): Handle failure on parent configs + parentRes, _ := loadExportBlocks(rootdir, filepath.Dir(cfgdir)) + + merge(res, parentRes) return res, nil } + +func merge(target, src map[string]*hclsyntax.Block) { + for k, v := range src { + if _, ok := target[k]; ok { + continue + } + target[k] = v + } +} diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index 4f2d610f9..b36814707 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -207,6 +207,238 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, }, + { + name: "multiple exported terraform blocks on stack", + stack: "/stack", + configs: []hclconfig{ + { + path: "/stack", + add: globals( + str("some_string", "string"), + number("some_number", 666), + boolean("some_bool", true), + ), + }, + { + path: "/stack", + add: hcldoc( + exportAsTerraform("exported_one", + block("block1", + expr("bool", "global.some_bool"), + block("block2", + expr("number", "global.some_number"), + ), + ), + ), + exportAsTerraform("exported_two", + block("yay", + expr("data", "global.some_string"), + ), + ), + exportAsTerraform("exported_three", + block("something", + expr("number", "global.some_number"), + ), + ), + ), + }, + }, + want: []result{ + { + name: "exported_one", + hcl: block("block1", + boolean("bool", true), + block("block2", + number("number", 666), + ), + ), + }, + { + name: "exported_two", + hcl: block("yay", + str("data", "string"), + ), + }, + { + name: "exported_three", + hcl: block("something", + number("number", 666), + ), + }, + }, + }, + { + name: "exported terraform on stack parent dir", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/", + add: globals( + str("some_string", "string"), + number("some_number", 777), + boolean("some_bool", true), + ), + }, + { + path: "/stacks", + add: exportAsTerraform("on_parent", + block("on_parent_block", + expr("obj", `{ + string = global.some_string + number = global.some_number + bool = global.some_bool + }`), + ), + ), + }, + }, + want: []result{ + { + name: "on_parent", + hcl: block("on_parent_block", + attr("obj", `{ + bool = true + number = 777 + string = "string" + }`), + ), + }, + }, + }, + { + name: "exporting on all dirs of the project with different names get merged", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/", + add: globals( + str("some_string", "string"), + number("some_number", 777), + boolean("some_bool", true), + ), + }, + { + path: "/", + add: exportAsTerraform("on_root", + block("on_root_block", + expr("obj", `{ + string = global.some_string + }`), + ), + ), + }, + { + path: "/stacks", + add: exportAsTerraform("on_parent", + block("on_parent_block", + expr("obj", `{ + number = global.some_number + }`), + ), + ), + }, + { + path: "/stacks/stack", + add: exportAsTerraform("on_stack", + block("on_stack_block", + expr("obj", `{ + bool = global.some_bool + }`), + ), + ), + }, + }, + want: []result{ + { + name: "on_root", + hcl: block("on_root_block", + attr("obj", `{ + string = "string" + }`), + ), + }, + { + name: "on_parent", + hcl: block("on_parent_block", + attr("obj", `{ + number = 777 + }`), + ), + }, + { + name: "on_stack", + hcl: block("on_stack_block", + attr("obj", `{ + bool = true + }`), + ), + }, + }, + }, + { + name: "specific config overrides general config", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/", + add: hcldoc( + exportAsTerraform("root", + block("block", + expr("root_stackpath", "terramate.path"), + ), + ), + exportAsTerraform("parent", + block("block", + expr("root_stackpath", "terramate.path"), + ), + ), + exportAsTerraform("stack", + block("block", + expr("root_stackpath", "terramate.path"), + ), + ), + ), + }, + { + path: "/stacks", + add: exportAsTerraform("parent", + block("block", + expr("parent_stackpath", "terramate.path"), + expr("parent_stackname", "terramate.name"), + ), + ), + }, + { + path: "/stacks/stack", + add: exportAsTerraform("stack", + block("block", + str("overridden", "some literal data"), + ), + ), + }, + }, + want: []result{ + { + name: "root", + hcl: block("block", + str("root_stackpath", "/stacks/stack"), + ), + }, + { + name: "parent", + hcl: block("block", + str("parent_stackpath", "/stacks/stack"), + str("parent_stackname", "stack"), + ), + }, + { + name: "stack", + hcl: block("block", + str("overridden", "some literal data"), + ), + }, + }, + }, } for _, tcase := range tcases { From e9d37e80e14be2986a1805665cddd51e2da1863b Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 21:33:48 +0100 Subject: [PATCH 12/48] test: add error handling on parsing --- generate/exportedtf/exportedtf.go | 15 ++++++++++-- generate/exportedtf/exportedtf_test.go | 34 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index a42350a44..81e9036cf 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -21,6 +21,7 @@ import ( "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/madlambda/spells/errutil" "github.com/mineiros-io/terramate" "github.com/mineiros-io/terramate/config" "github.com/mineiros-io/terramate/hcl" @@ -41,6 +42,10 @@ type Body struct { body []byte } +const ( + ErrInvalidBlock errutil.Error = "invalid export_as_terraform block" +) + // ExportedCode returns all exported code, mapping the name to its // equivalent generated code. func (s StackTf) ExportedCode() map[string]Body { @@ -81,7 +86,7 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackT exportBlocks, err := loadExportBlocks(rootdir, stackpath) if err != nil { - return StackTf{}, fmt.Errorf("loading exported terraform code: %v", err) + return StackTf{}, fmt.Errorf("loading exported terraform code: %w", err) } evalctx, err := newEvalCtx(stackpath, sm, globals) @@ -168,8 +173,14 @@ func loadExportBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Bloc res := map[string]*hclsyntax.Block{} for _, block := range blocks { - // TODO(katcipis): properly test wrong amount of labels // TODO(katcipis): properly test two blocks with same label on same config file + if len(block.Labels) != 1 { + return nil, fmt.Errorf( + "%w: want single label instead got %d", + ErrInvalidBlock, + len(block.Labels), + ) + } name := block.Labels[0] res[name] = block } diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index b36814707..a869f530b 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -67,6 +67,9 @@ func TestLoadExportedTerraform(t *testing.T) { t.Helper() return hclwrite.AttributeValue(t, name, expr) } + labels := func(labels ...string) hclwrite.BlockBuilder { + return hclwrite.Labels(labels...) + } expr := hclwrite.Expression str := hclwrite.String number := hclwrite.NumberInt @@ -439,6 +442,37 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, }, + { + name: "export block with no label on stack gives err", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/stacks/stack", + add: block("export_as_terraform", + block("block", + str("data", "some literal data"), + ), + ), + }, + }, + wantErr: exportedtf.ErrInvalidBlock, + }, + { + name: "export block with two labels on stack gives err", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/stacks/stack", + add: block("export_as_terraform", + labels("one", "two"), + block("block", + str("data", "some literal data"), + ), + ), + }, + }, + wantErr: exportedtf.ErrInvalidBlock, + }, } for _, tcase := range tcases { From 02e744e9d3a3556298845983a2db0a5e334ffa32 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 21:51:00 +0100 Subject: [PATCH 13/48] test: add further error handling + tests --- generate/exportedtf/exportedtf.go | 18 ++++++-- generate/exportedtf/exportedtf_test.go | 63 ++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index 81e9036cf..cb3380e03 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -16,6 +16,7 @@ package exportedtf import ( "fmt" + "os" "path/filepath" "strings" @@ -167,13 +168,15 @@ func loadExportBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Bloc cfgpath := filepath.Join(cfgdir, config.Filename) blocks, err := hcl.ParseExportAsTerraformBlocks(cfgpath) if err != nil { + if os.IsNotExist(err) { + return loadExportBlocks(rootdir, filepath.Dir(cfgdir)) + } return nil, fmt.Errorf("loading exported terraform code: %v", err) } res := map[string]*hclsyntax.Block{} for _, block := range blocks { - // TODO(katcipis): properly test two blocks with same label on same config file if len(block.Labels) != 1 { return nil, fmt.Errorf( "%w: want single label instead got %d", @@ -182,11 +185,20 @@ func loadExportBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Bloc ) } name := block.Labels[0] + if _, ok := res[name]; ok { + return nil, fmt.Errorf( + "%w: found two blocks with same label %q", + ErrInvalidBlock, + name, + ) + } res[name] = block } - // TODO(katcipis): Handle failure on parent configs - parentRes, _ := loadExportBlocks(rootdir, filepath.Dir(cfgdir)) + parentRes, err := loadExportBlocks(rootdir, filepath.Dir(cfgdir)) + if err != nil { + return nil, err + } merge(res, parentRes) return res, nil diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index a869f530b..6c7b355ae 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -308,6 +308,28 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, }, + { + name: "exported terraform project root dir", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/", + add: exportAsTerraform("root", + block("root", + expr("test", "terramate.path"), + ), + ), + }, + }, + want: []result{ + { + name: "root", + hcl: block("root", + str("test", "/stacks/stack"), + ), + }, + }, + }, { name: "exporting on all dirs of the project with different names get merged", stack: "/stacks/stack", @@ -473,6 +495,47 @@ func TestLoadExportedTerraform(t *testing.T) { }, wantErr: exportedtf.ErrInvalidBlock, }, + { + name: "export blocks with same label on same config gives err", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/stacks/stack", + add: hcldoc( + exportAsTerraform("duplicated", + str("data", "some literal data"), + ), + exportAsTerraform("duplicated", + str("data2", "some literal data2"), + ), + ), + }, + }, + wantErr: exportedtf.ErrInvalidBlock, + }, + { + name: "valid config on stack but invalid on parent gives err", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/stacks", + add: block("export_as_terraform", + block("block", + str("data", "some literal data"), + ), + ), + }, + { + path: "/stacks/stack", + add: hcldoc( + exportAsTerraform("valid", + str("data", "some literal data"), + ), + ), + }, + }, + wantErr: exportedtf.ErrInvalidBlock, + }, } for _, tcase := range tcases { From 392d3071789204c3426197e43ae397235a08668b Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Thu, 27 Jan 2022 22:24:59 +0100 Subject: [PATCH 14/48] test: eval + multiple labels + try --- generate/exportedtf/exportedtf.go | 4 +- generate/exportedtf/exportedtf_test.go | 55 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index cb3380e03..de1d3250c 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -45,6 +45,7 @@ type Body struct { const ( ErrInvalidBlock errutil.Error = "invalid export_as_terraform block" + ErrEval errutil.Error = "evaluating export_as_terraform block" ) // ExportedCode returns all exported code, mapping the name to its @@ -111,7 +112,8 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackT gen := hclwrite.NewEmptyFile() if err := hcl.CopyBody(gen.Body(), block.Body, evalctx); err != nil { return StackTf{}, fmt.Errorf( - "generating terraform code for stack %q block %q: %v", + "%w: stack %q block %q: %v", + ErrEval, stackpath, name, err, diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index 6c7b355ae..bb3f3997a 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -158,6 +158,46 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, }, + { + name: "exported terraform on stack using try and labelled block", + stack: "/stack", + configs: []hclconfig{ + { + path: "/stack", + add: globals( + attr("obj", `{ + field_a = "a" + field_b = "b" + field_c = "c" + }`), + ), + }, + { + path: "/stack", + add: exportAsTerraform("test", + block("labeled", + labels("label1", "label2"), + expr("field_a", "try(global.obj.field_a, null)"), + expr("field_b", "try(global.obj.field_b, null)"), + expr("field_c", "try(global.obj.field_c, null)"), + expr("field_d", "try(global.obj.field_d, null)"), + ), + ), + }, + }, + want: []result{ + { + name: "test", + hcl: block("labeled", + labels("label1", "label2"), + str("field_a", "a"), + str("field_b", "b"), + str("field_c", "c"), + attr("field_d", "null"), + ), + }, + }, + }, { name: "exported terraform on stack with single nested block", stack: "/stack", @@ -513,6 +553,21 @@ func TestLoadExportedTerraform(t *testing.T) { }, wantErr: exportedtf.ErrInvalidBlock, }, + { + name: "evaluation failure on stack config gives err", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/stacks/stack", + add: hcldoc( + exportAsTerraform("test", + expr("attr", "global.undefined"), + ), + ), + }, + }, + wantErr: exportedtf.ErrEval, + }, { name: "valid config on stack but invalid on parent gives err", stack: "/stacks/stack", From 124e86e57a7e46dd92e626afc2b12a49c07388d1 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Fri, 28 Jan 2022 12:00:12 +0100 Subject: [PATCH 15/48] feat: add proper eval error sentinel for eval ctx --- generate/exportedtf/exportedtf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index de1d3250c..5a0a88a4b 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -93,7 +93,7 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackT evalctx, err := newEvalCtx(stackpath, sm, globals) if err != nil { - return StackTf{}, fmt.Errorf("preparing to eval exported terraform code: %v", err) + return StackTf{}, fmt.Errorf("%w: creating eval context: %v", ErrEval, err) } logger.Trace().Msg("generating exported terraform code.") From 128fd856dcd0dd9e7044e511a702b3cccadcdf37 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Fri, 28 Jan 2022 12:02:19 +0100 Subject: [PATCH 16/48] refactor: renaming types, hopefully for increased clarity --- generate/exportedtf/exportedtf.go | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index 5a0a88a4b..b0d7db370 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -31,15 +31,15 @@ import ( "github.com/rs/zerolog/log" ) -// StackTf represents all exported terraform code for a stack, +// StackHCLs represents all exported terraform code for a stack, // mapping the exported code name to the actual Terraform code. -type StackTf struct { - tfcode map[string]Body +type StackHCLs struct { + hcls map[string]HCL } -// Body represents exported Terraform code from a single block. +// HCL represents exported Terraform code from a single block. // Is contains parsed and evaluated code on it. -type Body struct { +type HCL struct { body []byte } @@ -50,9 +50,9 @@ const ( // ExportedCode returns all exported code, mapping the name to its // equivalent generated code. -func (s StackTf) ExportedCode() map[string]Body { - cp := map[string]Body{} - for k, v := range s.tfcode { +func (s StackHCLs) ExportedCode() map[string]HCL { + cp := map[string]HCL{} + for k, v := range s.hcls { cp[k] = v } return cp @@ -60,7 +60,7 @@ func (s StackTf) ExportedCode() map[string]Body { // String returns a string representation of the Terraform code // or an empty string if the config itself is empty. -func (b Body) String() string { +func (b HCL) String() string { return string(b.body) } @@ -77,7 +77,7 @@ func (b Body) String() string { // The returned result only contains evaluated values. // // The rootdir MUST be an absolute path. -func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackTf, error) { +func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackHCLs, error) { stackpath := filepath.Join(rootdir, sm.Path) logger := log.With(). Str("action", "exportedtf.Load()"). @@ -88,18 +88,18 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackT exportBlocks, err := loadExportBlocks(rootdir, stackpath) if err != nil { - return StackTf{}, fmt.Errorf("loading exported terraform code: %w", err) + return StackHCLs{}, fmt.Errorf("loading exported terraform code: %w", err) } evalctx, err := newEvalCtx(stackpath, sm, globals) if err != nil { - return StackTf{}, fmt.Errorf("%w: creating eval context: %v", ErrEval, err) + return StackHCLs{}, fmt.Errorf("%w: creating eval context: %v", ErrEval, err) } logger.Trace().Msg("generating exported terraform code.") - res := StackTf{ - tfcode: map[string]Body{}, + res := StackHCLs{ + hcls: map[string]HCL{}, } for name, block := range exportBlocks { @@ -111,7 +111,7 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackT gen := hclwrite.NewEmptyFile() if err := hcl.CopyBody(gen.Body(), block.Body, evalctx); err != nil { - return StackTf{}, fmt.Errorf( + return StackHCLs{}, fmt.Errorf( "%w: stack %q block %q: %v", ErrEval, stackpath, @@ -119,7 +119,7 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackT err, ) } - res.tfcode[name] = Body{body: gen.Bytes()} + res.hcls[name] = HCL{body: gen.Bytes()} } return res, nil From e32f254ef23f2e02378a5cfde1847c6f892d82b4 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Fri, 28 Jan 2022 16:40:18 +0100 Subject: [PATCH 17/48] refactor: split different code gen tests --- generate/generate_backend_cfg_test.go | 1040 +++++++++++++++++ generate/generate_locals_test.go | 551 +++++++++ generate/generate_test.go | 1542 ------------------------- 3 files changed, 1591 insertions(+), 1542 deletions(-) create mode 100644 generate/generate_backend_cfg_test.go create mode 100644 generate/generate_locals_test.go diff --git a/generate/generate_backend_cfg_test.go b/generate/generate_backend_cfg_test.go new file mode 100644 index 000000000..4f3f23010 --- /dev/null +++ b/generate/generate_backend_cfg_test.go @@ -0,0 +1,1040 @@ +// Copyright 2021 Mineiros GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generate_test + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/madlambda/spells/assert" + "github.com/mineiros-io/terramate/config" + "github.com/mineiros-io/terramate/generate" + "github.com/mineiros-io/terramate/hcl" + "github.com/mineiros-io/terramate/test" + "github.com/mineiros-io/terramate/test/hclwrite" + "github.com/mineiros-io/terramate/test/sandbox" +) + +func TestBackendConfigGeneration(t *testing.T) { + type ( + stackcode struct { + relpath string + code string + } + + backendconfig struct { + relpath string + config string + } + + want struct { + err error + stacks []stackcode + } + + testcase struct { + name string + layout []string + configs []backendconfig + workingDir string + want want + } + ) + + // gen instead of generate because name conflicts with generate pkg + gen := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("generate", builders...) + } + // cfg instead of config because name conflicts with config pkg + cfg := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("config", builders...) + } + + tcases := []testcase{ + { + name: "multiple stacks with no config", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + "s:stacks/stack-3", + }, + }, + { + name: "fails on single stack with invalid config", + layout: []string{"s:stack"}, + configs: []backendconfig{ + { + relpath: "stack", + config: `terramate { + required_version = "~> 0.0.0" + backend {} +} + +stack{}`, + }, + }, + want: want{ + err: hcl.ErrMalformedTerramateConfig, + }, + }, + { + name: "multiple stacks and one has invalid config fails", + layout: []string{ + "s:stack-invalid-backend", + "s:stack-ok-backend", + }, + configs: []backendconfig{ + { + relpath: "stack-invalid-backend", + config: `terramate { + required_version = "~> 0.0.0" + backend {} +} + +stack{}`, + }, + { + relpath: "stack-ok-backend", + config: `terramate { + required_version = "~> 0.0.0" + backend "valid" {} +} + +stack{}`, + }, + }, + want: want{ + err: hcl.ErrMalformedTerramateConfig, + }, + }, + { + name: "single stack with config on stack and empty config", + layout: []string{"s:stack"}, + configs: []backendconfig{ + { + relpath: "stack", + config: `terramate { + required_version = "~> 0.0.0" + backend "sometype" {} +} + +stack {}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stack", + code: `terraform { + backend "sometype" { + } +} +`, + }, + }, + }, + }, + { + name: "single stack with config on stack and empty config label", + layout: []string{"s:stack"}, + configs: []backendconfig{ + { + relpath: "stack", + config: `terramate { + required_version = "~> 0.0.0" + backend "" {} +} + +stack {}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stack", + code: `terraform { + backend "" { + } +} +`, + }, + }, + }, + }, + { + name: "single stack with config on stack and config with 1 attr", + layout: []string{"s:stack"}, + configs: []backendconfig{ + { + relpath: "stack", + config: `terramate { + required_version = "~> 0.0.0" + backend "sometype" { + attr = "value" + } +} + +stack {}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stack", + code: `terraform { + backend "sometype" { + attr = "value" + } +} +`, + }, + }, + }, + }, + { + name: "multiple stacks with config on each stack", + layout: []string{"s:stack-1"}, + configs: []backendconfig{ + { + relpath: "stack-1", + config: `terramate { + required_version = "~> 0.0.0" + backend "1" { + attr = "hi" + } +} + +stack {}`, + }, + { + relpath: "stack-2", + config: `terramate { + required_version = "~> 0.0.0" + backend "2" { + somebool = true + } +} + +stack {}`, + }, + { + relpath: "stack-3", + config: `terramate { + required_version = "~> 0.0.0" + backend "3" { + somelist = ["m", "i", "n", "e", "i", "r", "o", "s"] + } +} + +stack {}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stack-1", + code: `terraform { + backend "1" { + attr = "hi" + } +} +`, + }, + { + relpath: "stack-2", + code: `terraform { + backend "2" { + somebool = true + } +} +`, + }, + { + relpath: "stack-3", + code: `terraform { + backend "3" { + somelist = ["m", "i", "n", "e", "i", "r", "o", "s"] + } +} +`, + }, + }, + }, + }, + { + name: "single stack with config on stack with config N attrs", + layout: []string{"s:stack"}, + configs: []backendconfig{ + { + relpath: "stack", + config: `terramate { + required_version = "~> 0.0.0" + backend "lotsoftypes" { + attr = "value" + attrnumber = 5 + attrbool = true + somelist = ["hi", "again"] + } +} + +stack {} +`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stack", + code: `terraform { + backend "lotsoftypes" { + attr = "value" + attrbool = true + attrnumber = 5 + somelist = ["hi", "again"] + } +} +`, + }, + }, + }, + }, + { + name: "single stack with config on stack with subblock", + layout: []string{"s:stack"}, + configs: []backendconfig{ + { + relpath: "stack", + config: `terramate { + required_version = "~> 0.0.0" + backend "lotsoftypes" { + attr = "value" + block { + attrbool = true + attrnumber = 5 + somelist = ["hi", "again"] + } + } +} + +stack {}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stack", + code: `terraform { + backend "lotsoftypes" { + attr = "value" + block { + attrbool = true + attrnumber = 5 + somelist = ["hi", "again"] + } + } +} +`, + }, + }, + }, + }, + { + name: "single stack with config parent dir", + layout: []string{"s:stacks/stack"}, + configs: []backendconfig{ + { + relpath: "stacks", + config: `terramate { + backend "fromparent" { + attr = "value" + } +}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack", + code: `terraform { + backend "fromparent" { + attr = "value" + } +} +`, + }, + }, + }, + }, + { + name: "single stack with config on basedir", + layout: []string{"s:stacks/stack"}, + configs: []backendconfig{ + { + relpath: ".", + config: `terramate { + backend "basedir_config" { + attr = 666 + } +} +`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack", + code: `terraform { + backend "basedir_config" { + attr = 666 + } +} +`, + }, + }, + }, + }, + { + name: "multiple stacks with config on basedir", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []backendconfig{ + { + relpath: ".", + config: `terramate { + backend "basedir_config" { + attr = "test" + } +} +`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack-1", + code: `terraform { + backend "basedir_config" { + attr = "test" + } +} +`, + }, + { + relpath: "stacks/stack-2", + code: `terraform { + backend "basedir_config" { + attr = "test" + } +} +`, + }, + }, + }, + }, + { + name: "stacks on different envs with per env config", + layout: []string{ + "s:envs/prod/stacks/stack", + "s:envs/staging/stacks/stack", + }, + configs: []backendconfig{ + { + relpath: "envs/prod", + config: `terramate { + backend "remote" { + environment = "prod" + } +} +`, + }, + { + relpath: "envs/staging", + config: `terramate { + backend "remote" { + environment = "staging" + } +} +`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "envs/prod/stacks/stack", + code: `terraform { + backend "remote" { + environment = "prod" + } +} +`, + }, + { + relpath: "envs/staging/stacks/stack", + code: `terraform { + backend "remote" { + environment = "staging" + } +} +`, + }, + }, + }, + }, + { + name: "single stack with config on stack and N attrs using metadata", + layout: []string{"s:stack-metadata"}, + configs: []backendconfig{ + { + relpath: "stack-metadata", + config: `terramate { + required_version = "~> 0.0.0" + backend "metadata" { + name = terramate.name + path = terramate.path + somelist = [terramate.name, terramate.path] + } +} +stack { + name = "custom-name" +}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stack-metadata", + code: `terraform { + backend "metadata" { + name = "custom-name" + path = "/stack-metadata" + somelist = ["custom-name", "/stack-metadata"] + } +} +`, + }, + }, + }, + }, + { + name: "multiple stacks with config on root dir using metadata", + layout: []string{"s:stacks/stack-1", "s:stacks/stack-2"}, + configs: []backendconfig{ + { + relpath: ".", + config: `terramate { + backend "metadata" { + name = terramate.name + path = terramate.path + interpolation = "interpolate-${terramate.name}-fun-${terramate.path}" + } +}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack-1", + code: `terraform { + backend "metadata" { + interpolation = "interpolate-stack-1-fun-/stacks/stack-1" + name = "stack-1" + path = "/stacks/stack-1" + } +} +`, + }, + { + relpath: "stacks/stack-2", + code: `terraform { + backend "metadata" { + interpolation = "interpolate-stack-2-fun-/stacks/stack-2" + name = "stack-2" + path = "/stacks/stack-2" + } +} +`, + }, + }, + }, + }, + { + name: "multiple stacks with config on root dir using metadata and tf functions", + layout: []string{"s:stacks/stack-1", "s:stacks/stack-2"}, + configs: []backendconfig{ + { + relpath: ".", + config: `terramate { + backend "metadata" { + funcfun = replace(terramate.path, "/","-") + funcfunb = "testing-funcs-${replace(terramate.path, "/",".")}" + name = terramate.name + path = terramate.path + } +}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack-1", + code: `terraform { + backend "metadata" { + funcfun = "-stacks-stack-1" + funcfunb = "testing-funcs-.stacks.stack-1" + name = "stack-1" + path = "/stacks/stack-1" + } +} +`, + }, + { + relpath: "stacks/stack-2", + code: `terraform { + backend "metadata" { + funcfun = "-stacks-stack-2" + funcfunb = "testing-funcs-.stacks.stack-2" + name = "stack-2" + path = "/stacks/stack-2" + } +} +`, + }, + }, + }, + }, + { + name: "multiple stacks with config on parent dir using globals from root", + layout: []string{"s:stacks/stack-1", "s:stacks/stack-2"}, + configs: []backendconfig{ + { + relpath: ".", + config: ` +globals { + bucket = "project-wide-bucket" +}`, + }, + { + relpath: "stacks", + config: `terramate { + backend "gcs" { + bucket = global.bucket + prefix = terramate.path + } +}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack-1", + code: `terraform { + backend "gcs" { + bucket = "project-wide-bucket" + prefix = "/stacks/stack-1" + } +} +`, + }, + { + relpath: "stacks/stack-2", + code: `terraform { + backend "gcs" { + bucket = "project-wide-bucket" + prefix = "/stacks/stack-2" + } +} +`, + }, + }, + }, + }, + { + name: "stack with global on parent dir using config from root", + layout: []string{"s:stacks/stack"}, + configs: []backendconfig{ + { + relpath: ".", + config: `terramate { + backend "gcs" { + bucket = global.bucket + prefix = terramate.path + } +}`, + }, + { + relpath: "stacks", + config: ` +globals { + bucket = "project-wide-bucket" +}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack", + code: `terraform { + backend "gcs" { + bucket = "project-wide-bucket" + prefix = "/stacks/stack" + } +} +`, + }, + }, + }, + }, + { + name: "stack overriding parent global", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []backendconfig{ + { + relpath: ".", + config: `terramate { + backend "gcs" { + bucket = global.bucket + prefix = terramate.path + } +}`, + }, + { + relpath: "stacks", + config: ` +globals { + bucket = "project-wide-bucket" +}`, + }, + { + relpath: "stacks/stack-1", + config: ` +terramate { + required_version = "~> 0.0.0" +} + +stack {} + +globals { + bucket = "stack-specific-bucket" +}`, + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack-1", + code: `terraform { + backend "gcs" { + bucket = "stack-specific-bucket" + prefix = "/stacks/stack-1" + } +} +`, + }, + { + relpath: "stacks/stack-2", + code: `terraform { + backend "gcs" { + bucket = "project-wide-bucket" + prefix = "/stacks/stack-2" + } +} +`, + }, + }, + }, + }, + { + name: "reference to undefined global fails", + layout: []string{"s:stack"}, + configs: []backendconfig{ + { + relpath: ".", + config: `terramate { + backend "gcs" { + bucket = global.bucket + } +}`, + }, + }, + want: want{ + err: generate.ErrBackendConfigGen, + }, + }, + { + name: "invalid global definition fails", + layout: []string{"s:stack"}, + configs: []backendconfig{ + { + relpath: ".", + config: `terramate { + backend "gcs" { + bucket = "all good" + } +} + +globals { + undefined_reference = global.undefined +} +`, + }, + }, + want: want{ + err: generate.ErrLoadingGlobals, + }, + }, + { + name: "multiple stacks selecting single stack with working dir", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + workingDir: "stacks/stack-1", + configs: []backendconfig{ + { + relpath: ".", + config: hcldoc( + terramate( + backend( + labels("gcs"), + expr("prefix", "terramate.path"), + ), + ), + ).String(), + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack-1", + code: hcldoc( + terraform( + backend( + labels("gcs"), + str("prefix", "/stacks/stack-1"), + ), + ), + ).String(), + }, + }, + }, + }, + { + name: "stacks using parent generated code filenames", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []backendconfig{ + { + relpath: "/stacks", + config: hcldoc( + + terramate( + backend( + labels("gcs"), + expr("prefix", "terramate.path"), + ), + cfg( + gen( + str("backend_config_filename", "backend.tf"), + ), + ), + ), + ).String(), + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack-1", + code: hcldoc( + terraform( + backend( + labels("gcs"), + str("prefix", "/stacks/stack-1"), + ), + ), + ).String(), + }, + { + relpath: "stacks/stack-2", + code: hcldoc( + terraform( + backend( + labels("gcs"), + str("prefix", "/stacks/stack-2"), + ), + ), + ).String(), + }, + }, + }, + }, + { + name: "stacks using parent generated code filenames filtered by working dir", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + workingDir: "stacks/stack-1", + configs: []backendconfig{ + { + relpath: "/stacks", + config: hcldoc( + + terramate( + backend( + labels("gcs"), + expr("prefix", "terramate.path"), + ), + cfg( + gen( + str("backend_config_filename", "backend.tf"), + ), + ), + ), + ).String(), + }, + }, + want: want{ + stacks: []stackcode{ + { + relpath: "stacks/stack-1", + code: hcldoc( + terraform( + backend( + labels("gcs"), + str("prefix", "/stacks/stack-1"), + ), + ), + ).String(), + }, + }, + }, + }, + { + name: "working dir has no stacks inside", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + "d:notstack", + }, + workingDir: "notstack", + configs: []backendconfig{ + { + relpath: ".", + config: hcldoc( + terramate( + backend( + labels("gcs"), + expr("prefix", "terramate.path"), + ), + ), + ).String(), + }, + }, + want: want{}, + }, + } + + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + s := sandbox.New(t) + s.BuildTree(tcase.layout) + + for _, cfg := range tcase.configs { + dir := filepath.Join(s.RootDir(), cfg.relpath) + test.WriteFile(t, dir, config.Filename, cfg.config) + } + + workingDir := filepath.Join(s.RootDir(), tcase.workingDir) + err := generate.Do(s.RootDir(), workingDir) + assert.IsError(t, err, tcase.want.err) + + for _, want := range tcase.want.stacks { + stack := s.StackEntry(want.relpath) + got := string(stack.ReadGeneratedBackendCfg()) + + assertHCLEquals(t, got, want.code) + } + // TODO(katcipis): Add proper checks for extraneous generated code. + // For now we validated wanted files are there, but not that + // we may have new unwanted files being generated by a bug. + }) + } +} + +func TestWontOverwriteManuallyDefinedBackendConfig(t *testing.T) { + const ( + manualContents = "some manual backend configs" + ) + + backend := func(label string) *hclwrite.Block { + b := hclwrite.BuildBlock("backend") + b.AddLabel(label) + return b + } + rootTerramateConfig := terramate(backend("test")) + + s := sandbox.New(t) + s.BuildTree([]string{ + fmt.Sprintf("f:%s:%s", config.Filename, rootTerramateConfig.String()), + "s:stack", + fmt.Sprintf("f:stack/%s:%s", generate.BackendCfgFilename, manualContents), + }) + + err := generate.Do(s.RootDir(), s.RootDir()) + assert.IsError(t, err, generate.ErrManualCodeExists) + + stack := s.StackEntry("stack") + + backendConfig := string(stack.ReadGeneratedBackendCfg()) + assert.EqualStrings(t, manualContents, backendConfig, "backend config altered by generate") +} + +func TestBackendConfigOverwriting(t *testing.T) { + backend := func(label string) *hclwrite.Block { + b := hclwrite.BuildBlock("backend") + b.AddLabel(label) + return b + } + firstConfig := terramate(backend("first")) + firstWant := terraform(backend("first")) + + s := sandbox.New(t) + stack := s.CreateStack("stack") + rootEntry := s.DirEntry(".") + rootConfig := rootEntry.CreateConfig(firstConfig.String()) + + assert.NoError(t, generate.Do(s.RootDir(), s.RootDir())) + + got := string(stack.ReadGeneratedBackendCfg()) + assertHCLEquals(t, got, firstWant.String()) + + secondConfig := terramate(backend("second")) + secondWant := terraform(backend("second")) + rootConfig.Write(secondConfig.String()) + + assert.NoError(t, generate.Do(s.RootDir(), s.RootDir())) + + got = string(stack.ReadGeneratedBackendCfg()) + assertHCLEquals(t, got, secondWant.String()) +} diff --git a/generate/generate_locals_test.go b/generate/generate_locals_test.go new file mode 100644 index 000000000..9041a0831 --- /dev/null +++ b/generate/generate_locals_test.go @@ -0,0 +1,551 @@ +// Copyright 2021 Mineiros GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generate_test + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/madlambda/spells/assert" + "github.com/mineiros-io/terramate/config" + "github.com/mineiros-io/terramate/generate" + "github.com/mineiros-io/terramate/test" + "github.com/mineiros-io/terramate/test/hclwrite" + "github.com/mineiros-io/terramate/test/sandbox" +) + +func TestLocalsGeneration(t *testing.T) { + // The test approach for locals generation already uses a new test package + // to help creating the HCL files instead of using plain raw strings. + // There are some trade-offs involved and we are assessing how to approach + // the testing, hence for now it is inconsistent between locals generation + // and backend configuration generation. The idea is to converge to a single + // approach ASAP. + type ( + // hclblock represents an HCL block that will be appended on + // the file path. + hclblock struct { + path string + add *hclwrite.Block + } + want struct { + err error + stacksLocals map[string]*hclwrite.Block + } + testcase struct { + name string + layout []string + configs []hclblock + workingDir string + want want + } + ) + + // gen instead of generate because name conflicts with generate pkg + gen := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("generate", builders...) + } + // cfg instead of config because name conflicts with config pkg + cfg := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("config", builders...) + } + + tcases := []testcase{ + { + name: "no stacks no exported locals", + layout: []string{}, + }, + { + name: "single stacks no exported locals", + layout: []string{"s:stack"}, + }, + { + name: "two stacks no exported locals", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + }, + { + name: "single stack with its own exported locals using own globals", + layout: []string{"s:stack"}, + configs: []hclblock{ + { + path: "/stack", + add: globals( + str("some_string", "string"), + number("some_number", 777), + boolean("some_bool", true), + ), + }, + { + path: "/stack", + add: exportAsLocals( + expr("string_local", "global.some_string"), + expr("number_local", "global.some_number"), + expr("bool_local", "global.some_bool"), + ), + }, + }, + want: want{ + stacksLocals: map[string]*hclwrite.Block{ + "/stack": locals( + str("string_local", "string"), + number("number_local", 777), + boolean("bool_local", true), + ), + }, + }, + }, + { + name: "single stack exporting metadata using functions", + layout: []string{"s:stack"}, + configs: []hclblock{ + { + path: "/stack", + add: exportAsLocals( + expr("funny_path", `replace(terramate.path, "/", "@")`), + ), + }, + }, + want: want{ + stacksLocals: map[string]*hclwrite.Block{ + "/stack": locals( + str("funny_path", "@stack"), + ), + }, + }, + }, + { + name: "single stack referencing undefined global fails", + layout: []string{"s:stack"}, + configs: []hclblock{ + { + path: "/stack", + add: exportAsLocals( + expr("undefined", "global.undefined"), + ), + }, + }, + want: want{ + err: generate.ErrExportingLocalsGen, + }, + }, + { + name: "multiple stack with exported locals being overridden", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []hclblock{ + { + path: "/", + add: globals( + str("attr1", "value1"), + str("attr2", "value2"), + str("attr3", "value3"), + ), + }, + { + path: "/", + add: exportAsLocals( + expr("string", "global.attr1"), + ), + }, + { + path: "/stacks", + add: exportAsLocals( + expr("string", "global.attr2"), + ), + }, + { + path: "/stacks/stack-1", + add: exportAsLocals( + expr("string", "global.attr3"), + ), + }, + }, + want: want{ + stacksLocals: map[string]*hclwrite.Block{ + "/stacks/stack-1": locals( + str("string", "value3"), + ), + "/stacks/stack-2": locals( + str("string", "value2"), + ), + }, + }, + }, + { + name: "single stack with exported locals and globals from parent dirs", + layout: []string{"s:stacks/stack"}, + configs: []hclblock{ + { + path: "/", + add: globals( + str("str", "string"), + ), + }, + { + path: "/", + add: exportAsLocals( + expr("num_local", "global.num"), + expr("path_local", "terramate.path"), + ), + }, + { + path: "/stacks", + add: globals( + number("num", 666), + ), + }, + { + path: "/stacks", + add: exportAsLocals( + expr("str_local", "global.str"), + expr("name_local", "terramate.name"), + ), + }, + }, + want: want{ + stacksLocals: map[string]*hclwrite.Block{ + "/stacks/stack": locals( + str("name_local", "stack"), + str("path_local", "/stacks/stack"), + str("str_local", "string"), + number("num_local", 666), + ), + }, + }, + }, + { + name: "multiple stacks with exported locals and globals from parent dirs", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []hclblock{ + { + path: "/", + add: globals( + str("str", "string"), + ), + }, + { + path: "/", + add: exportAsLocals( + expr("num_local", "global.num"), + expr("path_local", "terramate.path"), + ), + }, + { + path: "/stacks", + add: globals( + number("num", 666), + ), + }, + { + path: "/stacks", + add: exportAsLocals( + expr("str_local", "global.str"), + expr("name_local", "terramate.name"), + ), + }, + }, + want: want{ + stacksLocals: map[string]*hclwrite.Block{ + "/stacks/stack-1": locals( + str("name_local", "stack-1"), + str("path_local", "/stacks/stack-1"), + str("str_local", "string"), + number("num_local", 666), + ), + "/stacks/stack-2": locals( + str("name_local", "stack-2"), + str("path_local", "/stacks/stack-2"), + str("str_local", "string"), + number("num_local", 666), + ), + }, + }, + }, + { + name: "multiple stacks with specific exported locals and globals from parent dirs", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []hclblock{ + { + path: "/", + add: globals( + str("str", "string"), + ), + }, + { + path: "/stacks", + add: globals( + number("num", 666), + ), + }, + { + path: "/stacks/stack-1", + add: exportAsLocals( + expr("str_local", "global.str"), + expr("name_local", "terramate.name"), + ), + }, + { + path: "/stacks/stack-2", + add: exportAsLocals( + expr("num_local", "global.num"), + expr("path_local", "terramate.path"), + ), + }, + }, + want: want{ + stacksLocals: map[string]*hclwrite.Block{ + "/stacks/stack-1": locals( + str("name_local", "stack-1"), + str("str_local", "string"), + ), + "/stacks/stack-2": locals( + str("path_local", "/stacks/stack-2"), + number("num_local", 666), + ), + }, + }, + }, + { + name: "multiple stacks selecting single stack with working dir", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + workingDir: "stacks/stack-1", + configs: []hclblock{ + { + path: "/", + add: globals( + str("str", "string"), + ), + }, + { + path: "/stacks/stack-1", + add: exportAsLocals( + expr("str_local", "global.str"), + ), + }, + { + path: "/stacks/stack-2", + add: exportAsLocals( + expr("str_local", "global.str"), + ), + }, + }, + want: want{ + stacksLocals: map[string]*hclwrite.Block{ + "/stacks/stack-1": locals( + str("str_local", "string"), + ), + }, + }, + }, + { + name: "stacks getting code generation config from parent", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []hclblock{ + { + path: "/", + add: globals( + str("str", "string"), + ), + }, + { + path: "/stacks", + add: terramate( + cfg( + gen( + str("locals_filename", "locals.tf"), + ), + ), + ), + }, + { + path: "/stacks/stack-1", + add: exportAsLocals( + expr("str_local", "global.str"), + ), + }, + { + path: "/stacks/stack-2", + add: exportAsLocals( + expr("str_local", "global.str"), + ), + }, + }, + want: want{ + stacksLocals: map[string]*hclwrite.Block{ + "/stacks/stack-1": locals( + str("str_local", "string"), + ), + "/stacks/stack-2": locals( + str("str_local", "string"), + ), + }, + }, + }, + { + name: "stacks with code gen cfg filtered by working dir", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + workingDir: "stacks/stack-2", + configs: []hclblock{ + { + path: "/", + add: globals( + str("str", "string"), + ), + }, + { + path: "/stacks", + add: terramate( + cfg( + gen( + str("locals_filename", "locals.tf"), + ), + ), + ), + }, + { + path: "/stacks/stack-1", + add: exportAsLocals( + expr("str_local", "global.str"), + ), + }, + { + path: "/stacks/stack-2", + add: exportAsLocals( + expr("str_local", "global.str"), + ), + }, + }, + want: want{ + stacksLocals: map[string]*hclwrite.Block{ + "/stacks/stack-2": locals( + str("str_local", "string"), + ), + }, + }, + }, + { + name: "working dir has no stacks inside", + layout: []string{ + "s:stack", + "d:somedir", + }, + workingDir: "somedir", + configs: []hclblock{ + { + path: "/stack", + add: exportAsLocals( + expr("path", "terramate.path"), + ), + }, + }, + want: want{}, + }, + } + + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + s := sandbox.New(t) + s.BuildTree(tcase.layout) + + for _, cfg := range tcase.configs { + path := filepath.Join(s.RootDir(), cfg.path) + test.AppendFile(t, path, config.Filename, cfg.add.String()) + } + + workingDir := filepath.Join(s.RootDir(), tcase.workingDir) + err := generate.Do(s.RootDir(), workingDir) + assert.IsError(t, err, tcase.want.err) + + for stackPath, wantHCLBlock := range tcase.want.stacksLocals { + stackRelPath := stackPath[1:] + want := wantHCLBlock.String() + stack := s.StackEntry(stackRelPath) + got := string(stack.ReadGeneratedLocals()) + + assertHCLEquals(t, got, want) + } + // TODO(katcipis): Add proper checks for extraneous generated code. + // For now we validated wanted files are there, but not that + // we may have new unwanted files being generated by a bug. + }) + } +} + +func TestWontOverwriteManuallyDefinedLocals(t *testing.T) { + const ( + manualLocals = "some manual stuff" + ) + + exportLocalsCfg := exportAsLocals(expr("a", "terramate.path")) + + s := sandbox.New(t) + s.BuildTree([]string{ + fmt.Sprintf("f:%s:%s", config.Filename, exportLocalsCfg.String()), + "s:stack", + fmt.Sprintf("f:stack/%s:%s", generate.LocalsFilename, manualLocals), + }) + + err := generate.Do(s.RootDir(), s.RootDir()) + assert.IsError(t, err, generate.ErrManualCodeExists) + + stack := s.StackEntry("stack") + locals := string(stack.ReadGeneratedLocals()) + assert.EqualStrings(t, manualLocals, locals, "locals altered by generate") +} + +func TestExportedLocalsOverwriting(t *testing.T) { + firstConfig := exportAsLocals(expr("a", "terramate.path")) + firstWant := locals(str("a", "/stack")) + + s := sandbox.New(t) + stack := s.CreateStack("stack") + rootEntry := s.DirEntry(".") + rootConfig := rootEntry.CreateConfig(firstConfig.String()) + + assert.NoError(t, generate.Do(s.RootDir(), s.RootDir())) + + got := string(stack.ReadGeneratedLocals()) + assertHCLEquals(t, got, firstWant.String()) + + secondConfig := exportAsLocals(expr("b", "terramate.name")) + secondWant := locals(str("b", "stack")) + rootConfig.Write(secondConfig.String()) + + assert.NoError(t, generate.Do(s.RootDir(), s.RootDir())) + + got = string(stack.ReadGeneratedLocals()) + assertHCLEquals(t, got, secondWant.String()) +} diff --git a/generate/generate_test.go b/generate/generate_test.go index 1eeca2eba..2a8cdf313 100644 --- a/generate/generate_test.go +++ b/generate/generate_test.go @@ -15,1556 +15,14 @@ package generate_test import ( - "fmt" - "path/filepath" "strings" "testing" "github.com/google/go-cmp/cmp" - "github.com/madlambda/spells/assert" - "github.com/mineiros-io/terramate/config" "github.com/mineiros-io/terramate/generate" - "github.com/mineiros-io/terramate/hcl" - "github.com/mineiros-io/terramate/test" - "github.com/mineiros-io/terramate/test/hclwrite" - "github.com/mineiros-io/terramate/test/sandbox" "github.com/rs/zerolog" ) -func TestBackendConfigGeneration(t *testing.T) { - type ( - stackcode struct { - relpath string - code string - } - - backendconfig struct { - relpath string - config string - } - - want struct { - err error - stacks []stackcode - } - - testcase struct { - name string - layout []string - configs []backendconfig - workingDir string - want want - } - ) - - // gen instead of generate because name conflicts with generate pkg - gen := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("generate", builders...) - } - // cfg instead of config because name conflicts with config pkg - cfg := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("config", builders...) - } - - tcases := []testcase{ - { - name: "multiple stacks with no config", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - "s:stacks/stack-3", - }, - }, - { - name: "fails on single stack with invalid config", - layout: []string{"s:stack"}, - configs: []backendconfig{ - { - relpath: "stack", - config: `terramate { - required_version = "~> 0.0.0" - backend {} -} - -stack{}`, - }, - }, - want: want{ - err: hcl.ErrMalformedTerramateConfig, - }, - }, - { - name: "multiple stacks and one has invalid config fails", - layout: []string{ - "s:stack-invalid-backend", - "s:stack-ok-backend", - }, - configs: []backendconfig{ - { - relpath: "stack-invalid-backend", - config: `terramate { - required_version = "~> 0.0.0" - backend {} -} - -stack{}`, - }, - { - relpath: "stack-ok-backend", - config: `terramate { - required_version = "~> 0.0.0" - backend "valid" {} -} - -stack{}`, - }, - }, - want: want{ - err: hcl.ErrMalformedTerramateConfig, - }, - }, - { - name: "single stack with config on stack and empty config", - layout: []string{"s:stack"}, - configs: []backendconfig{ - { - relpath: "stack", - config: `terramate { - required_version = "~> 0.0.0" - backend "sometype" {} -} - -stack {}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stack", - code: `terraform { - backend "sometype" { - } -} -`, - }, - }, - }, - }, - { - name: "single stack with config on stack and empty config label", - layout: []string{"s:stack"}, - configs: []backendconfig{ - { - relpath: "stack", - config: `terramate { - required_version = "~> 0.0.0" - backend "" {} -} - -stack {}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stack", - code: `terraform { - backend "" { - } -} -`, - }, - }, - }, - }, - { - name: "single stack with config on stack and config with 1 attr", - layout: []string{"s:stack"}, - configs: []backendconfig{ - { - relpath: "stack", - config: `terramate { - required_version = "~> 0.0.0" - backend "sometype" { - attr = "value" - } -} - -stack {}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stack", - code: `terraform { - backend "sometype" { - attr = "value" - } -} -`, - }, - }, - }, - }, - { - name: "multiple stacks with config on each stack", - layout: []string{"s:stack-1"}, - configs: []backendconfig{ - { - relpath: "stack-1", - config: `terramate { - required_version = "~> 0.0.0" - backend "1" { - attr = "hi" - } -} - -stack {}`, - }, - { - relpath: "stack-2", - config: `terramate { - required_version = "~> 0.0.0" - backend "2" { - somebool = true - } -} - -stack {}`, - }, - { - relpath: "stack-3", - config: `terramate { - required_version = "~> 0.0.0" - backend "3" { - somelist = ["m", "i", "n", "e", "i", "r", "o", "s"] - } -} - -stack {}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stack-1", - code: `terraform { - backend "1" { - attr = "hi" - } -} -`, - }, - { - relpath: "stack-2", - code: `terraform { - backend "2" { - somebool = true - } -} -`, - }, - { - relpath: "stack-3", - code: `terraform { - backend "3" { - somelist = ["m", "i", "n", "e", "i", "r", "o", "s"] - } -} -`, - }, - }, - }, - }, - { - name: "single stack with config on stack with config N attrs", - layout: []string{"s:stack"}, - configs: []backendconfig{ - { - relpath: "stack", - config: `terramate { - required_version = "~> 0.0.0" - backend "lotsoftypes" { - attr = "value" - attrnumber = 5 - attrbool = true - somelist = ["hi", "again"] - } -} - -stack {} -`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stack", - code: `terraform { - backend "lotsoftypes" { - attr = "value" - attrbool = true - attrnumber = 5 - somelist = ["hi", "again"] - } -} -`, - }, - }, - }, - }, - { - name: "single stack with config on stack with subblock", - layout: []string{"s:stack"}, - configs: []backendconfig{ - { - relpath: "stack", - config: `terramate { - required_version = "~> 0.0.0" - backend "lotsoftypes" { - attr = "value" - block { - attrbool = true - attrnumber = 5 - somelist = ["hi", "again"] - } - } -} - -stack {}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stack", - code: `terraform { - backend "lotsoftypes" { - attr = "value" - block { - attrbool = true - attrnumber = 5 - somelist = ["hi", "again"] - } - } -} -`, - }, - }, - }, - }, - { - name: "single stack with config parent dir", - layout: []string{"s:stacks/stack"}, - configs: []backendconfig{ - { - relpath: "stacks", - config: `terramate { - backend "fromparent" { - attr = "value" - } -}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack", - code: `terraform { - backend "fromparent" { - attr = "value" - } -} -`, - }, - }, - }, - }, - { - name: "single stack with config on basedir", - layout: []string{"s:stacks/stack"}, - configs: []backendconfig{ - { - relpath: ".", - config: `terramate { - backend "basedir_config" { - attr = 666 - } -} -`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack", - code: `terraform { - backend "basedir_config" { - attr = 666 - } -} -`, - }, - }, - }, - }, - { - name: "multiple stacks with config on basedir", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - configs: []backendconfig{ - { - relpath: ".", - config: `terramate { - backend "basedir_config" { - attr = "test" - } -} -`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack-1", - code: `terraform { - backend "basedir_config" { - attr = "test" - } -} -`, - }, - { - relpath: "stacks/stack-2", - code: `terraform { - backend "basedir_config" { - attr = "test" - } -} -`, - }, - }, - }, - }, - { - name: "stacks on different envs with per env config", - layout: []string{ - "s:envs/prod/stacks/stack", - "s:envs/staging/stacks/stack", - }, - configs: []backendconfig{ - { - relpath: "envs/prod", - config: `terramate { - backend "remote" { - environment = "prod" - } -} -`, - }, - { - relpath: "envs/staging", - config: `terramate { - backend "remote" { - environment = "staging" - } -} -`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "envs/prod/stacks/stack", - code: `terraform { - backend "remote" { - environment = "prod" - } -} -`, - }, - { - relpath: "envs/staging/stacks/stack", - code: `terraform { - backend "remote" { - environment = "staging" - } -} -`, - }, - }, - }, - }, - { - name: "single stack with config on stack and N attrs using metadata", - layout: []string{"s:stack-metadata"}, - configs: []backendconfig{ - { - relpath: "stack-metadata", - config: `terramate { - required_version = "~> 0.0.0" - backend "metadata" { - name = terramate.name - path = terramate.path - somelist = [terramate.name, terramate.path] - } -} -stack { - name = "custom-name" -}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stack-metadata", - code: `terraform { - backend "metadata" { - name = "custom-name" - path = "/stack-metadata" - somelist = ["custom-name", "/stack-metadata"] - } -} -`, - }, - }, - }, - }, - { - name: "multiple stacks with config on root dir using metadata", - layout: []string{"s:stacks/stack-1", "s:stacks/stack-2"}, - configs: []backendconfig{ - { - relpath: ".", - config: `terramate { - backend "metadata" { - name = terramate.name - path = terramate.path - interpolation = "interpolate-${terramate.name}-fun-${terramate.path}" - } -}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack-1", - code: `terraform { - backend "metadata" { - interpolation = "interpolate-stack-1-fun-/stacks/stack-1" - name = "stack-1" - path = "/stacks/stack-1" - } -} -`, - }, - { - relpath: "stacks/stack-2", - code: `terraform { - backend "metadata" { - interpolation = "interpolate-stack-2-fun-/stacks/stack-2" - name = "stack-2" - path = "/stacks/stack-2" - } -} -`, - }, - }, - }, - }, - { - name: "multiple stacks with config on root dir using metadata and tf functions", - layout: []string{"s:stacks/stack-1", "s:stacks/stack-2"}, - configs: []backendconfig{ - { - relpath: ".", - config: `terramate { - backend "metadata" { - funcfun = replace(terramate.path, "/","-") - funcfunb = "testing-funcs-${replace(terramate.path, "/",".")}" - name = terramate.name - path = terramate.path - } -}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack-1", - code: `terraform { - backend "metadata" { - funcfun = "-stacks-stack-1" - funcfunb = "testing-funcs-.stacks.stack-1" - name = "stack-1" - path = "/stacks/stack-1" - } -} -`, - }, - { - relpath: "stacks/stack-2", - code: `terraform { - backend "metadata" { - funcfun = "-stacks-stack-2" - funcfunb = "testing-funcs-.stacks.stack-2" - name = "stack-2" - path = "/stacks/stack-2" - } -} -`, - }, - }, - }, - }, - { - name: "multiple stacks with config on parent dir using globals from root", - layout: []string{"s:stacks/stack-1", "s:stacks/stack-2"}, - configs: []backendconfig{ - { - relpath: ".", - config: ` -globals { - bucket = "project-wide-bucket" -}`, - }, - { - relpath: "stacks", - config: `terramate { - backend "gcs" { - bucket = global.bucket - prefix = terramate.path - } -}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack-1", - code: `terraform { - backend "gcs" { - bucket = "project-wide-bucket" - prefix = "/stacks/stack-1" - } -} -`, - }, - { - relpath: "stacks/stack-2", - code: `terraform { - backend "gcs" { - bucket = "project-wide-bucket" - prefix = "/stacks/stack-2" - } -} -`, - }, - }, - }, - }, - { - name: "stack with global on parent dir using config from root", - layout: []string{"s:stacks/stack"}, - configs: []backendconfig{ - { - relpath: ".", - config: `terramate { - backend "gcs" { - bucket = global.bucket - prefix = terramate.path - } -}`, - }, - { - relpath: "stacks", - config: ` -globals { - bucket = "project-wide-bucket" -}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack", - code: `terraform { - backend "gcs" { - bucket = "project-wide-bucket" - prefix = "/stacks/stack" - } -} -`, - }, - }, - }, - }, - { - name: "stack overriding parent global", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - configs: []backendconfig{ - { - relpath: ".", - config: `terramate { - backend "gcs" { - bucket = global.bucket - prefix = terramate.path - } -}`, - }, - { - relpath: "stacks", - config: ` -globals { - bucket = "project-wide-bucket" -}`, - }, - { - relpath: "stacks/stack-1", - config: ` -terramate { - required_version = "~> 0.0.0" -} - -stack {} - -globals { - bucket = "stack-specific-bucket" -}`, - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack-1", - code: `terraform { - backend "gcs" { - bucket = "stack-specific-bucket" - prefix = "/stacks/stack-1" - } -} -`, - }, - { - relpath: "stacks/stack-2", - code: `terraform { - backend "gcs" { - bucket = "project-wide-bucket" - prefix = "/stacks/stack-2" - } -} -`, - }, - }, - }, - }, - { - name: "reference to undefined global fails", - layout: []string{"s:stack"}, - configs: []backendconfig{ - { - relpath: ".", - config: `terramate { - backend "gcs" { - bucket = global.bucket - } -}`, - }, - }, - want: want{ - err: generate.ErrBackendConfigGen, - }, - }, - { - name: "invalid global definition fails", - layout: []string{"s:stack"}, - configs: []backendconfig{ - { - relpath: ".", - config: `terramate { - backend "gcs" { - bucket = "all good" - } -} - -globals { - undefined_reference = global.undefined -} -`, - }, - }, - want: want{ - err: generate.ErrLoadingGlobals, - }, - }, - { - name: "multiple stacks selecting single stack with working dir", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - workingDir: "stacks/stack-1", - configs: []backendconfig{ - { - relpath: ".", - config: hcldoc( - terramate( - backend( - labels("gcs"), - expr("prefix", "terramate.path"), - ), - ), - ).String(), - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack-1", - code: hcldoc( - terraform( - backend( - labels("gcs"), - str("prefix", "/stacks/stack-1"), - ), - ), - ).String(), - }, - }, - }, - }, - { - name: "stacks using parent generated code filenames", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - configs: []backendconfig{ - { - relpath: "/stacks", - config: hcldoc( - - terramate( - backend( - labels("gcs"), - expr("prefix", "terramate.path"), - ), - cfg( - gen( - str("backend_config_filename", "backend.tf"), - ), - ), - ), - ).String(), - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack-1", - code: hcldoc( - terraform( - backend( - labels("gcs"), - str("prefix", "/stacks/stack-1"), - ), - ), - ).String(), - }, - { - relpath: "stacks/stack-2", - code: hcldoc( - terraform( - backend( - labels("gcs"), - str("prefix", "/stacks/stack-2"), - ), - ), - ).String(), - }, - }, - }, - }, - { - name: "stacks using parent generated code filenames filtered by working dir", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - workingDir: "stacks/stack-1", - configs: []backendconfig{ - { - relpath: "/stacks", - config: hcldoc( - - terramate( - backend( - labels("gcs"), - expr("prefix", "terramate.path"), - ), - cfg( - gen( - str("backend_config_filename", "backend.tf"), - ), - ), - ), - ).String(), - }, - }, - want: want{ - stacks: []stackcode{ - { - relpath: "stacks/stack-1", - code: hcldoc( - terraform( - backend( - labels("gcs"), - str("prefix", "/stacks/stack-1"), - ), - ), - ).String(), - }, - }, - }, - }, - { - name: "working dir has no stacks inside", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - "d:notstack", - }, - workingDir: "notstack", - configs: []backendconfig{ - { - relpath: ".", - config: hcldoc( - terramate( - backend( - labels("gcs"), - expr("prefix", "terramate.path"), - ), - ), - ).String(), - }, - }, - want: want{}, - }, - } - - for _, tcase := range tcases { - t.Run(tcase.name, func(t *testing.T) { - s := sandbox.New(t) - s.BuildTree(tcase.layout) - - for _, cfg := range tcase.configs { - dir := filepath.Join(s.RootDir(), cfg.relpath) - test.WriteFile(t, dir, config.Filename, cfg.config) - } - - workingDir := filepath.Join(s.RootDir(), tcase.workingDir) - err := generate.Do(s.RootDir(), workingDir) - assert.IsError(t, err, tcase.want.err) - - for _, want := range tcase.want.stacks { - stack := s.StackEntry(want.relpath) - got := string(stack.ReadGeneratedBackendCfg()) - - assertHCLEquals(t, got, want.code) - } - // TODO(katcipis): Add proper checks for extraneous generated code. - // For now we validated wanted files are there, but not that - // we may have new unwanted files being generated by a bug. - }) - } -} - -func TestLocalsGeneration(t *testing.T) { - // The test approach for locals generation already uses a new test package - // to help creating the HCL files instead of using plain raw strings. - // There are some trade-offs involved and we are assessing how to approach - // the testing, hence for now it is inconsistent between locals generation - // and backend configuration generation. The idea is to converge to a single - // approach ASAP. - type ( - // hclblock represents an HCL block that will be appended on - // the file path. - hclblock struct { - path string - add *hclwrite.Block - } - want struct { - err error - stacksLocals map[string]*hclwrite.Block - } - testcase struct { - name string - layout []string - configs []hclblock - workingDir string - want want - } - ) - - // gen instead of generate because name conflicts with generate pkg - gen := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("generate", builders...) - } - // cfg instead of config because name conflicts with config pkg - cfg := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("config", builders...) - } - - tcases := []testcase{ - { - name: "no stacks no exported locals", - layout: []string{}, - }, - { - name: "single stacks no exported locals", - layout: []string{"s:stack"}, - }, - { - name: "two stacks no exported locals", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - }, - { - name: "single stack with its own exported locals using own globals", - layout: []string{"s:stack"}, - configs: []hclblock{ - { - path: "/stack", - add: globals( - str("some_string", "string"), - number("some_number", 777), - boolean("some_bool", true), - ), - }, - { - path: "/stack", - add: exportAsLocals( - expr("string_local", "global.some_string"), - expr("number_local", "global.some_number"), - expr("bool_local", "global.some_bool"), - ), - }, - }, - want: want{ - stacksLocals: map[string]*hclwrite.Block{ - "/stack": locals( - str("string_local", "string"), - number("number_local", 777), - boolean("bool_local", true), - ), - }, - }, - }, - { - name: "single stack exporting metadata using functions", - layout: []string{"s:stack"}, - configs: []hclblock{ - { - path: "/stack", - add: exportAsLocals( - expr("funny_path", `replace(terramate.path, "/", "@")`), - ), - }, - }, - want: want{ - stacksLocals: map[string]*hclwrite.Block{ - "/stack": locals( - str("funny_path", "@stack"), - ), - }, - }, - }, - { - name: "single stack referencing undefined global fails", - layout: []string{"s:stack"}, - configs: []hclblock{ - { - path: "/stack", - add: exportAsLocals( - expr("undefined", "global.undefined"), - ), - }, - }, - want: want{ - err: generate.ErrExportingLocalsGen, - }, - }, - { - name: "multiple stack with exported locals being overridden", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - configs: []hclblock{ - { - path: "/", - add: globals( - str("attr1", "value1"), - str("attr2", "value2"), - str("attr3", "value3"), - ), - }, - { - path: "/", - add: exportAsLocals( - expr("string", "global.attr1"), - ), - }, - { - path: "/stacks", - add: exportAsLocals( - expr("string", "global.attr2"), - ), - }, - { - path: "/stacks/stack-1", - add: exportAsLocals( - expr("string", "global.attr3"), - ), - }, - }, - want: want{ - stacksLocals: map[string]*hclwrite.Block{ - "/stacks/stack-1": locals( - str("string", "value3"), - ), - "/stacks/stack-2": locals( - str("string", "value2"), - ), - }, - }, - }, - { - name: "single stack with exported locals and globals from parent dirs", - layout: []string{"s:stacks/stack"}, - configs: []hclblock{ - { - path: "/", - add: globals( - str("str", "string"), - ), - }, - { - path: "/", - add: exportAsLocals( - expr("num_local", "global.num"), - expr("path_local", "terramate.path"), - ), - }, - { - path: "/stacks", - add: globals( - number("num", 666), - ), - }, - { - path: "/stacks", - add: exportAsLocals( - expr("str_local", "global.str"), - expr("name_local", "terramate.name"), - ), - }, - }, - want: want{ - stacksLocals: map[string]*hclwrite.Block{ - "/stacks/stack": locals( - str("name_local", "stack"), - str("path_local", "/stacks/stack"), - str("str_local", "string"), - number("num_local", 666), - ), - }, - }, - }, - { - name: "multiple stacks with exported locals and globals from parent dirs", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - configs: []hclblock{ - { - path: "/", - add: globals( - str("str", "string"), - ), - }, - { - path: "/", - add: exportAsLocals( - expr("num_local", "global.num"), - expr("path_local", "terramate.path"), - ), - }, - { - path: "/stacks", - add: globals( - number("num", 666), - ), - }, - { - path: "/stacks", - add: exportAsLocals( - expr("str_local", "global.str"), - expr("name_local", "terramate.name"), - ), - }, - }, - want: want{ - stacksLocals: map[string]*hclwrite.Block{ - "/stacks/stack-1": locals( - str("name_local", "stack-1"), - str("path_local", "/stacks/stack-1"), - str("str_local", "string"), - number("num_local", 666), - ), - "/stacks/stack-2": locals( - str("name_local", "stack-2"), - str("path_local", "/stacks/stack-2"), - str("str_local", "string"), - number("num_local", 666), - ), - }, - }, - }, - { - name: "multiple stacks with specific exported locals and globals from parent dirs", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - configs: []hclblock{ - { - path: "/", - add: globals( - str("str", "string"), - ), - }, - { - path: "/stacks", - add: globals( - number("num", 666), - ), - }, - { - path: "/stacks/stack-1", - add: exportAsLocals( - expr("str_local", "global.str"), - expr("name_local", "terramate.name"), - ), - }, - { - path: "/stacks/stack-2", - add: exportAsLocals( - expr("num_local", "global.num"), - expr("path_local", "terramate.path"), - ), - }, - }, - want: want{ - stacksLocals: map[string]*hclwrite.Block{ - "/stacks/stack-1": locals( - str("name_local", "stack-1"), - str("str_local", "string"), - ), - "/stacks/stack-2": locals( - str("path_local", "/stacks/stack-2"), - number("num_local", 666), - ), - }, - }, - }, - { - name: "multiple stacks selecting single stack with working dir", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - workingDir: "stacks/stack-1", - configs: []hclblock{ - { - path: "/", - add: globals( - str("str", "string"), - ), - }, - { - path: "/stacks/stack-1", - add: exportAsLocals( - expr("str_local", "global.str"), - ), - }, - { - path: "/stacks/stack-2", - add: exportAsLocals( - expr("str_local", "global.str"), - ), - }, - }, - want: want{ - stacksLocals: map[string]*hclwrite.Block{ - "/stacks/stack-1": locals( - str("str_local", "string"), - ), - }, - }, - }, - { - name: "stacks getting code generation config from parent", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - configs: []hclblock{ - { - path: "/", - add: globals( - str("str", "string"), - ), - }, - { - path: "/stacks", - add: terramate( - cfg( - gen( - str("locals_filename", "locals.tf"), - ), - ), - ), - }, - { - path: "/stacks/stack-1", - add: exportAsLocals( - expr("str_local", "global.str"), - ), - }, - { - path: "/stacks/stack-2", - add: exportAsLocals( - expr("str_local", "global.str"), - ), - }, - }, - want: want{ - stacksLocals: map[string]*hclwrite.Block{ - "/stacks/stack-1": locals( - str("str_local", "string"), - ), - "/stacks/stack-2": locals( - str("str_local", "string"), - ), - }, - }, - }, - { - name: "stacks with code gen cfg filtered by working dir", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - workingDir: "stacks/stack-2", - configs: []hclblock{ - { - path: "/", - add: globals( - str("str", "string"), - ), - }, - { - path: "/stacks", - add: terramate( - cfg( - gen( - str("locals_filename", "locals.tf"), - ), - ), - ), - }, - { - path: "/stacks/stack-1", - add: exportAsLocals( - expr("str_local", "global.str"), - ), - }, - { - path: "/stacks/stack-2", - add: exportAsLocals( - expr("str_local", "global.str"), - ), - }, - }, - want: want{ - stacksLocals: map[string]*hclwrite.Block{ - "/stacks/stack-2": locals( - str("str_local", "string"), - ), - }, - }, - }, - { - name: "working dir has no stacks inside", - layout: []string{ - "s:stack", - "d:somedir", - }, - workingDir: "somedir", - configs: []hclblock{ - { - path: "/stack", - add: exportAsLocals( - expr("path", "terramate.path"), - ), - }, - }, - want: want{}, - }, - } - - for _, tcase := range tcases { - t.Run(tcase.name, func(t *testing.T) { - s := sandbox.New(t) - s.BuildTree(tcase.layout) - - for _, cfg := range tcase.configs { - path := filepath.Join(s.RootDir(), cfg.path) - test.AppendFile(t, path, config.Filename, cfg.add.String()) - } - - workingDir := filepath.Join(s.RootDir(), tcase.workingDir) - err := generate.Do(s.RootDir(), workingDir) - assert.IsError(t, err, tcase.want.err) - - for stackPath, wantHCLBlock := range tcase.want.stacksLocals { - stackRelPath := stackPath[1:] - want := wantHCLBlock.String() - stack := s.StackEntry(stackRelPath) - got := string(stack.ReadGeneratedLocals()) - - assertHCLEquals(t, got, want) - } - // TODO(katcipis): Add proper checks for extraneous generated code. - // For now we validated wanted files are there, but not that - // we may have new unwanted files being generated by a bug. - }) - } -} - -func TestWontOverwriteManuallyDefinedBackendConfig(t *testing.T) { - const ( - manualContents = "some manual backend configs" - ) - - backend := func(label string) *hclwrite.Block { - b := hclwrite.BuildBlock("backend") - b.AddLabel(label) - return b - } - rootTerramateConfig := terramate(backend("test")) - - s := sandbox.New(t) - s.BuildTree([]string{ - fmt.Sprintf("f:%s:%s", config.Filename, rootTerramateConfig.String()), - "s:stack", - fmt.Sprintf("f:stack/%s:%s", generate.BackendCfgFilename, manualContents), - }) - - err := generate.Do(s.RootDir(), s.RootDir()) - assert.IsError(t, err, generate.ErrManualCodeExists) - - stack := s.StackEntry("stack") - - backendConfig := string(stack.ReadGeneratedBackendCfg()) - assert.EqualStrings(t, manualContents, backendConfig, "backend config altered by generate") -} - -func TestBackendConfigOverwriting(t *testing.T) { - backend := func(label string) *hclwrite.Block { - b := hclwrite.BuildBlock("backend") - b.AddLabel(label) - return b - } - firstConfig := terramate(backend("first")) - firstWant := terraform(backend("first")) - - s := sandbox.New(t) - stack := s.CreateStack("stack") - rootEntry := s.DirEntry(".") - rootConfig := rootEntry.CreateConfig(firstConfig.String()) - - assert.NoError(t, generate.Do(s.RootDir(), s.RootDir())) - - got := string(stack.ReadGeneratedBackendCfg()) - assertHCLEquals(t, got, firstWant.String()) - - secondConfig := terramate(backend("second")) - secondWant := terraform(backend("second")) - rootConfig.Write(secondConfig.String()) - - assert.NoError(t, generate.Do(s.RootDir(), s.RootDir())) - - got = string(stack.ReadGeneratedBackendCfg()) - assertHCLEquals(t, got, secondWant.String()) -} - -func TestWontOverwriteManuallyDefinedLocals(t *testing.T) { - const ( - manualLocals = "some manual stuff" - ) - - exportLocalsCfg := exportAsLocals(expr("a", "terramate.path")) - - s := sandbox.New(t) - s.BuildTree([]string{ - fmt.Sprintf("f:%s:%s", config.Filename, exportLocalsCfg.String()), - "s:stack", - fmt.Sprintf("f:stack/%s:%s", generate.LocalsFilename, manualLocals), - }) - - err := generate.Do(s.RootDir(), s.RootDir()) - assert.IsError(t, err, generate.ErrManualCodeExists) - - stack := s.StackEntry("stack") - locals := string(stack.ReadGeneratedLocals()) - assert.EqualStrings(t, manualLocals, locals, "locals altered by generate") -} - -func TestExportedLocalsOverwriting(t *testing.T) { - firstConfig := exportAsLocals(expr("a", "terramate.path")) - firstWant := locals(str("a", "/stack")) - - s := sandbox.New(t) - stack := s.CreateStack("stack") - rootEntry := s.DirEntry(".") - rootConfig := rootEntry.CreateConfig(firstConfig.String()) - - assert.NoError(t, generate.Do(s.RootDir(), s.RootDir())) - - got := string(stack.ReadGeneratedLocals()) - assertHCLEquals(t, got, firstWant.String()) - - secondConfig := exportAsLocals(expr("b", "terramate.name")) - secondWant := locals(str("b", "stack")) - rootConfig.Write(secondConfig.String()) - - assert.NoError(t, generate.Do(s.RootDir(), s.RootDir())) - - got = string(stack.ReadGeneratedLocals()) - assertHCLEquals(t, got, secondWant.String()) -} - func assertHCLEquals(t *testing.T, got string, want string) { t.Helper() From 059ebbac14956740c115be73301cb52f5acf04a9 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Fri, 28 Jan 2022 18:05:17 +0100 Subject: [PATCH 18/48] test: add initial test fixture + core + first test --- generate/config.go | 5 ++ generate/generate_terraform_test.go | 102 ++++++++++++++++++++++++++++ test/os.go | 8 +++ test/sandbox/sandbox.go | 28 ++++++++ 4 files changed, 143 insertions(+) create mode 100644 generate/generate_terraform_test.go diff --git a/generate/config.go b/generate/config.go index 29ab3ad24..789b88fad 100644 --- a/generate/config.go +++ b/generate/config.go @@ -43,6 +43,11 @@ func LoadStackCfg(root string, stack stack.S) (StackCfg, error) { return loadStackCfg(root, stack.AbsPath()) } +func (s StackCfg) ExportedTfFilename(name string) string { + // We may have customized configuration someday + return fmt.Sprintf("_gen_terramate_%s.tf", name) +} + func loadStackCfg(root string, configdir string) (StackCfg, error) { logger := log.With(). Str("action", "loadStackCfg()"). diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go new file mode 100644 index 000000000..e96642989 --- /dev/null +++ b/generate/generate_terraform_test.go @@ -0,0 +1,102 @@ +package generate_test + +import ( + "fmt" + "io/fs" + "path/filepath" + "testing" + + "github.com/madlambda/spells/assert" + "github.com/mineiros-io/terramate/config" + "github.com/mineiros-io/terramate/generate" + "github.com/mineiros-io/terramate/test" + "github.com/mineiros-io/terramate/test/sandbox" +) + +// Test +// +// - Overwriting Behavior (manual tf code already exists) + +func TestTerraformGeneration(t *testing.T) { + type ( + hclconfig struct { + path string + add fmt.Stringer + } + want struct { + stack string + hcls map[string]fmt.Stringer + } + testcase struct { + name string + layout []string + configs []hclconfig + workingDir string + want []want + wantErr error + } + ) + + tcases := []testcase{ + { + name: "no exported terraform", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + }, + } + + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + s := sandbox.New(t) + s.BuildTree(tcase.layout) + + for _, cfg := range tcase.configs { + path := filepath.Join(s.RootDir(), cfg.path) + test.AppendFile(t, path, config.Filename, cfg.add.String()) + } + + workingDir := filepath.Join(s.RootDir(), tcase.workingDir) + err := generate.Do(s.RootDir(), workingDir) + assert.IsError(t, err, tcase.wantErr) + + for _, wantDesc := range tcase.want { + stackRelPath := wantDesc.stack[1:] + stack := s.StackEntry(stackRelPath) + + for name, wantHCL := range wantDesc.hcls { + want := wantHCL.String() + got := string(stack.ReadGeneratedTerraform(name)) + + assertHCLEquals(t, got, want) + + stack.RemoveGeneratedTerraform(name) + } + } + + // Check we don't have extraneous/unwanted files + // Wanted/expected generated code was removed by this point + // So we should have only basic terramate configs left + filepath.WalkDir(s.RootDir(), func(path string, d fs.DirEntry, err error) error { + t.Helper() + + assert.NoError(t, err, "checking for unwanted generated files") + if d.IsDir() { + if d.Name() == ".git" { + return filepath.SkipDir + } + return nil + } + + // sandbox create README.md inside test dirs + if d.Name() == config.Filename || d.Name() == "README.md" { + return nil + } + + t.Errorf("expected only basic terramate config at %q, got %q", path, d.Name()) + return nil + }) + }) + } +} diff --git a/test/os.go b/test/os.go index bac3b0547..a4130a4b0 100644 --- a/test/os.go +++ b/test/os.go @@ -77,6 +77,14 @@ func ReadFile(t *testing.T, dir, fname string) []byte { return data } +// RemoveFile removes the file fname from dir directory. +// If the files doesn't exists, it succeeds. +func RemoveFile(t *testing.T, dir, fname string) { + t.Helper() + err := os.Remove(filepath.Join(dir, fname)) + assert.NoError(t, err) +} + // Mkdir creates a directory inside base. func Mkdir(t *testing.T, base string, name string) string { path := filepath.Join(base, name) diff --git a/test/sandbox/sandbox.go b/test/sandbox/sandbox.go index 271dbaf7b..b72b5636f 100644 --- a/test/sandbox/sandbox.go +++ b/test/sandbox/sandbox.go @@ -355,6 +355,13 @@ func (de DirEntry) ReadFile(name string) []byte { return test.ReadFile(de.t, de.abspath, name) } +// RemoveFile will delete a file inside this dir entry with the given name. +// It will succeeds if the file already doesn't exist. +func (de DirEntry) RemoveFile(name string) { + de.t.Helper() + test.RemoveFile(de.t, de.abspath, name) +} + // Path returns the absolute path of the directory entry. func (de DirEntry) Path() string { return de.abspath @@ -420,6 +427,27 @@ func (se StackEntry) ReadGeneratedLocals() []byte { return se.DirEntry.ReadFile(cfg.LocalsFilename) } +// ReadGeneratedTerraform will read code that was generated by Terramate for this stack +// using export_as_terraform blocks. +// The given name is the name of the export_as_terraform block as indicated by its label. +// +// It will fail the test if there is no generated code available on the stack, +// since it assumes generated code is expected to be there. +func (se StackEntry) ReadGeneratedTerraform(name string) []byte { + se.t.Helper() + cfg := se.LoadStackCodeGenCfg() + return se.DirEntry.ReadFile(cfg.ExportedTfFilename(name)) +} + +// RemoveGeneratedTerraform will delete the file with generated code from +// export_as_terraform blocks. +// The given name is the name of the export_as_terraform block as indicated by its label. +func (se StackEntry) RemoveGeneratedTerraform(name string) { + se.t.Helper() + cfg := se.LoadStackCodeGenCfg() + se.DirEntry.RemoveFile(cfg.ExportedTfFilename(name)) +} + // LoadStackCodeGenCfg will load the stack code generation configuration. func (se StackEntry) LoadStackCodeGenCfg() generate.StackCfg { se.t.Helper() From 623d78864a4c6aa75093a9f05149bbd6a229b38a Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Fri, 28 Jan 2022 18:06:44 +0100 Subject: [PATCH 19/48] chore: add missing license info --- generate/generate_terraform_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go index e96642989..3bed4d290 100644 --- a/generate/generate_terraform_test.go +++ b/generate/generate_terraform_test.go @@ -1,3 +1,17 @@ +// Copyright 2022 Mineiros GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package generate_test import ( @@ -78,6 +92,7 @@ func TestTerraformGeneration(t *testing.T) { // Check we don't have extraneous/unwanted files // Wanted/expected generated code was removed by this point // So we should have only basic terramate configs left + // There is potential to extract this for other code generation tests. filepath.WalkDir(s.RootDir(), func(path string, d fs.DirEntry, err error) error { t.Helper() From 7996560ec26b442239aeb855ee8db74b804fd024 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Fri, 28 Jan 2022 18:12:03 +0100 Subject: [PATCH 20/48] feat: fail if export_as_terraform label is empty --- generate/exportedtf/exportedtf.go | 4 ++++ generate/exportedtf/exportedtf_test.go | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index b0d7db370..871828859 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -187,6 +187,10 @@ func loadExportBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Bloc ) } name := block.Labels[0] + if name == "" { + return nil, fmt.Errorf("%w: label can't be empty", ErrInvalidBlock) + } + if _, ok := res[name]; ok { return nil, fmt.Errorf( "%w: found two blocks with same label %q", diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index bb3f3997a..5b3186960 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -535,6 +535,22 @@ func TestLoadExportedTerraform(t *testing.T) { }, wantErr: exportedtf.ErrInvalidBlock, }, + { + name: "export block with empty label on stack gives err", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/stacks/stack", + add: block("export_as_terraform", + labels(""), + block("block", + str("data", "some literal data"), + ), + ), + }, + }, + wantErr: exportedtf.ErrInvalidBlock, + }, { name: "export blocks with same label on same config gives err", stack: "/stacks/stack", From 08c19428ad584709945023b3d76fbfd887bf0740 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Fri, 28 Jan 2022 18:26:51 +0100 Subject: [PATCH 21/48] test: fix error checking on walk --- generate/generate_terraform_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go index 3bed4d290..cdd9f3ea5 100644 --- a/generate/generate_terraform_test.go +++ b/generate/generate_terraform_test.go @@ -93,7 +93,7 @@ func TestTerraformGeneration(t *testing.T) { // Wanted/expected generated code was removed by this point // So we should have only basic terramate configs left // There is potential to extract this for other code generation tests. - filepath.WalkDir(s.RootDir(), func(path string, d fs.DirEntry, err error) error { + err = filepath.WalkDir(s.RootDir(), func(path string, d fs.DirEntry, err error) error { t.Helper() assert.NoError(t, err, "checking for unwanted generated files") @@ -112,6 +112,8 @@ func TestTerraformGeneration(t *testing.T) { t.Errorf("expected only basic terramate config at %q, got %q", path, d.Name()) return nil }) + + assert.NoError(t, err) }) } } From 03f50fe6728add17b6a2363c744de02f3ebb1ef5 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Fri, 28 Jan 2022 19:52:47 +0100 Subject: [PATCH 22/48] feat: add export as terraform code generation --- generate/generate.go | 88 +++++++++++---- generate/generate_terraform_test.go | 159 ++++++++++++++++++++++++++++ generate/generate_test.go | 2 +- hcl/hcl.go | 5 +- 4 files changed, 232 insertions(+), 22 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 9f9c040ff..c842fd9f1 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -27,6 +27,7 @@ import ( "github.com/madlambda/spells/errutil" "github.com/mineiros-io/terramate" "github.com/mineiros-io/terramate/config" + "github.com/mineiros-io/terramate/generate/exportedtf" "github.com/mineiros-io/terramate/hcl" "github.com/mineiros-io/terramate/hcl/eval" "github.com/mineiros-io/terramate/stack" @@ -85,7 +86,9 @@ func Do(root string, workingDir string) error { if err != nil { return err } - return nil + + logger.Debug().Msg("Generate stack terraform.") + return writeStackTerraformCode(root, stackpath, stackMeta, globals, cfg) }) // FIXME(katcipis): errutil.Chain produces a very hard to read string representation @@ -172,6 +175,55 @@ func CheckStack(root string, stack stack.S) ([]string, error) { return outdated, nil } +func writeStackTerraformCode( + root string, + stackpath string, + meta stack.Metadata, + globals *terramate.Globals, + cfg StackCfg, +) error { + logger := log.With(). + Str("action", "writeStackTerraformCode()"). + Str("root", root). + Str("stackpath", stackpath). + Logger() + + logger.Trace().Msg("generating terraform code.") + + loadedStackTf, err := exportedtf.Load(root, meta, globals) + if err != nil { + return err + } + + logger.Trace().Msg("generated terraform code.") + + for name, tf := range loadedStackTf.ExportedCode() { + targetpath := filepath.Join(stackpath, cfg.ExportedTfFilename(name)) + logger := logger.With(). + Str("blockName", name). + Str("targetpath", targetpath). + Logger() + + tfcode := tf.String() + if tfcode == "" { + logger.Debug().Msg("ignoring empty export_as_terraform block.") + continue + } + + logger.Debug().Msg("Stack has exported terraform, saving generated code.") + + tfcode = PrependHeader(tfcode) + if err := writeGeneratedCode(targetpath, tfcode); err != nil { + return fmt.Errorf("stack %q: writing code at %q: %v", stackpath, targetpath, err) + } + + logger.Debug().Msg("Saved stack generated code.") + } + + logger.Trace().Msg("all terraform code has been saved.") + return nil +} + func writeStackLocalsCode( root string, stackpath string, @@ -218,7 +270,7 @@ func generateStackLocalsCode( stackpath string, metadata stack.Metadata, globals *terramate.Globals, -) ([]byte, error) { +) (string, error) { logger := log.With(). Str("action", "generateStackLocals()"). Str("stack", stackpath). @@ -228,14 +280,14 @@ func generateStackLocalsCode( stackLocals, err := terramate.LoadStackExportedLocals(rootdir, metadata, globals) if err != nil { - return nil, err + return "", err } logger.Trace().Msg("Get stack attributes.") localsAttrs := stackLocals.Attributes() if len(localsAttrs) == 0 { - return nil, nil + return "", nil } logger.Trace().Msg("Sort attributes.") @@ -260,7 +312,7 @@ func generateStackLocalsCode( localsBody.SetAttributeValue(name, localsAttrs[name]) } - tfcode := PrependHeaderBytes(gen.Bytes()) + tfcode := PrependHeader(string(gen.Bytes())) return tfcode, nil } @@ -308,7 +360,7 @@ func generateBackendCfgCode( stackMetadata stack.Metadata, globals *terramate.Globals, configdir string, -) ([]byte, error) { +) (string, error) { logger := log.With(). Str("action", "loadStackBackendConfig()"). Str("configDir", configdir). @@ -319,7 +371,7 @@ func generateBackendCfgCode( if !strings.HasPrefix(configdir, root) { // check if we are outside of project's root, time to stop - return nil, nil + return "", nil } logger.Trace(). @@ -342,14 +394,14 @@ func generateBackendCfgCode( Msg("Read config file.") config, err := os.ReadFile(configfile) if err != nil { - return nil, fmt.Errorf("reading config: %v", err) + return "", fmt.Errorf("reading config: %v", err) } logger.Debug(). Msg("Parse config file.") parsedConfig, err := hcl.Parse(configfile, config) if err != nil { - return nil, fmt.Errorf("parsing config: %w", err) + return "", fmt.Errorf("parsing config: %w", err) } logger.Trace(). @@ -365,14 +417,14 @@ func generateBackendCfgCode( err = evalctx.SetNamespace("terramate", stackMetadata.ToCtyMap()) if err != nil { - return nil, fmt.Errorf("setting terramate namespace on eval context for stack %q: %v", + return "", fmt.Errorf("setting terramate namespace on eval context for stack %q: %v", stackpath, err) } logger.Trace().Msg("Add global evaluation namespace.") if err := evalctx.SetNamespace("global", globals.Attributes()); err != nil { - return nil, fmt.Errorf("setting global namespace on eval context for stack %q: %v", + return "", fmt.Errorf("setting global namespace on eval context for stack %q: %v", stackpath, err) } @@ -386,21 +438,21 @@ func generateBackendCfgCode( backendBody := backendBlock.Body() if err := hcl.CopyBody(backendBody, parsed.Backend.Body, evalctx); err != nil { - return nil, err + return "", err } - return PrependHeaderBytes(gen.Bytes()), nil + return PrependHeader(string(gen.Bytes())), nil } -// PrependHeaderBytes will add a proper Terramate header indicating that code +// PrependHeader will add a proper Terramate header indicating that code // was generated by Terramate. -func PrependHeaderBytes(code []byte) []byte { - return append([]byte(codeHeader+"\n\n"), code...) +func PrependHeader(code string) string { + return codeHeader + "\n\n" + code } const codeHeader = "// GENERATED BY TERRAMATE: DO NOT EDIT" -func writeGeneratedCode(target string, code []byte) error { +func writeGeneratedCode(target string, code string) error { logger := log.With(). Str("action", "writeGeneratedCode()"). Str("file", target). @@ -413,7 +465,7 @@ func writeGeneratedCode(target string, code []byte) error { } logger.Trace().Msg("Writing code") - return os.WriteFile(target, code, 0666) + return os.WriteFile(target, []byte(code), 0666) } func checkFileCanBeOverwritten(path string) error { diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go index cdd9f3ea5..f5e575ad4 100644 --- a/generate/generate_terraform_test.go +++ b/generate/generate_terraform_test.go @@ -24,6 +24,7 @@ import ( "github.com/mineiros-io/terramate/config" "github.com/mineiros-io/terramate/generate" "github.com/mineiros-io/terramate/test" + "github.com/mineiros-io/terramate/test/hclwrite" "github.com/mineiros-io/terramate/test/sandbox" ) @@ -51,6 +52,22 @@ func TestTerraformGeneration(t *testing.T) { } ) + exportAsTerraform := func(label string, builders ...hclwrite.BlockBuilder) *hclwrite.Block { + b := hclwrite.BuildBlock("export_as_terraform", builders...) + b.AddLabel(label) + return b + } + provider := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("provider", builders...) + } + required_providers := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("required_providers", builders...) + } + attr := func(name, expr string) hclwrite.BlockBuilder { + t.Helper() + return hclwrite.AttributeValue(t, name, expr) + } + tcases := []testcase{ { name: "no exported terraform", @@ -59,6 +76,148 @@ func TestTerraformGeneration(t *testing.T) { "s:stacks/stack-2", }, }, + { + name: "export terraform for all stacks on parent", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []hclconfig{ + { + path: "/stacks", + add: hcldoc( + exportAsTerraform("backend", + backend( + labels("test"), + expr("prefix", "global.backend_prefix"), + ), + ), + exportAsTerraform("locals", + locals( + expr("stackpath", "terramate.path"), + expr("local_a", "global.local_a"), + expr("local_b", "global.local_b"), + expr("local_c", "global.local_c"), + expr("local_d", "try(global.local_d.field, null)"), + ), + ), + exportAsTerraform("provider", + provider( + labels("name"), + expr("data", "global.provider_data"), + ), + terraform( + required_providers( + expr("name", `{ + source = "integrations/name" + version = global.provider_version + }`), + ), + ), + terraform( + expr("required_version", "global.terraform_version"), + ), + ), + ), + }, + { + path: "/stacks/stack-1", + add: globals( + str("local_a", "stack-1-local"), + boolean("local_b", true), + number("local_c", 666), + attr("local_d", `{ field = "local_d_field"}`), + str("backend_prefix", "stack-1-backend"), + str("provider_data", "stack-1-provider-data"), + str("provider_version", "stack-1-provider-version"), + str("terraform_version", "stack-1-terraform-version"), + ), + }, + { + path: "/stacks/stack-2", + add: globals( + str("local_a", "stack-2-local"), + boolean("local_b", false), + number("local_c", 777), + attr("local_d", `{ oopsie = "local_d_field"}`), + str("backend_prefix", "stack-2-backend"), + str("provider_data", "stack-2-provider-data"), + str("provider_version", "stack-2-provider-version"), + str("terraform_version", "stack-2-terraform-version"), + ), + }, + }, + want: []want{ + { + stack: "/stacks/stack-1", + hcls: map[string]fmt.Stringer{ + "backend": backend( + labels("test"), + str("prefix", "stack-1-backend"), + ), + + "locals": locals( + str("stackpath", "/stacks/stack-1"), + str("local_a", "stack-1-local"), + boolean("local_b", true), + number("local_c", 666), + str("local_d", "local_d_field"), + ), + + "provider": hcldoc( + provider( + labels("name"), + str("data", "stack-1-provider-data"), + ), + terraform( + required_providers( + expr("name", `{ + source = "integrations/name" + version = "stack-1-provider-version" + }`), + ), + ), + terraform( + str("required_version", "stack-1-terraform-version"), + ), + ), + }, + }, + { + stack: "/stacks/stack-2", + hcls: map[string]fmt.Stringer{ + "backend": backend( + labels("test"), + str("prefix", "stack-2-backend"), + ), + "locals": locals( + str("stackpath", "/stacks/stack-2"), + str("local_a", "stack-2-local"), + boolean("local_b", false), + number("local_c", 777), + attr("local_d", "null"), + ), + "provider": hcldoc( + provider( + labels("name"), + str("data", "stack-2-provider-data"), + ), + terraform( + required_providers( + expr("name", `{ + source = "integrations/name" + version = "stack-2-provider-version" + }`), + ), + ), + terraform( + str("required_version", "stack-2-terraform-version"), + ), + ), + }, + }, + }, + }, } for _, tcase := range tcases { diff --git a/generate/generate_test.go b/generate/generate_test.go index 2a8cdf313..850b4a469 100644 --- a/generate/generate_test.go +++ b/generate/generate_test.go @@ -31,7 +31,7 @@ func assertHCLEquals(t *testing.T, got string, want string) { // (but we test the formatting too... so maybe that is good ? =P) const trimmedChars = "\n " - want = string(generate.PrependHeaderBytes([]byte(want))) + want = string(generate.PrependHeader(want)) got = strings.Trim(got, trimmedChars) want = strings.Trim(want, trimmedChars) diff --git a/hcl/hcl.go b/hcl/hcl.go index 581e63503..12f9343a1 100644 --- a/hcl/hcl.go +++ b/hcl/hcl.go @@ -950,9 +950,8 @@ func blockIsAllowed(name string) bool { Logger() switch name { - case "terramate", "stack", "backend", "globals", "export_as_locals": - logger.Trace(). - Msg("Block name was allowed.") + case "terramate", "stack", "backend", "globals", "export_as_locals", "export_as_terraform": + logger.Trace().Msg("Block name was allowed.") return true default: return false From 107eeeb205f9dae198d2bd93cc0b07324e033b9d Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Fri, 28 Jan 2022 20:04:08 +0100 Subject: [PATCH 23/48] test: empty export_as_terraform block is ignored --- generate/generate_terraform_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go index f5e575ad4..4c1a73aff 100644 --- a/generate/generate_terraform_test.go +++ b/generate/generate_terraform_test.go @@ -76,6 +76,19 @@ func TestTerraformGeneration(t *testing.T) { "s:stacks/stack-2", }, }, + { + name: "empty export_as_terraform block is ignored", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []hclconfig{ + { + path: "/stacks", + add: exportAsTerraform("empty"), + }, + }, + }, { name: "export terraform for all stacks on parent", layout: []string{ From 2a096f3f0835ead15886825bb175607383076b47 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 13:35:36 +0100 Subject: [PATCH 24/48] test: add initial failing test for outdated detection --- generate/generate_check_test.go | 83 ++++++++++++++++++++++++++++- generate/generate_hcl_test.go | 12 +++++ generate/generate_terraform_test.go | 16 +++--- 3 files changed, 101 insertions(+), 10 deletions(-) diff --git a/generate/generate_check_test.go b/generate/generate_check_test.go index a04275229..c10baf6ae 100644 --- a/generate/generate_check_test.go +++ b/generate/generate_check_test.go @@ -23,7 +23,87 @@ import ( "github.com/mineiros-io/terramate/test/sandbox" ) -func TestCheckReturnsOutdatedStackFilenames(t *testing.T) { +func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { + s := sandbox.New(t) + + stackEntry := s.CreateStack("stacks/stack") + stack := stackEntry.Load() + + assertOutdated := func(want []string) { + t.Helper() + + got, err := generate.CheckStack(s.RootDir(), stack) + assert.NoError(t, err) + assertStringsEquals(t, got, want) + } + + // Checking detection when there is no config generated yet + assertOutdated([]string{}) + stackEntry.CreateConfig( + stackConfig( + exportAsTerraform( + labels("test.tf"), + str("required_version", "1.10"), + ), + ).String()) + assertOutdated([]string{"test.tf"}) + + s.Generate() + + assertOutdated([]string{}) + + // Now checking when we have code + it gets outdated. + stackEntry.CreateConfig( + stackConfig( + exportAsTerraform( + labels("test.tf"), + str("required_version", "1.11"), + ), + ).String()) + + assertOutdated([]string{"test.tf"}) + + s.Generate() + + // Changing generated filenames will trigger detection, with new filenames + stackEntry.CreateConfig( + stackConfig( + exportAsTerraform( + labels("testnew.tf"), + str("required_version", "1.11"), + ), + ).String()) + + // TODO(katcipis): detect the old test.tf generated file. + // It is stale but it doesn't map to code generation anymore so + // we need extra steps to detect it that are not done today. + assertOutdated([]string{"testnew.tf"}) + + // TODO(katcipis): cleanup the old test.tf + + // Adding new filename to generation trigger detection + stackEntry.CreateConfig( + stackConfig( + exportAsTerraform( + labels("testnew.tf"), + str("required_version", "1.11"), + ), + exportAsTerraform( + labels("another.tf"), + backend( + labels("type"), + ), + ), + ).String()) + + assertOutdated([]string{"testnew.tf", "another.tf"}) + + s.Generate() + + assertOutdated([]string{}) +} + +func TestCheckReturnsOutdatedStackFilenamesForBackendAndLocals(t *testing.T) { s := sandbox.New(t) stack1 := s.CreateStack("stacks/stack-1") @@ -161,6 +241,7 @@ func TestCheckReturnsOutdatedStackFilenames(t *testing.T) { } func TestCheckFailsWithInvalidConfig(t *testing.T) { + // TODO(katcipis): add export_as_terraform invalidConfigs := []string{ hcldoc( terramate( diff --git a/generate/generate_hcl_test.go b/generate/generate_hcl_test.go index 422b2f4b1..59ced370a 100644 --- a/generate/generate_hcl_test.go +++ b/generate/generate_hcl_test.go @@ -22,6 +22,18 @@ func hcldoc(blocks ...*hclwrite.Block) hclwrite.HCL { return hclwrite.NewHCL(blocks...) } +// stackConfig wraps the blocks built by the given builders in a valid stack +// configuration +func stackConfig(blocks ...*hclwrite.Block) hclwrite.HCL { + b := []*hclwrite.Block{stack()} + b = append(b, blocks...) + return hcldoc(b...) +} + +func exportAsTerraform(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("export_as_terraform", builders...) +} + func expr(name string, expr string) hclwrite.BlockBuilder { return hclwrite.Expression(name, expr) } diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go index 4c1a73aff..d2bed5639 100644 --- a/generate/generate_terraform_test.go +++ b/generate/generate_terraform_test.go @@ -52,11 +52,6 @@ func TestTerraformGeneration(t *testing.T) { } ) - exportAsTerraform := func(label string, builders ...hclwrite.BlockBuilder) *hclwrite.Block { - b := hclwrite.BuildBlock("export_as_terraform", builders...) - b.AddLabel(label) - return b - } provider := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { return hclwrite.BuildBlock("provider", builders...) } @@ -85,7 +80,7 @@ func TestTerraformGeneration(t *testing.T) { configs: []hclconfig{ { path: "/stacks", - add: exportAsTerraform("empty"), + add: exportAsTerraform(labels("empty")), }, }, }, @@ -99,13 +94,15 @@ func TestTerraformGeneration(t *testing.T) { { path: "/stacks", add: hcldoc( - exportAsTerraform("backend", + exportAsTerraform( + labels("backend"), backend( labels("test"), expr("prefix", "global.backend_prefix"), ), ), - exportAsTerraform("locals", + exportAsTerraform( + labels("locals"), locals( expr("stackpath", "terramate.path"), expr("local_a", "global.local_a"), @@ -114,7 +111,8 @@ func TestTerraformGeneration(t *testing.T) { expr("local_d", "try(global.local_d.field, null)"), ), ), - exportAsTerraform("provider", + exportAsTerraform( + labels("provider"), provider( labels("name"), expr("data", "global.provider_data"), From 674f67e199c401de8fceee84c839b1c586a772b3 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 13:51:52 +0100 Subject: [PATCH 25/48] refactor: extract backend cfg outdated detection --- generate/generate.go | 53 +++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index c842fd9f1..14720d653 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -132,27 +132,14 @@ func CheckStack(root string, stack stack.S) ([]string, error) { return nil, fmt.Errorf("checking for outdated code: %v", err) } - logger.Trace().Msg("Generating backend cfg code for stack.") - stackpath := stack.AbsPath() stackMeta := stack.Meta() - genbackend, err := generateBackendCfgCode(root, stackpath, stackMeta, globals, stackpath) - if err != nil { - return nil, fmt.Errorf("checking for outdated code: %v", err) - } - stackBackendCfgFile := filepath.Join(stackpath, cfg.BackendCfgFilename) - currentbackend, err := loadGeneratedCode(stackBackendCfgFile) + outdatedBackendFiles, err := backendConfigIsOutdated(root, stackpath, stackMeta, globals, cfg) if err != nil { - return nil, fmt.Errorf("checking for outdated code: %v", err) - } - - logger.Trace().Msg("Checking for outdated backend cfg code on stack.") - - if string(genbackend) != string(currentbackend) { - logger.Trace().Msg("Detected outdated backend config.") - outdated = append(outdated, cfg.BackendCfgFilename) + return nil, fmt.Errorf("checking for outdated backend config: %v", err) } + outdated = append(outdated, outdatedBackendFiles...) logger.Trace().Msg("Checking for outdated exported locals code on stack.") @@ -175,6 +162,40 @@ func CheckStack(root string, stack stack.S) ([]string, error) { return outdated, nil } +func backendConfigIsOutdated( + root, stackpath string, + stackMeta stack.Metadata, + globals *terramate.Globals, + cfg StackCfg, +) ([]string, error) { + logger := log.With(). + Str("action", "generate.backendConfigIsOutdated()"). + Str("root", root). + Str("stackpath", stackpath). + Logger() + + logger.Trace().Msg("Generating backend cfg code for stack.") + + genbackend, err := generateBackendCfgCode(root, stackpath, stackMeta, globals, stackpath) + if err != nil { + return nil, err + } + + stackBackendCfgFile := filepath.Join(stackpath, cfg.BackendCfgFilename) + currentbackend, err := loadGeneratedCode(stackBackendCfgFile) + if err != nil { + return nil, err + } + + logger.Trace().Msg("Checking for outdated backend cfg code on stack.") + + if string(genbackend) != string(currentbackend) { + return []string{cfg.BackendCfgFilename}, nil + } + + return nil, nil +} + func writeStackTerraformCode( root string, stackpath string, From df57ae1d81f48057625cf8209646c3ad49ad9a57 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 13:57:06 +0100 Subject: [PATCH 26/48] refactor: extract exported locals outdated detection --- generate/generate.go | 61 +++++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 14720d653..2bc764000 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -135,41 +135,29 @@ func CheckStack(root string, stack stack.S) ([]string, error) { stackpath := stack.AbsPath() stackMeta := stack.Meta() - outdatedBackendFiles, err := backendConfigIsOutdated(root, stackpath, stackMeta, globals, cfg) + outdatedBackendFiles, err := backendConfigOutdatedFiles(root, stackpath, stackMeta, globals, cfg) if err != nil { return nil, fmt.Errorf("checking for outdated backend config: %v", err) } outdated = append(outdated, outdatedBackendFiles...) - logger.Trace().Msg("Checking for outdated exported locals code on stack.") - - genlocals, err := generateStackLocalsCode(root, stackpath, stackMeta, globals) - if err != nil { - return nil, fmt.Errorf("checking for outdated code: %v", err) - } - - stackLocalsFile := filepath.Join(stackpath, cfg.LocalsFilename) - currentlocals, err := loadGeneratedCode(stackLocalsFile) + outdatedLocalsFiles, err := exportedLocalsOutdatedFiles(root, stackpath, stackMeta, globals, cfg) if err != nil { - return nil, fmt.Errorf("checking for outdated code: %v", err) - } - - if string(genlocals) != string(currentlocals) { - logger.Trace().Msg("Detected outdated exported locals.") - outdated = append(outdated, cfg.LocalsFilename) + return nil, fmt.Errorf("checking for outdated exported locals: %v", err) } + outdated = append(outdated, outdatedLocalsFiles...) return outdated, nil } -func backendConfigIsOutdated( +func backendConfigOutdatedFiles( root, stackpath string, stackMeta stack.Metadata, globals *terramate.Globals, cfg StackCfg, ) ([]string, error) { logger := log.With(). - Str("action", "generate.backendConfigIsOutdated()"). + Str("action", "generate.backendConfigOutdatedFiles()"). Str("root", root). Str("stackpath", stackpath). Logger() @@ -190,9 +178,46 @@ func backendConfigIsOutdated( logger.Trace().Msg("Checking for outdated backend cfg code on stack.") if string(genbackend) != string(currentbackend) { + logger.Trace().Msg("Detected outdated backend cfg.") return []string{cfg.BackendCfgFilename}, nil } + logger.Trace().Msg("backend cfg is updated.") + return nil, nil +} + +func exportedLocalsOutdatedFiles( + root, stackpath string, + stackMeta stack.Metadata, + globals *terramate.Globals, + cfg StackCfg, +) ([]string, error) { + logger := log.With(). + Str("action", "generate.exportedLocalsOutdatedFiles()"). + Str("root", root). + Str("stackpath", stackpath). + Logger() + + logger.Trace().Msg("Checking for outdated exported locals code on stack.") + + genlocals, err := generateStackLocalsCode(root, stackpath, stackMeta, globals) + if err != nil { + return nil, err + } + + stackLocalsFile := filepath.Join(stackpath, cfg.LocalsFilename) + currentlocals, err := loadGeneratedCode(stackLocalsFile) + if err != nil { + return nil, err + } + + if string(genlocals) != string(currentlocals) { + logger.Trace().Msg("Detected outdated exported locals.") + return []string{cfg.LocalsFilename}, nil + } + + logger.Trace().Msg("exported locals are updated.") + return nil, nil } From a4eafc12e81be99dc6dc80b2d104dc5986631c07 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 16:52:04 +0100 Subject: [PATCH 27/48] feat: add support to export tf on generate.CheckStack --- generate/config.go | 5 --- generate/generate.go | 63 +++++++++++++++++++++++++++++++-- generate/generate_check_test.go | 2 +- test/sandbox/sandbox.go | 6 ++-- 4 files changed, 63 insertions(+), 13 deletions(-) diff --git a/generate/config.go b/generate/config.go index 789b88fad..29ab3ad24 100644 --- a/generate/config.go +++ b/generate/config.go @@ -43,11 +43,6 @@ func LoadStackCfg(root string, stack stack.S) (StackCfg, error) { return loadStackCfg(root, stack.AbsPath()) } -func (s StackCfg) ExportedTfFilename(name string) string { - // We may have customized configuration someday - return fmt.Sprintf("_gen_terramate_%s.tf", name) -} - func loadStackCfg(root string, configdir string) (StackCfg, error) { logger := log.With(). Str("action", "loadStackCfg()"). diff --git a/generate/generate.go b/generate/generate.go index 2bc764000..fdfc76e2a 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -103,8 +103,8 @@ func Do(root string, workingDir string) error { } // CheckStack will verify if a given stack has outdated code and return a list -// of filenames that are outdated. If the stack has invalid configuration -// it will return an error. +// of filenames that are outdated, ordered lexicographically. +// If the stack has an invalid configuration it will return an error. // // The provided root must be the project's root directory as an absolute path. // The provided stack dir must be the stack dir relative to the project @@ -147,6 +147,14 @@ func CheckStack(root string, stack stack.S) ([]string, error) { } outdated = append(outdated, outdatedLocalsFiles...) + outdatedTerraformFiles, err := exportedTerraformOutdatedFiles(root, stackpath, stackMeta, globals, cfg) + if err != nil { + return nil, fmt.Errorf("checking for outdated exported terraform: %v", err) + } + outdated = append(outdated, outdatedTerraformFiles...) + + sort.Strings(outdated) + return outdated, nil } @@ -186,6 +194,55 @@ func backendConfigOutdatedFiles( return nil, nil } +func exportedTerraformOutdatedFiles( + root, stackpath string, + stackMeta stack.Metadata, + globals *terramate.Globals, + cfg StackCfg, +) ([]string, error) { + logger := log.With(). + Str("action", "generate.exportedTerraformOutdatedFiles()"). + Str("root", root). + Str("stackpath", stackpath). + Logger() + + logger.Trace().Msg("Checking for outdated exported terraform code on stack.") + + loadedStackTf, err := exportedtf.Load(root, stackMeta, globals) + if err != nil { + return nil, err + } + + logger.Trace().Msg("Loaded exported terraform code, checking") + + outdated := []string{} + + for name, tf := range loadedStackTf.ExportedCode() { + exportedTfFilename := name + targetpath := filepath.Join(stackpath, exportedTfFilename) + logger := logger.With(). + Str("blockName", name). + Str("targetpath", targetpath). + Logger() + + logger.Trace().Msg("checking if code is updated.") + + tfcode := PrependHeader(tf.String()) + currentTfCode, err := loadGeneratedCode(targetpath) + if err != nil { + return nil, err + } + + if tfcode != string(currentTfCode) { + logger.Trace().Msg("Outdated code detected.") + outdated = append(outdated, exportedTfFilename) + } + + } + + return outdated, nil +} + func exportedLocalsOutdatedFiles( root, stackpath string, stackMeta stack.Metadata, @@ -244,7 +301,7 @@ func writeStackTerraformCode( logger.Trace().Msg("generated terraform code.") for name, tf := range loadedStackTf.ExportedCode() { - targetpath := filepath.Join(stackpath, cfg.ExportedTfFilename(name)) + targetpath := filepath.Join(stackpath, name) logger := logger.With(). Str("blockName", name). Str("targetpath", targetpath). diff --git a/generate/generate_check_test.go b/generate/generate_check_test.go index c10baf6ae..6af7aff86 100644 --- a/generate/generate_check_test.go +++ b/generate/generate_check_test.go @@ -96,7 +96,7 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { ), ).String()) - assertOutdated([]string{"testnew.tf", "another.tf"}) + assertOutdated([]string{"another.tf", "testnew.tf"}) s.Generate() diff --git a/test/sandbox/sandbox.go b/test/sandbox/sandbox.go index b72b5636f..72afc1429 100644 --- a/test/sandbox/sandbox.go +++ b/test/sandbox/sandbox.go @@ -435,8 +435,7 @@ func (se StackEntry) ReadGeneratedLocals() []byte { // since it assumes generated code is expected to be there. func (se StackEntry) ReadGeneratedTerraform(name string) []byte { se.t.Helper() - cfg := se.LoadStackCodeGenCfg() - return se.DirEntry.ReadFile(cfg.ExportedTfFilename(name)) + return se.DirEntry.ReadFile(name) } // RemoveGeneratedTerraform will delete the file with generated code from @@ -444,8 +443,7 @@ func (se StackEntry) ReadGeneratedTerraform(name string) []byte { // The given name is the name of the export_as_terraform block as indicated by its label. func (se StackEntry) RemoveGeneratedTerraform(name string) { se.t.Helper() - cfg := se.LoadStackCodeGenCfg() - se.DirEntry.RemoveFile(cfg.ExportedTfFilename(name)) + se.DirEntry.RemoveFile(name) } // LoadStackCodeGenCfg will load the stack code generation configuration. From 870cc0e41a97d87eb36af3c09797c9be9c080f08 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 17:03:41 +0100 Subject: [PATCH 28/48] refactor: rename block labels for clarity --- generate/generate_check_test.go | 2 +- generate/generate_terraform_test.go | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/generate/generate_check_test.go b/generate/generate_check_test.go index 6af7aff86..e6c5783fd 100644 --- a/generate/generate_check_test.go +++ b/generate/generate_check_test.go @@ -76,7 +76,7 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { // TODO(katcipis): detect the old test.tf generated file. // It is stale but it doesn't map to code generation anymore so - // we need extra steps to detect it that are not done today. + // we need extra steps to detect it that are not done right now. assertOutdated([]string{"testnew.tf"}) // TODO(katcipis): cleanup the old test.tf diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go index d2bed5639..d946db2ce 100644 --- a/generate/generate_terraform_test.go +++ b/generate/generate_terraform_test.go @@ -95,14 +95,14 @@ func TestTerraformGeneration(t *testing.T) { path: "/stacks", add: hcldoc( exportAsTerraform( - labels("backend"), + labels("backend.tf"), backend( labels("test"), expr("prefix", "global.backend_prefix"), ), ), exportAsTerraform( - labels("locals"), + labels("locals.tf"), locals( expr("stackpath", "terramate.path"), expr("local_a", "global.local_a"), @@ -112,7 +112,7 @@ func TestTerraformGeneration(t *testing.T) { ), ), exportAsTerraform( - labels("provider"), + labels("provider.tf"), provider( labels("name"), expr("data", "global.provider_data"), @@ -162,20 +162,18 @@ func TestTerraformGeneration(t *testing.T) { { stack: "/stacks/stack-1", hcls: map[string]fmt.Stringer{ - "backend": backend( + "backend.tf": backend( labels("test"), str("prefix", "stack-1-backend"), ), - - "locals": locals( + "locals.tf": locals( str("stackpath", "/stacks/stack-1"), str("local_a", "stack-1-local"), boolean("local_b", true), number("local_c", 666), str("local_d", "local_d_field"), ), - - "provider": hcldoc( + "provider.tf": hcldoc( provider( labels("name"), str("data", "stack-1-provider-data"), @@ -197,18 +195,18 @@ func TestTerraformGeneration(t *testing.T) { { stack: "/stacks/stack-2", hcls: map[string]fmt.Stringer{ - "backend": backend( + "backend.tf": backend( labels("test"), str("prefix", "stack-2-backend"), ), - "locals": locals( + "locals.tf": locals( str("stackpath", "/stacks/stack-2"), str("local_a", "stack-2-local"), boolean("local_b", false), number("local_c", 777), attr("local_d", "null"), ), - "provider": hcldoc( + "provider.tf": hcldoc( provider( labels("name"), str("data", "stack-2-provider-data"), From afc1a457251a009ce9e22f33b887cc5ff90034ea Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 17:15:57 +0100 Subject: [PATCH 29/48] refactor: generated code is always string --- generate/generate_terraform_test.go | 4 ++-- test/sandbox/sandbox.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go index d946db2ce..abaf2cb7d 100644 --- a/generate/generate_terraform_test.go +++ b/generate/generate_terraform_test.go @@ -249,7 +249,7 @@ func TestTerraformGeneration(t *testing.T) { for name, wantHCL := range wantDesc.hcls { want := wantHCL.String() - got := string(stack.ReadGeneratedTerraform(name)) + got := stack.ReadGeneratedTerraform(name) assertHCLEquals(t, got, want) @@ -277,7 +277,7 @@ func TestTerraformGeneration(t *testing.T) { return nil } - t.Errorf("expected only basic terramate config at %q, got %q", path, d.Name()) + t.Errorf("unwanted file at %q, got %q", path, d.Name()) return nil }) diff --git a/test/sandbox/sandbox.go b/test/sandbox/sandbox.go index 72afc1429..aa5788147 100644 --- a/test/sandbox/sandbox.go +++ b/test/sandbox/sandbox.go @@ -433,9 +433,9 @@ func (se StackEntry) ReadGeneratedLocals() []byte { // // It will fail the test if there is no generated code available on the stack, // since it assumes generated code is expected to be there. -func (se StackEntry) ReadGeneratedTerraform(name string) []byte { +func (se StackEntry) ReadGeneratedTerraform(name string) string { se.t.Helper() - return se.DirEntry.ReadFile(name) + return string(se.DirEntry.ReadFile(name)) } // RemoveGeneratedTerraform will delete the file with generated code from From 0340581f1c2a2024b7df78bc337f847dec254e52 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 17:16:14 +0100 Subject: [PATCH 30/48] refactor: avoid using callback, confusing on this ctx --- generate/generate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index fdfc76e2a..234684511 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -608,9 +608,9 @@ func loadGeneratedCode(path string) ([]byte, error) { return data, nil } -type forEachStackCallback func(stack.S, *terramate.Globals, StackCfg) error +type forEachStackFunc func(stack.S, *terramate.Globals, StackCfg) error -func forEachStack(root, workingDir string, callback forEachStackCallback) []error { +func forEachStack(root, workingDir string, fn forEachStackFunc) []error { logger := log.With(). Str("action", "generate.forEachStack()"). Str("root", root). @@ -663,7 +663,7 @@ func forEachStack(root, workingDir string, callback forEachStackCallback) []erro } logger.Trace().Msg("Calling stack callback.") - if err := callback(stack, globals, cfg); err != nil { + if err := fn(stack, globals, cfg); err != nil { errs = append(errs, err) } } From 2d9c9ec5012c6e146b2efbbe067cc489c3539c43 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 17:43:55 +0100 Subject: [PATCH 31/48] test: add overwritting/manual code exists tests --- generate/generate.go | 2 +- generate/generate_terraform_test.go | 67 +++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/generate/generate.go b/generate/generate.go index 234684511..3b1c598c5 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -317,7 +317,7 @@ func writeStackTerraformCode( tfcode = PrependHeader(tfcode) if err := writeGeneratedCode(targetpath, tfcode); err != nil { - return fmt.Errorf("stack %q: writing code at %q: %v", stackpath, targetpath, err) + return fmt.Errorf("stack %q: writing code at %q: %w", stackpath, targetpath, err) } logger.Debug().Msg("Saved stack generated code.") diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go index abaf2cb7d..28d07f545 100644 --- a/generate/generate_terraform_test.go +++ b/generate/generate_terraform_test.go @@ -285,3 +285,70 @@ func TestTerraformGeneration(t *testing.T) { }) } } + +func TestWontOverwriteManuallyDefinedTerraform(t *testing.T) { + const ( + genFilename = "test.tf" + manualTfCode = "some manual stuff, doesn't matter" + ) + + exportTfConfig := exportAsTerraform( + labels(genFilename), + str("required_version", "1.11"), + ) + + s := sandbox.New(t) + s.BuildTree([]string{ + fmt.Sprintf("f:%s:%s", config.Filename, exportTfConfig.String()), + "s:stack", + fmt.Sprintf("f:stack/%s:%s", genFilename, manualTfCode), + }) + + err := generate.Do(s.RootDir(), s.RootDir()) + assert.IsError(t, err, generate.ErrManualCodeExists) + + stack := s.StackEntry("stack") + actualTfCode := stack.ReadGeneratedTerraform(genFilename) + assert.EqualStrings(t, manualTfCode, actualTfCode, "tf code altered by generate") +} + +func TestExportedTerraformOverwriting(t *testing.T) { + const genFilename = "test.tf" + + firstConfig := exportAsTerraform( + labels(genFilename), + terraform( + str("required_version", "1.11"), + ), + ) + firstWant := terraform( + str("required_version", "1.11"), + ) + + s := sandbox.New(t) + stack := s.CreateStack("stack") + rootEntry := s.DirEntry(".") + rootConfig := rootEntry.CreateConfig(firstConfig.String()) + + s.Generate() + + got := stack.ReadGeneratedTerraform(genFilename) + assertHCLEquals(t, got, firstWant.String()) + + secondConfig := exportAsTerraform( + labels(genFilename), + terraform( + str("required_version", "2.0"), + ), + ) + secondWant := terraform( + str("required_version", "2.0"), + ) + + rootConfig.Write(secondConfig.String()) + + s.Generate() + + got = stack.ReadGeneratedTerraform(genFilename) + assertHCLEquals(t, got, secondWant.String()) +} From a8448b14f9de4ccbfba706d8568b4de3979dde0f Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 19:22:32 +0100 Subject: [PATCH 32/48] feat: add check for attributes on export_as_terraform --- generate/exportedtf/exportedtf.go | 4 ++++ generate/exportedtf/exportedtf_test.go | 31 ++++++++++++++++++++++---- generate/generate_check_test.go | 16 +++++++++---- generate/generate_terraform_test.go | 8 +++---- 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index 871828859..ac294ca15 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -191,6 +191,10 @@ func loadExportBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Bloc return nil, fmt.Errorf("%w: label can't be empty", ErrInvalidBlock) } + if len(block.Body.Attributes) != 0 { + return nil, fmt.Errorf("%w: attributes are not allowed", ErrInvalidBlock) + } + if _, ok := res[name]; ok { return nil, fmt.Errorf( "%w: found two blocks with same label %q", diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index 5b3186960..b2a0a469b 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -60,6 +60,9 @@ func TestLoadExportedTerraform(t *testing.T) { block := func(name string, builders ...hclwrite.BlockBuilder) *hclwrite.Block { return hclwrite.BuildBlock(name, builders...) } + terraform := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("terraform", builders...) + } globals := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { return hclwrite.BuildBlock("globals", builders...) } @@ -159,7 +162,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, { - name: "exported terraform on stack using try and labelled block", + name: "exported terraform on stack using try and labeled block", stack: "/stack", configs: []hclconfig{ { @@ -570,14 +573,16 @@ func TestLoadExportedTerraform(t *testing.T) { wantErr: exportedtf.ErrInvalidBlock, }, { - name: "evaluation failure on stack config gives err", + name: "evaluation failure on stack config fails", stack: "/stacks/stack", configs: []hclconfig{ { path: "/stacks/stack", add: hcldoc( exportAsTerraform("test", - expr("attr", "global.undefined"), + terraform( + expr("required_version", "global.undefined"), + ), ), ), }, @@ -585,7 +590,7 @@ func TestLoadExportedTerraform(t *testing.T) { wantErr: exportedtf.ErrEval, }, { - name: "valid config on stack but invalid on parent gives err", + name: "valid config on stack but invalid on parent fails", stack: "/stacks/stack", configs: []hclconfig{ { @@ -607,6 +612,24 @@ func TestLoadExportedTerraform(t *testing.T) { }, wantErr: exportedtf.ErrInvalidBlock, }, + { + name: "attributes on export_as_terraform block fails", + stack: "/stacks/stack", + configs: []hclconfig{ + { + path: "/stacks/stack", + add: hcldoc( + exportAsTerraform("test", + str("some_attribute", "whatever"), + terraform( + str("required_version", "1.11"), + ), + ), + ), + }, + }, + wantErr: exportedtf.ErrInvalidBlock, + }, } for _, tcase := range tcases { diff --git a/generate/generate_check_test.go b/generate/generate_check_test.go index e6c5783fd..fdf3b9a64 100644 --- a/generate/generate_check_test.go +++ b/generate/generate_check_test.go @@ -43,7 +43,9 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { stackConfig( exportAsTerraform( labels("test.tf"), - str("required_version", "1.10"), + terraform( + str("required_version", "1.10"), + ), ), ).String()) assertOutdated([]string{"test.tf"}) @@ -57,7 +59,9 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { stackConfig( exportAsTerraform( labels("test.tf"), - str("required_version", "1.11"), + terraform( + str("required_version", "1.11"), + ), ), ).String()) @@ -70,7 +74,9 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { stackConfig( exportAsTerraform( labels("testnew.tf"), - str("required_version", "1.11"), + terraform( + str("required_version", "1.11"), + ), ), ).String()) @@ -86,7 +92,9 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { stackConfig( exportAsTerraform( labels("testnew.tf"), - str("required_version", "1.11"), + terraform( + str("required_version", "1.11"), + ), ), exportAsTerraform( labels("another.tf"), diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go index 28d07f545..669f764b2 100644 --- a/generate/generate_terraform_test.go +++ b/generate/generate_terraform_test.go @@ -28,10 +28,6 @@ import ( "github.com/mineiros-io/terramate/test/sandbox" ) -// Test -// -// - Overwriting Behavior (manual tf code already exists) - func TestTerraformGeneration(t *testing.T) { type ( hclconfig struct { @@ -294,7 +290,9 @@ func TestWontOverwriteManuallyDefinedTerraform(t *testing.T) { exportTfConfig := exportAsTerraform( labels(genFilename), - str("required_version", "1.11"), + terraform( + str("required_version", "1.11"), + ), ) s := sandbox.New(t) From 95c583b128054a484ac9430141171679dd9d7ab9 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 20:27:59 +0100 Subject: [PATCH 33/48] test: improve tests --- generate/exportedtf/exportedtf.go | 2 +- generate/exportedtf/exportedtf_test.go | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index ac294ca15..d431e73e5 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -173,7 +173,7 @@ func loadExportBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Bloc if os.IsNotExist(err) { return loadExportBlocks(rootdir, filepath.Dir(cfgdir)) } - return nil, fmt.Errorf("loading exported terraform code: %v", err) + return nil, fmt.Errorf("parsing exported terraform code: %v", err) } res := map[string]*hclsyntax.Block{} diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index b2a0a469b..480fcb1ad 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -562,10 +562,14 @@ func TestLoadExportedTerraform(t *testing.T) { path: "/stacks/stack", add: hcldoc( exportAsTerraform("duplicated", - str("data", "some literal data"), + terraform( + str("data", "some literal data"), + ), ), exportAsTerraform("duplicated", - str("data2", "some literal data2"), + terraform( + str("data2", "some literal data2"), + ), ), ), }, @@ -605,7 +609,9 @@ func TestLoadExportedTerraform(t *testing.T) { path: "/stacks/stack", add: hcldoc( exportAsTerraform("valid", - str("data", "some literal data"), + terraform( + str("data", "some literal data"), + ), ), ), }, From 023d77f27aeba95e84a2cffeb512ed2512976b92 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Tue, 1 Feb 2022 20:36:49 +0100 Subject: [PATCH 34/48] docs: update spec for label=filename --- docs/terraform-generation.md | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/docs/terraform-generation.md b/docs/terraform-generation.md index 97cf76944..04dec4830 100644 --- a/docs/terraform-generation.md +++ b/docs/terraform-generation.md @@ -28,11 +28,8 @@ file reading functions, references to globals/metadata, will all be evaluated at code generation time and the generated code will only have literals like strings, numbers, lists, maps, objects, etc. -Each `export_as_terraform` block requires a label. This label is part of the identity -of the block and is also used as a default for which filename will be used when -code is generated. Given a label `x` the filename will be `_gen_terramate_x.tf`. The labels are -also used to configure different filenames for each block if the default names are -undesired. More details on how to configure this can be checked [here](todo-docs-for-config). +Each `export_as_terraform` block requires a single label. +This label is the filename of the generated code. Now lets jump to some examples. Lets generate backend and provider configurations for all stacks inside a project. @@ -53,14 +50,14 @@ stacks by defining a `export_as_terraform` block in the root of the project: ```hcl -export_as_terraform "backend" { +export_as_terraform "backend.tf" { backend "local" { param = global.backend_data } } ``` -Which will generate code for all stacks using the filename `_gen_terramate_backend.tf`: +Which will generate code for all stacks using the filename `backend.tf`: ```hcl backend "local" { @@ -68,11 +65,11 @@ backend "local" { } ``` -To generate provider/Terraform configuration for all stacks we can add +To generate provider/terraform configuration for all stacks we can add in the root configuration: ```hcl -export_as_terraform "provider" { +export_as_terraform "provider.tf" { provider "name" { param = global.provider_data @@ -93,7 +90,7 @@ export_as_terraform "provider" { } ``` -Which will generate code for all stacks using the filename `_gen_terramate_provider.tf`: +Which will generate code for all stacks using the filename `provider.tf`: ```hcl provider "name" { @@ -150,7 +147,7 @@ previously mentioned `stacks/stack-1`. Given this configuration at `stacks/terramate.tm.hcl`: ```hcl -export_as_terraform "provider" { +export_as_terraform "provider.tf" { terraform { required_version = "1.1.13" } @@ -160,7 +157,7 @@ export_as_terraform "provider" { And this configuration at `stacks/stack-1/terramate.tm.hcl`: ```hcl -export_as_terraform "backend" { +export_as_terraform "backend.tf" { backend "local" { param = "example" } @@ -173,7 +170,7 @@ label and will generate its own code in a separated file. But if we had this configuration at `stacks/stack-1/terramate.tm.hcl`: ```hcl -export_as_terraform "provider" { +export_as_terraform "provider.tf" { terraform { required_version = "overriden" } @@ -197,7 +194,7 @@ The overriding is total, there is no merging involved on the blocks inside configuration like this: ```hcl -export_as_terraform "name" { +export_as_terraform "name.tf" { block1 { } block2 { @@ -210,7 +207,7 @@ export_as_terraform "name" { And a more specific configuration redefines it like this: ```hcl -export_as_terraform "name" { +export_as_terraform "name.tf" { block4 { } } From dbc9d9d1ca871ae0906ffc8b68fa815a2f1f5dcd Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 13:32:29 +0100 Subject: [PATCH 35/48] chore: remove newline --- generate/generate.go | 1 - 1 file changed, 1 deletion(-) diff --git a/generate/generate.go b/generate/generate.go index 3b1c598c5..51d87f21c 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -237,7 +237,6 @@ func exportedTerraformOutdatedFiles( logger.Trace().Msg("Outdated code detected.") outdated = append(outdated, exportedTfFilename) } - } return outdated, nil From 667cae09d506db56140ff4b007719cebe89a48cd Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 14:18:37 +0100 Subject: [PATCH 36/48] feat: change export_as_terraform to export_as_hcl --- hcl/hcl.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hcl/hcl.go b/hcl/hcl.go index 12f9343a1..f0a3c9cd6 100644 --- a/hcl/hcl.go +++ b/hcl/hcl.go @@ -469,7 +469,7 @@ func ParseExportAsLocalsBlocks(path string) ([]*hclsyntax.Block, error) { // ParseExportAsTerraformBlocks parses export_as_terraform blocks, ignoring other blocks func ParseExportAsTerraformBlocks(path string) ([]*hclsyntax.Block, error) { - return parseBlocksOfType(path, "export_as_terraform") + return parseBlocksOfType(path, "export_as_hcl") } // CopyBody will copy the src body to the given target, evaluating attributes using the @@ -950,7 +950,7 @@ func blockIsAllowed(name string) bool { Logger() switch name { - case "terramate", "stack", "backend", "globals", "export_as_locals", "export_as_terraform": + case "terramate", "stack", "backend", "globals", "export_as_locals", "export_as_hcl": logger.Trace().Msg("Block name was allowed.") return true default: From 26cf72afb54388f80b2c0b9c614043eb20bb548d Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 14:44:21 +0100 Subject: [PATCH 37/48] feat: rename to generate_hcl --- hcl/hcl.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hcl/hcl.go b/hcl/hcl.go index f0a3c9cd6..ca7d8b900 100644 --- a/hcl/hcl.go +++ b/hcl/hcl.go @@ -469,7 +469,7 @@ func ParseExportAsLocalsBlocks(path string) ([]*hclsyntax.Block, error) { // ParseExportAsTerraformBlocks parses export_as_terraform blocks, ignoring other blocks func ParseExportAsTerraformBlocks(path string) ([]*hclsyntax.Block, error) { - return parseBlocksOfType(path, "export_as_hcl") + return parseBlocksOfType(path, "generate_hcl") } // CopyBody will copy the src body to the given target, evaluating attributes using the @@ -950,7 +950,7 @@ func blockIsAllowed(name string) bool { Logger() switch name { - case "terramate", "stack", "backend", "globals", "export_as_locals", "export_as_hcl": + case "terramate", "stack", "backend", "globals", "export_as_locals", "generate_hcl": logger.Trace().Msg("Block name was allowed.") return true default: From 01e84fc65df0229c412ff7f02bb8a3de985a6e32 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 17:35:51 +0100 Subject: [PATCH 38/48] refactor: rename from export_as_terraform to generate_hcl --- generate/exportedtf/exportedtf.go | 50 +++++++++++++++---------------- hcl/hcl.go | 4 +-- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/generate/exportedtf/exportedtf.go b/generate/exportedtf/exportedtf.go index d431e73e5..a5fd67d1a 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/exportedtf/exportedtf.go @@ -31,24 +31,24 @@ import ( "github.com/rs/zerolog/log" ) -// StackHCLs represents all exported terraform code for a stack, -// mapping the exported code name to the actual Terraform code. +// StackHCLs represents all generated HCL code for a stack, +// mapping the generated code name to the actual HCL code. type StackHCLs struct { hcls map[string]HCL } -// HCL represents exported Terraform code from a single block. +// HCL represents generated HCL code from a single block. // Is contains parsed and evaluated code on it. type HCL struct { body []byte } const ( - ErrInvalidBlock errutil.Error = "invalid export_as_terraform block" - ErrEval errutil.Error = "evaluating export_as_terraform block" + ErrInvalidBlock errutil.Error = "invalid generate_hcl block" + ErrEval errutil.Error = "evaluating generate_hcl block" ) -// ExportedCode returns all exported code, mapping the name to its +// ExportedCode returns all generated code, mapping the name to its // equivalent generated code. func (s StackHCLs) ExportedCode() map[string]HCL { cp := map[string]HCL{} @@ -58,21 +58,21 @@ func (s StackHCLs) ExportedCode() map[string]HCL { return cp } -// String returns a string representation of the Terraform code +// String returns a string representation of the HCL code // or an empty string if the config itself is empty. func (b HCL) String() string { return string(b.body) } -// Load loads from the file system all export_as_terraform for +// Load loads from the file system all generate_hcl for // a given stack. It will navigate the file system from the stack dir until -// it reaches rootdir, loading export_as_terraform and merging them appropriately. +// it reaches rootdir, loading generate_hcl and merging them appropriately. // // More specific definitions (closer or at the stack) have precedence over // less specific ones (closer or at the root dir). // // Metadata and globals for the stack are used on the evaluation of the -// export_as_terramate blocks. +// generate_hcl blocks. // // The returned result only contains evaluated values. // @@ -80,15 +80,15 @@ func (b HCL) String() string { func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackHCLs, error) { stackpath := filepath.Join(rootdir, sm.Path) logger := log.With(). - Str("action", "exportedtf.Load()"). + Str("action", "genhcl.Load()"). Str("path", stackpath). Logger() - logger.Trace().Msg("loading export_as_terraform blocks.") + logger.Trace().Msg("loading generate_hcl blocks.") - exportBlocks, err := loadExportBlocks(rootdir, stackpath) + generatedBlocks, err := loadGenHCLBlocks(rootdir, stackpath) if err != nil { - return StackHCLs{}, fmt.Errorf("loading exported terraform code: %w", err) + return StackHCLs{}, fmt.Errorf("loading generated HCL code: %w", err) } evalctx, err := newEvalCtx(stackpath, sm, globals) @@ -96,13 +96,13 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackH return StackHCLs{}, fmt.Errorf("%w: creating eval context: %v", ErrEval, err) } - logger.Trace().Msg("generating exported terraform code.") + logger.Trace().Msg("generating HCL code.") res := StackHCLs{ hcls: map[string]HCL{}, } - for name, block := range exportBlocks { + for name, block := range generatedBlocks { logger := logger.With(). Str("block", name). Logger() @@ -127,7 +127,7 @@ func Load(rootdir string, sm stack.Metadata, globals *terramate.Globals) (StackH func newEvalCtx(stackpath string, sm stack.Metadata, globals *terramate.Globals) (*eval.Context, error) { logger := log.With(). - Str("action", "exportedtf.newEvalCtx()"). + Str("action", "genhcl.newEvalCtx()"). Str("path", stackpath). Logger() @@ -151,16 +151,16 @@ func newEvalCtx(stackpath string, sm stack.Metadata, globals *terramate.Globals) return evalctx, nil } -// loadExportBlocks will load all export_as_terraform blocks applying overriding +// loadGenHCLBlocks will load all generate_hcl blocks applying overriding // as it goes, the returned map maps the name of the block (its label) to the original block -func loadExportBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Block, error) { +func loadGenHCLBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Block, error) { logger := log.With(). - Str("action", "exportedtf.loadExportBlocks()"). + Str("action", "genhcl.loadGenHCLBlocks()"). Str("root", rootdir). Str("configDir", cfgdir). Logger() - logger.Trace().Msg("Parsing export_as_terraform blocks.") + logger.Trace().Msg("Parsing generate_hcl blocks.") if !strings.HasPrefix(cfgdir, rootdir) { logger.Trace().Msg("config dir outside root, nothing to do") @@ -168,12 +168,12 @@ func loadExportBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Bloc } cfgpath := filepath.Join(cfgdir, config.Filename) - blocks, err := hcl.ParseExportAsTerraformBlocks(cfgpath) + blocks, err := hcl.ParseGenerateHCLBlocks(cfgpath) if err != nil { if os.IsNotExist(err) { - return loadExportBlocks(rootdir, filepath.Dir(cfgdir)) + return loadGenHCLBlocks(rootdir, filepath.Dir(cfgdir)) } - return nil, fmt.Errorf("parsing exported terraform code: %v", err) + return nil, fmt.Errorf("parsing generate_hcl code: %v", err) } res := map[string]*hclsyntax.Block{} @@ -205,7 +205,7 @@ func loadExportBlocks(rootdir string, cfgdir string) (map[string]*hclsyntax.Bloc res[name] = block } - parentRes, err := loadExportBlocks(rootdir, filepath.Dir(cfgdir)) + parentRes, err := loadGenHCLBlocks(rootdir, filepath.Dir(cfgdir)) if err != nil { return nil, err } diff --git a/hcl/hcl.go b/hcl/hcl.go index ca7d8b900..7caf43cce 100644 --- a/hcl/hcl.go +++ b/hcl/hcl.go @@ -467,8 +467,8 @@ func ParseExportAsLocalsBlocks(path string) ([]*hclsyntax.Block, error) { return parseBlocksOfType(path, "export_as_locals") } -// ParseExportAsTerraformBlocks parses export_as_terraform blocks, ignoring other blocks -func ParseExportAsTerraformBlocks(path string) ([]*hclsyntax.Block, error) { +// ParseGenerateHCLBlocks parses export_as_terraform blocks, ignoring other blocks +func ParseGenerateHCLBlocks(path string) ([]*hclsyntax.Block, error) { return parseBlocksOfType(path, "generate_hcl") } From da3feca5f83f01e8d5428c1a17ac32c0c52eef95 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 17:42:29 +0100 Subject: [PATCH 39/48] refactor: tests use the new generate_hcl block --- generate/exportedtf/exportedtf_test.go | 90 +++++++++++++------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/generate/exportedtf/exportedtf_test.go b/generate/exportedtf/exportedtf_test.go index 480fcb1ad..c9e2950e7 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/exportedtf/exportedtf_test.go @@ -30,7 +30,7 @@ import ( "github.com/rs/zerolog" ) -func TestLoadExportedTerraform(t *testing.T) { +func TestLoadGeneratedHCL(t *testing.T) { type ( hclconfig struct { path string @@ -52,8 +52,8 @@ func TestLoadExportedTerraform(t *testing.T) { hcldoc := func(blocks ...*hclwrite.Block) hclwrite.HCL { return hclwrite.NewHCL(blocks...) } - exportAsTerraform := func(label string, builders ...hclwrite.BlockBuilder) *hclwrite.Block { - b := hclwrite.BuildBlock("export_as_terraform", builders...) + generateHCL := func(label string, builders ...hclwrite.BlockBuilder) *hclwrite.Block { + b := hclwrite.BuildBlock("generate_hcl", builders...) b.AddLabel(label) return b } @@ -80,16 +80,16 @@ func TestLoadExportedTerraform(t *testing.T) { tcases := []testcase{ { - name: "no exported terraform", + name: "no generation", stack: "/stack", }, { - name: "empty export_as_terraform block generates empty code", + name: "empty block generates empty code", stack: "/stack", configs: []hclconfig{ { path: "/stack", - add: exportAsTerraform("empty"), + add: generateHCL("empty"), }, }, want: []result{ @@ -100,12 +100,12 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, { - name: "exported terraform on stack with single empty block", + name: "generate hcl on stack with single empty block", stack: "/stack", configs: []hclconfig{ { path: "/stack", - add: exportAsTerraform("emptytest", + add: generateHCL("emptytest", block("empty"), ), }, @@ -118,7 +118,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, { - name: "exported terraform on stack with single block", + name: "generate HCL on stack with single block", stack: "/stack", configs: []hclconfig{ { @@ -131,7 +131,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, { path: "/stack", - add: exportAsTerraform("test", + add: generateHCL("test", block("testblock", expr("bool", "global.some_bool"), expr("number", "global.some_number"), @@ -162,7 +162,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, { - name: "exported terraform on stack using try and labeled block", + name: "generate HCL on stack using try and labeled block", stack: "/stack", configs: []hclconfig{ { @@ -177,7 +177,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, { path: "/stack", - add: exportAsTerraform("test", + add: generateHCL("test", block("labeled", labels("label1", "label2"), expr("field_a", "try(global.obj.field_a, null)"), @@ -202,7 +202,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, { - name: "exported terraform on stack with single nested block", + name: "generate HCL on stack with single nested block", stack: "/stack", configs: []hclconfig{ { @@ -215,7 +215,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, { path: "/stack", - add: exportAsTerraform("nesting", + add: generateHCL("nesting", block("block1", expr("bool", "global.some_bool"), block("block2", @@ -254,7 +254,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, { - name: "multiple exported terraform blocks on stack", + name: "multiple generate HCL blocks on stack", stack: "/stack", configs: []hclconfig{ { @@ -268,7 +268,7 @@ func TestLoadExportedTerraform(t *testing.T) { { path: "/stack", add: hcldoc( - exportAsTerraform("exported_one", + generateHCL("exported_one", block("block1", expr("bool", "global.some_bool"), block("block2", @@ -276,12 +276,12 @@ func TestLoadExportedTerraform(t *testing.T) { ), ), ), - exportAsTerraform("exported_two", + generateHCL("exported_two", block("yay", expr("data", "global.some_string"), ), ), - exportAsTerraform("exported_three", + generateHCL("exported_three", block("something", expr("number", "global.some_number"), ), @@ -314,7 +314,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, { - name: "exported terraform on stack parent dir", + name: "generate HCL on stack parent dir", stack: "/stacks/stack", configs: []hclconfig{ { @@ -327,7 +327,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, { path: "/stacks", - add: exportAsTerraform("on_parent", + add: generateHCL("on_parent", block("on_parent_block", expr("obj", `{ string = global.some_string @@ -352,12 +352,12 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, { - name: "exported terraform project root dir", + name: "generate HCL on project root dir", stack: "/stacks/stack", configs: []hclconfig{ { path: "/", - add: exportAsTerraform("root", + add: generateHCL("root", block("root", expr("test", "terramate.path"), ), @@ -374,7 +374,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, { - name: "exporting on all dirs of the project with different names get merged", + name: "generate HCL on all dirs of the project with different labels", stack: "/stacks/stack", configs: []hclconfig{ { @@ -387,7 +387,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, { path: "/", - add: exportAsTerraform("on_root", + add: generateHCL("on_root", block("on_root_block", expr("obj", `{ string = global.some_string @@ -397,7 +397,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, { path: "/stacks", - add: exportAsTerraform("on_parent", + add: generateHCL("on_parent", block("on_parent_block", expr("obj", `{ number = global.some_number @@ -407,7 +407,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, { path: "/stacks/stack", - add: exportAsTerraform("on_stack", + add: generateHCL("on_stack", block("on_stack_block", expr("obj", `{ bool = global.some_bool @@ -450,17 +450,17 @@ func TestLoadExportedTerraform(t *testing.T) { { path: "/", add: hcldoc( - exportAsTerraform("root", + generateHCL("root", block("block", expr("root_stackpath", "terramate.path"), ), ), - exportAsTerraform("parent", + generateHCL("parent", block("block", expr("root_stackpath", "terramate.path"), ), ), - exportAsTerraform("stack", + generateHCL("stack", block("block", expr("root_stackpath", "terramate.path"), ), @@ -469,7 +469,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, { path: "/stacks", - add: exportAsTerraform("parent", + add: generateHCL("parent", block("block", expr("parent_stackpath", "terramate.path"), expr("parent_stackname", "terramate.name"), @@ -478,7 +478,7 @@ func TestLoadExportedTerraform(t *testing.T) { }, { path: "/stacks/stack", - add: exportAsTerraform("stack", + add: generateHCL("stack", block("block", str("overridden", "some literal data"), ), @@ -508,12 +508,12 @@ func TestLoadExportedTerraform(t *testing.T) { }, }, { - name: "export block with no label on stack gives err", + name: "block with no label on stack gives err", stack: "/stacks/stack", configs: []hclconfig{ { path: "/stacks/stack", - add: block("export_as_terraform", + add: block("generate_hcl", block("block", str("data", "some literal data"), ), @@ -523,12 +523,12 @@ func TestLoadExportedTerraform(t *testing.T) { wantErr: exportedtf.ErrInvalidBlock, }, { - name: "export block with two labels on stack gives err", + name: "block with two labels on stack gives err", stack: "/stacks/stack", configs: []hclconfig{ { path: "/stacks/stack", - add: block("export_as_terraform", + add: block("generate_hcl", labels("one", "two"), block("block", str("data", "some literal data"), @@ -539,12 +539,12 @@ func TestLoadExportedTerraform(t *testing.T) { wantErr: exportedtf.ErrInvalidBlock, }, { - name: "export block with empty label on stack gives err", + name: "block with empty label on stack gives err", stack: "/stacks/stack", configs: []hclconfig{ { path: "/stacks/stack", - add: block("export_as_terraform", + add: block("generate_hcl", labels(""), block("block", str("data", "some literal data"), @@ -555,18 +555,18 @@ func TestLoadExportedTerraform(t *testing.T) { wantErr: exportedtf.ErrInvalidBlock, }, { - name: "export blocks with same label on same config gives err", + name: "blocks with same label on same config gives err", stack: "/stacks/stack", configs: []hclconfig{ { path: "/stacks/stack", add: hcldoc( - exportAsTerraform("duplicated", + generateHCL("duplicated", terraform( str("data", "some literal data"), ), ), - exportAsTerraform("duplicated", + generateHCL("duplicated", terraform( str("data2", "some literal data2"), ), @@ -583,7 +583,7 @@ func TestLoadExportedTerraform(t *testing.T) { { path: "/stacks/stack", add: hcldoc( - exportAsTerraform("test", + generateHCL("test", terraform( expr("required_version", "global.undefined"), ), @@ -599,7 +599,7 @@ func TestLoadExportedTerraform(t *testing.T) { configs: []hclconfig{ { path: "/stacks", - add: block("export_as_terraform", + add: block("generate_hcl", block("block", str("data", "some literal data"), ), @@ -608,7 +608,7 @@ func TestLoadExportedTerraform(t *testing.T) { { path: "/stacks/stack", add: hcldoc( - exportAsTerraform("valid", + generateHCL("valid", terraform( str("data", "some literal data"), ), @@ -619,13 +619,13 @@ func TestLoadExportedTerraform(t *testing.T) { wantErr: exportedtf.ErrInvalidBlock, }, { - name: "attributes on export_as_terraform block fails", + name: "attributes on generate_hcl block fails", stack: "/stacks/stack", configs: []hclconfig{ { path: "/stacks/stack", add: hcldoc( - exportAsTerraform("test", + generateHCL("test", str("some_attribute", "whatever"), terraform( str("required_version", "1.11"), From 3d2e50279aee7fc17978bb2bdca2a46b2e390624 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 17:48:51 +0100 Subject: [PATCH 40/48] refactor: rename export_as_terraform to generate_hcl --- generate/generate_check_test.go | 10 +- generate/generate_hcl_test.go | 367 +++++++++++++++++++---- generate/generate_hclwrite_utils_test.go | 87 ++++++ generate/generate_terraform_test.go | 352 ---------------------- test/sandbox/sandbox.go | 16 +- 5 files changed, 416 insertions(+), 416 deletions(-) create mode 100644 generate/generate_hclwrite_utils_test.go delete mode 100644 generate/generate_terraform_test.go diff --git a/generate/generate_check_test.go b/generate/generate_check_test.go index fdf3b9a64..879421324 100644 --- a/generate/generate_check_test.go +++ b/generate/generate_check_test.go @@ -41,7 +41,7 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { assertOutdated([]string{}) stackEntry.CreateConfig( stackConfig( - exportAsTerraform( + generateHCL( labels("test.tf"), terraform( str("required_version", "1.10"), @@ -57,7 +57,7 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { // Now checking when we have code + it gets outdated. stackEntry.CreateConfig( stackConfig( - exportAsTerraform( + generateHCL( labels("test.tf"), terraform( str("required_version", "1.11"), @@ -72,7 +72,7 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { // Changing generated filenames will trigger detection, with new filenames stackEntry.CreateConfig( stackConfig( - exportAsTerraform( + generateHCL( labels("testnew.tf"), terraform( str("required_version", "1.11"), @@ -90,13 +90,13 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { // Adding new filename to generation trigger detection stackEntry.CreateConfig( stackConfig( - exportAsTerraform( + generateHCL( labels("testnew.tf"), terraform( str("required_version", "1.11"), ), ), - exportAsTerraform( + generateHCL( labels("another.tf"), backend( labels("type"), diff --git a/generate/generate_hcl_test.go b/generate/generate_hcl_test.go index 59ced370a..9f7db5ada 100644 --- a/generate/generate_hcl_test.go +++ b/generate/generate_hcl_test.go @@ -14,74 +14,339 @@ package generate_test -import "github.com/mineiros-io/terramate/test/hclwrite" +import ( + "fmt" + "io/fs" + "path/filepath" + "testing" -// useful function aliases to build HCL documents + "github.com/madlambda/spells/assert" + "github.com/mineiros-io/terramate/config" + "github.com/mineiros-io/terramate/generate" + "github.com/mineiros-io/terramate/test" + "github.com/mineiros-io/terramate/test/hclwrite" + "github.com/mineiros-io/terramate/test/sandbox" +) -func hcldoc(blocks ...*hclwrite.Block) hclwrite.HCL { - return hclwrite.NewHCL(blocks...) -} +func TestHCLGeneration(t *testing.T) { + type ( + hclconfig struct { + path string + add fmt.Stringer + } + want struct { + stack string + hcls map[string]fmt.Stringer + } + testcase struct { + name string + layout []string + configs []hclconfig + workingDir string + want []want + wantErr error + } + ) -// stackConfig wraps the blocks built by the given builders in a valid stack -// configuration -func stackConfig(blocks ...*hclwrite.Block) hclwrite.HCL { - b := []*hclwrite.Block{stack()} - b = append(b, blocks...) - return hcldoc(b...) -} + provider := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("provider", builders...) + } + required_providers := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("required_providers", builders...) + } + attr := func(name, expr string) hclwrite.BlockBuilder { + t.Helper() + return hclwrite.AttributeValue(t, name, expr) + } -func exportAsTerraform(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("export_as_terraform", builders...) -} + tcases := []testcase{ + { + name: "no generated HCL", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + }, + { + name: "empty generate_hcl block is ignored", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []hclconfig{ + { + path: "/stacks", + add: generateHCL(labels("empty")), + }, + }, + }, + { + name: "generate HCL for all stacks on parent", + layout: []string{ + "s:stacks/stack-1", + "s:stacks/stack-2", + }, + configs: []hclconfig{ + { + path: "/stacks", + add: hcldoc( + generateHCL( + labels("backend.tf"), + backend( + labels("test"), + expr("prefix", "global.backend_prefix"), + ), + ), + generateHCL( + labels("locals.tf"), + locals( + expr("stackpath", "terramate.path"), + expr("local_a", "global.local_a"), + expr("local_b", "global.local_b"), + expr("local_c", "global.local_c"), + expr("local_d", "try(global.local_d.field, null)"), + ), + ), + generateHCL( + labels("provider.tf"), + provider( + labels("name"), + expr("data", "global.provider_data"), + ), + terraform( + required_providers( + expr("name", `{ + source = "integrations/name" + version = global.provider_version + }`), + ), + ), + terraform( + expr("required_version", "global.terraform_version"), + ), + ), + ), + }, + { + path: "/stacks/stack-1", + add: globals( + str("local_a", "stack-1-local"), + boolean("local_b", true), + number("local_c", 666), + attr("local_d", `{ field = "local_d_field"}`), + str("backend_prefix", "stack-1-backend"), + str("provider_data", "stack-1-provider-data"), + str("provider_version", "stack-1-provider-version"), + str("terraform_version", "stack-1-terraform-version"), + ), + }, + { + path: "/stacks/stack-2", + add: globals( + str("local_a", "stack-2-local"), + boolean("local_b", false), + number("local_c", 777), + attr("local_d", `{ oopsie = "local_d_field"}`), + str("backend_prefix", "stack-2-backend"), + str("provider_data", "stack-2-provider-data"), + str("provider_version", "stack-2-provider-version"), + str("terraform_version", "stack-2-terraform-version"), + ), + }, + }, + want: []want{ + { + stack: "/stacks/stack-1", + hcls: map[string]fmt.Stringer{ + "backend.tf": backend( + labels("test"), + str("prefix", "stack-1-backend"), + ), + "locals.tf": locals( + str("stackpath", "/stacks/stack-1"), + str("local_a", "stack-1-local"), + boolean("local_b", true), + number("local_c", 666), + str("local_d", "local_d_field"), + ), + "provider.tf": hcldoc( + provider( + labels("name"), + str("data", "stack-1-provider-data"), + ), + terraform( + required_providers( + expr("name", `{ + source = "integrations/name" + version = "stack-1-provider-version" + }`), + ), + ), + terraform( + str("required_version", "stack-1-terraform-version"), + ), + ), + }, + }, + { + stack: "/stacks/stack-2", + hcls: map[string]fmt.Stringer{ + "backend.tf": backend( + labels("test"), + str("prefix", "stack-2-backend"), + ), + "locals.tf": locals( + str("stackpath", "/stacks/stack-2"), + str("local_a", "stack-2-local"), + boolean("local_b", false), + number("local_c", 777), + attr("local_d", "null"), + ), + "provider.tf": hcldoc( + provider( + labels("name"), + str("data", "stack-2-provider-data"), + ), + terraform( + required_providers( + expr("name", `{ + source = "integrations/name" + version = "stack-2-provider-version" + }`), + ), + ), + terraform( + str("required_version", "stack-2-terraform-version"), + ), + ), + }, + }, + }, + }, + } -func expr(name string, expr string) hclwrite.BlockBuilder { - return hclwrite.Expression(name, expr) -} + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + s := sandbox.New(t) + s.BuildTree(tcase.layout) -func str(name string, val string) hclwrite.BlockBuilder { - return hclwrite.String(name, val) -} + for _, cfg := range tcase.configs { + path := filepath.Join(s.RootDir(), cfg.path) + test.AppendFile(t, path, config.Filename, cfg.add.String()) + } -func number(name string, val int64) hclwrite.BlockBuilder { - return hclwrite.NumberInt(name, val) -} + workingDir := filepath.Join(s.RootDir(), tcase.workingDir) + err := generate.Do(s.RootDir(), workingDir) + assert.IsError(t, err, tcase.wantErr) -func boolean(name string, val bool) hclwrite.BlockBuilder { - return hclwrite.Boolean(name, val) -} + for _, wantDesc := range tcase.want { + stackRelPath := wantDesc.stack[1:] + stack := s.StackEntry(stackRelPath) -func labels(labels ...string) hclwrite.BlockBuilder { - return hclwrite.Labels(labels...) -} + for name, wantHCL := range wantDesc.hcls { + want := wantHCL.String() + got := stack.ReadGeneratedHCL(name) -func stack(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("stack", builders...) -} + assertHCLEquals(t, got, want) -func backend(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("backend", builders...) -} + stack.RemoveGeneratedHCL(name) + } + } -func block(name string, builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock(name, builders...) -} + // Check we don't have extraneous/unwanted files + // Wanted/expected generated code was removed by this point + // So we should have only basic terramate configs left + // There is potential to extract this for other code generation tests. + err = filepath.WalkDir(s.RootDir(), func(path string, d fs.DirEntry, err error) error { + t.Helper() -func globals(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("globals", builders...) -} + assert.NoError(t, err, "checking for unwanted generated files") + if d.IsDir() { + if d.Name() == ".git" { + return filepath.SkipDir + } + return nil + } -func locals(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("locals", builders...) -} + // sandbox create README.md inside test dirs + if d.Name() == config.Filename || d.Name() == "README.md" { + return nil + } -func terramate(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("terramate", builders...) + t.Errorf("unwanted file at %q, got %q", path, d.Name()) + return nil + }) + + assert.NoError(t, err) + }) + } } -func terraform(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("terraform", builders...) +func TestWontOverwriteManuallyDefinedTerraform(t *testing.T) { + const ( + genFilename = "test.tf" + manualTfCode = "some manual stuff, doesn't matter" + ) + + generateHCLConfig := generateHCL( + labels(genFilename), + terraform( + str("required_version", "1.11"), + ), + ) + + s := sandbox.New(t) + s.BuildTree([]string{ + fmt.Sprintf("f:%s:%s", config.Filename, generateHCLConfig.String()), + "s:stack", + fmt.Sprintf("f:stack/%s:%s", genFilename, manualTfCode), + }) + + err := generate.Do(s.RootDir(), s.RootDir()) + assert.IsError(t, err, generate.ErrManualCodeExists) + + stack := s.StackEntry("stack") + actualTfCode := stack.ReadGeneratedHCL(genFilename) + assert.EqualStrings(t, manualTfCode, actualTfCode, "tf code altered by generate") } -func exportAsLocals(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("export_as_locals", builders...) +func TestGenerateHCLOverwriting(t *testing.T) { + const genFilename = "test.tf" + + firstConfig := generateHCL( + labels(genFilename), + terraform( + str("required_version", "1.11"), + ), + ) + firstWant := terraform( + str("required_version", "1.11"), + ) + + s := sandbox.New(t) + stack := s.CreateStack("stack") + rootEntry := s.DirEntry(".") + rootConfig := rootEntry.CreateConfig(firstConfig.String()) + + s.Generate() + + got := stack.ReadGeneratedHCL(genFilename) + assertHCLEquals(t, got, firstWant.String()) + + secondConfig := generateHCL( + labels(genFilename), + terraform( + str("required_version", "2.0"), + ), + ) + secondWant := terraform( + str("required_version", "2.0"), + ) + + rootConfig.Write(secondConfig.String()) + + s.Generate() + + got = stack.ReadGeneratedHCL(genFilename) + assertHCLEquals(t, got, secondWant.String()) } diff --git a/generate/generate_hclwrite_utils_test.go b/generate/generate_hclwrite_utils_test.go new file mode 100644 index 000000000..1b2c21955 --- /dev/null +++ b/generate/generate_hclwrite_utils_test.go @@ -0,0 +1,87 @@ +// Copyright 2022 Mineiros GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generate_test + +import "github.com/mineiros-io/terramate/test/hclwrite" + +// useful function aliases to build HCL documents + +func hcldoc(blocks ...*hclwrite.Block) hclwrite.HCL { + return hclwrite.NewHCL(blocks...) +} + +// stackConfig wraps the blocks built by the given builders in a valid stack +// configuration +func stackConfig(blocks ...*hclwrite.Block) hclwrite.HCL { + b := []*hclwrite.Block{stack()} + b = append(b, blocks...) + return hcldoc(b...) +} + +func generateHCL(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("generate_hcl", builders...) +} + +func expr(name string, expr string) hclwrite.BlockBuilder { + return hclwrite.Expression(name, expr) +} + +func str(name string, val string) hclwrite.BlockBuilder { + return hclwrite.String(name, val) +} + +func number(name string, val int64) hclwrite.BlockBuilder { + return hclwrite.NumberInt(name, val) +} + +func boolean(name string, val bool) hclwrite.BlockBuilder { + return hclwrite.Boolean(name, val) +} + +func labels(labels ...string) hclwrite.BlockBuilder { + return hclwrite.Labels(labels...) +} + +func stack(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("stack", builders...) +} + +func backend(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("backend", builders...) +} + +func block(name string, builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock(name, builders...) +} + +func globals(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("globals", builders...) +} + +func locals(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("locals", builders...) +} + +func terramate(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("terramate", builders...) +} + +func terraform(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("terraform", builders...) +} + +func exportAsLocals(builders ...hclwrite.BlockBuilder) *hclwrite.Block { + return hclwrite.BuildBlock("export_as_locals", builders...) +} diff --git a/generate/generate_terraform_test.go b/generate/generate_terraform_test.go deleted file mode 100644 index 669f764b2..000000000 --- a/generate/generate_terraform_test.go +++ /dev/null @@ -1,352 +0,0 @@ -// Copyright 2022 Mineiros GmbH -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package generate_test - -import ( - "fmt" - "io/fs" - "path/filepath" - "testing" - - "github.com/madlambda/spells/assert" - "github.com/mineiros-io/terramate/config" - "github.com/mineiros-io/terramate/generate" - "github.com/mineiros-io/terramate/test" - "github.com/mineiros-io/terramate/test/hclwrite" - "github.com/mineiros-io/terramate/test/sandbox" -) - -func TestTerraformGeneration(t *testing.T) { - type ( - hclconfig struct { - path string - add fmt.Stringer - } - want struct { - stack string - hcls map[string]fmt.Stringer - } - testcase struct { - name string - layout []string - configs []hclconfig - workingDir string - want []want - wantErr error - } - ) - - provider := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("provider", builders...) - } - required_providers := func(builders ...hclwrite.BlockBuilder) *hclwrite.Block { - return hclwrite.BuildBlock("required_providers", builders...) - } - attr := func(name, expr string) hclwrite.BlockBuilder { - t.Helper() - return hclwrite.AttributeValue(t, name, expr) - } - - tcases := []testcase{ - { - name: "no exported terraform", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - }, - { - name: "empty export_as_terraform block is ignored", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - configs: []hclconfig{ - { - path: "/stacks", - add: exportAsTerraform(labels("empty")), - }, - }, - }, - { - name: "export terraform for all stacks on parent", - layout: []string{ - "s:stacks/stack-1", - "s:stacks/stack-2", - }, - configs: []hclconfig{ - { - path: "/stacks", - add: hcldoc( - exportAsTerraform( - labels("backend.tf"), - backend( - labels("test"), - expr("prefix", "global.backend_prefix"), - ), - ), - exportAsTerraform( - labels("locals.tf"), - locals( - expr("stackpath", "terramate.path"), - expr("local_a", "global.local_a"), - expr("local_b", "global.local_b"), - expr("local_c", "global.local_c"), - expr("local_d", "try(global.local_d.field, null)"), - ), - ), - exportAsTerraform( - labels("provider.tf"), - provider( - labels("name"), - expr("data", "global.provider_data"), - ), - terraform( - required_providers( - expr("name", `{ - source = "integrations/name" - version = global.provider_version - }`), - ), - ), - terraform( - expr("required_version", "global.terraform_version"), - ), - ), - ), - }, - { - path: "/stacks/stack-1", - add: globals( - str("local_a", "stack-1-local"), - boolean("local_b", true), - number("local_c", 666), - attr("local_d", `{ field = "local_d_field"}`), - str("backend_prefix", "stack-1-backend"), - str("provider_data", "stack-1-provider-data"), - str("provider_version", "stack-1-provider-version"), - str("terraform_version", "stack-1-terraform-version"), - ), - }, - { - path: "/stacks/stack-2", - add: globals( - str("local_a", "stack-2-local"), - boolean("local_b", false), - number("local_c", 777), - attr("local_d", `{ oopsie = "local_d_field"}`), - str("backend_prefix", "stack-2-backend"), - str("provider_data", "stack-2-provider-data"), - str("provider_version", "stack-2-provider-version"), - str("terraform_version", "stack-2-terraform-version"), - ), - }, - }, - want: []want{ - { - stack: "/stacks/stack-1", - hcls: map[string]fmt.Stringer{ - "backend.tf": backend( - labels("test"), - str("prefix", "stack-1-backend"), - ), - "locals.tf": locals( - str("stackpath", "/stacks/stack-1"), - str("local_a", "stack-1-local"), - boolean("local_b", true), - number("local_c", 666), - str("local_d", "local_d_field"), - ), - "provider.tf": hcldoc( - provider( - labels("name"), - str("data", "stack-1-provider-data"), - ), - terraform( - required_providers( - expr("name", `{ - source = "integrations/name" - version = "stack-1-provider-version" - }`), - ), - ), - terraform( - str("required_version", "stack-1-terraform-version"), - ), - ), - }, - }, - { - stack: "/stacks/stack-2", - hcls: map[string]fmt.Stringer{ - "backend.tf": backend( - labels("test"), - str("prefix", "stack-2-backend"), - ), - "locals.tf": locals( - str("stackpath", "/stacks/stack-2"), - str("local_a", "stack-2-local"), - boolean("local_b", false), - number("local_c", 777), - attr("local_d", "null"), - ), - "provider.tf": hcldoc( - provider( - labels("name"), - str("data", "stack-2-provider-data"), - ), - terraform( - required_providers( - expr("name", `{ - source = "integrations/name" - version = "stack-2-provider-version" - }`), - ), - ), - terraform( - str("required_version", "stack-2-terraform-version"), - ), - ), - }, - }, - }, - }, - } - - for _, tcase := range tcases { - t.Run(tcase.name, func(t *testing.T) { - s := sandbox.New(t) - s.BuildTree(tcase.layout) - - for _, cfg := range tcase.configs { - path := filepath.Join(s.RootDir(), cfg.path) - test.AppendFile(t, path, config.Filename, cfg.add.String()) - } - - workingDir := filepath.Join(s.RootDir(), tcase.workingDir) - err := generate.Do(s.RootDir(), workingDir) - assert.IsError(t, err, tcase.wantErr) - - for _, wantDesc := range tcase.want { - stackRelPath := wantDesc.stack[1:] - stack := s.StackEntry(stackRelPath) - - for name, wantHCL := range wantDesc.hcls { - want := wantHCL.String() - got := stack.ReadGeneratedTerraform(name) - - assertHCLEquals(t, got, want) - - stack.RemoveGeneratedTerraform(name) - } - } - - // Check we don't have extraneous/unwanted files - // Wanted/expected generated code was removed by this point - // So we should have only basic terramate configs left - // There is potential to extract this for other code generation tests. - err = filepath.WalkDir(s.RootDir(), func(path string, d fs.DirEntry, err error) error { - t.Helper() - - assert.NoError(t, err, "checking for unwanted generated files") - if d.IsDir() { - if d.Name() == ".git" { - return filepath.SkipDir - } - return nil - } - - // sandbox create README.md inside test dirs - if d.Name() == config.Filename || d.Name() == "README.md" { - return nil - } - - t.Errorf("unwanted file at %q, got %q", path, d.Name()) - return nil - }) - - assert.NoError(t, err) - }) - } -} - -func TestWontOverwriteManuallyDefinedTerraform(t *testing.T) { - const ( - genFilename = "test.tf" - manualTfCode = "some manual stuff, doesn't matter" - ) - - exportTfConfig := exportAsTerraform( - labels(genFilename), - terraform( - str("required_version", "1.11"), - ), - ) - - s := sandbox.New(t) - s.BuildTree([]string{ - fmt.Sprintf("f:%s:%s", config.Filename, exportTfConfig.String()), - "s:stack", - fmt.Sprintf("f:stack/%s:%s", genFilename, manualTfCode), - }) - - err := generate.Do(s.RootDir(), s.RootDir()) - assert.IsError(t, err, generate.ErrManualCodeExists) - - stack := s.StackEntry("stack") - actualTfCode := stack.ReadGeneratedTerraform(genFilename) - assert.EqualStrings(t, manualTfCode, actualTfCode, "tf code altered by generate") -} - -func TestExportedTerraformOverwriting(t *testing.T) { - const genFilename = "test.tf" - - firstConfig := exportAsTerraform( - labels(genFilename), - terraform( - str("required_version", "1.11"), - ), - ) - firstWant := terraform( - str("required_version", "1.11"), - ) - - s := sandbox.New(t) - stack := s.CreateStack("stack") - rootEntry := s.DirEntry(".") - rootConfig := rootEntry.CreateConfig(firstConfig.String()) - - s.Generate() - - got := stack.ReadGeneratedTerraform(genFilename) - assertHCLEquals(t, got, firstWant.String()) - - secondConfig := exportAsTerraform( - labels(genFilename), - terraform( - str("required_version", "2.0"), - ), - ) - secondWant := terraform( - str("required_version", "2.0"), - ) - - rootConfig.Write(secondConfig.String()) - - s.Generate() - - got = stack.ReadGeneratedTerraform(genFilename) - assertHCLEquals(t, got, secondWant.String()) -} diff --git a/test/sandbox/sandbox.go b/test/sandbox/sandbox.go index aa5788147..94e0b13bd 100644 --- a/test/sandbox/sandbox.go +++ b/test/sandbox/sandbox.go @@ -427,21 +427,21 @@ func (se StackEntry) ReadGeneratedLocals() []byte { return se.DirEntry.ReadFile(cfg.LocalsFilename) } -// ReadGeneratedTerraform will read code that was generated by Terramate for this stack -// using export_as_terraform blocks. -// The given name is the name of the export_as_terraform block as indicated by its label. +// ReadGeneratedHCL will read code that was generated by Terramate for this stack +// using generate_hcl blocks. +// The given name is the name of the generate_hcl block as indicated by its label. // // It will fail the test if there is no generated code available on the stack, // since it assumes generated code is expected to be there. -func (se StackEntry) ReadGeneratedTerraform(name string) string { +func (se StackEntry) ReadGeneratedHCL(name string) string { se.t.Helper() return string(se.DirEntry.ReadFile(name)) } -// RemoveGeneratedTerraform will delete the file with generated code from -// export_as_terraform blocks. -// The given name is the name of the export_as_terraform block as indicated by its label. -func (se StackEntry) RemoveGeneratedTerraform(name string) { +// RemoveGeneratedHCL will delete the file with generated code from +// generate_hcl blocks. +// The given name is the name of the generate_hcl block as indicated by its label. +func (se StackEntry) RemoveGeneratedHCL(name string) { se.t.Helper() se.DirEntry.RemoveFile(name) } From cd7e319553e977dd04e3ebf72fa92e7b21b76af1 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 17:53:27 +0100 Subject: [PATCH 41/48] refactor: rename export_as_terraform to generate_hcl --- docs/terraform-generation.md | 32 ++++++++++++++++---------------- generate/generate.go | 2 +- generate/generate_check_test.go | 2 +- hcl/hcl.go | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/terraform-generation.md b/docs/terraform-generation.md index 04dec4830..688d03e20 100644 --- a/docs/terraform-generation.md +++ b/docs/terraform-generation.md @@ -5,7 +5,7 @@ both [globals](globals.md) and [metadata](metadata.md). The generated code can then be composed/referenced by any Terraform code inside a stack. -Terraform code generation starts with the definition of a `export_as_terraform` +Terraform code generation starts with the definition of a `generate_hcl` block in a [Terramate configuration file](config.md) defining the code you want to generate inside the block. The code may include: @@ -15,7 +15,7 @@ want to generate inside the block. The code may include: * Terramate Metadata references * Expressions using interpolation, functions, etc -Most of what you can do in Terraform can be done in a `export_as_terraform` +Most of what you can do in Terraform can be done in a `generate_hcl` block. For now, only the following is not allowed: * References to variables on the form `var.name` @@ -28,7 +28,7 @@ file reading functions, references to globals/metadata, will all be evaluated at code generation time and the generated code will only have literals like strings, numbers, lists, maps, objects, etc. -Each `export_as_terraform` block requires a single label. +Each `generate_hcl` block requires a single label. This label is the filename of the generated code. Now lets jump to some examples. Lets generate backend and provider configurations @@ -46,11 +46,11 @@ globals { ``` We can define the generation of a backend configuration for all -stacks by defining a `export_as_terraform` block in the root +stacks by defining a `generate_hcl` block in the root of the project: ```hcl -export_as_terraform "backend.tf" { +generate_hcl "backend.tf" { backend "local" { param = global.backend_data } @@ -69,7 +69,7 @@ To generate provider/terraform configuration for all stacks we can add in the root configuration: ```hcl -export_as_terraform "provider.tf" { +generate_hcl "provider.tf" { provider "name" { param = global.provider_data @@ -133,21 +133,21 @@ to more general: * `stacks` * `/` which means the project root -Given this definition, the behavior of `export_as_terraform` blocks is that +Given this definition, the behavior of `generate_hcl` blocks is that more specific configuration always override general purpose configuration. There is no merge strategy/ composition involved, the configuration found closest to a stack on the file system, or directly at the stack directory, is the one used, ignoring more general configuration. -It is important to note that overriding happens when `export_as_terraform` -blocks are considered the same, and the identity of a `export_as_terraform` +It is important to note that overriding happens when `generate_hcl` +blocks are considered the same, and the identity of a `generate_hcl` block includes its label. Lets use as an example the previously mentioned `stacks/stack-1`. Given this configuration at `stacks/terramate.tm.hcl`: ```hcl -export_as_terraform "provider.tf" { +generate_hcl "provider.tf" { terraform { required_version = "1.1.13" } @@ -157,20 +157,20 @@ export_as_terraform "provider.tf" { And this configuration at `stacks/stack-1/terramate.tm.hcl`: ```hcl -export_as_terraform "backend.tf" { +generate_hcl "backend.tf" { backend "local" { param = "example" } } ``` -No overriding happens since each `export_as_terraform` block has a different +No overriding happens since each `generate_hcl` block has a different label and will generate its own code in a separated file. But if we had this configuration at `stacks/stack-1/terramate.tm.hcl`: ```hcl -export_as_terraform "provider.tf" { +generate_hcl "provider.tf" { terraform { required_version = "overriden" } @@ -190,11 +190,11 @@ definition at `stacks`. Any other stack under `stacks` would remain with the configuration defined in the parent dir `stacks`. The overriding is total, there is no merging involved on the blocks inside -`export_as_terraform`, so if a parent directory defines a +`generate_hcl`, so if a parent directory defines a configuration like this: ```hcl -export_as_terraform "name.tf" { +generate_hcl "name.tf" { block1 { } block2 { @@ -207,7 +207,7 @@ export_as_terraform "name.tf" { And a more specific configuration redefines it like this: ```hcl -export_as_terraform "name.tf" { +generate_hcl "name.tf" { block4 { } } diff --git a/generate/generate.go b/generate/generate.go index 51d87f21c..f4c254ea1 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -308,7 +308,7 @@ func writeStackTerraformCode( tfcode := tf.String() if tfcode == "" { - logger.Debug().Msg("ignoring empty export_as_terraform block.") + logger.Debug().Msg("ignoring empty generate_hcl block.") continue } diff --git a/generate/generate_check_test.go b/generate/generate_check_test.go index 879421324..2916042ea 100644 --- a/generate/generate_check_test.go +++ b/generate/generate_check_test.go @@ -249,7 +249,7 @@ func TestCheckReturnsOutdatedStackFilenamesForBackendAndLocals(t *testing.T) { } func TestCheckFailsWithInvalidConfig(t *testing.T) { - // TODO(katcipis): add export_as_terraform + // TODO(katcipis): add generate_hcl invalidConfigs := []string{ hcldoc( terramate( diff --git a/hcl/hcl.go b/hcl/hcl.go index 7caf43cce..e082e0b32 100644 --- a/hcl/hcl.go +++ b/hcl/hcl.go @@ -467,7 +467,7 @@ func ParseExportAsLocalsBlocks(path string) ([]*hclsyntax.Block, error) { return parseBlocksOfType(path, "export_as_locals") } -// ParseGenerateHCLBlocks parses export_as_terraform blocks, ignoring other blocks +// ParseGenerateHCLBlocks parses generate_hcl blocks, ignoring other blocks func ParseGenerateHCLBlocks(path string) ([]*hclsyntax.Block, error) { return parseBlocksOfType(path, "generate_hcl") } From 908395a927aa56a3382b822b54897379c68ca009 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 17:57:17 +0100 Subject: [PATCH 42/48] test: add generate_hcl to invalid config check --- generate/generate_check_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/generate/generate_check_test.go b/generate/generate_check_test.go index 2916042ea..5750baa5b 100644 --- a/generate/generate_check_test.go +++ b/generate/generate_check_test.go @@ -249,7 +249,6 @@ func TestCheckReturnsOutdatedStackFilenamesForBackendAndLocals(t *testing.T) { } func TestCheckFailsWithInvalidConfig(t *testing.T) { - // TODO(katcipis): add generate_hcl invalidConfigs := []string{ hcldoc( terramate( @@ -266,6 +265,12 @@ func TestCheckFailsWithInvalidConfig(t *testing.T) { ), stack(), ).String(), + hcldoc( + generateHCL( + expr("undefined", "terramate.undefined"), + ), + stack(), + ).String(), } for _, invalidConfig := range invalidConfigs { From 26c0380c8535816aa89c838afb0f59111429081c Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 21:04:31 +0100 Subject: [PATCH 43/48] refactor: rename exportedtf to genhcl --- generate/generate.go | 6 +++--- .../exportedtf.go => genhcl/genhcl.go} | 2 +- .../genhcl_test.go} | 20 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) rename generate/{exportedtf/exportedtf.go => genhcl/genhcl.go} (99%) rename generate/{exportedtf/exportedtf_test.go => genhcl/genhcl_test.go} (97%) diff --git a/generate/generate.go b/generate/generate.go index f4c254ea1..1fe83610e 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -27,7 +27,7 @@ import ( "github.com/madlambda/spells/errutil" "github.com/mineiros-io/terramate" "github.com/mineiros-io/terramate/config" - "github.com/mineiros-io/terramate/generate/exportedtf" + "github.com/mineiros-io/terramate/generate/genhcl" "github.com/mineiros-io/terramate/hcl" "github.com/mineiros-io/terramate/hcl/eval" "github.com/mineiros-io/terramate/stack" @@ -208,7 +208,7 @@ func exportedTerraformOutdatedFiles( logger.Trace().Msg("Checking for outdated exported terraform code on stack.") - loadedStackTf, err := exportedtf.Load(root, stackMeta, globals) + loadedStackTf, err := genhcl.Load(root, stackMeta, globals) if err != nil { return nil, err } @@ -292,7 +292,7 @@ func writeStackTerraformCode( logger.Trace().Msg("generating terraform code.") - loadedStackTf, err := exportedtf.Load(root, meta, globals) + loadedStackTf, err := genhcl.Load(root, meta, globals) if err != nil { return err } diff --git a/generate/exportedtf/exportedtf.go b/generate/genhcl/genhcl.go similarity index 99% rename from generate/exportedtf/exportedtf.go rename to generate/genhcl/genhcl.go index a5fd67d1a..c755ee66b 100644 --- a/generate/exportedtf/exportedtf.go +++ b/generate/genhcl/genhcl.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package exportedtf +package genhcl import ( "fmt" diff --git a/generate/exportedtf/exportedtf_test.go b/generate/genhcl/genhcl_test.go similarity index 97% rename from generate/exportedtf/exportedtf_test.go rename to generate/genhcl/genhcl_test.go index c9e2950e7..fd66d47bf 100644 --- a/generate/exportedtf/exportedtf_test.go +++ b/generate/genhcl/genhcl_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package exportedtf_test +package genhcl_test import ( "fmt" @@ -23,7 +23,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/madlambda/spells/assert" "github.com/mineiros-io/terramate/config" - "github.com/mineiros-io/terramate/generate/exportedtf" + "github.com/mineiros-io/terramate/generate/genhcl" "github.com/mineiros-io/terramate/test" "github.com/mineiros-io/terramate/test/hclwrite" "github.com/mineiros-io/terramate/test/sandbox" @@ -520,7 +520,7 @@ func TestLoadGeneratedHCL(t *testing.T) { ), }, }, - wantErr: exportedtf.ErrInvalidBlock, + wantErr: genhcl.ErrInvalidBlock, }, { name: "block with two labels on stack gives err", @@ -536,7 +536,7 @@ func TestLoadGeneratedHCL(t *testing.T) { ), }, }, - wantErr: exportedtf.ErrInvalidBlock, + wantErr: genhcl.ErrInvalidBlock, }, { name: "block with empty label on stack gives err", @@ -552,7 +552,7 @@ func TestLoadGeneratedHCL(t *testing.T) { ), }, }, - wantErr: exportedtf.ErrInvalidBlock, + wantErr: genhcl.ErrInvalidBlock, }, { name: "blocks with same label on same config gives err", @@ -574,7 +574,7 @@ func TestLoadGeneratedHCL(t *testing.T) { ), }, }, - wantErr: exportedtf.ErrInvalidBlock, + wantErr: genhcl.ErrInvalidBlock, }, { name: "evaluation failure on stack config fails", @@ -591,7 +591,7 @@ func TestLoadGeneratedHCL(t *testing.T) { ), }, }, - wantErr: exportedtf.ErrEval, + wantErr: genhcl.ErrEval, }, { name: "valid config on stack but invalid on parent fails", @@ -616,7 +616,7 @@ func TestLoadGeneratedHCL(t *testing.T) { ), }, }, - wantErr: exportedtf.ErrInvalidBlock, + wantErr: genhcl.ErrInvalidBlock, }, { name: "attributes on generate_hcl block fails", @@ -634,7 +634,7 @@ func TestLoadGeneratedHCL(t *testing.T) { ), }, }, - wantErr: exportedtf.ErrInvalidBlock, + wantErr: genhcl.ErrInvalidBlock, }, } @@ -651,7 +651,7 @@ func TestLoadGeneratedHCL(t *testing.T) { meta := stack.Meta() globals := s.LoadStackGlobals(meta) - res, err := exportedtf.Load(s.RootDir(), meta, globals) + res, err := genhcl.Load(s.RootDir(), meta, globals) assert.IsError(t, err, tcase.wantErr) got := res.ExportedCode() From 4f75d50be01341ff4c45f7825e990011e9cf28f6 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 23:31:36 +0100 Subject: [PATCH 44/48] refactor: improve variable names --- generate/generate.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 1fe83610e..1beee8211 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -217,17 +217,16 @@ func exportedTerraformOutdatedFiles( outdated := []string{} - for name, tf := range loadedStackTf.ExportedCode() { - exportedTfFilename := name - targetpath := filepath.Join(stackpath, exportedTfFilename) + for filename, hclcode := range loadedStackTf.ExportedCode() { + targetpath := filepath.Join(stackpath, filename) logger := logger.With(). - Str("blockName", name). + Str("blockName", filename). Str("targetpath", targetpath). Logger() logger.Trace().Msg("checking if code is updated.") - tfcode := PrependHeader(tf.String()) + tfcode := PrependHeader(hclcode.String()) currentTfCode, err := loadGeneratedCode(targetpath) if err != nil { return nil, err @@ -235,7 +234,7 @@ func exportedTerraformOutdatedFiles( if tfcode != string(currentTfCode) { logger.Trace().Msg("Outdated code detected.") - outdated = append(outdated, exportedTfFilename) + outdated = append(outdated, filename) } } From febdd7219cb98d97f050b4ae4c85060ecc14d2db Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 23:32:17 +0100 Subject: [PATCH 45/48] chore: improve logging --- generate/generate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 1beee8211..436a5bde3 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -224,7 +224,7 @@ func exportedTerraformOutdatedFiles( Str("targetpath", targetpath). Logger() - logger.Trace().Msg("checking if code is updated.") + logger.Trace().Msg("Checking if code is updated.") tfcode := PrependHeader(hclcode.String()) currentTfCode, err := loadGeneratedCode(targetpath) @@ -233,7 +233,7 @@ func exportedTerraformOutdatedFiles( } if tfcode != string(currentTfCode) { - logger.Trace().Msg("Outdated code detected.") + logger.Trace().Msg("Outdated HCL code detected.") outdated = append(outdated, filename) } } From 74e119975f452e303c31bb336aa0e3b063d0b3dd Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 23:34:31 +0100 Subject: [PATCH 46/48] refactor: improve naming --- generate/generate_check_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/generate/generate_check_test.go b/generate/generate_check_test.go index 5750baa5b..6ec11ff9a 100644 --- a/generate/generate_check_test.go +++ b/generate/generate_check_test.go @@ -29,7 +29,7 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { stackEntry := s.CreateStack("stacks/stack") stack := stackEntry.Load() - assertOutdated := func(want []string) { + assertOutdatedFiles := func(want []string) { t.Helper() got, err := generate.CheckStack(s.RootDir(), stack) @@ -38,7 +38,7 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { } // Checking detection when there is no config generated yet - assertOutdated([]string{}) + assertOutdatedFiles([]string{}) stackEntry.CreateConfig( stackConfig( generateHCL( @@ -48,11 +48,11 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { ), ), ).String()) - assertOutdated([]string{"test.tf"}) + assertOutdatedFiles([]string{"test.tf"}) s.Generate() - assertOutdated([]string{}) + assertOutdatedFiles([]string{}) // Now checking when we have code + it gets outdated. stackEntry.CreateConfig( @@ -65,7 +65,7 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { ), ).String()) - assertOutdated([]string{"test.tf"}) + assertOutdatedFiles([]string{"test.tf"}) s.Generate() @@ -83,7 +83,7 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { // TODO(katcipis): detect the old test.tf generated file. // It is stale but it doesn't map to code generation anymore so // we need extra steps to detect it that are not done right now. - assertOutdated([]string{"testnew.tf"}) + assertOutdatedFiles([]string{"testnew.tf"}) // TODO(katcipis): cleanup the old test.tf @@ -104,11 +104,11 @@ func TestCheckReturnsOutdatedStackFilenamesForExportedTf(t *testing.T) { ), ).String()) - assertOutdated([]string{"another.tf", "testnew.tf"}) + assertOutdatedFiles([]string{"another.tf", "testnew.tf"}) s.Generate() - assertOutdated([]string{}) + assertOutdatedFiles([]string{}) } func TestCheckReturnsOutdatedStackFilenamesForBackendAndLocals(t *testing.T) { From ad148a17de74f6f7b6e4594c69188f920f265483 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 23:38:19 +0100 Subject: [PATCH 47/48] docs: improve --- generate/genhcl/genhcl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/genhcl/genhcl.go b/generate/genhcl/genhcl.go index c755ee66b..6967f3b2d 100644 --- a/generate/genhcl/genhcl.go +++ b/generate/genhcl/genhcl.go @@ -32,7 +32,7 @@ import ( ) // StackHCLs represents all generated HCL code for a stack, -// mapping the generated code name to the actual HCL code. +// mapping the generated code filename to the actual HCL code. type StackHCLs struct { hcls map[string]HCL } From 3390f21a74ae1010977f633c20cee6ade5ba4fa4 Mon Sep 17 00:00:00 2001 From: Tiago Katcipis Date: Wed, 2 Feb 2022 23:44:08 +0100 Subject: [PATCH 48/48] refactor: remove unused wantErr --- generate/generate_hcl_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/generate/generate_hcl_test.go b/generate/generate_hcl_test.go index 9f7db5ada..f84284c5e 100644 --- a/generate/generate_hcl_test.go +++ b/generate/generate_hcl_test.go @@ -44,7 +44,6 @@ func TestHCLGeneration(t *testing.T) { configs []hclconfig workingDir string want []want - wantErr error } ) @@ -237,7 +236,7 @@ func TestHCLGeneration(t *testing.T) { workingDir := filepath.Join(s.RootDir(), tcase.workingDir) err := generate.Do(s.RootDir(), workingDir) - assert.IsError(t, err, tcase.wantErr) + assert.NoError(t, err) for _, wantDesc := range tcase.want { stackRelPath := wantDesc.stack[1:]