From bf7b1a12e60740a284b6296449f31f539bf6dc0c Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Thu, 5 May 2022 20:54:57 +0100 Subject: [PATCH] module: Decouple installed and declared modules --- earlydecoder/decoder.go | 2 +- earlydecoder/decoder_test.go | 70 ++++++++++++++++++------------------ earlydecoder/load_module.go | 20 +++++++---- module/meta.go | 9 +---- module/module_calls.go | 21 +++++++++++ schema/module_schema.go | 13 ++++--- schema/module_schema_test.go | 21 +++++------ schema/schema_merge.go | 7 ++-- schema/schema_merge_test.go | 28 ++++++++------- 9 files changed, 111 insertions(+), 80 deletions(-) create mode 100644 module/module_calls.go diff --git a/earlydecoder/decoder.go b/earlydecoder/decoder.go index efe2d5d2..53576538 100644 --- a/earlydecoder/decoder.go +++ b/earlydecoder/decoder.go @@ -163,7 +163,7 @@ func LoadModule(path string, files map[string]*hcl.File) (*module.Meta, hcl.Diag outputs[key] = *output } - modulesCalls := make(map[string]module.ModuleCall) + modulesCalls := make(map[string]module.DeclaredModuleCall) for key, moduleCall := range mod.ModuleCalls { modulesCalls[key] = *moduleCall } diff --git a/earlydecoder/decoder_test.go b/earlydecoder/decoder_test.go index 0d6b33b6..97b747f9 100644 --- a/earlydecoder/decoder_test.go +++ b/earlydecoder/decoder_test.go @@ -41,7 +41,7 @@ func TestLoadModule(t *testing.T) { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -59,7 +59,7 @@ terraform { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -100,7 +100,7 @@ provider "grafana" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -141,7 +141,7 @@ provider "grafana" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -186,7 +186,7 @@ provider "grafana" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -261,7 +261,7 @@ provider "grafana" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -326,7 +326,7 @@ resource "google_storage_bucket" "bucket" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -392,7 +392,7 @@ resource "google_storage_bucket" "bucket" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -450,7 +450,7 @@ provider "aws" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -514,7 +514,7 @@ provider "aws" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -553,7 +553,7 @@ resource "google_something" "test" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -578,7 +578,7 @@ variable "" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -594,7 +594,7 @@ variable { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, hcl.Diagnostics{ &hcl.Diagnostic{ @@ -642,7 +642,7 @@ variable "one" "two" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, hcl.Diagnostics{ &hcl.Diagnostic{ @@ -694,7 +694,7 @@ variable "name" { }, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -715,7 +715,7 @@ variable "name" { }, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -737,7 +737,7 @@ variable "name" { }, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -759,7 +759,7 @@ variable "name" { }, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -784,7 +784,7 @@ variable "name" { }, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -806,7 +806,7 @@ variable "name" { }, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -825,7 +825,7 @@ output "name" { "name": {Value: cty.NilVal}, }, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -852,7 +852,7 @@ terraform { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -875,7 +875,7 @@ terraform { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -896,7 +896,7 @@ terraform { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -919,7 +919,7 @@ terraform { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -947,7 +947,7 @@ terraform { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -972,7 +972,7 @@ module "" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, nil, }, @@ -988,7 +988,7 @@ module { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, hcl.Diagnostics{ &hcl.Diagnostic{ @@ -1036,7 +1036,7 @@ module "one" "two" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{}, + ModuleCalls: map[string]module.DeclaredModuleCall{}, }, hcl.Diagnostics{ &hcl.Diagnostic{ @@ -1084,7 +1084,7 @@ module "name" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{ + ModuleCalls: map[string]module.DeclaredModuleCall{ "name": { LocalName: "name", }, @@ -1105,7 +1105,7 @@ module "name" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{ + ModuleCalls: map[string]module.DeclaredModuleCall{ "name": { LocalName: "name", SourceAddr: "registry.terraform.io/terraform-aws-modules/vpc/aws", @@ -1127,10 +1127,10 @@ module "name" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{ + ModuleCalls: map[string]module.DeclaredModuleCall{ "name": { LocalName: "name", - Version: "> 3.0.0, < 4.0.0", + Version: version.MustConstraints(version.NewConstraint("> 3.0.0, < 4.0.0")), }, }, }, @@ -1150,11 +1150,11 @@ module "name" { Variables: map[string]module.Variable{}, Outputs: map[string]module.Output{}, Filenames: []string{"test.tf"}, - ModuleCalls: map[string]module.ModuleCall{ + ModuleCalls: map[string]module.DeclaredModuleCall{ "name": { LocalName: "name", SourceAddr: "terraform-aws-modules/vpc/aws", - Version: "1.0.0", + Version: version.MustConstraints(version.NewConstraint("1.0.0")), }, }, }, diff --git a/earlydecoder/load_module.go b/earlydecoder/load_module.go index 82331b71..f958e4b3 100644 --- a/earlydecoder/load_module.go +++ b/earlydecoder/load_module.go @@ -3,6 +3,7 @@ package earlydecoder import ( "fmt" + "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" @@ -23,7 +24,7 @@ type decodedModule struct { DataSources map[string]*dataSource Variables map[string]*module.Variable Outputs map[string]*module.Output - ModuleCalls map[string]*module.ModuleCall + ModuleCalls map[string]*module.DeclaredModuleCall } func newDecodedModule() *decodedModule { @@ -36,7 +37,7 @@ func newDecodedModule() *decodedModule { DataSources: make(map[string]*dataSource), Variables: make(map[string]*module.Variable), Outputs: make(map[string]*module.Output), - ModuleCalls: make(map[string]*module.ModuleCall), + ModuleCalls: make(map[string]*module.DeclaredModuleCall), } } @@ -285,7 +286,7 @@ func loadModuleFromFile(file *hcl.File, mod *decodedModule) hcl.Diagnostics { } name := block.Labels[0] source := "" - version := "" + var versionCons version.Constraints var valDiags hcl.Diagnostics if attr, defined := content.Attributes["source"]; defined { @@ -293,14 +294,21 @@ func loadModuleFromFile(file *hcl.File, mod *decodedModule) hcl.Diagnostics { diags = append(diags, valDiags...) } if attr, defined := content.Attributes["version"]; defined { - valDiags = gohcl.DecodeExpression(attr.Expr, nil, &version) + var versionStr string + valDiags = gohcl.DecodeExpression(attr.Expr, nil, &versionStr) diags = append(diags, valDiags...) + if versionStr != "" { + vc, err := version.NewConstraint(versionStr) + if err == nil { + versionCons = vc + } + } } - mod.ModuleCalls[name] = &module.ModuleCall{ + mod.ModuleCalls[name] = &module.DeclaredModuleCall{ LocalName: name, SourceAddr: source, - Version: version, + Version: versionCons, } } diff --git a/module/meta.go b/module/meta.go index 0e399e0f..dc17ada5 100644 --- a/module/meta.go +++ b/module/meta.go @@ -16,7 +16,7 @@ type Meta struct { CoreRequirements version.Constraints Variables map[string]Variable Outputs map[string]Output - ModuleCalls map[string]ModuleCall + ModuleCalls map[string]DeclaredModuleCall } type ProviderRequirements map[tfaddr.Provider]version.Constraints @@ -67,10 +67,3 @@ type ProviderRef struct { // configuration this address refers to. Alias string } - -type ModuleCall struct { - LocalName string - SourceAddr string - Version string - Path string -} diff --git a/module/module_calls.go b/module/module_calls.go new file mode 100644 index 00000000..cda2101d --- /dev/null +++ b/module/module_calls.go @@ -0,0 +1,21 @@ +package module + +import "github.com/hashicorp/go-version" + +type ModuleCalls struct { + Installed map[string]InstalledModuleCall + Declared map[string]DeclaredModuleCall +} + +type InstalledModuleCall struct { + LocalName string + SourceAddr string + Version *version.Version + Path string +} + +type DeclaredModuleCall struct { + LocalName string + SourceAddr string + Version version.Constraints +} diff --git a/schema/module_schema.go b/schema/module_schema.go index 20f7886d..3e7a0a08 100644 --- a/schema/module_schema.go +++ b/schema/module_schema.go @@ -13,7 +13,7 @@ import ( "github.com/zclconf/go-cty/cty" ) -func schemaForDependentModuleBlock(module module.ModuleCall, modMeta *module.Meta) (*schema.BodySchema, error) { +func schemaForDependentModuleBlock(module module.InstalledModuleCall, modMeta *module.Meta) (*schema.BodySchema, error) { attributes := make(map[string]*schema.AttributeSchema, 0) for name, modVar := range modMeta.Variables { @@ -117,15 +117,18 @@ func schemaForDependentModuleBlock(module module.ModuleCall, modMeta *module.Met moduleSourceRegistry, err := tfaddr.ParseRawModuleSourceRegistry(module.SourceAddr) if err == nil && moduleSourceRegistry.PackageAddr.Host == "registry.terraform.io" { - version := module.Version - if version == "" { - version = "latest" + versionStr := "" + if module.Version == nil { + versionStr = "latest" + } else { + versionStr = module.Version.String() } + bodySchema.DocsLink = &schema.DocsLink{ URL: fmt.Sprintf( `https://registry.terraform.io/modules/%s/%s`, moduleSourceRegistry.PackageAddr.ForRegistryProtocol(), - version, + versionStr, ), } } diff --git a/schema/module_schema_test.go b/schema/module_schema_test.go index c639fdb1..c998cd63 100644 --- a/schema/module_schema_test.go +++ b/schema/module_schema_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/hashicorp/go-version" "github.com/hashicorp/hcl-lang/lang" "github.com/hashicorp/hcl-lang/schema" "github.com/hashicorp/hcl/v2" @@ -15,7 +16,7 @@ import ( func TestSchemaForDependentModuleBlock_emptyMeta(t *testing.T) { meta := &module.Meta{} - module := module.ModuleCall{ + module := module.InstalledModuleCall{ LocalName: "refname", } depSchema, err := schemaForDependentModuleBlock(module, meta) @@ -66,7 +67,7 @@ func TestSchemaForDependentModuleBlock_basic(t *testing.T) { }, }, } - module := module.ModuleCall{ + module := module.InstalledModuleCall{ LocalName: "refname", } depSchema, err := schemaForDependentModuleBlock(module, meta) @@ -269,7 +270,7 @@ func TestSchemaForDependentModuleBlock_Target(t *testing.T) { }, }, } - module := module.ModuleCall{ + module := module.InstalledModuleCall{ LocalName: "refname", } @@ -288,7 +289,7 @@ func TestSchemaForDependentModuleBlock_DocsLink(t *testing.T) { type testCase struct { name string meta *module.Meta - module module.ModuleCall + module module.InstalledModuleCall expectedSchema *schema.BodySchema } @@ -301,7 +302,7 @@ func TestSchemaForDependentModuleBlock_DocsLink(t *testing.T) { Outputs: map[string]module.Output{}, Filenames: nil, }, - module.ModuleCall{ + module.InstalledModuleCall{ LocalName: "refname", SourceAddr: "./local", }, @@ -329,7 +330,7 @@ func TestSchemaForDependentModuleBlock_DocsLink(t *testing.T) { Outputs: map[string]module.Output{}, Filenames: nil, }, - module.ModuleCall{ + module.InstalledModuleCall{ LocalName: "vpc", SourceAddr: "registry.terraform.io/terraform-aws-modules/vpc/aws", }, @@ -359,10 +360,10 @@ func TestSchemaForDependentModuleBlock_DocsLink(t *testing.T) { Outputs: map[string]module.Output{}, Filenames: nil, }, - module.ModuleCall{ + module.InstalledModuleCall{ LocalName: "vpc", SourceAddr: "registry.terraform.io/terraform-aws-modules/vpc/aws", - Version: "1.33.7", + Version: version.Must(version.NewVersion("1.33.7")), }, &schema.BodySchema{ Attributes: map[string]*schema.AttributeSchema{}, @@ -390,10 +391,10 @@ func TestSchemaForDependentModuleBlock_DocsLink(t *testing.T) { Outputs: map[string]module.Output{}, Filenames: nil, }, - module.ModuleCall{ + module.InstalledModuleCall{ LocalName: "vpc", SourceAddr: "example.com/terraform-aws-modules/vpc/aws", - Version: "1.33.7", + Version: version.Must(version.NewVersion("1.33.7")), }, &schema.BodySchema{ Attributes: map[string]*schema.AttributeSchema{}, diff --git a/schema/schema_merge.go b/schema/schema_merge.go index c501c2d9..5430fea0 100644 --- a/schema/schema_merge.go +++ b/schema/schema_merge.go @@ -20,7 +20,7 @@ type SchemaMerger struct { } type ModuleReader interface { - ModuleCalls(modPath string) ([]module.ModuleCall, error) + ModuleCalls(modPath string) (module.ModuleCalls, error) ModuleMeta(modPath string) (*module.Meta, error) } @@ -182,12 +182,13 @@ func (m *SchemaMerger) SchemaForModule(meta *module.Meta) (*schema.BodySchema, e if m.moduleReader != nil { reader := m.moduleReader - modules, err := reader.ModuleCalls(meta.Path) + mc, err := reader.ModuleCalls(meta.Path) if err != nil { return mergedSchema, nil } - for _, module := range modules { + // TODO: mc.Declared + for _, module := range mc.Installed { modMeta, err := reader.ModuleMeta(module.Path) if err != nil { continue diff --git a/schema/schema_merge_test.go b/schema/schema_merge_test.go index d6cd6332..2c5b15cf 100644 --- a/schema/schema_merge_test.go +++ b/schema/schema_merge_test.go @@ -392,12 +392,14 @@ func testModuleReader() ModuleReader { type testModuleReaderStruct struct { } -func (m *testModuleReaderStruct) ModuleCalls(modPath string) ([]module.ModuleCall, error) { - return []module.ModuleCall{ - { - LocalName: "example", - SourceAddr: "source", - Path: "path", +func (m *testModuleReaderStruct) ModuleCalls(modPath string) (module.ModuleCalls, error) { + return module.ModuleCalls{ + Installed: map[string]module.InstalledModuleCall{ + "example": { + LocalName: "example", + SourceAddr: "source", + Path: "path", + }, }, }, nil } @@ -424,12 +426,14 @@ func testRegistryModuleReader() ModuleReader { type testRegistryModuleReaderStruct struct { } -func (m *testRegistryModuleReaderStruct) ModuleCalls(modPath string) ([]module.ModuleCall, error) { - return []module.ModuleCall{ - { - LocalName: "remote-example", - SourceAddr: "registry.terraform.io/namespace/foobar", - Path: ".terraform/modules/remote-example", +func (m *testRegistryModuleReaderStruct) ModuleCalls(modPath string) (module.ModuleCalls, error) { + return module.ModuleCalls{ + Installed: map[string]module.InstalledModuleCall{ + "remote-example": { + LocalName: "remote-example", + SourceAddr: "registry.terraform.io/namespace/foobar", + Path: ".terraform/modules/remote-example", + }, }, }, nil }