Add varsFromEnvFiles config option

.goreleaser.yml

@@ -38,7 +38,7 @@ brews:
       name: craftypath-ci-bot
       email: craftypath-ci-bot@users.noreply.github.com
     folder: Formula
-    homepage:  https://github.com/craftypath/gotf/
+    homepage: https://github.com/craftypath/gotf/
     description: Handling multiple environments with Terraform made easy
     install: |
       bin.install "gotf"

README.md

@@ -101,6 +101,31 @@ Variables which are added to the Terraform environment via `TF_VAR_<var>=value`
 Module-specific variables which are added to the Terraform environment if the corresponding module is run via `TF_VAR_<var>=value` for commands that support them.
 Module-specific variables override global ones.
 
+#### `varsFromEnvFiles`
+
+Allows variables to be configured as env files (`name=value` per line) which can also be sourced from a shell script.
+`gotf` interprets these files and passes each entry via `TF_VAR_` environment variable.
+Names are automatically lower-cased to match the common Terraform style.
+This feature can be quite convenient if you create parts of your infrastructure via Terraform and other parts via shell scripts
+but want to have a common source for shared variables.
+This is also a workaround for getting rid of Terraform warnings in case a variable is not declared, which might happen if you use global var files for different modules.
+Comments and variable expansion are supported.
+
+```shell
+# comment for FOO
+FOO=foo
+
+# comment for BAR
+BAR="bar is just a $FOO"
+```
+
+This would then set as follows:
+
+```shell
+TF_VAR_foo=foo
+TV_VAR_bar="bar is just a foo"
+```
+
 #### `envs`
 
 Environment variables to be added to the Terraform process.
@@ -159,6 +184,9 @@ moduleVars:
   02_compute:
     myvar: value for compute
 
+varsFromEnvFiles:
+  - '{{ .Params.environment }}.env'
+
 envs:
   BAR: barvalue
   TEMPLATED_ENV: "{{ .Params.param }}"

cmd/gotf/gotf_test.go

@@ -54,9 +54,9 @@ func TestExecute(t *testing.T) {
 					want: []string{
 						".terraform/terraform-networking-prod.tfstate",
 						`bar = "module1_prod"
-envSpecificVar = "prodvalue"
+env_specific_var = "prodvalue"
 foo = "42"
-globalVar = "globalvalue"
+global_var = "globalvalue"
 mapvar = <<EOT
 {
   entry1 = {
@@ -70,6 +70,7 @@ mapvar = <<EOT
 }
 EOT
 myvar = "value for networking"
+var_from_env_file = "prod-env"
 `},
 				},
 			},
@@ -93,9 +94,9 @@ myvar = "value for networking"
 					want: []string{
 						".terraform/terraform-compute-dev.tfstate",
 						`bar = "module2_dev"
-envSpecificVar = "devvalue"
+env_specific_var = "devvalue"
 foo = "42"
-globalVar = "globalvalue"
+global_var = "globalvalue"
 mapvar = <<EOT
 {
   entry1 = {
@@ -109,6 +110,7 @@ mapvar = <<EOT
 }
 EOT
 myvar = "value for compute"
+var_from_env_file = "dev-env"
 `},
 				},
 			},

cmd/gotf/testdata/01_networking/test.tf

@@ -10,9 +10,11 @@ variable "mapvar" {}
 
 variable "myvar" {}
 
-variable "globalVar" {}
+variable "global_var" {}
 
-variable "envSpecificVar" {}
+variable "env_specific_var" {}
+
+variable "var_from_env_file" {}
 
 output "bar" {
   value = var.bar
@@ -23,19 +25,23 @@ output "foo" {
 }
 
 output "mapvar" {
-  value   = var.mapvar
+  value = var.mapvar
+}
+
+output "global_var" {
+  value = var.global_var
 }
 
-output "globalVar" {
-  value   = var.globalVar
+output "env_specific_var" {
+  value = var.env_specific_var
 }
 
-output "envSpecificVar" {
-  value   = var.envSpecificVar
+output "var_from_env_file" {
+  value = var.var_from_env_file
 }
 
 output "myvar" {
-  value   = var.myvar
+  value = var.myvar
 }
 
 resource "null_resource" "echo" {

cmd/gotf/testdata/02_compute/test.tf

@@ -10,9 +10,11 @@ variable "mapvar" {}
 
 variable "myvar" {}
 
-variable "globalVar" {}
+variable "global_var" {}
 
-variable "envSpecificVar" {}
+variable "env_specific_var" {}
+
+variable "var_from_env_file" {}
 
 output "bar" {
   value = var.bar
@@ -23,19 +25,23 @@ output "foo" {
 }
 
 output "mapvar" {
-  value   = var.mapvar
+  value = var.mapvar
+}
+
+output "global_var" {
+  value = var.global_var
 }
 
-output "globalVar" {
-  value   = var.globalVar
+output "env_specific_var" {
+  value = var.env_specific_var
 }
 
-output "envSpecificVar" {
-  value   = var.envSpecificVar
+output "var_from_env_file" {
+  value = var.var_from_env_file
 }
 
 output "myvar" {
-  value   = var.myvar
+  value = var.myvar
 }
 
 resource "null_resource" "echo" {

cmd/gotf/testdata/dev.env

@@ -0,0 +1 @@
+VAR_FROM_ENV_FILE=dev-env

cmd/gotf/testdata/global-dev.tfvars

@@ -1 +1 @@
-envSpecificVar = "devvalue"
+env_specific_var = "devvalue"

cmd/gotf/testdata/global-prod.tfvars

@@ -1 +1 @@
-envSpecificVar = "prodvalue"
+env_specific_var = "prodvalue"

cmd/gotf/testdata/global.tfvars

@@ -1 +1 @@
-globalVar = "globalvalue"
+global_var = "globalvalue"

cmd/gotf/testdata/prod.env

@@ -0,0 +1 @@
+VAR_FROM_ENV_FILE=prod-env

cmd/gotf/testdata/test-config.yaml

@@ -45,6 +45,9 @@ moduleVars:
   02_compute:
     myvar: value for compute
 
+varsFromEnvFiles:
+  - '{{ .Params.environment }}.env'
+
 envs:
   BAR: barvalue
   TEMPLATED_ENV: "{{ .Params.param }}"

demo/01_first/variables.tf

@@ -10,5 +10,5 @@ variable "use_special_chars" {
 
 variable "module_specific_messages" {
   description = "Messages to print to the console"
-  type = list(string)
+  type        = list(string)
 }

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/golangci/golangci-lint v1.44.0
 	github.com/goreleaser/goreleaser v1.4.1
 	github.com/hashicorp/go-multierror v1.1.1
+	github.com/joho/godotenv v1.4.0
 	github.com/magefile/mage v1.11.0
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/spf13/cobra v1.3.0

go.sum

@@ -780,6 +780,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.2.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/txtarfs v0.0.0-20210218200122-0702f000015a/go.mod h1:izVPOvVRsHiKkeGCT6tYBNWyDVuzj9wAaBb5R9qamfw=

pkg/config/config.go

@@ -24,6 +24,7 @@ import (
 	"text/template"
 
 	"github.com/Masterminds/sprig/v3"
+	"github.com/joho/godotenv"
 	"gopkg.in/yaml.v2"
 )
 
@@ -38,6 +39,7 @@ type fileConfig struct {
 	GlobalVars            map[string]interface{}            `yaml:"globalVars"`
 	ModuleVars            map[string]map[string]interface{} `yaml:"moduleVars"`
 	Envs                  map[string]string                 `yaml:"envs"`
+	VarsFromEnvFiles      []string                          `yaml:"varsFromEnvFiles"`
 	BackendConfigs        map[string]string                 `yaml:"backendConfigs"`
 	IgnoreMissingVarFiles bool                              `yaml:"ignoreMissingVarFiles"`
 }
@@ -94,9 +96,8 @@ func Load(configFile string, modulePath string, cliParams map[string]string) (*C
 		BackendConfigs:   make(map[string]string),
 	}
 
-	log.Println("Processing global var files...")
 	for _, f := range fileCfg.GlobalVarFiles {
-		varFilePath, err := computeModuleRelativeVarFilePath(f, params, cfgFileDir, modulePath)
+		varFilePath, err := computeModuleRelativePath(f, params, cfgFileDir, modulePath)
 		if err != nil {
 			return nil, err
 		}
@@ -105,17 +106,26 @@ func Load(configFile string, modulePath string, cliParams map[string]string) (*C
 		}
 	}
 
+	log.Println("Processing module var files...")
 	moduleDir := params[moduleDirParamName].(string)
-	if moduleVarFiles, ok := fileCfg.ModuleVarFiles[moduleDir]; ok {
-		log.Println("Processing module var files...")
-		for _, f := range moduleVarFiles {
-			varFilePath, err := computeModuleRelativeVarFilePath(f, params, cfgFileDir, modulePath)
-			if err != nil {
-				return nil, err
-			}
-			if err := maybeAppendValFile(cfg, fileCfg.IgnoreMissingVarFiles, varFilePath, modulePath); err != nil {
-				return nil, err
-			}
+	for _, f := range fileCfg.ModuleVarFiles[moduleDir] {
+		varFilePath, err := computeModuleRelativePath(f, params, cfgFileDir, modulePath)
+		if err != nil {
+			return nil, err
+		}
+		if err := maybeAppendValFile(cfg, fileCfg.IgnoreMissingVarFiles, varFilePath, modulePath); err != nil {
+			return nil, err
+		}
+	}
+
+	log.Println("Processing vars from env files...")
+	for _, f := range fileCfg.VarsFromEnvFiles {
+		path, err := computeModuleRelativePath(f, params, cfgFileDir, modulePath)
+		if err != nil {
+			return nil, err
+		}
+		if err := maybeAppendVarsFromEnvFile(cfg, fileCfg.IgnoreMissingVarFiles, path, modulePath); err != nil {
+			return nil, err
 		}
 	}
 
@@ -128,15 +138,13 @@ func Load(configFile string, modulePath string, cliParams map[string]string) (*C
 		cfg.Vars[key] = result
 	}
 
-	if moduleVars, ok := fileCfg.ModuleVars[moduleDir]; ok {
-		log.Println("Processing module vars...")
-		for key, value := range moduleVars {
-			result, err := computeValue(value, params)
-			if err != nil {
-				return nil, err
-			}
-			cfg.Vars[key] = result
+	log.Println("Processing module vars...")
+	for key, value := range fileCfg.ModuleVars[moduleDir] {
+		result, err := computeValue(value, params)
+		if err != nil {
+			return nil, err
 		}
+		cfg.Vars[key] = result
 	}
 
 	log.Println("Processing envs...")
@@ -187,6 +195,34 @@ func maybeAppendValFile(cfg *Config, ignoreMissingVarFiles bool, varFilePath str
 	return nil
 }
 
+func maybeAppendVarsFromEnvFile(cfg *Config, IgnoreMissingVarFiles bool, varFilePath string, modulePath string) error {
+	var path string
+	if filepath.IsAbs(varFilePath) {
+		path = varFilePath
+	} else {
+		path = filepath.Join(modulePath, varFilePath)
+	}
+
+	if IgnoreMissingVarFiles {
+		_, err := os.Stat(path)
+		if err != nil {
+			if os.IsNotExist(err) {
+				log.Println(fmt.Sprintf("File %s does not exist. Ignoring it.", path))
+				return nil
+			}
+		}
+	}
+
+	envs, err := godotenv.Read(path)
+	if err != nil {
+		return err
+	}
+	for key, value := range envs {
+		cfg.Vars[strings.ToLower(key)] = value
+	}
+	return nil
+}
+
 func checkRequiredParams(fileCfg *fileConfig, cliParams map[string]string) error {
 	for k, v := range fileCfg.RequiredParams {
 		value, ok := cliParams[k]
@@ -230,22 +266,22 @@ func renderTemplate(data map[string]interface{}, tmpl string) (string, error) {
 	return wr.String(), nil
 }
 
-func computeModuleRelativeVarFilePath(varFilePathTemplate string, params map[string]interface{}, cfgFileDir string, modulePath string) (string, error) {
+func computeModuleRelativePath(pathTemplate string, params map[string]interface{}, cfgFileDir string, modulePath string) (string, error) {
 	templatingInput := map[string]interface{}{
 		"Params": params,
 	}
-	varFilePath, err := renderTemplate(templatingInput, varFilePathTemplate)
+	path, err := renderTemplate(templatingInput, pathTemplate)
 	if err != nil {
 		return "", err
 	}
-	if !filepath.IsAbs(varFilePath) {
-		varFilePath := filepath.Join(cfgFileDir, varFilePath)
+	if !filepath.IsAbs(path) {
+		varFilePath := filepath.Join(cfgFileDir, path)
 		if varFilePath, err = filepath.Rel(modulePath, varFilePath); err != nil {
 			return "", err
 		}
 		return varFilePath, nil
 	}
-	return varFilePath, nil
+	return path, nil
 }
 
 func computeValue(valueTemplate interface{}, params map[string]interface{}) (string, error) {