From 2597df798fd1b8541cd14aa8e55c7394718f045a Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 16 Aug 2019 13:25:07 -0700 Subject: [PATCH 01/24] Better error message for unapplied dependencies, as well as an out so that you can run validate --- README.md | 61 +++++++++ config/dependency.go | 123 ++++++++++++++++-- config/dependency_test.go | 40 ++++++ .../default-outputs/dependent1/main.tf | 5 + .../default-outputs/dependent1/terragrunt.hcl | 10 ++ .../default-outputs/dependent2/main.tf | 5 + .../default-outputs/dependent2/terragrunt.hcl | 11 ++ .../default-outputs/source/main.tf | 3 + .../default-outputs/source/terragrunt.hcl | 1 + test/integration_test.go | 118 +++++++++++++++++ 10 files changed, 369 insertions(+), 8 deletions(-) create mode 100644 test/fixture-get-output/default-outputs/dependent1/main.tf create mode 100644 test/fixture-get-output/default-outputs/dependent1/terragrunt.hcl create mode 100644 test/fixture-get-output/default-outputs/dependent2/main.tf create mode 100644 test/fixture-get-output/default-outputs/dependent2/terragrunt.hcl create mode 100644 test/fixture-get-output/default-outputs/source/main.tf create mode 100644 test/fixture-get-output/default-outputs/source/terragrunt.hcl diff --git a/README.md b/README.md index d847fcb00c..4cce763e00 100644 --- a/README.md +++ b/README.md @@ -1114,6 +1114,67 @@ If any of the modules failed to deploy, then Terragrunt will not attempt to depl **Note**: Not all blocks are able to access outputs passed by `dependency` blocks. See the section on [Configuration parsing order](#configuration-parsing-order) in this README for more information. +##### Unapplied dependency and default outputs + +Terragrunt will return an error indicating the dependency hasn't been applied yet if the config referenced in a +`dependency` block has not been applied yet. This is because you cannot actually fetch outputs out of an unapplied +Terraform module, even if there are no resources being created in the module. + +This is most prominent when running commands that do not modify state (e.g `plan-all` and `validate-all`) on a +completely new setup where no infrastructure has been deployed. You won't be able to `plan` or `validate` a module if +you can't determine the `inputs`. If the module depends on the outputs of another module that hasn't been applied +yet, you won't be able to compute the `inputs` unless the dependencies are all applied. + +To address this, you can provide default outputs to use when a module hasn't been applied yet. This is configured using +the `default_outputs` attribute on the `dependency` block and it corresponds to a map that will be injected in place of +the actual dependency outputs if the target config hasn't been applied yet. + +For example, in the previous example with a `mysql` module and `vpc` module, suppose you wanted to place in a temporary, +dummy value for the `vpc_id` during a `validate-all` for the `mysql` module. You can specify in `mysql/terragrunt.hcl`: + +``` +dependency "vpc" { + config_path = "../vpc" + + default_outputs = { + vpc_id = "temporary-dummy-id" + } +} + +inputs = { + vpc_id = dependency.vpc.outputs.vpc_id +} +``` + +You can now run `validate` on this config before the `vpc` module is applied because Terragrunt will use the map +`{vpc_id = "temporary-dummy-id"}` as the `outputs` attribute on the dependency instead of erroring out. + +What if you wanted to restrict this behavior to only the `validate` command? For example, you might not want to use +the defaults for a `plan` operation because the plan doesn't give you any indication of what is actually going to be +created. + +You can use the `default_outputs_allowed_terraform_commands` attribute to indicate that the `default_outputs` should +only be used when running those Terraform commands. So to restrict the `default_outputs` to only when `validate` is +being run, you can modify the above `terragrunt.hcl` file to: + +``` +dependency "vpc" { + config_path = "../vpc" + + default_outputs = { + vpc_id = "temporary-dummy-id" + } + default_outputs_allowed_terraform_commands = ["validate"] +} + +inputs = { + vpc_id = dependency.vpc.outputs.vpc_id +} +``` + +Note that indicating `validate` means that the `default_outputs` will be used either with `validate` or with +`validate-all`. + #### Dependencies between modules diff --git a/config/dependency.go b/config/dependency.go index 31337b45ee..06d7f2e5c0 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "path/filepath" + "strings" "github.com/hashicorp/hcl2/hcl" "github.com/zclconf/go-cty/cty" @@ -18,8 +19,10 @@ import ( ) type Dependency struct { - Name string `hcl:",label"` - ConfigPath string `hcl:"config_path,attr"` + Name string `hcl:",label"` + ConfigPath string `hcl:"config_path,attr"` + DefaultOutputs *cty.Value `hcl:"default_outputs,attr"` + DefaultOutputsAllowedTerraformCommands *[]string `hcl:"default_outputs_allowed_terraform_commands,attr"` } // Decode the dependency blocks from the file, and then retrieve all the outputs from the remote state. Then encode the @@ -76,7 +79,7 @@ func dependencyBlocksToCtyValue(dependencyConfigs []Dependency, terragruntOption // Encode the outputs and nest under `outputs` attribute paths = append(paths, dependencyConfig.ConfigPath) - outputVal, err := getTerragruntOutput(dependencyConfig, terragruntOptions) + outputVal, err := getTerragruntOutputIfAppliedElseConfiguredDefault(dependencyConfig, terragruntOptions) if err != nil { return nil, err } @@ -101,11 +104,9 @@ func dependencyBlocksToCtyValue(dependencyConfigs []Dependency, terragruntOption return &convertedOutput, errors.WithStackTrace(err) } -// Return the output from the state of another module, managed by terragrunt. This function will parse the provided -// terragrunt config and extract the desired output from the remote state. Note that this will error if the targetted -// module hasn't been applied yet. -func getTerragruntOutput(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) (*cty.Value, error) { - // target config check: make sure the target config exists +// Returns a cleaned path to the target config (the `terragrunt.hcl` file) encoded in the Dependency, handling relative +// paths correctly. This will automatically append `terragrunt.hcl` to the path if the target path is a directory. +func getCleanedTargetConfigPath(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) string { cwd := filepath.Dir(terragruntOptions.TerragruntConfigPath) targetConfig := dependencyConfig.ConfigPath if !filepath.IsAbs(targetConfig) { @@ -114,6 +115,86 @@ func getTerragruntOutput(dependencyConfig Dependency, terragruntOptions *options if util.IsDir(targetConfig) { targetConfig = util.JoinPath(targetConfig, DefaultTerragruntConfigPath) } + return targetConfig +} + +// This will attempt to get the outputs from the target terragrunt config if it is applied. If it is not applied, the +// behavior is different depending on the configuration of the dependency: +// - If the dependency block indicates a default_outputs attribute, this will return that. +// - If the dependency block does NOT indicate a default_outputs attribute, this will return an error. +func getTerragruntOutputIfAppliedElseConfiguredDefault(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) (*cty.Value, error) { + isApplied, err := isTerragruntApplied(dependencyConfig, terragruntOptions) + if err != nil { + return nil, err + } + + if isApplied { + return getTerragruntOutput(dependencyConfig, terragruntOptions) + } + + // When the module is not applied, check if there is a default outputs value to return. If yes, return that. Else, + // return error. + targetConfig := getCleanedTargetConfigPath(dependencyConfig, terragruntOptions) + currentConfig := terragruntOptions.TerragruntConfigPath + if shouldReturnDefaultOutputs(dependencyConfig, terragruntOptions) { + util.Debugf( + terragruntOptions.Logger, + "WARNING: config %s is a dependency of %s that hasn't been applied, but default outputs provided and returning those in dependency output.", + targetConfig, + currentConfig, + ) + return dependencyConfig.DefaultOutputs, nil + } + + err = TerragruntOutputTargetNotApplied{ + targetConfig: targetConfig, + currentConfig: currentConfig, + } + return nil, err +} + +// We should only return default outputs if the default_outputs attribute is set, and if we are running one of the +// allowed commands when `default_outputs_allowed_terraform_commands` is set as well. +func shouldReturnDefaultOutputs(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) bool { + defaultOutputsSet := dependencyConfig.DefaultOutputs != nil + allowedCommand := + dependencyConfig.DefaultOutputsAllowedTerraformCommands == nil || + len(*dependencyConfig.DefaultOutputsAllowedTerraformCommands) == 0 || + util.ListContainsElement(*dependencyConfig.DefaultOutputsAllowedTerraformCommands, terragruntOptions.TerraformCommand) + return defaultOutputsSet && allowedCommand +} + +// Return whether or not the target config has been applied. Returns false when the target config hasn't been applied, +// and true when it has. +func isTerragruntApplied(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) (bool, error) { + // target config check: make sure the target config exists + targetConfig := getCleanedTargetConfigPath(dependencyConfig, terragruntOptions) + if !util.FileExists(targetConfig) { + return false, errors.WithStackTrace(DependencyConfigNotFound{Path: targetConfig}) + } + + // Check if the target terraform module (managed by terragrunt target config) has been applied. + // When a module hasn't been applied yet, there will be no state object, so `terragrunt show` will indicate that. + stdout, err := runTerragruntShow(terragruntOptions, targetConfig) + stdout = strings.TrimSpace(stdout) + isApplied := stdout != "No state." + util.Debugf( + terragruntOptions.Logger, + "Dependency %s of %s is applied: %v (output of terragrunt show: %s)", + targetConfig, + terragruntOptions.TerragruntConfigPath, + isApplied, + stdout, + ) + return isApplied, err +} + +// Return the output from the state of another module, managed by terragrunt. This function will parse the provided +// terragrunt config and extract the desired output from the remote state. Note that this will error if the targetted +// module hasn't been applied yet. +func getTerragruntOutput(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) (*cty.Value, error) { + // target config check: make sure the target config exists + targetConfig := getCleanedTargetConfigPath(dependencyConfig, terragruntOptions) if !util.FileExists(targetConfig) { return nil, errors.WithStackTrace(DependencyConfigNotFound{Path: targetConfig}) } @@ -136,6 +217,23 @@ func getTerragruntOutput(dependencyConfig Dependency, terragruntOptions *options return &convertedOutput, errors.WithStackTrace(err) } +// runTerragruntShow uses terragrunt running functions to show the current state of the target config. +func runTerragruntShow(terragruntOptions *options.TerragruntOptions, targetConfig string) (string, error) { + var stdoutBuffer bytes.Buffer + stdoutBufferWriter := bufio.NewWriter(&stdoutBuffer) + targetOptions := terragruntOptions.Clone(targetConfig) + targetOptions.TerraformCliArgs = []string{"show", "-no-color"} + targetOptions.Writer = stdoutBufferWriter + err := targetOptions.RunTerragrunt(targetOptions) + if err != nil { + return "", errors.WithStackTrace(err) + } + + stdoutBufferWriter.Flush() + outputString := stdoutBuffer.String() + return outputString, nil +} + // runTerragruntOutputJson uses terragrunt running functions to extract the json output from the target config. Make a // copy of the terragruntOptions so that we can reuse the same execution environment. func runTerragruntOutputJson(terragruntOptions *options.TerragruntOptions, targetConfig string) ([]byte, error) { @@ -223,3 +321,12 @@ type TerragruntOutputListEncodingError struct { func (err TerragruntOutputListEncodingError) Error() string { return fmt.Sprintf("Could not encode output from list of terragrunt configs %v. Underlying error: %s", err.Paths, err.Err) } + +type TerragruntOutputTargetNotApplied struct { + targetConfig string + currentConfig string +} + +func (err TerragruntOutputTargetNotApplied) Error() string { + return fmt.Sprintf("%s is a dependency of %s but is not applied yet so can not get outputs.", err.targetConfig, err.currentConfig) +} diff --git a/config/dependency_test.go b/config/dependency_test.go index d7b807d01c..9e338b8390 100644 --- a/config/dependency_test.go +++ b/config/dependency_test.go @@ -6,6 +6,7 @@ import ( "github.com/hashicorp/hcl2/hclparse" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/zclconf/go-cty/cty/gocty" ) func TestDecodeDependencyBlockMultiple(t *testing.T) { @@ -69,3 +70,42 @@ dependency { decoded := terragruntDependency{} require.Error(t, decodeHcl(file, filename, &decoded, mockOptionsForTest(t), EvalContextExtensions{})) } + +func TestDecodeDependencyDefaultOutputs(t *testing.T) { + t.Parallel() + + config := ` +dependency "hitchhiker" { + config_path = "../answers" + default_outputs = { + the_answer = 42 + } + default_outputs_allowed_terraform_commands = ["validate", "apply"] +} +` + filename := DefaultTerragruntConfigPath + parser := hclparse.NewParser() + file, err := parseHcl(parser, config, filename) + require.NoError(t, err) + + decoded := terragruntDependency{} + require.NoError(t, decodeHcl(file, filename, &decoded, mockOptionsForTest(t), EvalContextExtensions{})) + + assert.Equal(t, len(decoded.Dependencies), 1) + dependency := decoded.Dependencies[0] + assert.Equal(t, dependency.Name, "hitchhiker") + assert.Equal(t, dependency.ConfigPath, "../answers") + + ctyValueDefault := dependency.DefaultOutputs + require.NotNil(t, ctyValueDefault) + + var actualDefault struct { + TheAnswer int `cty:"the_answer"` + } + require.NoError(t, gocty.FromCtyValue(*ctyValueDefault, &actualDefault)) + assert.Equal(t, actualDefault.TheAnswer, 42) + + defaultAllowedCommands := dependency.DefaultOutputsAllowedTerraformCommands + require.NotNil(t, defaultAllowedCommands) + assert.Equal(t, *defaultAllowedCommands, []string{"validate", "apply"}) +} diff --git a/test/fixture-get-output/default-outputs/dependent1/main.tf b/test/fixture-get-output/default-outputs/dependent1/main.tf new file mode 100644 index 0000000000..472c109d8f --- /dev/null +++ b/test/fixture-get-output/default-outputs/dependent1/main.tf @@ -0,0 +1,5 @@ +variable "the_answer" {} + +output "truth" { + value = "The answer is ${var.the_answer}" +} diff --git a/test/fixture-get-output/default-outputs/dependent1/terragrunt.hcl b/test/fixture-get-output/default-outputs/dependent1/terragrunt.hcl new file mode 100644 index 0000000000..7aaf849e81 --- /dev/null +++ b/test/fixture-get-output/default-outputs/dependent1/terragrunt.hcl @@ -0,0 +1,10 @@ +dependency "source" { + config_path = "../source" + default_outputs = { + the_answer = "0" + } +} + +inputs = { + the_answer = dependency.source.outputs.the_answer +} diff --git a/test/fixture-get-output/default-outputs/dependent2/main.tf b/test/fixture-get-output/default-outputs/dependent2/main.tf new file mode 100644 index 0000000000..f450730b6d --- /dev/null +++ b/test/fixture-get-output/default-outputs/dependent2/main.tf @@ -0,0 +1,5 @@ +variable "the_answer" {} + +output "fake" { + value = "never ${var.the_answer}" +} diff --git a/test/fixture-get-output/default-outputs/dependent2/terragrunt.hcl b/test/fixture-get-output/default-outputs/dependent2/terragrunt.hcl new file mode 100644 index 0000000000..d3d7438ba1 --- /dev/null +++ b/test/fixture-get-output/default-outputs/dependent2/terragrunt.hcl @@ -0,0 +1,11 @@ +dependency "source" { + config_path = "../source" + default_outputs = { + the_answer = "0" + } + default_outputs_allowed_terraform_commands = ["validate"] +} + +inputs = { + the_answer = dependency.source.outputs.the_answer +} diff --git a/test/fixture-get-output/default-outputs/source/main.tf b/test/fixture-get-output/default-outputs/source/main.tf new file mode 100644 index 0000000000..34e03fc2b6 --- /dev/null +++ b/test/fixture-get-output/default-outputs/source/main.tf @@ -0,0 +1,3 @@ +output "the_answer" { + value = 42 +} diff --git a/test/fixture-get-output/default-outputs/source/terragrunt.hcl b/test/fixture-get-output/default-outputs/source/terragrunt.hcl new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/default-outputs/source/terragrunt.hcl @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/integration_test.go b/test/integration_test.go index c5f995e44c..99f0726ad9 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "math/rand" "os" + "path/filepath" "regexp" "strings" "testing" @@ -1605,6 +1606,123 @@ func TestDependencyOutput(t *testing.T) { assert.Equal(t, int(outputs["z"].Value.(float64)), 42) } +func TestDependencyOutputErrorBeforeApply(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) + rootPath := filepath.Join(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "integration") + app3Path := filepath.Join(rootPath, "app3") + + showStdout := bytes.Buffer{} + showStderr := bytes.Buffer{} + + err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-working-dir %s", app3Path), &showStdout, &showStderr) + assert.Error(t, err) + // Verify that we fail because the dependency is not applied yet + assert.Contains(t, err.Error(), "is not applied yet") + + logBufferContentsLineByLine(t, showStdout, "show stdout") + logBufferContentsLineByLine(t, showStderr, "show stderr") +} + +// Test that when you have an output default on a dependency, the dependency will use the default as the output instead +// of erroring out. +func TestDependencyOutputDefaults(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) + rootPath := filepath.Join(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "default-outputs") + dependent1Path := filepath.Join(rootPath, "dependent1") + + showStdout := bytes.Buffer{} + showStderr := bytes.Buffer{} + + err := runTerragruntCommand(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", dependent1Path), &showStdout, &showStderr) + assert.NoError(t, err) + + logBufferContentsLineByLine(t, showStdout, "show stdout") + logBufferContentsLineByLine(t, showStderr, "show stderr") + + // verify expected output when defaults are used: The answer is 0 + stdout := bytes.Buffer{} + stderr := bytes.Buffer{} + require.NoError( + t, + runTerragruntCommand(t, fmt.Sprintf("terragrunt output -no-color -json --terragrunt-non-interactive --terragrunt-working-dir %s", dependent1Path), &stdout, &stderr), + ) + outputs := map[string]TerraformOutput{} + require.NoError(t, json.Unmarshal([]byte(stdout.String()), &outputs)) + assert.Equal(t, outputs["truth"].Value, "The answer is 0") + + // Now apply-all so that the dependency is applied, and verify it uses the dependency output + err = runTerragruntCommand(t, fmt.Sprintf("terragrunt apply-all --terragrunt-non-interactive --terragrunt-working-dir %s", rootPath), &showStdout, &showStderr) + assert.NoError(t, err) + + logBufferContentsLineByLine(t, showStdout, "show stdout") + logBufferContentsLineByLine(t, showStderr, "show stderr") + + // verify expected output when defaults are used: The answer is 0 + stdout = bytes.Buffer{} + stderr = bytes.Buffer{} + require.NoError( + t, + runTerragruntCommand(t, fmt.Sprintf("terragrunt output -no-color -json --terragrunt-non-interactive --terragrunt-working-dir %s", dependent1Path), &stdout, &stderr), + ) + outputs = map[string]TerraformOutput{} + require.NoError(t, json.Unmarshal([]byte(stdout.String()), &outputs)) + assert.Equal(t, outputs["truth"].Value, "The answer is 42") +} + +// Test that when you have an output default on a dependency, the dependency will use the default as the output instead +// of erroring out when running an allowed command. +func TestDependencyOutputDefaultsRestricted(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) + rootPath := filepath.Join(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "default-outputs") + dependent2Path := filepath.Join(rootPath, "dependent2") + + showStdout := bytes.Buffer{} + showStderr := bytes.Buffer{} + + err := runTerragruntCommand(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", dependent2Path), &showStdout, &showStderr) + assert.Error(t, err) + // Verify that we fail because the dependency is not applied yet + assert.Contains(t, err.Error(), "is not applied yet") + + logBufferContentsLineByLine(t, showStdout, "show stdout") + logBufferContentsLineByLine(t, showStderr, "show stderr") + + // Verify we can run when using one of the allowed commands + showStdout.Reset() + showStderr.Reset() + err = runTerragruntCommand(t, fmt.Sprintf("terragrunt validate --terragrunt-non-interactive --terragrunt-working-dir %s", dependent2Path), &showStdout, &showStderr) + assert.NoError(t, err) + + logBufferContentsLineByLine(t, showStdout, "show stdout") + logBufferContentsLineByLine(t, showStderr, "show stderr") + + // Verify that validate-all works as well. + showStdout.Reset() + showStderr.Reset() + err = runTerragruntCommand(t, fmt.Sprintf("terragrunt validate-all --terragrunt-non-interactive --terragrunt-working-dir %s", dependent2Path), &showStdout, &showStderr) + assert.NoError(t, err) + + logBufferContentsLineByLine(t, showStdout, "show stdout") + logBufferContentsLineByLine(t, showStderr, "show stderr") + + showStdout.Reset() + showStderr.Reset() + err = runTerragruntCommand(t, fmt.Sprintf("terragrunt validate-all --terragrunt-non-interactive --terragrunt-working-dir %s", rootPath), &showStdout, &showStderr) + assert.NoError(t, err) + + logBufferContentsLineByLine(t, showStdout, "show stdout") + logBufferContentsLineByLine(t, showStderr, "show stderr") +} + func TestDependencyOutputTypeConversion(t *testing.T) { t.Parallel() From 637115fbb8fdc3ff34b81e39737579314cf59325 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 16 Aug 2019 13:25:30 -0700 Subject: [PATCH 02/24] Color the error at the end in red to highlight it --- Gopkg.lock | 25 +++++++++++++++++++++++++ main.go | 8 ++++++-- util/logger.go | 13 +++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 4a1c875dfc..895bfa1c05 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -104,6 +104,14 @@ revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" version = "v1.1.1" +[[projects]] + digest = "1:865079840386857c809b72ce300be7580cb50d3d3129ce11bf9aa6ca2bc1934a" + name = "github.com/fatih/color" + packages = ["."] + pruneopts = "UT" + revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" + version = "v1.7.0" + [[projects]] digest = "1:aacef5f5e45685f2aeda5534d0a750dee6859de7e9088cdd06192787bb01ae6d" name = "github.com/go-errors/errors" @@ -264,6 +272,22 @@ pruneopts = "UT" revision = "c2b33e84" +[[projects]] + digest = "1:c658e84ad3916da105a761660dcaeb01e63416c8ec7bc62256a9b411a05fcd67" + name = "github.com/mattn/go-colorable" + packages = ["."] + pruneopts = "UT" + revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" + version = "v0.0.9" + +[[projects]] + digest = "1:36325ebb862e0382f2f14feef409ba9351271b89ada286ae56836c603d43b59c" + name = "github.com/mattn/go-isatty" + packages = ["."] + pruneopts = "UT" + revision = "e1f7b56ace729e4a73a29a6b4fac6cd5fcda7ab3" + version = "v0.0.9" + [[projects]] digest = "1:36aebe90a13cf9128280ac834399b8bebf83685283c78df279d61c46bb2a8d83" name = "github.com/mattn/go-zglob" @@ -589,6 +613,7 @@ "github.com/aws/aws-sdk-go/service/dynamodb", "github.com/aws/aws-sdk-go/service/s3", "github.com/aws/aws-sdk-go/service/sts", + "github.com/fatih/color", "github.com/go-errors/errors", "github.com/gruntwork-io/terratest/modules/files", "github.com/hashicorp/go-getter", diff --git a/main.go b/main.go index 28eb49990b..e8a31b9294 100644 --- a/main.go +++ b/main.go @@ -1,11 +1,14 @@ package main import ( + "os" + + "github.com/fatih/color" + "github.com/gruntwork-io/terragrunt/cli" "github.com/gruntwork-io/terragrunt/errors" "github.com/gruntwork-io/terragrunt/shell" "github.com/gruntwork-io/terragrunt/util" - "os" ) // This variable is set at build time using -ldflags parameters. For more info, see: @@ -31,7 +34,8 @@ func checkForErrorsAndExit(err error) { if os.Getenv("TERRAGRUNT_DEBUG") != "" { logger.Println(errors.PrintErrorWithStackTrace(err)) } else { - logger.Println(err) + // Log error in red so that it is highlighted + util.ColorLogf(logger, color.New(color.FgRed), err.Error()) } // exit with the underlying error code exitCode, exitCodeErr := shell.GetExitCode(err) diff --git a/util/logger.go b/util/logger.go index 31ae2f7fda..a49384ff69 100644 --- a/util/logger.go +++ b/util/logger.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/fatih/color" "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hclparse" "golang.org/x/crypto/ssh/terminal" @@ -34,6 +35,18 @@ func Debugf(logger *log.Logger, fmtString string, fmtArgs ...interface{}) { } } +// ColorLogf +func ColorLogf(logger *log.Logger, colorCode *color.Color, fmtString string, fmtArgs ...interface{}) { + logOut := fmt.Sprintf(fmtString, fmtArgs...) + + loggerIsStderr := logger.Writer() == os.Stderr + allowColor := terminal.IsTerminal(int(os.Stderr.Fd())) + if loggerIsStderr && allowColor { + logOut = colorCode.SprintFunc()(logOut) + } + logger.Println(logOut) +} + // GetDiagnosticsWriter returns a hcl2 parsing diagnostics emitter for the current terminal. func GetDiagnosticsWriter(parser *hclparse.Parser) hcl.DiagnosticWriter { termColor := terminal.IsTerminal(int(os.Stderr.Fd())) From ed5e38469e0a19e5b7559ea7ed2c16d4b916d927 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 16 Aug 2019 13:31:02 -0700 Subject: [PATCH 03/24] Use go1.11 container --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4764ee2e2b..015570d608 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ workspace_root: &workspace_root defaults: &defaults working_directory: *workspace_root docker: - - image: 087285199408.dkr.ecr.us-east-1.amazonaws.com/circle-ci-test-image-base:tf-0.12 + - image: 087285199408.dkr.ecr.us-east-1.amazonaws.com/circle-ci-test-image-base:go1.11 version: 2 jobs: From e9f5aa24da9d995583c48089f761e709fa55179b Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 16 Aug 2019 13:37:27 -0700 Subject: [PATCH 04/24] logger.Writer is not available until go1.12 --- util/logger.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/util/logger.go b/util/logger.go index a49384ff69..dd43ddb2f3 100644 --- a/util/logger.go +++ b/util/logger.go @@ -39,9 +39,8 @@ func Debugf(logger *log.Logger, fmtString string, fmtArgs ...interface{}) { func ColorLogf(logger *log.Logger, colorCode *color.Color, fmtString string, fmtArgs ...interface{}) { logOut := fmt.Sprintf(fmtString, fmtArgs...) - loggerIsStderr := logger.Writer() == os.Stderr allowColor := terminal.IsTerminal(int(os.Stderr.Fd())) - if loggerIsStderr && allowColor { + if allowColor { logOut = colorCode.SprintFunc()(logOut) } logger.Println(logOut) From 6755bd68f555595aabf8ea00314b668f1878bbda Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 16 Aug 2019 14:00:53 -0700 Subject: [PATCH 05/24] Fix a few typos in the readme --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4cce763e00..78e7a10c9e 100644 --- a/README.md +++ b/README.md @@ -1116,14 +1116,15 @@ If any of the modules failed to deploy, then Terragrunt will not attempt to depl ##### Unapplied dependency and default outputs -Terragrunt will return an error indicating the dependency hasn't been applied yet if the config referenced in a -`dependency` block has not been applied yet. This is because you cannot actually fetch outputs out of an unapplied -Terraform module, even if there are no resources being created in the module. +Terragrunt will return an error indicating the dependency hasn't been applied yet if the terraform module managed by the +terragrunt config referenced in a `dependency` block has not been applied yet. This is because you cannot actually fetch +outputs out of an unapplied Terraform module, even if there are no resources being created in the module. -This is most prominent when running commands that do not modify state (e.g `plan-all` and `validate-all`) on a +This is most problematic when running commands that do not modify state (e.g `plan-all` and `validate-all`) on a completely new setup where no infrastructure has been deployed. You won't be able to `plan` or `validate` a module if you can't determine the `inputs`. If the module depends on the outputs of another module that hasn't been applied -yet, you won't be able to compute the `inputs` unless the dependencies are all applied. +yet, you won't be able to compute the `inputs` unless the dependencies are all applied. However, in real life usage, you +would want to run `validate-all` or `plan-all` on a completely new set of infrastructure. To address this, you can provide default outputs to use when a module hasn't been applied yet. This is configured using the `default_outputs` attribute on the `dependency` block and it corresponds to a map that will be injected in place of From e5a08bfc5b87ad0a63467612e623120d49bda648 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 16 Aug 2019 17:26:29 -0700 Subject: [PATCH 06/24] Fix isApplied logic to just introspect the output json to avoid the extra show call --- config/dependency.go | 78 +++++-------------- .../integration/empty/main.tf | 0 .../integration/empty/terragrunt.hcl | 1 + test/integration_test.go | 4 +- 4 files changed, 24 insertions(+), 59 deletions(-) create mode 100644 test/fixture-get-output/integration/empty/main.tf create mode 100644 test/fixture-get-output/integration/empty/terragrunt.hcl diff --git a/config/dependency.go b/config/dependency.go index 06d7f2e5c0..4f1ca3a972 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -123,30 +123,31 @@ func getCleanedTargetConfigPath(dependencyConfig Dependency, terragruntOptions * // - If the dependency block indicates a default_outputs attribute, this will return that. // - If the dependency block does NOT indicate a default_outputs attribute, this will return an error. func getTerragruntOutputIfAppliedElseConfiguredDefault(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) (*cty.Value, error) { - isApplied, err := isTerragruntApplied(dependencyConfig, terragruntOptions) + outputVal, isEmpty, err := getTerragruntOutput(dependencyConfig, terragruntOptions) if err != nil { return nil, err } - if isApplied { - return getTerragruntOutput(dependencyConfig, terragruntOptions) + if !isEmpty { + return outputVal, err } - // When the module is not applied, check if there is a default outputs value to return. If yes, return that. Else, + // When we get no output, it can be an indication that either the module has no outputs or the module is not + // applied. In either case, check if there are default output values to return. If yes, return that. Else, // return error. targetConfig := getCleanedTargetConfigPath(dependencyConfig, terragruntOptions) currentConfig := terragruntOptions.TerragruntConfigPath if shouldReturnDefaultOutputs(dependencyConfig, terragruntOptions) { util.Debugf( terragruntOptions.Logger, - "WARNING: config %s is a dependency of %s that hasn't been applied, but default outputs provided and returning those in dependency output.", + "WARNING: config %s is a dependency of %s that has no outputs, but default outputs provided and returning those in dependency output.", targetConfig, currentConfig, ) return dependencyConfig.DefaultOutputs, nil } - err = TerragruntOutputTargetNotApplied{ + err = TerragruntOutputTargetNoOutputs{ targetConfig: targetConfig, currentConfig: currentConfig, } @@ -164,49 +165,25 @@ func shouldReturnDefaultOutputs(dependencyConfig Dependency, terragruntOptions * return defaultOutputsSet && allowedCommand } -// Return whether or not the target config has been applied. Returns false when the target config hasn't been applied, -// and true when it has. -func isTerragruntApplied(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) (bool, error) { - // target config check: make sure the target config exists - targetConfig := getCleanedTargetConfigPath(dependencyConfig, terragruntOptions) - if !util.FileExists(targetConfig) { - return false, errors.WithStackTrace(DependencyConfigNotFound{Path: targetConfig}) - } - - // Check if the target terraform module (managed by terragrunt target config) has been applied. - // When a module hasn't been applied yet, there will be no state object, so `terragrunt show` will indicate that. - stdout, err := runTerragruntShow(terragruntOptions, targetConfig) - stdout = strings.TrimSpace(stdout) - isApplied := stdout != "No state." - util.Debugf( - terragruntOptions.Logger, - "Dependency %s of %s is applied: %v (output of terragrunt show: %s)", - targetConfig, - terragruntOptions.TerragruntConfigPath, - isApplied, - stdout, - ) - return isApplied, err -} - // Return the output from the state of another module, managed by terragrunt. This function will parse the provided // terragrunt config and extract the desired output from the remote state. Note that this will error if the targetted // module hasn't been applied yet. -func getTerragruntOutput(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) (*cty.Value, error) { +func getTerragruntOutput(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) (*cty.Value, bool, error) { // target config check: make sure the target config exists targetConfig := getCleanedTargetConfigPath(dependencyConfig, terragruntOptions) if !util.FileExists(targetConfig) { - return nil, errors.WithStackTrace(DependencyConfigNotFound{Path: targetConfig}) + return nil, true, errors.WithStackTrace(DependencyConfigNotFound{Path: targetConfig}) } jsonBytes, err := runTerragruntOutputJson(terragruntOptions, targetConfig) if err != nil { - return nil, err + return nil, true, err } + isEmpty := string(jsonBytes) == "{}" outputMap, err := terraformOutputJsonToCtyValueMap(targetConfig, jsonBytes) if err != nil { - return nil, err + return nil, isEmpty, err } // We need to convert the value map to a single cty.Value at the end for use in the terragrunt config. @@ -214,24 +191,7 @@ func getTerragruntOutput(dependencyConfig Dependency, terragruntOptions *options if err != nil { err = TerragruntOutputEncodingError{Path: targetConfig, Err: err} } - return &convertedOutput, errors.WithStackTrace(err) -} - -// runTerragruntShow uses terragrunt running functions to show the current state of the target config. -func runTerragruntShow(terragruntOptions *options.TerragruntOptions, targetConfig string) (string, error) { - var stdoutBuffer bytes.Buffer - stdoutBufferWriter := bufio.NewWriter(&stdoutBuffer) - targetOptions := terragruntOptions.Clone(targetConfig) - targetOptions.TerraformCliArgs = []string{"show", "-no-color"} - targetOptions.Writer = stdoutBufferWriter - err := targetOptions.RunTerragrunt(targetOptions) - if err != nil { - return "", errors.WithStackTrace(err) - } - - stdoutBufferWriter.Flush() - outputString := stdoutBuffer.String() - return outputString, nil + return &convertedOutput, isEmpty, errors.WithStackTrace(err) } // runTerragruntOutputJson uses terragrunt running functions to extract the json output from the target config. Make a @@ -249,7 +209,7 @@ func runTerragruntOutputJson(terragruntOptions *options.TerragruntOptions, targe stdoutBufferWriter.Flush() jsonString := stdoutBuffer.String() - jsonBytes := []byte(jsonString) + jsonBytes := []byte(strings.TrimSpace(jsonString)) util.Debugf(terragruntOptions.Logger, "Retrieved output from %s as json: %s", targetConfig, jsonString) return jsonBytes, nil } @@ -322,11 +282,15 @@ func (err TerragruntOutputListEncodingError) Error() string { return fmt.Sprintf("Could not encode output from list of terragrunt configs %v. Underlying error: %s", err.Paths, err.Err) } -type TerragruntOutputTargetNotApplied struct { +type TerragruntOutputTargetNoOutputs struct { targetConfig string currentConfig string } -func (err TerragruntOutputTargetNotApplied) Error() string { - return fmt.Sprintf("%s is a dependency of %s but is not applied yet so can not get outputs.", err.targetConfig, err.currentConfig) +func (err TerragruntOutputTargetNoOutputs) Error() string { + return fmt.Sprintf( + "%s is a dependency of %s but detected no outputs. Either the target module has not been applied yet, or the module has no outputs. If this is expected, set the skip_outputs flag to true on the dependency block.", + err.targetConfig, + err.currentConfig, + ) } diff --git a/test/fixture-get-output/integration/empty/main.tf b/test/fixture-get-output/integration/empty/main.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/fixture-get-output/integration/empty/terragrunt.hcl b/test/fixture-get-output/integration/empty/terragrunt.hcl new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/integration/empty/terragrunt.hcl @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/integration_test.go b/test/integration_test.go index 99f0726ad9..642947f912 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1620,7 +1620,7 @@ func TestDependencyOutputErrorBeforeApply(t *testing.T) { err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-working-dir %s", app3Path), &showStdout, &showStderr) assert.Error(t, err) // Verify that we fail because the dependency is not applied yet - assert.Contains(t, err.Error(), "is not applied yet") + assert.Contains(t, err.Error(), "has not been applied yet") logBufferContentsLineByLine(t, showStdout, "show stdout") logBufferContentsLineByLine(t, showStderr, "show stderr") @@ -1691,7 +1691,7 @@ func TestDependencyOutputDefaultsRestricted(t *testing.T) { err := runTerragruntCommand(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", dependent2Path), &showStdout, &showStderr) assert.Error(t, err) // Verify that we fail because the dependency is not applied yet - assert.Contains(t, err.Error(), "is not applied yet") + assert.Contains(t, err.Error(), "has not been applied yet") logBufferContentsLineByLine(t, showStdout, "show stdout") logBufferContentsLineByLine(t, showStderr, "show stderr") From f557763b82cc508b88c2401c2ddf6a3d50349115 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 16 Aug 2019 17:33:51 -0700 Subject: [PATCH 07/24] Add regression test that should fail on this commit --- config/dependency.go | 1 + .../integration/empty/terragrunt.hcl | 6 +++++- test/integration_test.go | 20 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/config/dependency.go b/config/dependency.go index 4f1ca3a972..b16427347d 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -21,6 +21,7 @@ import ( type Dependency struct { Name string `hcl:",label"` ConfigPath string `hcl:"config_path,attr"` + SkipOutputs *bool `hcl:"skip_outputs,attr"` DefaultOutputs *cty.Value `hcl:"default_outputs,attr"` DefaultOutputsAllowedTerraformCommands *[]string `hcl:"default_outputs_allowed_terraform_commands,attr"` } diff --git a/test/fixture-get-output/integration/empty/terragrunt.hcl b/test/fixture-get-output/integration/empty/terragrunt.hcl index bb7b160deb..cc7aada779 100644 --- a/test/fixture-get-output/integration/empty/terragrunt.hcl +++ b/test/fixture-get-output/integration/empty/terragrunt.hcl @@ -1 +1,5 @@ -# Intentionally empty +# Intentionally only has one dependency with skip_outputs to test logic that it doesn't attempt to pull the outputs. +dependency "app1" { + config_path = "../app1" + skip_outputs = true +} diff --git a/test/integration_test.go b/test/integration_test.go index 642947f912..b3fe6c4e97 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1626,6 +1626,26 @@ func TestDependencyOutputErrorBeforeApply(t *testing.T) { logBufferContentsLineByLine(t, showStderr, "show stderr") } +func TestDependencyOutputSkipOutputs(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) + rootPath := filepath.Join(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "integration") + emptyPath := filepath.Join(rootPath, "empty") + + showStdout := bytes.Buffer{} + showStderr := bytes.Buffer{} + + // Test that even if the dependency (app1) is not applied, using skip_outputs will skip pulling the outputs so there + // will be no errors. + err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-working-dir %s", emptyPath), &showStdout, &showStderr) + assert.NoError(t, err) + + logBufferContentsLineByLine(t, showStdout, "show stdout") + logBufferContentsLineByLine(t, showStderr, "show stderr") +} + // Test that when you have an output default on a dependency, the dependency will use the default as the output instead // of erroring out. func TestDependencyOutputDefaults(t *testing.T) { From de93c9a9b9300c358d12b900945f0a147a6319c0 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 16 Aug 2019 17:36:44 -0700 Subject: [PATCH 08/24] Implement skip_outputs --- config/dependency.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/config/dependency.go b/config/dependency.go index b16427347d..33de55e677 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -26,6 +26,11 @@ type Dependency struct { DefaultOutputsAllowedTerraformCommands *[]string `hcl:"default_outputs_allowed_terraform_commands,attr"` } +// Given a dependency config, we should only attempt to get the outputs if SkipOutputs is nil or false +func (dependencyConfig Dependency) shouldGetOutputs() bool { + return dependencyConfig.SkipOutputs == nil || !(*dependencyConfig.SkipOutputs) +} + // Decode the dependency blocks from the file, and then retrieve all the outputs from the remote state. Then encode the // resulting map as a cty.Value object. // TODO: In the future, consider allowing importing dependency blocks from included config @@ -78,13 +83,15 @@ func dependencyBlocksToCtyValue(dependencyConfigs []Dependency, terragruntOption // - outputs: The module outputs of the target config dependencyEncodingMap := map[string]cty.Value{} - // Encode the outputs and nest under `outputs` attribute - paths = append(paths, dependencyConfig.ConfigPath) - outputVal, err := getTerragruntOutputIfAppliedElseConfiguredDefault(dependencyConfig, terragruntOptions) - if err != nil { - return nil, err + // Encode the outputs and nest under `outputs` attribute if we should get the outputs + if dependencyConfig.shouldGetOutputs() { + paths = append(paths, dependencyConfig.ConfigPath) + outputVal, err := getTerragruntOutputIfAppliedElseConfiguredDefault(dependencyConfig, terragruntOptions) + if err != nil { + return nil, err + } + dependencyEncodingMap["outputs"] = *outputVal } - dependencyEncodingMap["outputs"] = *outputVal // Once the dependency is encoded into a map, we need to conver to a cty.Value again so that it can be fed to // the higher order dependency map. From 95cbfb31b8ed4b5cb16cbf41010bb42c3b5b6ac3 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 16 Aug 2019 17:45:10 -0700 Subject: [PATCH 09/24] Add note about skip_outputs to the readme --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 78e7a10c9e..6406849a75 100644 --- a/README.md +++ b/README.md @@ -1176,6 +1176,20 @@ inputs = { Note that indicating `validate` means that the `default_outputs` will be used either with `validate` or with `validate-all`. +You can also use `skip_outputs` on the `dependency` block to specify the dependency without pulling in the outputs: + +``` +dependency "vpc" { + config_path = "../vpc" + skip_outputs = true +} +``` + + + #### Dependencies between modules From c210c4adbba47a18c5e9c4e20ca3e25e38ed2215 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Tue, 3 Sep 2019 16:10:10 -0700 Subject: [PATCH 10/24] Rename default_output to mock_output --- README.md | 18 +++++----- config/dependency.go | 34 +++++++++---------- config/dependency_test.go | 10 +++--- .../dependent1/main.tf | 0 .../dependent1/terragrunt.hcl | 2 +- .../dependent2/main.tf | 0 .../dependent2/terragrunt.hcl | 4 +-- .../source/main.tf | 0 .../source/terragrunt.hcl | 0 9 files changed, 34 insertions(+), 34 deletions(-) rename test/fixture-get-output/{default-outputs => mock-outputs}/dependent1/main.tf (100%) rename test/fixture-get-output/{default-outputs => mock-outputs}/dependent1/terragrunt.hcl (86%) rename test/fixture-get-output/{default-outputs => mock-outputs}/dependent2/main.tf (100%) rename test/fixture-get-output/{default-outputs => mock-outputs}/dependent2/terragrunt.hcl (63%) rename test/fixture-get-output/{default-outputs => mock-outputs}/source/main.tf (100%) rename test/fixture-get-output/{default-outputs => mock-outputs}/source/terragrunt.hcl (100%) diff --git a/README.md b/README.md index 6406849a75..7d902b1edf 100644 --- a/README.md +++ b/README.md @@ -1114,7 +1114,7 @@ If any of the modules failed to deploy, then Terragrunt will not attempt to depl **Note**: Not all blocks are able to access outputs passed by `dependency` blocks. See the section on [Configuration parsing order](#configuration-parsing-order) in this README for more information. -##### Unapplied dependency and default outputs +##### Unapplied dependency and mock outputs Terragrunt will return an error indicating the dependency hasn't been applied yet if the terraform module managed by the terragrunt config referenced in a `dependency` block has not been applied yet. This is because you cannot actually fetch @@ -1126,8 +1126,8 @@ you can't determine the `inputs`. If the module depends on the outputs of anothe yet, you won't be able to compute the `inputs` unless the dependencies are all applied. However, in real life usage, you would want to run `validate-all` or `plan-all` on a completely new set of infrastructure. -To address this, you can provide default outputs to use when a module hasn't been applied yet. This is configured using -the `default_outputs` attribute on the `dependency` block and it corresponds to a map that will be injected in place of +To address this, you can provide mock outputs to use when a module hasn't been applied yet. This is configured using +the `mock_outputs` attribute on the `dependency` block and it corresponds to a map that will be injected in place of the actual dependency outputs if the target config hasn't been applied yet. For example, in the previous example with a `mysql` module and `vpc` module, suppose you wanted to place in a temporary, @@ -1137,7 +1137,7 @@ dummy value for the `vpc_id` during a `validate-all` for the `mysql` module. You dependency "vpc" { config_path = "../vpc" - default_outputs = { + mock_outputs = { vpc_id = "temporary-dummy-id" } } @@ -1154,18 +1154,18 @@ What if you wanted to restrict this behavior to only the `validate` command? For the defaults for a `plan` operation because the plan doesn't give you any indication of what is actually going to be created. -You can use the `default_outputs_allowed_terraform_commands` attribute to indicate that the `default_outputs` should -only be used when running those Terraform commands. So to restrict the `default_outputs` to only when `validate` is +You can use the `mock_outputs_allowed_terraform_commands` attribute to indicate that the `mock_outputs` should +only be used when running those Terraform commands. So to restrict the `mock_outputs` to only when `validate` is being run, you can modify the above `terragrunt.hcl` file to: ``` dependency "vpc" { config_path = "../vpc" - default_outputs = { + mock_outputs = { vpc_id = "temporary-dummy-id" } - default_outputs_allowed_terraform_commands = ["validate"] + mock_outputs_allowed_terraform_commands = ["validate"] } inputs = { @@ -1173,7 +1173,7 @@ inputs = { } ``` -Note that indicating `validate` means that the `default_outputs` will be used either with `validate` or with +Note that indicating `validate` means that the `mock_outputs` will be used either with `validate` or with `validate-all`. You can also use `skip_outputs` on the `dependency` block to specify the dependency without pulling in the outputs: diff --git a/config/dependency.go b/config/dependency.go index 33de55e677..1d20ce8db7 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -19,11 +19,11 @@ import ( ) type Dependency struct { - Name string `hcl:",label"` - ConfigPath string `hcl:"config_path,attr"` - SkipOutputs *bool `hcl:"skip_outputs,attr"` - DefaultOutputs *cty.Value `hcl:"default_outputs,attr"` - DefaultOutputsAllowedTerraformCommands *[]string `hcl:"default_outputs_allowed_terraform_commands,attr"` + Name string `hcl:",label"` + ConfigPath string `hcl:"config_path,attr"` + SkipOutputs *bool `hcl:"skip_outputs,attr"` + MockOutputs *cty.Value `hcl:"mock_outputs,attr"` + MockOutputsAllowedTerraformCommands *[]string `hcl:"mock_outputs_allowed_terraform_commands,attr"` } // Given a dependency config, we should only attempt to get the outputs if SkipOutputs is nil or false @@ -128,8 +128,8 @@ func getCleanedTargetConfigPath(dependencyConfig Dependency, terragruntOptions * // This will attempt to get the outputs from the target terragrunt config if it is applied. If it is not applied, the // behavior is different depending on the configuration of the dependency: -// - If the dependency block indicates a default_outputs attribute, this will return that. -// - If the dependency block does NOT indicate a default_outputs attribute, this will return an error. +// - If the dependency block indicates a mock_outputs attribute, this will return that. +// - If the dependency block does NOT indicate a mock_outputs attribute, this will return an error. func getTerragruntOutputIfAppliedElseConfiguredDefault(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) (*cty.Value, error) { outputVal, isEmpty, err := getTerragruntOutput(dependencyConfig, terragruntOptions) if err != nil { @@ -145,14 +145,14 @@ func getTerragruntOutputIfAppliedElseConfiguredDefault(dependencyConfig Dependen // return error. targetConfig := getCleanedTargetConfigPath(dependencyConfig, terragruntOptions) currentConfig := terragruntOptions.TerragruntConfigPath - if shouldReturnDefaultOutputs(dependencyConfig, terragruntOptions) { + if shouldReturnMockOutputs(dependencyConfig, terragruntOptions) { util.Debugf( terragruntOptions.Logger, - "WARNING: config %s is a dependency of %s that has no outputs, but default outputs provided and returning those in dependency output.", + "WARNING: config %s is a dependency of %s that has no outputs, but mock outputs provided and returning those in dependency output.", targetConfig, currentConfig, ) - return dependencyConfig.DefaultOutputs, nil + return dependencyConfig.MockOutputs, nil } err = TerragruntOutputTargetNoOutputs{ @@ -162,14 +162,14 @@ func getTerragruntOutputIfAppliedElseConfiguredDefault(dependencyConfig Dependen return nil, err } -// We should only return default outputs if the default_outputs attribute is set, and if we are running one of the -// allowed commands when `default_outputs_allowed_terraform_commands` is set as well. -func shouldReturnDefaultOutputs(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) bool { - defaultOutputsSet := dependencyConfig.DefaultOutputs != nil +// We should only return default outputs if the mock_outputs attribute is set, and if we are running one of the +// allowed commands when `mock_outputs_allowed_terraform_commands` is set as well. +func shouldReturnMockOutputs(dependencyConfig Dependency, terragruntOptions *options.TerragruntOptions) bool { + defaultOutputsSet := dependencyConfig.MockOutputs != nil allowedCommand := - dependencyConfig.DefaultOutputsAllowedTerraformCommands == nil || - len(*dependencyConfig.DefaultOutputsAllowedTerraformCommands) == 0 || - util.ListContainsElement(*dependencyConfig.DefaultOutputsAllowedTerraformCommands, terragruntOptions.TerraformCommand) + dependencyConfig.MockOutputsAllowedTerraformCommands == nil || + len(*dependencyConfig.MockOutputsAllowedTerraformCommands) == 0 || + util.ListContainsElement(*dependencyConfig.MockOutputsAllowedTerraformCommands, terragruntOptions.TerraformCommand) return defaultOutputsSet && allowedCommand } diff --git a/config/dependency_test.go b/config/dependency_test.go index 9e338b8390..36b28199e3 100644 --- a/config/dependency_test.go +++ b/config/dependency_test.go @@ -71,16 +71,16 @@ dependency { require.Error(t, decodeHcl(file, filename, &decoded, mockOptionsForTest(t), EvalContextExtensions{})) } -func TestDecodeDependencyDefaultOutputs(t *testing.T) { +func TestDecodeDependencyMockOutputs(t *testing.T) { t.Parallel() config := ` dependency "hitchhiker" { config_path = "../answers" - default_outputs = { + mock_outputs = { the_answer = 42 } - default_outputs_allowed_terraform_commands = ["validate", "apply"] + mock_outputs_allowed_terraform_commands = ["validate", "apply"] } ` filename := DefaultTerragruntConfigPath @@ -96,7 +96,7 @@ dependency "hitchhiker" { assert.Equal(t, dependency.Name, "hitchhiker") assert.Equal(t, dependency.ConfigPath, "../answers") - ctyValueDefault := dependency.DefaultOutputs + ctyValueDefault := dependency.MockOutputs require.NotNil(t, ctyValueDefault) var actualDefault struct { @@ -105,7 +105,7 @@ dependency "hitchhiker" { require.NoError(t, gocty.FromCtyValue(*ctyValueDefault, &actualDefault)) assert.Equal(t, actualDefault.TheAnswer, 42) - defaultAllowedCommands := dependency.DefaultOutputsAllowedTerraformCommands + defaultAllowedCommands := dependency.MockOutputsAllowedTerraformCommands require.NotNil(t, defaultAllowedCommands) assert.Equal(t, *defaultAllowedCommands, []string{"validate", "apply"}) } diff --git a/test/fixture-get-output/default-outputs/dependent1/main.tf b/test/fixture-get-output/mock-outputs/dependent1/main.tf similarity index 100% rename from test/fixture-get-output/default-outputs/dependent1/main.tf rename to test/fixture-get-output/mock-outputs/dependent1/main.tf diff --git a/test/fixture-get-output/default-outputs/dependent1/terragrunt.hcl b/test/fixture-get-output/mock-outputs/dependent1/terragrunt.hcl similarity index 86% rename from test/fixture-get-output/default-outputs/dependent1/terragrunt.hcl rename to test/fixture-get-output/mock-outputs/dependent1/terragrunt.hcl index 7aaf849e81..447e329204 100644 --- a/test/fixture-get-output/default-outputs/dependent1/terragrunt.hcl +++ b/test/fixture-get-output/mock-outputs/dependent1/terragrunt.hcl @@ -1,6 +1,6 @@ dependency "source" { config_path = "../source" - default_outputs = { + mock_outputs = { the_answer = "0" } } diff --git a/test/fixture-get-output/default-outputs/dependent2/main.tf b/test/fixture-get-output/mock-outputs/dependent2/main.tf similarity index 100% rename from test/fixture-get-output/default-outputs/dependent2/main.tf rename to test/fixture-get-output/mock-outputs/dependent2/main.tf diff --git a/test/fixture-get-output/default-outputs/dependent2/terragrunt.hcl b/test/fixture-get-output/mock-outputs/dependent2/terragrunt.hcl similarity index 63% rename from test/fixture-get-output/default-outputs/dependent2/terragrunt.hcl rename to test/fixture-get-output/mock-outputs/dependent2/terragrunt.hcl index d3d7438ba1..7e6d7392a7 100644 --- a/test/fixture-get-output/default-outputs/dependent2/terragrunt.hcl +++ b/test/fixture-get-output/mock-outputs/dependent2/terragrunt.hcl @@ -1,9 +1,9 @@ dependency "source" { config_path = "../source" - default_outputs = { + mock_outputs = { the_answer = "0" } - default_outputs_allowed_terraform_commands = ["validate"] + mock_outputs_allowed_terraform_commands = ["validate"] } inputs = { diff --git a/test/fixture-get-output/default-outputs/source/main.tf b/test/fixture-get-output/mock-outputs/source/main.tf similarity index 100% rename from test/fixture-get-output/default-outputs/source/main.tf rename to test/fixture-get-output/mock-outputs/source/main.tf diff --git a/test/fixture-get-output/default-outputs/source/terragrunt.hcl b/test/fixture-get-output/mock-outputs/source/terragrunt.hcl similarity index 100% rename from test/fixture-get-output/default-outputs/source/terragrunt.hcl rename to test/fixture-get-output/mock-outputs/source/terragrunt.hcl From c5a033bdc7f6430a890a78852540e42c5845dafc Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Tue, 3 Sep 2019 16:22:36 -0700 Subject: [PATCH 11/24] Forgot to update the module path reference in the tests in the rename --- test/integration_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/integration_test.go b/test/integration_test.go index b3fe6c4e97..2f65c057fc 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1646,14 +1646,14 @@ func TestDependencyOutputSkipOutputs(t *testing.T) { logBufferContentsLineByLine(t, showStderr, "show stderr") } -// Test that when you have an output default on a dependency, the dependency will use the default as the output instead +// Test that when you have a mock_output on a dependency, the dependency will use the mock as the output instead // of erroring out. -func TestDependencyOutputDefaults(t *testing.T) { +func TestDependencyMockOutput(t *testing.T) { t.Parallel() cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT) tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) - rootPath := filepath.Join(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "default-outputs") + rootPath := filepath.Join(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "mock-outputs") dependent1Path := filepath.Join(rootPath, "dependent1") showStdout := bytes.Buffer{} @@ -1665,7 +1665,7 @@ func TestDependencyOutputDefaults(t *testing.T) { logBufferContentsLineByLine(t, showStdout, "show stdout") logBufferContentsLineByLine(t, showStderr, "show stderr") - // verify expected output when defaults are used: The answer is 0 + // verify expected output when mocks are used: The answer is 0 stdout := bytes.Buffer{} stderr := bytes.Buffer{} require.NoError( @@ -1683,7 +1683,7 @@ func TestDependencyOutputDefaults(t *testing.T) { logBufferContentsLineByLine(t, showStdout, "show stdout") logBufferContentsLineByLine(t, showStderr, "show stderr") - // verify expected output when defaults are used: The answer is 0 + // verify expected output when mocks are used: The answer is 0 stdout = bytes.Buffer{} stderr = bytes.Buffer{} require.NoError( @@ -1695,14 +1695,14 @@ func TestDependencyOutputDefaults(t *testing.T) { assert.Equal(t, outputs["truth"].Value, "The answer is 42") } -// Test that when you have an output default on a dependency, the dependency will use the default as the output instead +// Test that when you have a mock_output on a dependency, the dependency will use the mock as the output instead // of erroring out when running an allowed command. -func TestDependencyOutputDefaultsRestricted(t *testing.T) { +func TestDependencyMockOutputRestricted(t *testing.T) { t.Parallel() cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT) tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) - rootPath := filepath.Join(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "default-outputs") + rootPath := filepath.Join(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "mock-outputs") dependent2Path := filepath.Join(rootPath, "dependent2") showStdout := bytes.Buffer{} From 8810c8a1ee885e9eb923318623cb6b988557c676 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Thu, 8 Aug 2019 22:21:10 -0700 Subject: [PATCH 12/24] hclfmt should run before parsing the terragrunt config --- cli/cli_app.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/cli_app.go b/cli/cli_app.go index cfab947df3..79115c6c5d 100644 --- a/cli/cli_app.go +++ b/cli/cli_app.go @@ -232,6 +232,10 @@ func runTerragrunt(terragruntOptions *options.TerragruntOptions) error { return shell.RunTerraformCommand(terragruntOptions, terragruntOptions.TerraformCliArgs...) } + if shouldRunHCLFmt(terragruntOptions) { + return runHCLFmt(terragruntOptions) + } + terragruntConfig, err := config.ReadTerragruntConfig(terragruntOptions) if err != nil { @@ -294,10 +298,6 @@ func runTerragrunt(terragruntOptions *options.TerragruntOptions) error { return nil } - if shouldRunHCLFmt(terragruntOptions) { - return runHCLFmt(terragruntOptions) - } - if err := checkFolderContainsTerraformCode(terragruntOptions); err != nil { return err } From a6f8843da4efc9d0810f4c43dbc5f49ca6b4c2e7 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Tue, 3 Sep 2019 14:47:08 -0700 Subject: [PATCH 13/24] Detect cycles in dependency block parsing --- config/dependency.go | 76 ++++++++++++++++++- test/fixture-get-output/cycle/bar/main.tf | 1 + .../cycle/bar/terragrunt.hcl | 3 + test/fixture-get-output/cycle/baz/main.tf | 1 + .../cycle/baz/terragrunt.hcl | 3 + test/fixture-get-output/cycle/car/main.tf | 1 + .../cycle/car/terragrunt.hcl | 3 + test/fixture-get-output/cycle/foo/main.tf | 1 + .../cycle/foo/terragrunt.hcl | 7 ++ test/integration_test.go | 24 ++++++ 10 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 test/fixture-get-output/cycle/bar/main.tf create mode 100644 test/fixture-get-output/cycle/bar/terragrunt.hcl create mode 100644 test/fixture-get-output/cycle/baz/main.tf create mode 100644 test/fixture-get-output/cycle/baz/terragrunt.hcl create mode 100644 test/fixture-get-output/cycle/car/main.tf create mode 100644 test/fixture-get-output/cycle/car/terragrunt.hcl create mode 100644 test/fixture-get-output/cycle/foo/main.tf create mode 100644 test/fixture-get-output/cycle/foo/terragrunt.hcl diff --git a/config/dependency.go b/config/dependency.go index 1d20ce8db7..56eda91cfe 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -34,15 +34,20 @@ func (dependencyConfig Dependency) shouldGetOutputs() bool { // Decode the dependency blocks from the file, and then retrieve all the outputs from the remote state. Then encode the // resulting map as a cty.Value object. // TODO: In the future, consider allowing importing dependency blocks from included config +// NOTE FOR MAINTAINER: When implementing importation of other config blocks (e.g referencing inputs), carefully +// consider whether or not the implementation of the cyclic dependency detection still makes sense. func decodeAndRetrieveOutputs( file *hcl.File, filename string, terragruntOptions *options.TerragruntOptions, extensions EvalContextExtensions, ) (*cty.Value, error) { + if err := checkForDependencyBlockCyclesUsingDFS(filename, &[]string{}, &[]string{}, terragruntOptions); err != nil { + return nil, err + } + decodedDependency := terragruntDependency{} - err := decodeHcl(file, filename, &decodedDependency, terragruntOptions, extensions) - if err != nil { + if err := decodeHcl(file, filename, &decodedDependency, terragruntOptions, extensions); err != nil { return nil, err } return dependencyBlocksToCtyValue(decodedDependency.Dependencies, terragruntOptions) @@ -67,6 +72,67 @@ func dependencyBlocksToModuleDependencies(decodedDependencyBlocks []Dependency) return &ModuleDependencies{Paths: paths} } +// Check for cyclic dependency blocks to avoid infinite `terragrunt output` loops. +// +// Same implementation as configstack/graph.go:checkForCyclesUsingDepthFirstSearch, except walks the graph of +// dependencies by `dependency` blocks (which make explicit `terragrunt output` calls) instead of explicit dependencies. +func checkForDependencyBlockCyclesUsingDFS( + currentConfigPath string, + visitedPaths *[]string, + currentTraversalPaths *[]string, + terragruntOptions *options.TerragruntOptions, +) error { + if util.ListContainsElement(*visitedPaths, currentConfigPath) { + return nil + } + + if util.ListContainsElement(*currentTraversalPaths, currentConfigPath) { + return errors.WithStackTrace(DependencyCycle(append(*currentTraversalPaths, currentConfigPath))) + } + + *currentTraversalPaths = append(*currentTraversalPaths, currentConfigPath) + dependencyPaths, err := getDependencyBlockConfigPathsByFilepath(currentConfigPath, terragruntOptions) + if err != nil { + return err + } + for _, dependency := range dependencyPaths { + // Dependency paths are relative to the config, so we convert to absolute paths while we still have the proper + // context. + dependency = filepath.Clean(filepath.Join(filepath.Dir(currentConfigPath), dependency)) + // Dependency blocks can be the directory holding a terragrunt config, but we want to read the actual config + // file here. So if the dependency path is a directory, we assume the default config filename exists in the + // directory. + if util.IsDir(dependency) { + dependency = filepath.Join(dependency, DefaultTerragruntConfigPath) + } + + if err := checkForDependencyBlockCyclesUsingDFS(dependency, visitedPaths, currentTraversalPaths, terragruntOptions); err != nil { + return err + } + } + + *visitedPaths = append(*visitedPaths, currentConfigPath) + *currentTraversalPaths = util.RemoveElementFromList(*currentTraversalPaths, currentConfigPath) + + return nil +} + +// Given the config path, return the list of config paths that are specified as dependency blocks in the config +func getDependencyBlockConfigPathsByFilepath(configPath string, terragruntOptions *options.TerragruntOptions) ([]string, error) { + // This will automatically parse everything needed to parse the dependency block configs, and load them as + // TerragruntConfig.Dependencies. Note that since we aren't passing in `DependenciesBlock` to the + // PartialDecodeSectionType list, the Dependencies attribute will not include any dependencies specified via the + // dependencies block. + tgConfig, err := PartialParseConfigFile(configPath, terragruntOptions, nil, []PartialDecodeSectionType{DependencyBlock}) + if err != nil { + return nil, err + } + if tgConfig.Dependencies == nil { + return []string{}, nil + } + return tgConfig.Dependencies.Paths, nil +} + // Encode the list of dependency blocks into a single cty.Value object that maps the dependency block name to the // encoded dependency mapping. The encoded dependency mapping should have the attributes: // - outputs: The map of outputs of the corresponding terraform module that lives at the target config of the @@ -302,3 +368,9 @@ func (err TerragruntOutputTargetNoOutputs) Error() string { err.currentConfig, ) } + +type DependencyCycle []string + +func (err DependencyCycle) Error() string { + return fmt.Sprintf("Found a dependency cycle between modules: %s", strings.Join([]string(err), " -> ")) +} diff --git a/test/fixture-get-output/cycle/bar/main.tf b/test/fixture-get-output/cycle/bar/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/bar/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/bar/terragrunt.hcl b/test/fixture-get-output/cycle/bar/terragrunt.hcl new file mode 100644 index 0000000000..aefd2efde2 --- /dev/null +++ b/test/fixture-get-output/cycle/bar/terragrunt.hcl @@ -0,0 +1,3 @@ +dependency "baz" { + config_path = "../baz" +} diff --git a/test/fixture-get-output/cycle/baz/main.tf b/test/fixture-get-output/cycle/baz/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/baz/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/baz/terragrunt.hcl b/test/fixture-get-output/cycle/baz/terragrunt.hcl new file mode 100644 index 0000000000..c58ef65925 --- /dev/null +++ b/test/fixture-get-output/cycle/baz/terragrunt.hcl @@ -0,0 +1,3 @@ +dependency "foo" { + config_path = "../foo" +} diff --git a/test/fixture-get-output/cycle/car/main.tf b/test/fixture-get-output/cycle/car/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/car/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/car/terragrunt.hcl b/test/fixture-get-output/cycle/car/terragrunt.hcl new file mode 100644 index 0000000000..aefd2efde2 --- /dev/null +++ b/test/fixture-get-output/cycle/car/terragrunt.hcl @@ -0,0 +1,3 @@ +dependency "baz" { + config_path = "../baz" +} diff --git a/test/fixture-get-output/cycle/foo/main.tf b/test/fixture-get-output/cycle/foo/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/foo/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/foo/terragrunt.hcl b/test/fixture-get-output/cycle/foo/terragrunt.hcl new file mode 100644 index 0000000000..2e40ea6b88 --- /dev/null +++ b/test/fixture-get-output/cycle/foo/terragrunt.hcl @@ -0,0 +1,7 @@ +dependency "bar" { + config_path = "../bar" +} + +dependency "car" { + config_path = "../car" +} diff --git a/test/integration_test.go b/test/integration_test.go index 2f65c057fc..5772865bbf 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1790,7 +1790,31 @@ func TestDependencyOutputTypeConversion(t *testing.T) { assert.Equal(t, outputs["object"].Value, map[string]interface{}{"list": []interface{}{1.0, 2.0, 3.0}, "map": map[string]interface{}{"foo": "bar"}, "num": 42.0, "str": "string"}) assert.Equal(t, outputs["string"].Value, "string") assert.Equal(t, outputs["from_env"].Value, "default") +} + +// Test that we get the expected error message about dependency cycles when there is a cycle in the dependency chain +func TestDependencyOutputCycleHandling(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) + rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "cycle") + fooPath := util.JoinPath(rootPath, "foo") + var ( + planStdout bytes.Buffer + planStderr bytes.Buffer + ) + err := runTerragruntCommand( + t, + fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-working-dir %s", fooPath), + &planStdout, + &planStderr, + ) + logBufferContentsLineByLine(t, planStdout, "plan stdout") + logBufferContentsLineByLine(t, planStderr, "plan stderr") + assert.Error(t, err) + assert.True(t, strings.Contains(planStderr.String(), "Found a dependency cycle between modules")) } func logBufferContentsLineByLine(t *testing.T, out bytes.Buffer, label string) { From 98f597cbc93f4823ccc06e07e6f36f7c7efc70a5 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Tue, 3 Sep 2019 15:04:15 -0700 Subject: [PATCH 14/24] Tweak implementation of cyclic dependency detection to support better unit testing of ParseConfigString --- config/dependency.go | 51 ++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/config/dependency.go b/config/dependency.go index 56eda91cfe..eeb0961b64 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -42,14 +42,13 @@ func decodeAndRetrieveOutputs( terragruntOptions *options.TerragruntOptions, extensions EvalContextExtensions, ) (*cty.Value, error) { - if err := checkForDependencyBlockCyclesUsingDFS(filename, &[]string{}, &[]string{}, terragruntOptions); err != nil { - return nil, err - } - decodedDependency := terragruntDependency{} if err := decodeHcl(file, filename, &decodedDependency, terragruntOptions, extensions); err != nil { return nil, err } + if err := checkForDependencyBlockCycles(filename, decodedDependency, terragruntOptions); err != nil { + return nil, err + } return dependencyBlocksToCtyValue(decodedDependency.Dependencies, terragruntOptions) } @@ -72,7 +71,21 @@ func dependencyBlocksToModuleDependencies(decodedDependencyBlocks []Dependency) return &ModuleDependencies{Paths: paths} } -// Check for cyclic dependency blocks to avoid infinite `terragrunt output` loops. +// Check for cyclic dependency blocks to avoid infinite `terragrunt output` loops. To avoid reparsing the config, we +// kickstart the initial loop using what we already decoded. +func checkForDependencyBlockCycles(filename string, decodedDependency terragruntDependency, terragruntOptions *options.TerragruntOptions) error { + visitedPaths := []string{} + currentTraversalPaths := []string{filename} + for _, dependency := range decodedDependency.Dependencies { + dependencyPath := cleanDependencyTerragruntConfigPath(filename, dependency.ConfigPath) + if err := checkForDependencyBlockCyclesUsingDFS(dependencyPath, &visitedPaths, ¤tTraversalPaths, terragruntOptions); err != nil { + return err + } + } + return nil +} + +// Helper function for checkForDependencyBlockCycles. // // Same implementation as configstack/graph.go:checkForCyclesUsingDepthFirstSearch, except walks the graph of // dependencies by `dependency` blocks (which make explicit `terragrunt output` calls) instead of explicit dependencies. @@ -96,17 +109,7 @@ func checkForDependencyBlockCyclesUsingDFS( return err } for _, dependency := range dependencyPaths { - // Dependency paths are relative to the config, so we convert to absolute paths while we still have the proper - // context. - dependency = filepath.Clean(filepath.Join(filepath.Dir(currentConfigPath), dependency)) - // Dependency blocks can be the directory holding a terragrunt config, but we want to read the actual config - // file here. So if the dependency path is a directory, we assume the default config filename exists in the - // directory. - if util.IsDir(dependency) { - dependency = filepath.Join(dependency, DefaultTerragruntConfigPath) - } - - if err := checkForDependencyBlockCyclesUsingDFS(dependency, visitedPaths, currentTraversalPaths, terragruntOptions); err != nil { + if err := checkForDependencyBlockCyclesUsingDFS(cleanDependencyTerragruntConfigPath(currentConfigPath, dependency), visitedPaths, currentTraversalPaths, terragruntOptions); err != nil { return err } } @@ -117,6 +120,22 @@ func checkForDependencyBlockCyclesUsingDFS( return nil } +// Ensures the dependency path points to the right terragrunt config path. +func cleanDependencyTerragruntConfigPath(currentConfigPath string, dependencyPath string) string { + // Dependency paths are relative to the config, so we convert to absolute paths while we still have the proper + // context. + if !filepath.IsAbs(dependencyPath) { + dependencyPath = filepath.Clean(filepath.Join(filepath.Dir(currentConfigPath), dependencyPath)) + } + // Dependency blocks can be the directory holding a terragrunt config, but we want to read the actual config + // file here. So if the dependency path is a directory, we assume the default config filename exists in the + // directory. + if util.IsDir(dependencyPath) { + dependencyPath = filepath.Join(dependencyPath, DefaultTerragruntConfigPath) + } + return dependencyPath +} + // Given the config path, return the list of config paths that are specified as dependency blocks in the config func getDependencyBlockConfigPathsByFilepath(configPath string, terragruntOptions *options.TerragruntOptions) ([]string, error) { // This will automatically parse everything needed to parse the dependency block configs, and load them as From b4364856cbf33478d364fb579c40212995642b82 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Tue, 3 Sep 2019 15:37:57 -0700 Subject: [PATCH 15/24] Checking wrong thing --- test/integration_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/integration_test.go b/test/integration_test.go index 5772865bbf..3992db7bb8 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1801,10 +1801,8 @@ func TestDependencyOutputCycleHandling(t *testing.T) { rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "cycle") fooPath := util.JoinPath(rootPath, "foo") - var ( - planStdout bytes.Buffer - planStderr bytes.Buffer - ) + planStdout := bytes.Buffer{} + planStderr := bytes.Buffer{} err := runTerragruntCommand( t, fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-working-dir %s", fooPath), @@ -1814,7 +1812,7 @@ func TestDependencyOutputCycleHandling(t *testing.T) { logBufferContentsLineByLine(t, planStdout, "plan stdout") logBufferContentsLineByLine(t, planStderr, "plan stderr") assert.Error(t, err) - assert.True(t, strings.Contains(planStderr.String(), "Found a dependency cycle between modules")) + assert.True(t, strings.Contains(err.Error(), "Found a dependency cycle between modules")) } func logBufferContentsLineByLine(t *testing.T, out bytes.Buffer, label string) { From 9f4758e58c1276e57b7bcba043445e92dd7e8bc1 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Wed, 4 Sep 2019 08:52:29 -0700 Subject: [PATCH 16/24] Better test cases --- .../cycle/{bar => aa/foo}/main.tf | 0 .../cycle/{baz => aa/foo}/terragrunt.hcl | 0 .../cycle/{baz => aba/bar}/main.tf | 0 .../cycle/aba/bar/terragrunt.hcl | 3 ++ .../cycle/{car => aba/foo}/main.tf | 0 .../cycle/aba/foo/terragrunt.hcl | 3 ++ .../cycle/{foo => abca/bar}/main.tf | 0 .../cycle/abca/bar/terragrunt.hcl | 3 ++ .../fixture-get-output/cycle/abca/baz/main.tf | 1 + .../cycle/abca/baz/terragrunt.hcl | 1 + .../fixture-get-output/cycle/abca/foo/main.tf | 1 + .../cycle/{ => abca}/foo/terragrunt.hcl | 4 +- .../cycle/abcda/bar/main.tf | 1 + .../cycle/{ => abcda}/bar/terragrunt.hcl | 0 .../cycle/abcda/baz/main.tf | 1 + .../cycle/abcda/baz/terragrunt.hcl | 3 ++ .../cycle/abcda/car/main.tf | 1 + .../cycle/abcda/car/terragrunt.hcl | 3 ++ .../cycle/abcda/foo/main.tf | 1 + .../cycle/abcda/foo/terragrunt.hcl | 3 ++ .../cycle/car/terragrunt.hcl | 3 -- test/integration_test.go | 47 +++++++++++++------ 22 files changed, 59 insertions(+), 20 deletions(-) rename test/fixture-get-output/cycle/{bar => aa/foo}/main.tf (100%) rename test/fixture-get-output/cycle/{baz => aa/foo}/terragrunt.hcl (100%) rename test/fixture-get-output/cycle/{baz => aba/bar}/main.tf (100%) create mode 100644 test/fixture-get-output/cycle/aba/bar/terragrunt.hcl rename test/fixture-get-output/cycle/{car => aba/foo}/main.tf (100%) create mode 100644 test/fixture-get-output/cycle/aba/foo/terragrunt.hcl rename test/fixture-get-output/cycle/{foo => abca/bar}/main.tf (100%) create mode 100644 test/fixture-get-output/cycle/abca/bar/terragrunt.hcl create mode 100644 test/fixture-get-output/cycle/abca/baz/main.tf create mode 100644 test/fixture-get-output/cycle/abca/baz/terragrunt.hcl create mode 100644 test/fixture-get-output/cycle/abca/foo/main.tf rename test/fixture-get-output/cycle/{ => abca}/foo/terragrunt.hcl (52%) create mode 100644 test/fixture-get-output/cycle/abcda/bar/main.tf rename test/fixture-get-output/cycle/{ => abcda}/bar/terragrunt.hcl (100%) create mode 100644 test/fixture-get-output/cycle/abcda/baz/main.tf create mode 100644 test/fixture-get-output/cycle/abcda/baz/terragrunt.hcl create mode 100644 test/fixture-get-output/cycle/abcda/car/main.tf create mode 100644 test/fixture-get-output/cycle/abcda/car/terragrunt.hcl create mode 100644 test/fixture-get-output/cycle/abcda/foo/main.tf create mode 100644 test/fixture-get-output/cycle/abcda/foo/terragrunt.hcl delete mode 100644 test/fixture-get-output/cycle/car/terragrunt.hcl diff --git a/test/fixture-get-output/cycle/bar/main.tf b/test/fixture-get-output/cycle/aa/foo/main.tf similarity index 100% rename from test/fixture-get-output/cycle/bar/main.tf rename to test/fixture-get-output/cycle/aa/foo/main.tf diff --git a/test/fixture-get-output/cycle/baz/terragrunt.hcl b/test/fixture-get-output/cycle/aa/foo/terragrunt.hcl similarity index 100% rename from test/fixture-get-output/cycle/baz/terragrunt.hcl rename to test/fixture-get-output/cycle/aa/foo/terragrunt.hcl diff --git a/test/fixture-get-output/cycle/baz/main.tf b/test/fixture-get-output/cycle/aba/bar/main.tf similarity index 100% rename from test/fixture-get-output/cycle/baz/main.tf rename to test/fixture-get-output/cycle/aba/bar/main.tf diff --git a/test/fixture-get-output/cycle/aba/bar/terragrunt.hcl b/test/fixture-get-output/cycle/aba/bar/terragrunt.hcl new file mode 100644 index 0000000000..c58ef65925 --- /dev/null +++ b/test/fixture-get-output/cycle/aba/bar/terragrunt.hcl @@ -0,0 +1,3 @@ +dependency "foo" { + config_path = "../foo" +} diff --git a/test/fixture-get-output/cycle/car/main.tf b/test/fixture-get-output/cycle/aba/foo/main.tf similarity index 100% rename from test/fixture-get-output/cycle/car/main.tf rename to test/fixture-get-output/cycle/aba/foo/main.tf diff --git a/test/fixture-get-output/cycle/aba/foo/terragrunt.hcl b/test/fixture-get-output/cycle/aba/foo/terragrunt.hcl new file mode 100644 index 0000000000..5fee20517c --- /dev/null +++ b/test/fixture-get-output/cycle/aba/foo/terragrunt.hcl @@ -0,0 +1,3 @@ +dependency "bar" { + config_path = "../bar" +} diff --git a/test/fixture-get-output/cycle/foo/main.tf b/test/fixture-get-output/cycle/abca/bar/main.tf similarity index 100% rename from test/fixture-get-output/cycle/foo/main.tf rename to test/fixture-get-output/cycle/abca/bar/main.tf diff --git a/test/fixture-get-output/cycle/abca/bar/terragrunt.hcl b/test/fixture-get-output/cycle/abca/bar/terragrunt.hcl new file mode 100644 index 0000000000..c58ef65925 --- /dev/null +++ b/test/fixture-get-output/cycle/abca/bar/terragrunt.hcl @@ -0,0 +1,3 @@ +dependency "foo" { + config_path = "../foo" +} diff --git a/test/fixture-get-output/cycle/abca/baz/main.tf b/test/fixture-get-output/cycle/abca/baz/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/abca/baz/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/abca/baz/terragrunt.hcl b/test/fixture-get-output/cycle/abca/baz/terragrunt.hcl new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/abca/baz/terragrunt.hcl @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/abca/foo/main.tf b/test/fixture-get-output/cycle/abca/foo/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/abca/foo/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/foo/terragrunt.hcl b/test/fixture-get-output/cycle/abca/foo/terragrunt.hcl similarity index 52% rename from test/fixture-get-output/cycle/foo/terragrunt.hcl rename to test/fixture-get-output/cycle/abca/foo/terragrunt.hcl index 2e40ea6b88..de056e2a2b 100644 --- a/test/fixture-get-output/cycle/foo/terragrunt.hcl +++ b/test/fixture-get-output/cycle/abca/foo/terragrunt.hcl @@ -2,6 +2,6 @@ dependency "bar" { config_path = "../bar" } -dependency "car" { - config_path = "../car" +dependency "baz" { + config_path = "../baz" } diff --git a/test/fixture-get-output/cycle/abcda/bar/main.tf b/test/fixture-get-output/cycle/abcda/bar/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/abcda/bar/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/bar/terragrunt.hcl b/test/fixture-get-output/cycle/abcda/bar/terragrunt.hcl similarity index 100% rename from test/fixture-get-output/cycle/bar/terragrunt.hcl rename to test/fixture-get-output/cycle/abcda/bar/terragrunt.hcl diff --git a/test/fixture-get-output/cycle/abcda/baz/main.tf b/test/fixture-get-output/cycle/abcda/baz/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/abcda/baz/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/abcda/baz/terragrunt.hcl b/test/fixture-get-output/cycle/abcda/baz/terragrunt.hcl new file mode 100644 index 0000000000..eeb73e40a4 --- /dev/null +++ b/test/fixture-get-output/cycle/abcda/baz/terragrunt.hcl @@ -0,0 +1,3 @@ +dependency "car" { + config_path = "../car" +} diff --git a/test/fixture-get-output/cycle/abcda/car/main.tf b/test/fixture-get-output/cycle/abcda/car/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/abcda/car/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/abcda/car/terragrunt.hcl b/test/fixture-get-output/cycle/abcda/car/terragrunt.hcl new file mode 100644 index 0000000000..c58ef65925 --- /dev/null +++ b/test/fixture-get-output/cycle/abcda/car/terragrunt.hcl @@ -0,0 +1,3 @@ +dependency "foo" { + config_path = "../foo" +} diff --git a/test/fixture-get-output/cycle/abcda/foo/main.tf b/test/fixture-get-output/cycle/abcda/foo/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/cycle/abcda/foo/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-get-output/cycle/abcda/foo/terragrunt.hcl b/test/fixture-get-output/cycle/abcda/foo/terragrunt.hcl new file mode 100644 index 0000000000..5fee20517c --- /dev/null +++ b/test/fixture-get-output/cycle/abcda/foo/terragrunt.hcl @@ -0,0 +1,3 @@ +dependency "bar" { + config_path = "../bar" +} diff --git a/test/fixture-get-output/cycle/car/terragrunt.hcl b/test/fixture-get-output/cycle/car/terragrunt.hcl deleted file mode 100644 index aefd2efde2..0000000000 --- a/test/fixture-get-output/cycle/car/terragrunt.hcl +++ /dev/null @@ -1,3 +0,0 @@ -dependency "baz" { - config_path = "../baz" -} diff --git a/test/integration_test.go b/test/integration_test.go index 3992db7bb8..de346d6cd9 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1797,22 +1797,39 @@ func TestDependencyOutputCycleHandling(t *testing.T) { t.Parallel() cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT) - tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) - rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "cycle") - fooPath := util.JoinPath(rootPath, "foo") - planStdout := bytes.Buffer{} - planStderr := bytes.Buffer{} - err := runTerragruntCommand( - t, - fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-working-dir %s", fooPath), - &planStdout, - &planStderr, - ) - logBufferContentsLineByLine(t, planStdout, "plan stdout") - logBufferContentsLineByLine(t, planStderr, "plan stderr") - assert.Error(t, err) - assert.True(t, strings.Contains(err.Error(), "Found a dependency cycle between modules")) + testCases := []string{ + "aa", + "aba", + "abca", + "abcda", + } + + for _, testCase := range testCases { + // Capture range variable into forloop so that the binding is consistent across runs. + testCase := testCase + + t.Run(testCase, func(t *testing.T) { + t.Parallel() + + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) + rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "cycle", testCase) + fooPath := util.JoinPath(rootPath, "foo") + + planStdout := bytes.Buffer{} + planStderr := bytes.Buffer{} + err := runTerragruntCommand( + t, + fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-working-dir %s", fooPath), + &planStdout, + &planStderr, + ) + logBufferContentsLineByLine(t, planStdout, "plan stdout") + logBufferContentsLineByLine(t, planStderr, "plan stderr") + assert.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "Found a dependency cycle between modules")) + }) + } } func logBufferContentsLineByLine(t *testing.T, out bytes.Buffer, label string) { From 11dd3430d7a7631baf828a0f0dbd9bbcb265d718 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Wed, 4 Sep 2019 16:58:20 -0700 Subject: [PATCH 17/24] RFC template --- CONTRIBUTING.md | 19 ++++++++++++++++--- _docs/rfc/TEMPLATE.md | 44 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 _docs/rfc/TEMPLATE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4658fcd77a..a04e5869f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,19 +3,32 @@ Contributions to this repo are very welcome! We follow a fairly standard [pull request process](https://help.github.com/articles/about-pull-requests/) for contributions, subject to the following guidelines: -1. [File a GitHub issue](#file-a-github-issue) +1. [File a GitHub issue or write an RFC](#file-a-github-issue-or-write-an-rfc) 1. [Update the documentation](#update-the-documentation) 1. [Update the tests](#update-the-tests) 1. [Update the code](#update-the-code) 1. [Create a pull request](#create-a-pull-request) 1. [Merge and release](#merge-and-release) -## File a GitHub issue +## File a GitHub issue or write an RFC Before starting any work, we recommend filing a GitHub issue in this repo. This is your chance to ask questions and get feedback from the maintainers and the community before you sink a lot of time into writing (possibly the wrong) code. If there is anything you're unsure about, just ask! +Sometimes, the scope of the feature proposal is large enough that it requires major updates to the code base to +implement. In these situations, a maintainer may suggest writing up an RFC that describes the feature in more details +than what can be reasonably captured in a Github Issue. RFCs are written in markdown and live in the directory +`_docs/rfc`. + +To write an RFC: + +- Clone the repository +- Create a new branch +- Copy the template (`_docs/rfc/TEMPLATE.md`) to a new file in the same directory. +- Fill out the template +- Open a PR for comments, prefixing the title with the term `[RFC]` to indicate that it is an RFC PR. + ## Update the documentation We recommend updating the documentation *before* updating any code (see [Readme Driven @@ -52,4 +65,4 @@ to include the following: ## Merge and release The maintainers for this repo will review your code and provide feedback. If everything looks good, they will merge the -code and release a new version, which you'll be able to find in the [releases page](../../releases). \ No newline at end of file +code and release a new version, which you'll be able to find in the [releases page](../../releases). diff --git a/_docs/rfc/TEMPLATE.md b/_docs/rfc/TEMPLATE.md new file mode 100644 index 0000000000..2a114e1f7d --- /dev/null +++ b/_docs/rfc/TEMPLATE.md @@ -0,0 +1,44 @@ +# RFC Template + +This is a template you can use for proposing new major features to Terragrunt. When creating a new RFC, copy this +template and fill in each respective section. + +**STATUS**: In proposal _(This should be updated when you open a PR for the implementation)_ + + +## Background + +This section should describe why you need this feature in Terragrunt. This should include a description of: + +- The problem you are trying to solve. +- The reason why Terraform can't solve this problem, or data points that suggest there is a long enough time horizon for + implementation that it makes sense to workaround it in Terragrunt. +- Use cases for the problem. Why is it important that we have this feature? + + +## Proposed solution + +This section should describe which solution you are ultimately picking for implementation. This should describe in +detail how this solution addresses the problem. Additionally, this section should include implementation details. Be +sure to include code samples so that it is clear how users are intended to use the solution you are proposing! + +Note: This section can be left blank in the initial PR, if you are unsure what the best solution is. In this case, +include all the possible solutions you can think of as a part of the `Alternatives` section below. You can fill this +section out once you feel confident in a potential approach after discussion on the PR. + + +## Alternatives + +This section should describe various options for resolving the problem stated in the `Background` section. Be sure to +include various alternatives here and not just the one you are ultimately proposing. This helps communicate what +tradeoffs you are making in picking your solution. Any reasonably complex problem that requires an RFC have multiple +solutions that appear to be valid, so it helps to be explicit about why certain solutions were not chosen. + + +## References + +This section should include any links that are helpful for background reading such as: + +- Relevant issues +- Links to PRs: at a minimum, the initial PR for the RFC, and the implementation PR. +- Links to Terragrunt releases, if the proposed solution has been implemented. From b787dba874226b47e0422a7d6fa3f815bf78c027 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Thu, 5 Sep 2019 11:47:41 -0700 Subject: [PATCH 18/24] RFC for iteration of modules --- _docs/rfc/for_each_iteration.md | 218 ++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 _docs/rfc/for_each_iteration.md diff --git a/_docs/rfc/for_each_iteration.md b/_docs/rfc/for_each_iteration.md new file mode 100644 index 0000000000..8a48b6efa1 --- /dev/null +++ b/_docs/rfc/for_each_iteration.md @@ -0,0 +1,218 @@ +# RFC: for_each - looping variables to call module multiple times + +**STATUS**: In proposal _(This should be updated when you open a PR for the implementation)_ + + +## Background + +Oftentimes it is desirable to repeatedly apply a module multiple times with only a few modifications. This is common +when you have a canonical way to provision resources and need to quickly produce copies with one or two modifications. +For example, in Kubernetes, you might have a Namespace factory module that provisions Namespaces in a canonical way that +you configure with a common set of variables, and the only thing that differs across multiple Namespaces is the name. + +Another use case is in achieving compliance with the CIS benchmark for AWS. One of the requirements in the benchmark is +that you need to enable AWS Config in all enabled regions on your account. In this case, you would want to loop over +each enabled region and run the module that provisions and configures AWS Config using the same args. + +Ideally, we can handle this in the Terraform module by looping over `module` blocks: + +``` +module "aws_config" { + for_each = var.all_enabled_regions + region = each.key + + # additional arguments omitted for brevity +} +``` + +However, [as of 0.12.7, this is still not available](https://github.com/hashicorp/terraform/issues/17519). That said, +this is being developed and there is reason to believe that this will eventually be available, especially since, +starting with Terraform 0.12.0, `count` and `for_each` has been reserved on `module` blocks. What is not known is how +long it will take before `for_each` is implemented on modules. + +There are a few advantages to a Terragrunt implementation of `for_each` to call modules repeatedly with different +parameters: + +- Provide a workaround sooner than Terraform might implement module `for_each` and `count`. +- You can keep separate state files for each module call. While this does not help with isolation of blast radius (due + to the looping interface), this helps with isolating access to individual state files. For example, you can adjust the + remote state config such that the bucket is parameterized by the looping argument, allowing you to store the state of + each module call in a different S3 bucket. +- You can allow separate IAM roles for each loop iteration. This can be done if you parameterize the `iam_role` + property with the looping argument. + +For this reason, it is worth contemplating a possible implementation path for supporting iteration in Terragrunt to +repeatedly call a module with different parameters. Note when considering an implementation, it is important to optimize +for **speed of implementation and simplicity**, since it is very likely that this feature will be made obsolete by +`for_each` on `module` blocks. + + +## Proposed solution + +_Intentionally blank, as the solution is not obvious._ + + +## Alternatives + +### Option 1: Null option - do nothing + +Since we already know Terraform wants to support looping `module` blocks, we can make the conclusion to wait for +Terraform to implement it as opposed to implementing a workaround in Terragrunt. + +**Pros**: + +- Keeps Terragrunt clean, since no new code is introduced. +- Relies on official solution. We don't want to introduce workarounds in Terragrunt unless it is absolutely necessary + and unlikely to be supported officially in Terraform. + +**Cons**: + +- We must wait for Hashicorp to implement the feature. Given how the resources, data sources, and providers work in + `module` blocks, looping modules is a reasonably complex feature, both in terms of state representation and resource + management. As such, this is expected to take some time, as described in [the 0.12 + preview](https://www.hashicorp.com/blog/hashicorp-terraform-0-12-preview-for-and-for-each#module-count-and-for_each). + +### Option 2: for_each attribute in terragrunt config that "generates" multiple copies of the config + +##### Description + +In this approach, looping is handled at the top level using a new attribute named `for_each` to mimic the `for_each` +construct in Terraform. For example: + +``` +# terragrunt.hcl +for_each = ["us-east-1", "us-west-1"] + +inputs = { + region = each.value +} +``` + +Internally, this will translate to multiple copies of `config.TerragruntConfig` structs, one for each item in the +`for_each` collection with `each.value` interpolated to the value for that item. Then, Terragrunt will run the command +on each config in a loop. + +One challenge with this approach is that the state needs to be unique for each item in the `for_each` collection to +avoid conflict in the state file. Since each call is a separate module call, having each one share state will result in +each call stepping over each other. Therefore, it is important that this implementation **add a guard to ensure the +remote state file is unique per iteration**. + +To support this, there should be a way to partially patch the `remote_state` config that is inherited from a parent. For +example, we should be able to do the following: + +_Parent terragrunt.hcl_ + +``` +remote_state { + backend = "s3" + config = { + bucket = "my-terraform-state" + key = "${path_relative_to_include()}/terraform.tfstate" + region = "us-east-1" + encrypt = true + dynamodb_table = "my-lock-table" + } +} +``` + +_Child terragrunt.hcl_ + +``` +for_each = ["us-east-1", "us-west-1"] + +remote_state { + # Added for backwards compatibility. This should default to false, so that existing config overrides completely as + # opposed to partially. + merge_parent = true + + config = { + key = "dev/${each.value}/_global/aws-config/terraform.tfstate" + } +} + +inputs = { + region = each.value +} +``` + +In this setup, the `remote_state` block in the child `terragrunt.hcl` should be used to patch the parent `remote_state` +block, as opposed to completely replacing it as it is now in the current behavior. This way, you can still keep your +`remote_state` config DRY while only updating the relevant pieces to support iteration. + +##### Implementation detail + +To implement this, the following changes need to be made: + +- `ParseConfigFile`, `ParseConfigString`, `PartialParseConfigFile`, and `PartialParseConfigString` need to be updated to + return list of `TerragruntConfig` structs, as opposed to a single one. +- Each caller of parsing functions need to accordingly handle the list of `TerragruntConfig` objects. As a part of this, + the caller should loop through each config to run the commands in the context of each iteration of the expanded loop. +- The terragrunt workspace logic needs to be updated to ensure a separate module folder is created for each loop + iteration. +- The parsing functions need to be updated to first parse the `for_each` attribute. Then, the parser should iterate each + item in the list and set the `each` variable accordingly as it parses the rest of the config. This means the parsing + order is now adjusted as: + - `for_each` and expand loop. + - `locals` + - `include` + - `dependency` + - Everything else +- Support `merge_parent` in `remote_state`, which ultimately adjusts how `mergeConfigWithIncludedConfig` merges the + child `remote_state` config with the parent. + + +##### Pros and Cons of approach + +**Pros**: + +- Parallels `for_each` in Terraform. In the examples I used `list`, but the implementation details can mimic that of + Terraform (requiring `set` or `map`). That way, the syntax closely resembles an existing construct and thus it will be + easier to pick up. +- Limited changes are necessary in the user config to add in iteration. +- The config parser implementation is relatively simple. + +**Cons**: + +- The implementation of the runners can become complex, especially in the dependency detection for the `xxx-all` + commands. If `dependency` blocks are parameterized by looping, this can be hairy to unpack and expand into the + dependency graph if the terragrunt files don't exist. +- The logic for expanding out the loop for the terragrunt cache is complex as it is not immediately obvious what the + folder structure should be. Should it be a hash of the iteration value appended to the working path (e.g + `.terragrunt-cache/HASH_OF_EACH_VALUE/path/to/workspace`)? Or should it be one cache per iteration value (e.g + `HASH_OF_EACH_VALUE/.terragrunt-cache/path/to/workspace`)? +- Implementation requires a major refactor in the calling logic post parsing. +- It is not immediately obvious how to handle removing an item in the `for_each` list. A naive approach would not run + `terragrunt destroy` on that module. Additionally, as stated, it is impossible to run the command on a single item in + the list and thus you can't destroy just the items you want to remove without doing complex dancing (e.g update the + `for_each` list to only reference the items you want to destroy, then run `terragrunt destroy` to destroy just those + items, and then restore the list to the version with the items removed). + +### Option 3: scaffolding tool that code gens live config using a template + +Instead of implementing looping within the terragrunt execution logic, this approach proposes generating the live config +through templating. The idea here would be to use a templating tool (e.g +[cookiecutter](https://github.com/cookiecutter/cookiecutter)) to render each live config parameterized by some list +value that will then generate the folder structure. + +In this approach, no change is necessary within Terragrunt to support this other than perhaps documentation and examples +to demonstrate the approach. + +**Pros**: + +- Keeps Terragrunt clean, since no new code is introduced. +- Available workaround now, without any work. + +**Cons**: + +- Code generation is complex. When should you run the code generation step? How often? How do you know if there is + drift? +- Removing an iteration still requires a bit of a dance. You need to `terragrunt destroy` the relevant folder, then + remove that folder, and then remove the corresponding item from the iteration list. +- Users now need to learn a templating tool and language, on top of Terragrunt and Terraform. +- It is not immediately obvious where these templates should live. Is it a part of `infrastructure-modules`? + `infrastructure-live`? Or a completely separate repo? + + +## References + +- [Terraform Issue proposing for_each on modules](https://github.com/hashicorp/terraform/issues/17519) From 065d2fe5fbeefb3c86581229783ff89e0997d4e1 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 6 Sep 2019 09:40:05 -0700 Subject: [PATCH 19/24] Add in additional concerns highlighted from code review --- _docs/rfc/for_each_iteration.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/_docs/rfc/for_each_iteration.md b/_docs/rfc/for_each_iteration.md index 8a48b6efa1..4dd2975307 100644 --- a/_docs/rfc/for_each_iteration.md +++ b/_docs/rfc/for_each_iteration.md @@ -204,8 +204,13 @@ to demonstrate the approach. **Cons**: -- Code generation is complex. When should you run the code generation step? How often? How do you know if there is - drift? +- Code generation is complex. + - When should you run the code generation step? + - How often? + - How do you know if there is drift? + - Do you check in the generated code? If so, you have to maintain it (e.g., 19 sets of files/folders for 19 AWS + regions). If it's always generated at runtime, then interacting with just one of the generated things becomes + harder. - Removing an iteration still requires a bit of a dance. You need to `terragrunt destroy` the relevant folder, then remove that folder, and then remove the corresponding item from the iteration list. - Users now need to learn a templating tool and language, on top of Terragrunt and Terraform. From e281b4c38d09eae2e4c9c3b8f35d7f7901a4fdef Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 6 Sep 2019 15:48:18 -0700 Subject: [PATCH 20/24] Fix #854 --- config/dependency.go | 7 ++++-- .../root/environments/network/main.tf | 3 +++ .../root/environments/network/terragrunt.hcl | 3 +++ .../root/environments/web/main.tf | 3 +++ .../root/environments/web/sg/main.tf | 3 +++ .../root/environments/web/sg/terragrunt.hcl | 7 ++++++ .../root/environments/web/terragrunt.hcl | 11 +++++++++ .../regression-854/root/terragrunt.hcl | 1 + test/integration_test.go | 23 +++++++++++++++++++ 9 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 test/fixture-get-output/regression-854/root/environments/network/main.tf create mode 100644 test/fixture-get-output/regression-854/root/environments/network/terragrunt.hcl create mode 100644 test/fixture-get-output/regression-854/root/environments/web/main.tf create mode 100644 test/fixture-get-output/regression-854/root/environments/web/sg/main.tf create mode 100644 test/fixture-get-output/regression-854/root/environments/web/sg/terragrunt.hcl create mode 100644 test/fixture-get-output/regression-854/root/environments/web/terragrunt.hcl create mode 100644 test/fixture-get-output/regression-854/root/terragrunt.hcl diff --git a/config/dependency.go b/config/dependency.go index eeb0961b64..44223648de 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -78,7 +78,8 @@ func checkForDependencyBlockCycles(filename string, decodedDependency terragrunt currentTraversalPaths := []string{filename} for _, dependency := range decodedDependency.Dependencies { dependencyPath := cleanDependencyTerragruntConfigPath(filename, dependency.ConfigPath) - if err := checkForDependencyBlockCyclesUsingDFS(dependencyPath, &visitedPaths, ¤tTraversalPaths, terragruntOptions); err != nil { + dependencyOptions := terragruntOptions.Clone(dependencyPath) + if err := checkForDependencyBlockCyclesUsingDFS(dependencyPath, &visitedPaths, ¤tTraversalPaths, dependencyOptions); err != nil { return err } } @@ -109,7 +110,9 @@ func checkForDependencyBlockCyclesUsingDFS( return err } for _, dependency := range dependencyPaths { - if err := checkForDependencyBlockCyclesUsingDFS(cleanDependencyTerragruntConfigPath(currentConfigPath, dependency), visitedPaths, currentTraversalPaths, terragruntOptions); err != nil { + nextPath := cleanDependencyTerragruntConfigPath(currentConfigPath, dependency) + nextOptions := terragruntOptions.Clone(nextPath) + if err := checkForDependencyBlockCyclesUsingDFS(nextPath, visitedPaths, currentTraversalPaths, nextOptions); err != nil { return err } } diff --git a/test/fixture-get-output/regression-854/root/environments/network/main.tf b/test/fixture-get-output/regression-854/root/environments/network/main.tf new file mode 100644 index 0000000000..c917f7897c --- /dev/null +++ b/test/fixture-get-output/regression-854/root/environments/network/main.tf @@ -0,0 +1,3 @@ +output "id" { + value = 42 +} diff --git a/test/fixture-get-output/regression-854/root/environments/network/terragrunt.hcl b/test/fixture-get-output/regression-854/root/environments/network/terragrunt.hcl new file mode 100644 index 0000000000..db1e046238 --- /dev/null +++ b/test/fixture-get-output/regression-854/root/environments/network/terragrunt.hcl @@ -0,0 +1,3 @@ +include { + path = "../../terragrunt.hcl" +} diff --git a/test/fixture-get-output/regression-854/root/environments/web/main.tf b/test/fixture-get-output/regression-854/root/environments/web/main.tf new file mode 100644 index 0000000000..c917f7897c --- /dev/null +++ b/test/fixture-get-output/regression-854/root/environments/web/main.tf @@ -0,0 +1,3 @@ +output "id" { + value = 42 +} diff --git a/test/fixture-get-output/regression-854/root/environments/web/sg/main.tf b/test/fixture-get-output/regression-854/root/environments/web/sg/main.tf new file mode 100644 index 0000000000..c917f7897c --- /dev/null +++ b/test/fixture-get-output/regression-854/root/environments/web/sg/main.tf @@ -0,0 +1,3 @@ +output "id" { + value = 42 +} diff --git a/test/fixture-get-output/regression-854/root/environments/web/sg/terragrunt.hcl b/test/fixture-get-output/regression-854/root/environments/web/sg/terragrunt.hcl new file mode 100644 index 0000000000..4bea6955f9 --- /dev/null +++ b/test/fixture-get-output/regression-854/root/environments/web/sg/terragrunt.hcl @@ -0,0 +1,7 @@ +include { + path = "../../../terragrunt.hcl" +} + +dependency "network" { + config_path = "../../network" +} diff --git a/test/fixture-get-output/regression-854/root/environments/web/terragrunt.hcl b/test/fixture-get-output/regression-854/root/environments/web/terragrunt.hcl new file mode 100644 index 0000000000..fc6e4c0b33 --- /dev/null +++ b/test/fixture-get-output/regression-854/root/environments/web/terragrunt.hcl @@ -0,0 +1,11 @@ +include { + path = "../../terragrunt.hcl" +} + +dependency "network" { + config_path = "../network" +} + +dependency "sg" { + config_path = "./sg" +} diff --git a/test/fixture-get-output/regression-854/root/terragrunt.hcl b/test/fixture-get-output/regression-854/root/terragrunt.hcl new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-get-output/regression-854/root/terragrunt.hcl @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/integration_test.go b/test/integration_test.go index de346d6cd9..754eaac4e4 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1832,6 +1832,29 @@ func TestDependencyOutputCycleHandling(t *testing.T) { } } +// Regression testing for https://github.com/gruntwork-io/terragrunt/issues/854: Referencing a dependency that is a +// subdirectory of the current config, which includes an `include` block has problems resolving the correct relative +// path. +func TestDependencyOutputRegression854(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) + rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "regression-854", "root") + + stdout := bytes.Buffer{} + stderr := bytes.Buffer{} + err := runTerragruntCommand( + t, + fmt.Sprintf("terragrunt apply-all --terragrunt-non-interactive --terragrunt-working-dir %s", rootPath), + &stdout, + &stderr, + ) + logBufferContentsLineByLine(t, stdout, "stdout") + logBufferContentsLineByLine(t, stderr, "stderr") + require.NoError(t, err) +} + func logBufferContentsLineByLine(t *testing.T, out bytes.Buffer, label string) { t.Logf("[%s] Full contents of %s:", t.Name(), label) lines := strings.Split(out.String(), "\n") From 5ce7cd0833a9141c546b86497f05436a37e93c4a Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Fri, 13 Sep 2019 03:13:22 -0700 Subject: [PATCH 21/24] [skip ci] Write final conclusion and update state --- _docs/rfc/for_each_iteration.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/_docs/rfc/for_each_iteration.md b/_docs/rfc/for_each_iteration.md index 4dd2975307..de836ef28f 100644 --- a/_docs/rfc/for_each_iteration.md +++ b/_docs/rfc/for_each_iteration.md @@ -1,6 +1,6 @@ # RFC: for_each - looping variables to call module multiple times -**STATUS**: In proposal _(This should be updated when you open a PR for the implementation)_ +**STATUS: Won't Implement** ## Background @@ -49,7 +49,9 @@ for **speed of implementation and simplicity**, since it is very likely that thi ## Proposed solution -_Intentionally blank, as the solution is not obvious._ +We decided to go with Option 1 where we will not implement any new features in Terragrunt to specifically handle this +use case, given the complexity of the task involved and the relative short lifespan of such a feature. If you have a +need for replicating modules, we recommend using a code generation tool to templatize your Terraform modules. ## Alternatives @@ -221,3 +223,4 @@ to demonstrate the approach. ## References - [Terraform Issue proposing for_each on modules](https://github.com/hashicorp/terraform/issues/17519) +- [Original RFC PR](https://github.com/gruntwork-io/terragrunt/pull/853) From 2f9172769b8c5a0e998930642faf0527353ada69 Mon Sep 17 00:00:00 2001 From: Ilia Lazebnik Date: Fri, 13 Sep 2019 16:50:22 +0300 Subject: [PATCH 22/24] Add windows installation option in docs Added choco command for installing terragrunt in readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 7d902b1edf..f49b1b5d0b 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,9 @@ Note that third-party Terragrunt packages may not be updated with the latest ver Please check your version against the latest available on the [Releases Page](https://github.com/gruntwork-io/terragrunt/releases). +### Windows +You can install Terragrunt on Windows using [Chocolatey](https://chocolatey.org/): `choco install terragrunt`. + ### macOS You can install Terragrunt on macOS using [Homebrew](https://brew.sh/): `brew install terragrunt`. From 3bb5734a3f527389600c354c804ad3c4e1195e5f Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Mon, 16 Sep 2019 10:04:07 -0700 Subject: [PATCH 23/24] Regression test for bug where terragrunt-cache path is not correct for child dependencies --- .../localstate/live/child/terragrunt.hcl | 15 +++++++++++++ .../localstate/live/parent/terragrunt.hcl | 7 ++++++ .../localstate/live/terragrunt.hcl | 6 +++++ .../localstate/modules/child/main.tf | 9 ++++++++ .../localstate/modules/parent/main.tf | 7 ++++++ test/integration_test.go | 22 +++++++++++++++++++ 6 files changed, 66 insertions(+) create mode 100644 test/fixture-get-output/localstate/live/child/terragrunt.hcl create mode 100644 test/fixture-get-output/localstate/live/parent/terragrunt.hcl create mode 100644 test/fixture-get-output/localstate/live/terragrunt.hcl create mode 100644 test/fixture-get-output/localstate/modules/child/main.tf create mode 100644 test/fixture-get-output/localstate/modules/parent/main.tf diff --git a/test/fixture-get-output/localstate/live/child/terragrunt.hcl b/test/fixture-get-output/localstate/live/child/terragrunt.hcl new file mode 100644 index 0000000000..d518bbfdb4 --- /dev/null +++ b/test/fixture-get-output/localstate/live/child/terragrunt.hcl @@ -0,0 +1,15 @@ +include { + path = find_in_parent_folders() +} + +terraform { + source = "../../modules/child" +} + +dependency "x" { + config_path = "../parent" +} + +inputs = { + x = dependency.x.outputs.x +} diff --git a/test/fixture-get-output/localstate/live/parent/terragrunt.hcl b/test/fixture-get-output/localstate/live/parent/terragrunt.hcl new file mode 100644 index 0000000000..136641826d --- /dev/null +++ b/test/fixture-get-output/localstate/live/parent/terragrunt.hcl @@ -0,0 +1,7 @@ +include { + path = find_in_parent_folders() +} + +terraform { + source = "../../modules/parent" +} diff --git a/test/fixture-get-output/localstate/live/terragrunt.hcl b/test/fixture-get-output/localstate/live/terragrunt.hcl new file mode 100644 index 0000000000..c1df67e484 --- /dev/null +++ b/test/fixture-get-output/localstate/live/terragrunt.hcl @@ -0,0 +1,6 @@ +remote_state { + backend = "local" + config = { + path = "${path_relative_to_include()}/terraform.tfstate" + } +} diff --git a/test/fixture-get-output/localstate/modules/child/main.tf b/test/fixture-get-output/localstate/modules/child/main.tf new file mode 100644 index 0000000000..e513359ede --- /dev/null +++ b/test/fixture-get-output/localstate/modules/child/main.tf @@ -0,0 +1,9 @@ +terraform { + backend "local" {} +} + +variable "x" {} + +output "y" { + value = var.x * 3 +} diff --git a/test/fixture-get-output/localstate/modules/parent/main.tf b/test/fixture-get-output/localstate/modules/parent/main.tf new file mode 100644 index 0000000000..b90ac736a6 --- /dev/null +++ b/test/fixture-get-output/localstate/modules/parent/main.tf @@ -0,0 +1,7 @@ +terraform { + backend "local" {} +} + +output "x" { + value = 14 +} diff --git a/test/integration_test.go b/test/integration_test.go index 754eaac4e4..f94490dc2e 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1855,6 +1855,28 @@ func TestDependencyOutputRegression854(t *testing.T) { require.NoError(t, err) } +// Regression testing for bug where terragrunt output runs on dependency blocks are done in the terragrunt-cache for the +// child, not the parent. +func TestDependencyOutputCachePathBug(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT) + rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "localstate", "live") + + stdout := bytes.Buffer{} + stderr := bytes.Buffer{} + err := runTerragruntCommand( + t, + fmt.Sprintf("terragrunt apply-all --terragrunt-non-interactive --terragrunt-working-dir %s", rootPath), + &stdout, + &stderr, + ) + logBufferContentsLineByLine(t, stdout, "stdout") + logBufferContentsLineByLine(t, stderr, "stderr") + require.NoError(t, err) +} + func logBufferContentsLineByLine(t *testing.T, out bytes.Buffer, label string) { t.Logf("[%s] Full contents of %s:", t.Name(), label) lines := strings.Split(out.String(), "\n") From 41e0abd74accae4675c3f608c3fd42a0fa2a7a62 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Mon, 16 Sep 2019 10:14:55 -0700 Subject: [PATCH 24/24] Fix bug where the context for running terraform output on the child dependency was not fully setup: --- config/dependency.go | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/config/dependency.go b/config/dependency.go index 44223648de..02d2d49a4e 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -290,15 +290,43 @@ func getTerragruntOutput(dependencyConfig Dependency, terragruntOptions *options return &convertedOutput, isEmpty, errors.WithStackTrace(err) } +// Clone terragrunt options and update context for dependency block so that the outputs can be read correctly +func cloneTerragruntOptionsForDependencyOutput(terragruntOptions *options.TerragruntOptions, targetConfig string) (*options.TerragruntOptions, error) { + targetOptions := terragruntOptions.Clone(targetConfig) + targetOptions.TerraformCliArgs = []string{"output", "-json"} + + // DownloadDir needs to be updated to be in the context of the new config, if using default + _, originalDefaultDownloadDir, err := options.DefaultWorkingAndDownloadDirs(terragruntOptions.TerragruntConfigPath) + if err != nil { + return nil, errors.WithStackTrace(err) + } + + // Using default, so compute new download dir and update + if terragruntOptions.DownloadDir == originalDefaultDownloadDir { + _, downloadDir, err := options.DefaultWorkingAndDownloadDirs(targetConfig) + if err != nil { + return nil, errors.WithStackTrace(err) + } + targetOptions.DownloadDir = downloadDir + } + + return targetOptions, nil +} + // runTerragruntOutputJson uses terragrunt running functions to extract the json output from the target config. Make a // copy of the terragruntOptions so that we can reuse the same execution environment. func runTerragruntOutputJson(terragruntOptions *options.TerragruntOptions, targetConfig string) ([]byte, error) { + targetOptions, err := cloneTerragruntOptionsForDependencyOutput(terragruntOptions, targetConfig) + if err != nil { + return nil, err + } + + // Update the stdout buffer so we can capture the output var stdoutBuffer bytes.Buffer stdoutBufferWriter := bufio.NewWriter(&stdoutBuffer) - targetOptions := terragruntOptions.Clone(targetConfig) - targetOptions.TerraformCliArgs = []string{"output", "-json"} targetOptions.Writer = stdoutBufferWriter - err := targetOptions.RunTerragrunt(targetOptions) + + err = targetOptions.RunTerragrunt(targetOptions) if err != nil { return nil, errors.WithStackTrace(err) }