Merge pull request #519 from gruntwork-io/fix-all-commands-attempt-2

Fix xxx-all commands, attempt #2

config/config.go

@@ -142,27 +142,21 @@ func DefaultConfigPath(workingDir string) string {
 // Returns a list of all Terragrunt config files in the given path or any subfolder of the path. A file is a Terragrunt
 // config file if it has a name as returned by the DefaultConfigPath method and contains Terragrunt config contents
 // as returned by the IsTerragruntConfigFile method.
-func FindConfigFilesInPath(rootPath string) ([]string, error) {
+func FindConfigFilesInPath(rootPath string, terragruntOptions *options.TerragruntOptions) ([]string, error) {
 	configFiles := []string{}
 
 	err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
-		if info.IsDir() {
-			if strings.Contains(info.Name(), options.TerragruntCacheDir) {
-				return nil
-			}
+		isTerragruntModule, err := containsTerragruntModule(path, info, terragruntOptions)
+		if err != nil {
+			return err
+		}
 
-			configPath := DefaultConfigPath(path)
-			isTerragruntConfig, err := IsTerragruntConfigFile(configPath)
-			if err != nil {
-				return err
-			}
-			if isTerragruntConfig {
-				configFiles = append(configFiles, configPath)
-			}
+		if isTerragruntModule {
+			configFiles = append(configFiles, DefaultConfigPath(path))
 		}
 
 		return nil
@@ -171,6 +165,37 @@ func FindConfigFilesInPath(rootPath string) ([]string, error) {
 	return configFiles, err
 }
 
+// Returns true if the given path with the given FileInfo contains a Terragrunt module and false otherwise. A path
+// contains a Terragrunt module if it contains a Terragrunt configuration file (terraform.tfvars) and is not a cache
+// or download dir.
+func containsTerragruntModule(path string, info os.FileInfo, terragruntOptions *options.TerragruntOptions) (bool, error) {
+	if !info.IsDir() {
+		return false, nil
+	}
+
+	// Skip the Terragrunt cache dir
+	if strings.Contains(path, options.TerragruntCacheDir) {
+		return false, nil
+	}
+
+	canonicalPath, err := util.CanonicalPath(path, "")
+	if err != nil {
+		return false, err
+	}
+
+	canonicalDownloadPath, err := util.CanonicalPath(terragruntOptions.DownloadDir, "")
+	if err != nil {
+		return false, err
+	}
+
+	// Skip any custom download dir specified by the user
+	if strings.Contains(canonicalPath, canonicalDownloadPath) {
+		return false, err
+	}
+
+	return IsTerragruntConfigFile(DefaultConfigPath(path))
+}
+
 // Returns true if the given path corresponds to file that could be a Terragrunt config file. A file could be a
 // Terragrunt config file if:
 //

config/config_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/remote"
 	"github.com/gruntwork-io/terragrunt/util"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestParseTerragruntConfigRemoteStateMinimalConfig(t *testing.T) {
@@ -721,7 +722,10 @@ func TestFindConfigFilesInPathNone(t *testing.T) {
 	t.Parallel()
 
 	expected := []string{}
-	actual, err := FindConfigFilesInPath("../test/fixture-config-files/none")
+	terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
+	require.NoError(t, err)
+
+	actual, err := FindConfigFilesInPath("../test/fixture-config-files/none", terragruntOptions)
 
 	assert.Nil(t, err, "Unexpected error: %v", err)
 	assert.Equal(t, expected, actual)
@@ -731,7 +735,10 @@ func TestFindConfigFilesInPathOneNewConfig(t *testing.T) {
 	t.Parallel()
 
 	expected := []string{"../test/fixture-config-files/one-new-config/subdir/terraform.tfvars"}
-	actual, err := FindConfigFilesInPath("../test/fixture-config-files/one-new-config")
+	terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
+	require.NoError(t, err)
+
+	actual, err := FindConfigFilesInPath("../test/fixture-config-files/one-new-config", terragruntOptions)
 
 	assert.Nil(t, err, "Unexpected error: %v", err)
 	assert.Equal(t, expected, actual)
@@ -741,7 +748,10 @@ func TestFindConfigFilesInPathOneOldConfig(t *testing.T) {
 	t.Parallel()
 
 	expected := []string{"../test/fixture-config-files/one-old-config/subdir/.terragrunt"}
-	actual, err := FindConfigFilesInPath("../test/fixture-config-files/one-old-config")
+	terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
+	require.NoError(t, err)
+
+	actual, err := FindConfigFilesInPath("../test/fixture-config-files/one-old-config", terragruntOptions)
 
 	assert.Nil(t, err, "Unexpected error: %v", err)
 	assert.Equal(t, expected, actual)
@@ -755,7 +765,10 @@ func TestFindConfigFilesInPathMultipleConfigs(t *testing.T) {
 		"../test/fixture-config-files/multiple-configs/subdir-2/subdir/.terragrunt",
 		"../test/fixture-config-files/multiple-configs/subdir-3/terraform.tfvars",
 	}
-	actual, err := FindConfigFilesInPath("../test/fixture-config-files/multiple-configs")
+	terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
+	require.NoError(t, err)
+
+	actual, err := FindConfigFilesInPath("../test/fixture-config-files/multiple-configs", terragruntOptions)
 
 	assert.Nil(t, err, "Unexpected error: %v", err)
 	assert.Equal(t, expected, actual)
@@ -767,7 +780,27 @@ func TestFindConfigFilesIgnoresTerragruntCache(t *testing.T) {
 	expected := []string{
 		"../test/fixture-config-files/ignore-cached-config/terraform.tfvars",
 	}
-	actual, err := FindConfigFilesInPath("../test/fixture-config-files/ignore-cached-config")
+	terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
+	require.NoError(t, err)
+
+	actual, err := FindConfigFilesInPath("../test/fixture-config-files/ignore-cached-config", terragruntOptions)
+
+	assert.Nil(t, err, "Unexpected error: %v", err)
+	assert.Equal(t, expected, actual)
+}
+
+func TestFindConfigFilesIgnoresDownloadDir(t *testing.T) {
+	t.Parallel()
+
+	expected := []string{
+		"../test/fixture-config-files/multiple-configs/terraform.tfvars",
+		"../test/fixture-config-files/multiple-configs/subdir-3/terraform.tfvars",
+	}
+	terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
+	require.NoError(t, err)
+	terragruntOptions.DownloadDir = "../test/fixture-config-files/multiple-configs/subdir-2"
+
+	actual, err := FindConfigFilesInPath("../test/fixture-config-files/multiple-configs", terragruntOptions)
 
 	assert.Nil(t, err, "Unexpected error: %v", err)
 	assert.Equal(t, expected, actual)

configstack/module.go

@@ -95,6 +95,22 @@ func resolveTerraformModule(terragruntConfigPath string, terragruntOptions *opti
 	}
 	opts.Source = terragruntSource
 
+	_, defaultDownloadDir, err := options.DefaultWorkingAndDownloadDirs(terragruntOptions.TerragruntConfigPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// If we're using the default download directory, put it into the same folder as the Terragrunt configuration file.
+	// If we're not using the default, then the user has specified a custom download directory, and we leave it as-is.
+	if terragruntOptions.DownloadDir == defaultDownloadDir {
+		_, downloadDir, err := options.DefaultWorkingAndDownloadDirs(terragruntConfigPath)
+		if err != nil {
+			return nil, err
+		}
+		terragruntOptions.Logger.Printf("Setting download directory for module %s to %s", modulePath, downloadDir)
+		opts.DownloadDir = downloadDir
+	}
+
 	// Fix for https://github.com/gruntwork-io/terragrunt/issues/208
 	matches, err := filepath.Glob(filepath.Join(filepath.Dir(terragruntConfigPath), "*.tf"))
 	if err != nil {

configstack/stack.go

@@ -101,7 +101,7 @@ func (stack *Stack) CheckForCycles() error {
 // Find all the Terraform modules in the subfolders of the working directory of the given TerragruntOptions and
 // assemble them into a Stack object that can be applied or destroyed in a single command
 func FindStackInSubfolders(terragruntOptions *options.TerragruntOptions) (*Stack, error) {
-	terragruntConfigFiles, err := config.FindConfigFilesInPath(terragruntOptions.WorkingDir)
+	terragruntConfigFiles, err := config.FindConfigFilesInPath(terragruntOptions.WorkingDir, terragruntOptions)
 	if err != nil {
 		return nil, err
 	}

options/options.go

@@ -87,11 +87,9 @@ type TerragruntOptions struct {
 
 // Create a new TerragruntOptions object with reasonable defaults for real usage
 func NewTerragruntOptions(terragruntConfigPath string) (*TerragruntOptions, error) {
-	workingDir := filepath.Dir(terragruntConfigPath)
-
 	logger := util.CreateLogger("")
 
-	downloadDir, err := filepath.Abs(filepath.Join(workingDir, TerragruntCacheDir))
+	workingDir, downloadDir, err := DefaultWorkingAndDownloadDirs(terragruntConfigPath)
 	if err != nil {
 		return nil, errors.WithStackTrace(err)
 	}
@@ -118,6 +116,18 @@ func NewTerragruntOptions(terragruntConfigPath string) (*TerragruntOptions, erro
 	}, nil
 }
 
+// Get the default working and download directories for the given Terragrunt config path
+func DefaultWorkingAndDownloadDirs(terragruntConfigPath string) (string, string, error) {
+	workingDir := filepath.Dir(terragruntConfigPath)
+
+	downloadDir, err := filepath.Abs(filepath.Join(workingDir, TerragruntCacheDir))
+	if err != nil {
+		return "", "", errors.WithStackTrace(err)
+	}
+
+	return workingDir, downloadDir, nil
+}
+
 // Create a new TerragruntOptions object with reasonable defaults for test usage
 func NewTerragruntOptionsForTest(terragruntConfigPath string) (*TerragruntOptions, error) {
 	opts, err := NewTerragruntOptions(terragruntConfigPath)

test/fixture-config-files/ignore-cached-config/subfolder/.terragrunt-cache/subfolder/terraform.tfvars

@@ -0,0 +1,5 @@
+# This file is used to test that we ignore terraform.tfvars files in Terragrunt's cache dir. We had to manually git add
+# this file since .terragrunt-cache is in .gitignore
+terragrunt = {
+
+}