Engine tests improvements (#3995)

* Engine download fixes

* Updated test documentation

* Documentation update

* Engine log level update

.circleci/config.yml

@@ -60,16 +60,6 @@ install_tflint: &install_tflint
     popd
     tflint --version
 
-install_tofu_engine: &install_tofu_engine
-  name: Install OpenTofu Engine
-  command: |
-    # Download the OpenTofu Engine binary
-    set -x
-    export REPO="gruntwork-io/terragrunt-engine-opentofu"
-    export ASSET_NAME="terragrunt-iac-engine-opentofu_rpc_${TOFU_ENGINE_VERSION}_linux_amd64.zip"
-    wget -O "engine.zip" "https://github.com/${REPO}/releases/download/${TOFU_ENGINE_VERSION}/${ASSET_NAME}"
-    unzip -o "engine.zip"
-
 setup_test_environment: &setup_test_environment
   name: Setup integration tests environment
   command: |
@@ -395,8 +385,6 @@ jobs:
       - checkout
       - run:
           <<: *install_tofu
-      - run:
-          <<: *install_tofu_engine
       - run:
           <<: *setup_test_environment
       - run:

engine/engine.go

@@ -16,6 +16,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/gruntwork-io/terragrunt/pkg/log"
+
 	"github.com/gruntwork-io/terragrunt/internal/cache"
 
 	"github.com/hashicorp/go-getter"
@@ -504,7 +506,11 @@ func createEngine(terragruntOptions *options.TerragruntOptions) (*proto.EngineCl
 
 	engineLogLevel := terragruntOptions.EngineLogLevel
 	if len(engineLogLevel) == 0 {
-		engineLogLevel = terragruntOptions.Logger.Level().String()
+		engineLogLevel = hclog.Warn.String()
+		// update log level if it is different from info
+		if terragruntOptions.Logger.Level() != log.InfoLevel {
+			engineLogLevel = terragruntOptions.Logger.Level().String()
+		}
 		// turn off log formatting if disabled for Terragrunt
 		if terragruntOptions.Logger.Formatter().DisabledOutput() {
 			engineLogLevel = hclog.Off.String()

pkg/log/logger.go

@@ -26,7 +26,7 @@ type Logger interface {
 	// SetFormatter sets the logger formatter.
 	SetFormatter(formatter Formatter)
 
-	// SetFormatter returns the logger formatter.
+	// Formatter returns the logger formatter.
 	Formatter() Formatter
 
 	// WithOptions clones and sets the given options for the new instance.

test/integration_engine_test.go

@@ -3,7 +3,11 @@
 package test_test
 
 import (
+	"context"
 	"fmt"
+
+	"github.com/hashicorp/go-getter/v2"
+
 	"os"
 	"path/filepath"
 	"runtime"
@@ -29,17 +33,22 @@ const (
 	envVarExperimental = "TG_EXPERIMENTAL_ENGINE"
 )
 
-var LocalEngineBinaryPath = "terragrunt-iac-engine-opentofu_rpc_" + testEngineVersion() + "_" + runtime.GOOS + "_" + runtime.GOARCH
+var (
+	engineAssetName    = "terragrunt-iac-engine-opentofu_rpc_" + testEngineVersion() + "_" + runtime.GOOS + "_" + runtime.GOARCH
+	engineAssetArchive = engineAssetName + ".zip"
+	downloadUrl        = fmt.Sprintf("https://github.com/gruntwork-io/terragrunt-engine-opentofu/releases/download/%s/%s", testEngineVersion(), engineAssetArchive)
+)
 
 func TestEngineLocalPlan(t *testing.T) {
 	rootPath := setupLocalEngine(t)
 
 	stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s --terragrunt-log-level trace", rootPath))
 	require.NoError(t, err)
 
-	assert.Contains(t, stderr, LocalEngineBinaryPath)
-	assert.Contains(t, stderr, "[INFO]  plugin process exited:")
-	assert.Contains(t, stderr, "plugin process exited:")
+	assert.Contains(t, stderr, engineAssetName)
+	assert.Contains(t, stderr, "Tofu Initialization started")
+	assert.Contains(t, stderr, "Tofu Initialization completed")
+	assert.Contains(t, stderr, "Tofu Shutdown completed")
 	assert.Contains(t, stdout, "1 to add, 0 to change, 0 to destroy.")
 }
 
@@ -49,9 +58,10 @@ func TestEngineLocalApply(t *testing.T) {
 	stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
 	require.NoError(t, err)
 
-	assert.Contains(t, stderr, LocalEngineBinaryPath)
-	assert.Contains(t, stderr, "[INFO]  plugin process exited:")
-	assert.Contains(t, stderr, "plugin process exited:")
+	assert.Contains(t, stderr, engineAssetName)
+	assert.Contains(t, stderr, "Tofu Initialization started")
+	assert.Contains(t, stderr, "Tofu Initialization completed")
+	assert.Contains(t, stderr, "Tofu Shutdown completed")
 	assert.Contains(t, stdout, "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.")
 }
 
@@ -65,8 +75,9 @@ func TestEngineOpentofu(t *testing.T) {
 	stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
 	require.NoError(t, err)
 
-	assert.Contains(t, stderr, "[INFO]  plugin process exited:")
-	assert.Contains(t, stderr, "plugin process exited:")
+	assert.Contains(t, stderr, "Tofu Initialization started")
+	assert.Contains(t, stderr, "Tofu Initialization completed")
+	assert.Contains(t, stderr, "Tofu Shutdown completed")
 	assert.Contains(t, stdout, "OpenTofu has been successfully initialized")
 	assert.Contains(t, stdout, "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.")
 }
@@ -81,8 +92,7 @@ func TestEngineRunAllOpentofu(t *testing.T) {
 	stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all apply -no-color -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
 	require.NoError(t, err)
 
-	assert.Contains(t, stderr, "[INFO]  plugin process exited")
-	assert.Contains(t, stderr, "plugin process exited:")
+	assert.Contains(t, stderr, "Tofu Initialization started")
 	assert.Contains(t, stdout, "resource \"local_file\" \"test\"")
 	assert.Contains(t, stdout, "filename             = \"./test.txt\"\n")
 	assert.Contains(t, stdout, "OpenTofu has been successful")
@@ -98,8 +108,6 @@ func TestEngineRunAllOpentofuCustomPath(t *testing.T) {
 	stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all apply -no-color -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
 	require.NoError(t, err)
 
-	assert.Contains(t, stderr, "[INFO]  plugin process exited:")
-	assert.Contains(t, stderr, "plugin process exited:")
 	assert.Contains(t, stdout, "OpenTofu has been successful")
 	assert.Contains(t, stderr, "Tofu Shutdown completed")
 	assert.Contains(t, stdout, "Apply complete!")
@@ -127,8 +135,9 @@ func TestEngineDownloadOverHttp(t *testing.T) {
 	stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
 	require.NoError(t, err)
 
-	assert.Contains(t, stderr, "[INFO]  plugin process exited:")
-	assert.Contains(t, stderr, "plugin process exited:")
+	assert.Contains(t, stderr, "Tofu Initialization started")
+	assert.Contains(t, stderr, "Tofu Initialization completed")
+	assert.Contains(t, stderr, "Tofu Shutdown completed")
 	assert.Contains(t, stdout, "OpenTofu has been successfully initialized")
 	assert.Contains(t, stdout, "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.")
 }
@@ -249,12 +258,16 @@ func setupLocalEngine(t *testing.T) string {
 	tmpEnvPath := helpers.CopyEnvironment(t, testFixtureLocalEngine)
 	rootPath := util.JoinPath(tmpEnvPath, testFixtureLocalEngine)
 
-	// get pwd
-	pwd, err := os.Getwd()
+	// download engine to local directory
+	engineDir := util.JoinPath(rootPath, "engine")
+	if err := os.MkdirAll(engineDir, 0755); err != nil {
+		require.NoError(t, err)
+	}
+	_, err := getter.GetAny(context.TODO(), engineDir, downloadUrl)
 	require.NoError(t, err)
 
 	helpers.CopyAndFillMapPlaceholders(t, util.JoinPath(testFixtureLocalEngine, "terragrunt.hcl"), util.JoinPath(rootPath, config.DefaultTerragruntConfigPath), map[string]string{
-		"__engine_source__": pwd + "/../" + LocalEngineBinaryPath,
+		"__engine_source__": filepath.Join(engineDir, engineAssetName),
 	})
 	return rootPath
 }
@@ -263,7 +276,7 @@ func setupLocalEngine(t *testing.T) string {
 func testEngineVersion() string {
 	value, found := os.LookupEnv("TOFU_ENGINE_VERSION")
 	if !found {
-		return "v0.0.1"
+		return "v0.0.16"
 	}
 	return value
 }