feat(test): New UI implementation for ie test & refactor various parts of the engine

.github/workflows/scenario-testing.yaml

@@ -39,10 +39,11 @@ jobs:
     environment: ScenarioTesting
     steps:
       - uses: actions/checkout@v2
-      - name: Build all targets.
+      - name: Build & test all targets.
         run: |
           make build-all
           make test-all WITH_COVERAGE=true
+          make test-local-scenarios ENVIRONMENT=github-action
       - name: Upload test coverage
         uses: actions/upload-artifact@v2
         if: github.event_name == 'pull_request'
@@ -65,6 +66,6 @@ jobs:
             apk add --no-cache make git openssh openssl helm curl jq
             make test-upstream-scenarios SUBSCRIPTION=${{ secrets.AZURE_SUBSCRIPTION }}
       - name: Display ie.log file
-        if: (success() || failure()) && github.event_name != 'pull_request'
+        if: (success() || failure())
         run: |
           cat ie.log

Makefile

@@ -22,12 +22,11 @@ install-ie:
 # ------------------------------ Test targets ----------------------------------
 
 WITH_COVERAGE := false
-
 test-all:
 	@go clean -testcache
 ifeq ($(WITH_COVERAGE), true)
 	@echo "Running all tests with coverage..."
-	@go test -v -coverprofile=coverage.out ./...
+	@go test -v -coverpkg=./... -coverprofile=coverage.out ./...
 	@go tool cover -html=coverage.out -o coverage.html
 else
 	@echo "Running all tests..."
@@ -38,16 +37,27 @@ endif
 SUBSCRIPTION ?= 00000000-0000-0000-0000-000000000000
 SCENARIO ?= ./README.md
 WORKING_DIRECTORY ?= $(PWD)
+ENVIRONMENT ?= local
 test-scenario:
 	@echo "Running scenario $(SCENARIO)"
-	$(IE_BINARY) test $(SCENARIO) --subscription $(SUBSCRIPTION) --working-directory $(WORKING_DIRECTORY)
+ifeq ($(SUBSCRIPTION), 00000000-0000-0000-0000-000000000000)
+	$(IE_BINARY) test $(SCENARIO) --working-directory $(WORKING_DIRECTORY) --environment $(ENVIRONMENT)
+else
+	$(IE_BINARY) test $(SCENARIO) --subscription $(SUBSCRIPTION) --working-directory $(WORKING_DIRECTORY) --enviroment $(ENVIRONMENT)
+endif
 
 test-scenarios:
 	@echo "Testing out the scenarios"
 	for dir in ./scenarios/ocd/*/; do \
 		($(MAKE) test-scenario SCENARIO="$${dir}README.md" SUBCRIPTION="$(SUBSCRIPTION)") || exit $$?; \
 	done
 
+test-local-scenarios:
+	@echo "Testing out the local scenarios"
+	for file in ./scenarios/testing/*.md; do \
+		($(MAKE) test-scenario SCENARIO="$${file}") || exit $$?; \
+	done
+
 test-upstream-scenarios:
 	@echo "Pulling the upstream scenarios"
 	@git config --global --add safe.directory /home/runner/work/InnovationEngine/InnovationEngine

cmd/ie/commands/execute.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/Azure/InnovationEngine/internal/engine"
+	"github.com/Azure/InnovationEngine/internal/engine/common"
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/spf13/cobra"
 )
@@ -92,7 +93,7 @@ var executeCommand = &cobra.Command{
 		}
 
 		// Parse the markdown file and create a scenario
-		scenario, err := engine.CreateScenarioFromMarkdown(
+		scenario, err := common.CreateScenarioFromMarkdown(
 			markdownFile,
 			[]string{"bash", "azurecli", "azurecli-interactive", "terraform"},
 			cliEnvironmentVariables,
@@ -112,7 +113,6 @@ var executeCommand = &cobra.Command{
 			WorkingDirectory: workingDirectory,
 			RenderValues:     renderValues,
 		})
-
 		if err != nil {
 			logging.GlobalLogger.Errorf("Error creating engine: %s", err)
 			fmt.Printf("Error creating engine: %s", err)

cmd/ie/commands/inspect.go

@@ -5,7 +5,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/Azure/InnovationEngine/internal/engine"
+	"github.com/Azure/InnovationEngine/internal/engine/common"
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/Azure/InnovationEngine/internal/ui"
 	"github.com/spf13/cobra"
@@ -59,7 +59,7 @@ var inspectCommand = &cobra.Command{
 			cliEnvironmentVariables[keyValuePair[0]] = keyValuePair[1]
 		}
 		// Parse the markdown file and create a scenario
-		scenario, err := engine.CreateScenarioFromMarkdown(
+		scenario, err := common.CreateScenarioFromMarkdown(
 			markdownFile,
 			[]string{"bash", "azurecli", "azurecli-inspect", "terraform"},
 			cliEnvironmentVariables,
@@ -104,6 +104,5 @@ var inspectCommand = &cobra.Command{
 				fmt.Println()
 			}
 		}
-
 	},
 }

cmd/ie/commands/interactive.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/Azure/InnovationEngine/internal/engine"
+	"github.com/Azure/InnovationEngine/internal/engine/common"
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/spf13/cobra"
 )
@@ -69,7 +70,7 @@ var interactiveCommand = &cobra.Command{
 			cliEnvironmentVariables[keyValuePair[0]] = keyValuePair[1]
 		}
 		// Parse the markdown file and create a scenario
-		scenario, err := engine.CreateScenarioFromMarkdown(
+		scenario, err := common.CreateScenarioFromMarkdown(
 			markdownFile,
 			[]string{"bash", "azurecli", "azurecli-interactive", "terraform"},
 			cliEnvironmentVariables,
@@ -89,7 +90,6 @@ var interactiveCommand = &cobra.Command{
 			WorkingDirectory: workingDirectory,
 			RenderValues:     renderValues,
 		})
-
 		if err != nil {
 			logging.GlobalLogger.Errorf("Error creating engine: %s", err)
 			fmt.Printf("Error creating engine: %s", err)

cmd/ie/commands/test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/Azure/InnovationEngine/internal/engine"
+	"github.com/Azure/InnovationEngine/internal/engine/common"
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/spf13/cobra"
 )
@@ -23,9 +24,8 @@ func init() {
 var testCommand = &cobra.Command{
 	Use:   "test",
 	Args:  cobra.MinimumNArgs(1),
-	Short: "Test document commands against it's expected outputs.",
+	Short: "Test document commands against their expected outputs.",
 	Run: func(cmd *cobra.Command, args []string) {
-
 		markdownFile := args[0]
 		if markdownFile == "" {
 			cmd.Help()
@@ -35,22 +35,23 @@ var testCommand = &cobra.Command{
 		verbose, _ := cmd.Flags().GetBool("verbose")
 		subscription, _ := cmd.Flags().GetString("subscription")
 		workingDirectory, _ := cmd.Flags().GetString("working-directory")
+		environment, _ := cmd.Flags().GetString("environment")
 
 		innovationEngine, err := engine.NewEngine(engine.EngineConfiguration{
 			Verbose:          verbose,
 			DoNotDelete:      false,
 			Subscription:     subscription,
 			CorrelationId:    "",
 			WorkingDirectory: workingDirectory,
+			Environment:      environment,
 		})
-
 		if err != nil {
 			logging.GlobalLogger.Errorf("Error creating engine %s", err)
 			fmt.Printf("Error creating engine %s", err)
 			os.Exit(1)
 		}
 
-		scenario, err := engine.CreateScenarioFromMarkdown(
+		scenario, err := common.CreateScenarioFromMarkdown(
 			markdownFile,
 			[]string{"bash", "azurecli", "azurecli-interactive", "terraform"},
 			nil,

cmd/ie/commands/to-bash.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/Azure/InnovationEngine/internal/engine"
+	"github.com/Azure/InnovationEngine/internal/engine/common"
 	"github.com/Azure/InnovationEngine/internal/engine/environments"
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/spf13/cobra"
@@ -50,11 +50,10 @@ var toBashCommand = &cobra.Command{
 		}
 
 		// Parse the markdown file and create a scenario
-		scenario, err := engine.CreateScenarioFromMarkdown(
+		scenario, err := common.CreateScenarioFromMarkdown(
 			markdownFile,
 			[]string{"bash", "azurecli", "azurecli-interactive", "terraform"},
 			cliEnvironmentVariables)
-
 		if err != nil {
 			logging.GlobalLogger.Errorf("Error creating scenario: %s", err)
 			fmt.Printf("Error creating scenario: %s", err)
@@ -66,7 +65,6 @@ var toBashCommand = &cobra.Command{
 		if environments.IsAzureEnvironment(environment) {
 			script := AzureScript{Script: scenario.ToShellScript()}
 			scriptJson, err := json.Marshal(script)
-
 			if err != nil {
 				logging.GlobalLogger.Errorf("Error converting to json: %s", err)
 				fmt.Printf("Error converting to json: %s", err)
@@ -79,7 +77,6 @@ var toBashCommand = &cobra.Command{
 		}
 
 		return nil
-
 	},
 }
 

internal/az/group.go

@@ -17,7 +17,6 @@ func FindAllDeployedResourceURIs(resourceGroup string) []string {
 			WriteToHistory:       true,
 		},
 	)
-
 	if err != nil {
 		logging.GlobalLogger.Error("Failed to list deployments", err)
 	}

internal/az/group_test.go

@@ -0,0 +1,46 @@
+package az
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFindingResourceGroups(t *testing.T) {
+	testCases := []struct {
+		resourceGroupString       string
+		expectedResourceGroupName string
+	}{
+		// RG string that ends with a slash
+		{
+			resourceGroupString:       "resourceGroups/rg1/",
+			expectedResourceGroupName: "rg1",
+		},
+		// RG string that ends with a space and starts new text
+		{
+			resourceGroupString:       "resourceGroups/rg1 /subscriptions/",
+			expectedResourceGroupName: "rg1",
+		},
+
+		// RG string that includes nested resources and extraneous text.
+		{
+			resourceGroupString:       "/subscriptions/9b70acd9-975f-44ba-bad6-255a2c8bda37/resourceGroups/myResourceGroup-rg/providers/Microsoft.ContainerRegistry/registries/mydnsrandomnamebbbhe  ffc55a9e-ed2a-4b60-b034-45228dfe7db5  2024-06-11T09:41:36.631310+00:00",
+			expectedResourceGroupName: "myResourceGroup-rg",
+		},
+		// RG string that is surrounded by quotes.
+		{
+			resourceGroupString:       `"id": "/subscriptions/0a2c89a7-a44e-4cd0-b6ec-868432ad1d13/resourceGroups/myResourceGroup"`,
+			expectedResourceGroupName: "myResourceGroup",
+		},
+		// RG string that has no match.
+		{
+			resourceGroupString:       "NoMatch",
+			expectedResourceGroupName: "",
+		},
+	}
+
+	for _, tc := range testCases {
+		resourceGroupName := FindResourceGroupName(tc.resourceGroupString)
+		assert.Equal(t, tc.expectedResourceGroupName, resourceGroupName)
+	}
+}

internal/engine/common/codeblock.go

@@ -0,0 +1,22 @@
+package common
+
+import "github.com/Azure/InnovationEngine/internal/parsers"
+
+// State for the codeblock in interactive mode. Used to keep track of the
+// state of each codeblock.
+type StatefulCodeBlock struct {
+	CodeBlock       parsers.CodeBlock
+	CodeBlockNumber int
+	Error           error
+	StdErr          string
+	StdOut          string
+	StepName        string
+	StepNumber      int
+	Success         bool
+}
+
+// Checks if a codeblock was executed by looking at the
+// output, errors, and if success is true.
+func (s StatefulCodeBlock) WasExecuted() bool {
+	return s.StdOut != "" || s.StdErr != "" || s.Error != nil || s.Success
+}

internal/engine/commands.go → internal/engine/common/commands.go

@@ -1,4 +1,4 @@
-package engine
+package common
 
 import (
 	"fmt"
@@ -23,6 +23,16 @@ type FailedCommandMessage struct {
 	Error  error
 }
 
+type ExitMessage struct {
+	EncounteredFailure bool
+}
+
+func Exit(encounteredFailure bool) tea.Cmd {
+	return func() tea.Msg {
+		return ExitMessage{EncounteredFailure: encounteredFailure}
+	}
+}
+
 // Executes a bash command and returns a tea message with the output. This function
 // will be executed asycnhronously.
 func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) tea.Cmd {
@@ -52,7 +62,7 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t
 		expectedRegex := codeBlock.ExpectedOutput.ExpectedRegex
 		expectedOutputLanguage := codeBlock.ExpectedOutput.Language
 
-		outputComparisonError := compareCommandOutputs(
+		outputComparisonError := CompareCommandOutputs(
 			actualOutput,
 			expectedOutput,
 			expectedSimilarity,
@@ -86,16 +96,19 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t
 // finishes executing.
 func ExecuteCodeBlockSync(codeBlock parsers.CodeBlock, env map[string]string) tea.Msg {
 	logging.GlobalLogger.Info("Executing command synchronously: ", codeBlock.Content)
-	program.ReleaseTerminal()
+	Program.ReleaseTerminal()
 
-	output, err := shells.ExecuteBashCommand(codeBlock.Content, shells.BashCommandConfiguration{
-		EnvironmentVariables: env,
-		InheritEnvironment:   true,
-		InteractiveCommand:   true,
-		WriteToHistory:       true,
-	})
+	output, err := shells.ExecuteBashCommand(
+		codeBlock.Content,
+		shells.BashCommandConfiguration{
+			EnvironmentVariables: env,
+			InheritEnvironment:   true,
+			InteractiveCommand:   true,
+			WriteToHistory:       true,
+		},
+	)
 
-	program.RestoreTerminal()
+	Program.RestoreTerminal()
 
 	if err != nil {
 		return FailedCommandMessage{
@@ -113,7 +126,7 @@ func ExecuteCodeBlockSync(codeBlock parsers.CodeBlock, env map[string]string) te
 }
 
 // clearScreen returns a command that clears the terminal screen and positions the cursor at the top-left corner
-func clearScreen() tea.Cmd {
+func ClearScreen() tea.Cmd {
 	return func() tea.Msg {
 		fmt.Print(
 			"\033[H\033[2J",
@@ -124,13 +137,13 @@ func clearScreen() tea.Cmd {
 
 // Updates the azure status with the current state of the interactive mode
 // model.
-func updateAzureStatus(model InteractiveModeModel) tea.Cmd {
+func UpdateAzureStatus(azureStatus environments.AzureDeploymentStatus, environment string) tea.Cmd {
 	return func() tea.Msg {
 		logging.GlobalLogger.Tracef(
 			"Attempting to update the azure status: %+v",
-			model.azureStatus,
+			azureStatus,
 		)
-		environments.ReportAzureStatus(model.azureStatus, model.environment)
+		environments.ReportAzureStatus(azureStatus, environment)
 		return AzureStatusUpdatedMessage{}
 	}
 }

internal/engine/common/globals.go

@@ -0,0 +1,9 @@
+package common
+
+import tea "github.com/charmbracelet/bubbletea"
+
+// TODO: Ideally we won't need a global program variable. We should
+// refactor this in the future such that each tea program is localized to the
+// function that creates it and ExecuteCodeBlockSync doesn't mutate the global
+// program variable.
+var Program *tea.Program = nil