Nested step bundles (#1050)

* Rename 'with' group to 'with group' * Update step bundle's step list item type and move workflow run plan creation to the models package * Test step bundles in the workflow run execution plan * Fix bundle envs handling * Fix step list item getters * Code cleanup * Integration test nested bundles * Code cleanup * Code cleanup * Import list order * Remove obsolete tests * Validate circular step bundle reference * Validate step bundle reference cycle with a deterministic order
bitrise-io · Feb 12, 2025 · dfbd55f · dfbd55f
1 parent 2c928f5
commit dfbd55f
Show file tree

Hide file tree

Showing 10 changed files with 874 additions and 270 deletions.
diff --git a/_tests/integration/step_bundles_test.go b/_tests/integration/step_bundles_test.go
@@ -1,17 +1,85 @@
 package integration
 
 import (
+	"encoding/json"
+	"strings"
 	"testing"
 
 	"github.com/bitrise-io/go-utils/command"
 	"github.com/stretchr/testify/require"
 )
 
-func TestStepBundles(t *testing.T) {
+func TestStepBundleInputs(t *testing.T) {
 	configPth := "step_bundles_test_bitrise.yml"
 
 	cmd := command.New(binPath(), "run", "test_step_bundle_inputs", "-c", configPth)
 	out, err := cmd.RunAndReturnTrimmedCombinedOutput()
 	require.NoError(t, err, out)
 	require.Contains(t, out, "Hello Bitrise!")
 }
+
+func TestNestedStepBundle(t *testing.T) {
+	configPth := "step_bundles_test_bitrise.yml"
+
+	cmd := command.New(binPath(), "run", "--output-format", "json", "test_nested_step_bundle", "-c", configPth)
+	out, err := cmd.RunAndReturnTrimmedCombinedOutput()
+	require.NoError(t, err, out)
+	stepOutputs := collectStepOutputs(out, t)
+	require.Equal(t, stepOutputs, []string{
+		`bundle1
+bundle1_input1: bundle1_input1
+bundle2_input1: bundle2_input1
+`,
+		`bundle1
+bundle1_input1: bundle1_input1_override
+bundle2_input1: bundle2_input1
+`,
+		`bundle2
+bundle1_input1: 
+bundle2_input1: bundle2_input1
+`,
+		`workflow step
+bundle1_input1: 
+bundle2_input1: 
+`,
+	})
+}
+
+func collectStepOutputs(workflowRunOut string, t *testing.T) []string {
+	type messageLine struct {
+		Producer   string `json:"producer"`
+		ProducerID string `json:"producer_id"`
+		Message    string `json:"message"`
+	}
+
+	idxToStep := map[string]int{}
+	messageToStep := map[string]string{}
+	stepIDx := 0
+
+	for _, line := range strings.Split(workflowRunOut, "\n") {
+		var message messageLine
+		err := json.Unmarshal([]byte(line), &message)
+		require.NoError(t, err)
+
+		if message.Producer != "step" {
+			continue
+		}
+
+		stepMessage := messageToStep[message.ProducerID]
+		stepMessage += message.Message
+		messageToStep[message.ProducerID] = stepMessage
+
+		_, ok := idxToStep[message.ProducerID]
+		if !ok {
+			idxToStep[message.ProducerID] = stepIDx
+			stepIDx++
+		}
+	}
+
+	stepMessages := make([]string, len(idxToStep))
+	for step, idx := range idxToStep {
+		stepMessages[idx] = messageToStep[step]
+	}
+
+	return stepMessages
+}
diff --git a/_tests/integration/step_bundles_test_bitrise.yml b/_tests/integration/step_bundles_test_bitrise.yml
@@ -10,9 +10,45 @@ step_bundles:
         inputs:
         - content: echo "Hello $name!"
 
+  bundle1:
+    inputs:
+    - bundle1_input1: bundle1_input1
+    steps:
+    - script:
+        inputs:
+        - content: |-
+            echo "bundle1"
+            echo "bundle1_input1: $bundle1_input1"
+            echo "bundle2_input1: $bundle2_input1"
+
+  bundle2:
+    inputs:
+    - bundle2_input1: bundle2_input1
+    steps:
+    - bundle::bundle1: { }
+    - bundle::bundle1:
+        inputs:
+        - bundle1_input1: bundle1_input1_override
+    - script:
+        inputs:
+        - content: |-
+            echo "bundle2"
+            echo "bundle1_input1: $bundle1_input1"
+            echo "bundle2_input1: $bundle2_input1"
+
 workflows:
   test_step_bundle_inputs:
     steps:
     - bundle::print_hello:
         inputs:
         - name: Bitrise
+
+  test_nested_step_bundle:
+    steps:
+    - bundle::bundle2: { }
+    - script:
+        inputs:
+        - content: |-
+            echo "workflow step"
+            echo "bundle1_input1: $bundle1_input1"
+            echo "bundle2_input1: $bundle2_input1"
diff --git a/bitrise/util_test.go b/bitrise/util_test.go
@@ -312,18 +312,6 @@ func TestConfigModelFromYAMLFileContent_StepListValidation(t *testing.T) {
 		config  string
 		wantErr string
 	}{
-		{
-			name: "Invalid bitrise.yml: step bundle in a step bundle's steps list",
-			config: `
-format_version: '11'
-default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
-step_bundles:
-  build: {}
-  test:
-    steps:
-    - bundle::build: {}`,
-			wantErr: "step bundle (test) has config issue: step bundle is not allowed in a step bundle's step list",
-		},
 		{
 			name: "Invalid bitrise.yml: with group in a step bundle's steps list",
 			config: `
@@ -333,10 +321,10 @@ step_bundles:
   test:
     steps:
     - with: {}`,
-			wantErr: "step bundle (test) has config issue: 'with' group is not allowed in a step bundle's step list",
+			wantErr: "failed to normalize step_bundle: 'with group' is not allowed in a step bundle's step list",
 		},
 		{
-			name: "Invalid bitrise.yml: step bundle in a 'with' group's steps list",
+			name: "Invalid bitrise.yml: step bundle in a 'with group''s steps list",
 			config: `
 format_version: '11'
 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
@@ -351,10 +339,10 @@ workflows:
         - postgres
         steps:
         - bundle::test: {}`,
-			wantErr: "invalid 'with' group in workflow (primary): step bundle is not allowed in a 'with' group's step list",
+			wantErr: "invalid 'with group' in workflow (primary): step bundle is not allowed in a 'with group''s step list",
 		},
 		{
-			name: "Invalid bitrise.yml: with group in a 'with' group's steps list",
+			name: "Invalid bitrise.yml: with group in a 'with group''s steps list",
 			config: `
 format_version: '11'
 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
@@ -369,13 +357,12 @@ workflows:
         - postgres
         steps:
         - with: {}`,
-			wantErr: "invalid 'with' group in workflow (primary): 'with' group is not allowed in a 'with' group's step list",
+			wantErr: "invalid 'with group' in workflow (primary): 'with group' is not allowed in a 'with group''s step list",
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, warns, err := ConfigModelFromFileContent([]byte(tt.config), false, ValidationTypeFull)
-			require.Equal(t, []string(nil), warns)
+			_, _, err := ConfigModelFromFileContent([]byte(tt.config), false, ValidationTypeFull)
 			if tt.wantErr != "" {
 				require.EqualError(t, err, tt.wantErr)
 			} else {
@@ -391,24 +378,6 @@ func TestConfigModelFromJSONFileContent_StepListValidation(t *testing.T) {
 		config  string
 		wantErr string
 	}{
-		{
-			name: "Invalid bitrise.yml: step bundle in a step bundle's steps list",
-			config: `{
-  "format_version": "11",
-  "default_step_lib_source": "https://github.com/bitrise-io/bitrise-steplib.git",
-  "step_bundles": {
-    "build": {},
-    "test": {
-      "steps": [
-        {
-          "bundle::build": {}
-        }
-      ]
-    }
-  }
-}`,
-			wantErr: "step bundle (test) has config issue: step bundle is not allowed in a step bundle's step list",
-		},
 		{
 			name: "Invalid bitrise.yml: with group in a step bundle's steps list",
 			config: `{
@@ -424,10 +393,10 @@ func TestConfigModelFromJSONFileContent_StepListValidation(t *testing.T) {
     }
   }
 }`,
-			wantErr: "step bundle (test) has config issue: 'with' group is not allowed in a step bundle's step list",
+			wantErr: "failed to normalize step_bundle: 'with group' is not allowed in a step bundle's step list",
 		},
 		{
-			name: "Invalid bitrise.yml: step bundle in a 'with' group's steps list",
+			name: "Invalid bitrise.yml: step bundle in a 'with group''s steps list",
 			config: `{
   "format_version": "11",
   "default_step_lib_source": "https://github.com/bitrise-io/bitrise-steplib.git",
@@ -455,10 +424,10 @@ func TestConfigModelFromJSONFileContent_StepListValidation(t *testing.T) {
     }
   }
 }`,
-			wantErr: "invalid 'with' group in workflow (primary): step bundle is not allowed in a 'with' group's step list",
+			wantErr: "invalid 'with group' in workflow (primary): step bundle is not allowed in a 'with group''s step list",
 		},
 		{
-			name: "Invalid bitrise.yml: with group in a 'with' group's steps list",
+			name: "Invalid bitrise.yml: with group in a 'with group''s steps list",
 			config: `{
   "format_version": "11",
   "default_step_lib_source": "https://github.com/bitrise-io/bitrise-steplib.git",
@@ -486,13 +455,12 @@ func TestConfigModelFromJSONFileContent_StepListValidation(t *testing.T) {
     }
   }
 }`,
-			wantErr: "invalid 'with' group in workflow (primary): 'with' group is not allowed in a 'with' group's step list",
+			wantErr: "invalid 'with group' in workflow (primary): 'with group' is not allowed in a 'with group''s step list",
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, warns, err := ConfigModelFromFileContent([]byte(tt.config), true, ValidationTypeFull)
-			require.Equal(t, []string(nil), warns)
+			_, _, err := ConfigModelFromFileContent([]byte(tt.config), true, ValidationTypeFull)
 			if tt.wantErr != "" {
 				require.EqualError(t, err, tt.wantErr)
 			} else {