[CI-3404] Add graph pipeline support (#16)

* Create new matchers

* Use new matchers

* Add missing env var

* Update step/matcher/graph_pipeline.go

Co-authored-by: Gábor Szakács <gaborszakacs@users.noreply.github.com>

* Rename file

* Use proper naming

* Guard against both env var being set

* Add easter egg

* Add error test case

* Early loop exit

* Add env var existence tests

---------

Co-authored-by: Gábor Szakács <gaborszakacs@users.noreply.github.com>

model/finished_stages.go → model/model.go

@@ -7,6 +7,8 @@ type Stage struct {
 	Workflows []Workflow `json:"workflows"`
 }
 
+type FinishedWorkflows []Workflow
+
 type Workflow struct {
 	ExternalId string `json:"external_id"`
 	Name       string `json:"name"`

step.yml

@@ -82,8 +82,15 @@ inputs:
 
 - finished_stage: $BITRISEIO_FINISHED_STAGES
   opts:
-    title: The finished stages for which artifacts are available to download
-    summary: This is a JSON representation of the finished stages for which the step can download build artifacts.
+    title: The finished staged pipeline stages for which artifacts are available to download
+    summary: This is a JSON representation of the finished staged pipeline stages for which the step can download build artifacts.
+    is_required: true
+    is_dont_change_value: true
+
+- finished_workflows: $BITRISEIO_FINISHED_WORKFLOWS
+  opts:
+    title: The finished graph pipeline workflows for which artifacts are available to download
+    summary: This is a JSON representation of the finished graph pipeline workflows for which the step can download build artifacts.
     is_required: true
     is_dont_change_value: true
 

step/matcher/graph_pipeline.go

@@ -0,0 +1,59 @@
+package matcher
+
+import (
+	"regexp"
+
+	"github.com/bitrise-io/go-utils/log"
+	"github.com/bitrise-steplib/bitrise-step-pull-intermediate-files/model"
+)
+
+type graphPipelineMatcher struct {
+	finishedWorkflows model.FinishedWorkflows
+	targetNames       []string
+	logger            log.Logger
+}
+
+func newGraphPipelineMatcher(finishedWorkflows model.FinishedWorkflows, targetNames []string, logger log.Logger) graphPipelineMatcher {
+	return graphPipelineMatcher{
+		finishedWorkflows: finishedWorkflows,
+		targetNames:       targetNames,
+		logger:            logger,
+	}
+}
+
+func (g graphPipelineMatcher) Matches() ([]string, error) {
+	var executedWorkflows []model.Workflow
+	for _, workflow := range g.finishedWorkflows {
+		if workflow.ExternalId == "" {
+			g.logger.Printf("Skipping workflow %s because it was not executed.", workflow.Name)
+			continue
+		}
+		executedWorkflows = append(executedWorkflows, workflow)
+	}
+
+	identifiers := make(map[string]bool)
+
+	if len(g.targetNames) == 0 {
+		for _, workflow := range executedWorkflows {
+			identifiers[workflow.ExternalId] = true
+		}
+
+		return convertKeySetToSlice(identifiers), nil
+	}
+
+	for _, workflow := range executedWorkflows {
+		for _, target := range g.targetNames {
+			matched, err := regexp.MatchString(target, workflow.Name)
+			if err != nil {
+				return nil, err
+			}
+
+			if matched {
+				identifiers[workflow.ExternalId] = true
+				break
+			}
+		}
+	}
+
+	return convertKeySetToSlice(identifiers), nil
+}

step/matcher/graph_pipeline_test.go

@@ -0,0 +1,105 @@
+package matcher
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/bitrise-io/go-utils/log"
+	"github.com/bitrise-steplib/bitrise-step-pull-intermediate-files/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGraphMatcher(t *testing.T) {
+	finishedWorkflows := model.FinishedWorkflows{
+		{
+			Name:       "workflow1_1",
+			ExternalId: "build1_1",
+		},
+		{
+			Name:       "workflow1_2",
+			ExternalId: "build1_2",
+		},
+		{
+			Name:       "workflow2_1",
+			ExternalId: "build2_1",
+		},
+		{
+			Name:       "workflow3_1",
+			ExternalId: "build3_1",
+		},
+		{
+			Name:       "workflow3_2",
+			ExternalId: "",
+		},
+		{
+			Name:       "workflow4_1",
+			ExternalId: "build4_1",
+		},
+	}
+	testCases := []struct {
+		desc                 string
+		targetNames          []string
+		expectedBuildIDs     []string
+		expectedErrorMessage string
+	}{
+		{
+			desc:                 "download everything",
+			targetNames:          []string{".*"},
+			expectedBuildIDs:     []string{"build1_1", "build1_2", "build2_1", "build3_1", "build4_1"},
+			expectedErrorMessage: "",
+		},
+		{
+			desc:                 "exact workflow names",
+			targetNames:          []string{"workflow1_1", "workflow2_1"},
+			expectedBuildIDs:     []string{"build1_1", "build2_1"},
+			expectedErrorMessage: "",
+		},
+		{
+			desc:                 "partial workflow name",
+			targetNames:          []string{"workflow1.*"},
+			expectedBuildIDs:     []string{"build1_1", "build1_2"},
+			expectedErrorMessage: "",
+		},
+		{
+			desc:                 "multiple patterns with deduplicated results",
+			targetNames:          []string{"workflow1.*", "workflow1_1", "workflow.*"},
+			expectedBuildIDs:     []string{"build1_1", "build1_2", "build2_1", "build3_1", "build4_1"},
+			expectedErrorMessage: "",
+		},
+		{
+			desc:                 "missing target names",
+			targetNames:          []string{},
+			expectedBuildIDs:     []string{"build1_1", "build1_2", "build2_1", "build3_1", "build4_1"},
+			expectedErrorMessage: "",
+		},
+		{
+			desc:                 "not existing workflow name",
+			targetNames:          []string{"missing_workflow_name"},
+			expectedBuildIDs:     nil,
+			expectedErrorMessage: "",
+		},
+		{
+			desc:                 "invalid regex",
+			targetNames:          []string{"["},
+			expectedBuildIDs:     nil,
+			expectedErrorMessage: "error parsing regexp: missing closing ]: `[`",
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			matcher := newGraphPipelineMatcher(finishedWorkflows, tt.targetNames, log.NewLogger())
+
+			buildIDs, err := matcher.Matches()
+			if tt.expectedErrorMessage != "" {
+				assert.EqualError(t, err, tt.expectedErrorMessage)
+			} else {
+				assert.NoError(t, err)
+			}
+
+			sort.Strings(buildIDs)
+			sort.Strings(tt.expectedBuildIDs)
+
+			assert.Equal(t, tt.expectedBuildIDs, buildIDs)
+		})
+	}
+}

step/matcher/matcher.go

@@ -0,0 +1,28 @@
+package matcher
+
+import (
+	"github.com/bitrise-io/go-utils/log"
+	"github.com/bitrise-steplib/bitrise-step-pull-intermediate-files/model"
+)
+
+type BuildIDMatcher interface {
+	Matches() ([]string, error)
+}
+
+func NewBuildIDMatcher(finishedStages model.FinishedStages, finishedWorkflows model.FinishedWorkflows, targetNames []string, logger log.Logger) BuildIDMatcher {
+	if len(finishedStages) > 0 {
+		return newStagedPipelineMatcher(finishedStages, targetNames, logger)
+	}
+
+	return newGraphPipelineMatcher(finishedWorkflows, targetNames, logger)
+}
+
+func convertKeySetToSlice(set map[string]bool) []string {
+	var ids []string
+
+	for k := range set {
+		ids = append(ids, k)
+	}
+
+	return ids
+}

step/matcher/staged_pipeline.go

@@ -0,0 +1,76 @@
+package matcher
+
+import (
+	"regexp"
+
+	"github.com/bitrise-io/go-utils/log"
+	"github.com/bitrise-steplib/bitrise-step-pull-intermediate-files/model"
+)
+
+const DELIMITER = "."
+
+type stagedPipelineMatcher struct {
+	finishedStages model.FinishedStages
+	targetNames    []string
+	logger         log.Logger
+}
+
+type keyValuePair struct {
+	key   string
+	value string
+}
+
+func newStagedPipelineMatcher(finishedStages model.FinishedStages, targetNames []string, logger log.Logger) stagedPipelineMatcher {
+	return stagedPipelineMatcher{
+		finishedStages: finishedStages,
+		targetNames:    targetNames,
+		logger:         logger,
+	}
+}
+
+func (spm stagedPipelineMatcher) Matches() ([]string, error) {
+	buildIDsSet := make(map[string]bool)
+
+	kvpSlice := spm.createKeyValuePairSlice()
+
+	if len(spm.targetNames) == 0 {
+		for _, kvPair := range kvpSlice {
+			buildIDsSet[kvPair.value] = true
+		}
+
+		return convertKeySetToSlice(buildIDsSet), nil
+	}
+
+	for _, target := range spm.targetNames {
+		for _, kvPair := range kvpSlice {
+			matched, err := regexp.MatchString(target, kvPair.key)
+			if err != nil {
+				return nil, err
+			}
+
+			if matched {
+				buildIDsSet[kvPair.value] = true
+			}
+		}
+	}
+
+	return convertKeySetToSlice(buildIDsSet), nil
+}
+
+func (spm stagedPipelineMatcher) createKeyValuePairSlice() []keyValuePair {
+	var stageWorkflowMap []keyValuePair
+	for _, stage := range spm.finishedStages {
+		for _, wf := range stage.Workflows {
+			if wf.ExternalId == "" {
+				spm.logger.Printf("Skipping workflow %s in stage %s. Workflow was not executed.", wf.Name, stage.Name)
+				continue
+			}
+			stageWorkflowMap = append(stageWorkflowMap, keyValuePair{
+				key:   stage.Name + DELIMITER + wf.Name,
+				value: wf.ExternalId,
+			})
+		}
+	}
+
+	return stageWorkflowMap
+}