Populate the *Run.Status.Provenance.ConfigSource field

Prior, remote `ResolutionRequest` CRD supports **recording** the source information
in its Status that identifies where the remote resource came from.
Similarly, `TaskRun/PipelineRun` CRD supports **receiving** the source information
via a field in its status from remote ResolutionRequest. Specifically,
this field named `ConfigSource` is a subfield of the `Provenance` field in status.

In this PR, we are trying to pass the data from `ResolutionRequest` to pipeline
reconciler so that the data can be captured in TaskRun/PipelineRun's `Status.Provenance.ConfigSource`.
The implication of this change is that many functions' interface has been
changed because we are passing this extra source data alongside the remote resoure.

Note:
- The `provenance` field in Run.status is behind a feature flag named `enable-provenance-in-status`
, which was introduced in tektoncd#5670. The field will be populated iff
the flag is set to `true`.
- If a pipeline yaml is from remote place A, and the individual tasks
are from other remote places, pipelinerun status will only record the source
for the pipeline, and the individual taskrun status will record the
source for the corresponding task.

Related PRs/Issues:
- tektoncd#5580
- tektoncd#5551
- tektoncd#5670
- tektoncd#5522

Signed-off-by: Chuang Wang <chuangw@google.com>

pkg/internal/resolution/resolved_meta.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2022 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resolution
+
+import (
+	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ResolvedObjectMeta contains both ObjectMeta and the metadata that identifies the source where the resource came from.
+type ResolvedObjectMeta struct {
+	*metav1.ObjectMeta `json:",omitempty"`
+	// ConfigSource identifies where the spec came from.
+	ConfigSource *v1beta1.ConfigSource `json:",omitempty"`
+}

pkg/reconciler/pipelinerun/pipelinerun.go

@@ -40,6 +40,7 @@ import (
 	listersv1alpha1 "github.com/tektoncd/pipeline/pkg/client/listers/pipeline/v1alpha1"
 	listers "github.com/tektoncd/pipeline/pkg/client/listers/pipeline/v1beta1"
 	resourcelisters "github.com/tektoncd/pipeline/pkg/client/resource/listers/resource/v1alpha1"
+	resolutionutil "github.com/tektoncd/pipeline/pkg/internal/resolution"
 	"github.com/tektoncd/pipeline/pkg/matrix"
 	"github.com/tektoncd/pipeline/pkg/pipelinerunmetrics"
 	tknreconciler "github.com/tektoncd/pipeline/pkg/reconciler"
@@ -392,7 +393,7 @@ func (c *Reconciler) reconcile(ctx context.Context, pr *v1beta1.PipelineRun, get
 		return controller.NewPermanentError(err)
 	default:
 		// Store the fetched PipelineSpec on the PipelineRun for auditing
-		if err := storePipelineSpecAndMergeMeta(pr, pipelineSpec, pipelineMeta); err != nil {
+		if err := storePipelineSpecAndMergeMeta(ctx, pr, pipelineSpec, pipelineMeta); err != nil {
 			logger.Errorf("Failed to store PipelineSpec on PipelineRun.Status for pipelinerun %s: %v", pr.Name, err)
 		}
 	}
@@ -522,7 +523,7 @@ func (c *Reconciler) reconcile(ctx context.Context, pr *v1beta1.PipelineRun, get
 	if len(pipelineSpec.Finally) > 0 {
 		tasks = append(tasks, pipelineSpec.Finally...)
 	}
-	pipelineRunState, err := c.resolvePipelineState(ctx, tasks, pipelineMeta, pr, providedResources)
+	pipelineRunState, err := c.resolvePipelineState(ctx, tasks, pipelineMeta.ObjectMeta, pr, providedResources)
 	switch {
 	case errors.Is(err, remote.ErrorRequestInProgress):
 		message := fmt.Sprintf("PipelineRun %s/%s awaiting remote resource", pr.Namespace, pr.Name)
@@ -1234,10 +1235,13 @@ func (c *Reconciler) updateLabelsAndAnnotations(ctx context.Context, pr *v1beta1
 	return newPr, nil
 }
 
-func storePipelineSpecAndMergeMeta(pr *v1beta1.PipelineRun, ps *v1beta1.PipelineSpec, meta *metav1.ObjectMeta) error {
+func storePipelineSpecAndMergeMeta(ctx context.Context, pr *v1beta1.PipelineRun, ps *v1beta1.PipelineSpec, meta *resolutionutil.ResolvedObjectMeta) error {
 	// Only store the PipelineSpec once, if it has never been set before.
 	if pr.Status.PipelineSpec == nil {
 		pr.Status.PipelineSpec = ps
+		if meta == nil {
+			return nil
+		}
 
 		// Propagate labels from Pipeline to PipelineRun. PipelineRun labels take precedences over Pipeline.
 		pr.ObjectMeta.Labels = kmap.Union(meta.Labels, pr.ObjectMeta.Labels)
@@ -1247,6 +1251,19 @@ func storePipelineSpecAndMergeMeta(pr *v1beta1.PipelineRun, ps *v1beta1.Pipeline
 		pr.ObjectMeta.Annotations = kmap.Union(meta.Annotations, pr.ObjectMeta.Annotations)
 
 	}
+
+	// Propagate ConfigSource from remote resolution to PipelineRun Status
+	// This lives outside of the status.spec check to avoid the case where only the spec is available in the first reconcile and source comes in next reconcile.
+	cfg := config.FromContextOrDefaults(ctx)
+	if cfg.FeatureFlags.EnableProvenanceInStatus && meta != nil && meta.ConfigSource != nil {
+		if pr.Status.Provenance == nil {
+			pr.Status.Provenance = &v1beta1.Provenance{}
+		}
+		if pr.Status.Provenance.ConfigSource == nil {
+			pr.Status.Provenance.ConfigSource = meta.ConfigSource
+		}
+	}
+
 	return nil
 }
 

pkg/reconciler/pipelinerun/pipelinerun_test.go

@@ -37,6 +37,7 @@ import (
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
 	resourcev1alpha1 "github.com/tektoncd/pipeline/pkg/apis/resource/v1alpha1"
+	resolutionutil "github.com/tektoncd/pipeline/pkg/internal/resolution"
 	"github.com/tektoncd/pipeline/pkg/reconciler/events/cloudevent"
 	"github.com/tektoncd/pipeline/pkg/reconciler/pipelinerun/resources"
 	ttesting "github.com/tektoncd/pipeline/pkg/reconciler/testing"
@@ -5523,7 +5524,7 @@ status:
 	}
 }
 
-func Test_storePipelineSpec(t *testing.T) {
+func Test_storePipelineSpecAndConfigSource(t *testing.T) {
 	pr := parse.MustParseV1beta1PipelineRun(t, `
 metadata:
   name: test-pipeline-run-success
@@ -5532,6 +5533,13 @@ metadata:
   annotations:
     io.annotation: value
 `)
+	configSource := &v1beta1.ConfigSource{
+		URI: "abc.com",
+		Digest: map[string]string{
+			"sha1": "a123",
+		},
+		EntryPoint: "foo/bar",
+	}
 
 	ps := v1beta1.PipelineSpec{Description: "foo-pipeline"}
 	ps1 := v1beta1.PipelineSpec{Description: "bar-pipeline"}
@@ -5540,24 +5548,75 @@ metadata:
 	want.Status = v1beta1.PipelineRunStatus{
 		PipelineRunStatusFields: v1beta1.PipelineRunStatusFields{
 			PipelineSpec: ps.DeepCopy(),
+			Provenance: &v1beta1.Provenance{
+				ConfigSource: configSource.DeepCopy(),
+			},
 		},
 	}
 	want.ObjectMeta.Labels["tekton.dev/pipeline"] = pr.ObjectMeta.Name
 
-	// The first time we set it, it should get copied.
-	if err := storePipelineSpecAndMergeMeta(pr, &ps, &pr.ObjectMeta); err != nil {
-		t.Errorf("storePipelineSpec() error = %v", err)
-	}
-	if d := cmp.Diff(pr, want); d != "" {
-		t.Fatalf(diff.PrintWantGot(d))
+	type args struct {
+		pipelineSpec       *v1beta1.PipelineSpec
+		resolvedObjectMeta *resolutionutil.ResolvedObjectMeta
 	}
 
-	// The next time, it should not get overwritten
-	if err := storePipelineSpecAndMergeMeta(pr, &ps1, &metav1.ObjectMeta{}); err != nil {
-		t.Errorf("storePipelineSpec() error = %v", err)
+	var tests = []struct {
+		name            string
+		reconcile1Args  *args
+		reconcile2Args  *args
+		wantPipelineRun *v1beta1.PipelineRun
+	}{
+		{
+			name: "spec and source are available in the same reconcile",
+			reconcile1Args: &args{
+				pipelineSpec: &ps,
+				resolvedObjectMeta: &resolutionutil.ResolvedObjectMeta{
+					ObjectMeta:   &pr.ObjectMeta,
+					ConfigSource: configSource.DeepCopy(),
+				},
+			},
+			reconcile2Args: &args{
+				pipelineSpec:       &ps1,
+				resolvedObjectMeta: &resolutionutil.ResolvedObjectMeta{},
+			},
+			wantPipelineRun: want,
+		},
+		{
+			name: "spec comes in the first reconcile and source comes in next reconcile",
+			reconcile1Args: &args{
+				pipelineSpec: &ps,
+				resolvedObjectMeta: &resolutionutil.ResolvedObjectMeta{
+					ObjectMeta: &pr.ObjectMeta,
+				},
+			},
+			reconcile2Args: &args{
+				pipelineSpec: &ps,
+				resolvedObjectMeta: &resolutionutil.ResolvedObjectMeta{
+					ConfigSource: configSource.DeepCopy(),
+				},
+			},
+			wantPipelineRun: want,
+		},
 	}
-	if d := cmp.Diff(pr, want); d != "" {
-		t.Fatalf(diff.PrintWantGot(d))
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := ttesting.EnableFeatureFlagField(context.Background(), t, "enable-provenance-in-status")
+			// mock first reconcile
+			if err := storePipelineSpecAndMergeMeta(ctx, pr, tc.reconcile1Args.pipelineSpec, tc.reconcile1Args.resolvedObjectMeta); err != nil {
+				t.Errorf("storePipelineSpec() error = %v", err)
+			}
+			if d := cmp.Diff(pr, tc.wantPipelineRun); d != "" {
+				t.Fatalf(diff.PrintWantGot(d))
+			}
+
+			// mock second reconcile
+			if err := storePipelineSpecAndMergeMeta(ctx, pr, tc.reconcile2Args.pipelineSpec, tc.reconcile2Args.resolvedObjectMeta); err != nil {
+				t.Errorf("storePipelineSpec() error = %v", err)
+			}
+			if d := cmp.Diff(pr, tc.wantPipelineRun); d != "" {
+				t.Fatalf(diff.PrintWantGot(d))
+			}
+		})
 	}
 }
 
@@ -5573,7 +5632,9 @@ func Test_storePipelineSpec_metadata(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{Name: "foo", Labels: pipelinerunlabels, Annotations: pipelinerunannotations},
 	}
 	meta := metav1.ObjectMeta{Name: "bar", Labels: pipelinelabels, Annotations: pipelineannotations}
-	if err := storePipelineSpecAndMergeMeta(pr, &v1beta1.PipelineSpec{}, &meta); err != nil {
+	if err := storePipelineSpecAndMergeMeta(context.Background(), pr, &v1beta1.PipelineSpec{}, &resolutionutil.ResolvedObjectMeta{
+		ObjectMeta: &meta,
+	}); err != nil {
 		t.Errorf("storePipelineSpecAndMergeMeta error = %v", err)
 	}
 	if d := cmp.Diff(pr.ObjectMeta.Labels, wantedlabels); d != "" {

pkg/reconciler/pipelinerun/pipelinespec/pipelinespec.go

@@ -22,32 +22,37 @@ import (
 	"fmt"
 
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
+	resolutionutil "github.com/tektoncd/pipeline/pkg/internal/resolution"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // GetPipeline is a function used to retrieve Pipelines.
-type GetPipeline func(context.Context, string) (v1beta1.PipelineObject, error)
+type GetPipeline func(context.Context, string) (v1beta1.PipelineObject, *v1beta1.ConfigSource, error)
 
 // GetPipelineData will retrieve the Pipeline metadata and Spec associated with the
 // provided PipelineRun. This can come from a reference Pipeline or from the PipelineRun's
 // metadata and embedded PipelineSpec.
-func GetPipelineData(ctx context.Context, pipelineRun *v1beta1.PipelineRun, getPipeline GetPipeline) (*metav1.ObjectMeta, *v1beta1.PipelineSpec, error) {
+func GetPipelineData(ctx context.Context, pipelineRun *v1beta1.PipelineRun, getPipeline GetPipeline) (*resolutionutil.ResolvedObjectMeta, *v1beta1.PipelineSpec, error) {
 	pipelineMeta := metav1.ObjectMeta{}
+	var configSource *v1beta1.ConfigSource
 	pipelineSpec := v1beta1.PipelineSpec{}
 	switch {
 	case pipelineRun.Spec.PipelineRef != nil && pipelineRun.Spec.PipelineRef.Name != "":
 		// Get related pipeline for pipelinerun
-		t, err := getPipeline(ctx, pipelineRun.Spec.PipelineRef.Name)
+		p, source, err := getPipeline(ctx, pipelineRun.Spec.PipelineRef.Name)
 		if err != nil {
 			return nil, nil, fmt.Errorf("error when listing pipelines for pipelineRun %s: %w", pipelineRun.Name, err)
 		}
-		pipelineMeta = t.PipelineMetadata()
-		pipelineSpec = t.PipelineSpec()
+		pipelineMeta = p.PipelineMetadata()
+		pipelineSpec = p.PipelineSpec()
+		configSource = source
 	case pipelineRun.Spec.PipelineSpec != nil:
 		pipelineMeta = pipelineRun.ObjectMeta
 		pipelineSpec = *pipelineRun.Spec.PipelineSpec
+		// TODO: if we want to set source for embedded pipeline, set it here.
+		// https://github.com/tektoncd/pipeline/issues/5522
 	case pipelineRun.Spec.PipelineRef != nil && pipelineRun.Spec.PipelineRef.Resolver != "":
-		pipeline, err := getPipeline(ctx, "")
+		pipeline, source, err := getPipeline(ctx, "")
 		switch {
 		case err != nil:
 			return nil, nil, err
@@ -57,10 +62,14 @@ func GetPipelineData(ctx context.Context, pipelineRun *v1beta1.PipelineRun, getP
 			pipelineMeta = pipeline.PipelineMetadata()
 			pipelineSpec = pipeline.PipelineSpec()
 		}
+		configSource = source
 	default:
 		return nil, nil, fmt.Errorf("pipelineRun %s not providing PipelineRef or PipelineSpec", pipelineRun.Name)
 	}
 
 	pipelineSpec.SetDefaults(ctx)
-	return &pipelineMeta, &pipelineSpec, nil
+	return &resolutionutil.ResolvedObjectMeta{
+		ObjectMeta:   &pipelineMeta,
+		ConfigSource: configSource,
+	}, &pipelineSpec, nil
 }