Add subPath to WorkspacePipelineTaskBinding

Use case: Use two instances of a task that writes to the same workspace volume - but
they write to different path of the volume. A typical use case is two git-clone tasks
that clone two git repositories, but the files should be located in two different
directories on the workspace volume.

This commit solves this by adding the `subPath` field to the WorkspacePipelineTaskBinding.

examples/v1beta1/pipelineruns/pipelinerun-using-different-subpaths-of-workspace.yaml

@@ -0,0 +1,85 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+ name: writer
+spec:
+ steps:
+ - name: write
+ image: ubuntu
+ script: echo bar > $(workspaces.task-ws.path)/foo
+ workspaces:
+ - name: task-ws
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+ name: read-both
+spec:
+ params:
+ - name: directory1
+ type: string
+ - name: directory2
+ type: string
+ workspaces:
+ - name: local-ws
+ steps:
+ - name: read-1
+ image: ubuntu
+ script: cat $(workspaces.local-ws.path)/$(params.directory1)/foo | grep bar
+ - name: read-2
+ image: ubuntu
+ script: cat $(workspaces.local-ws.path)/$(params.directory2)/foo | grep bar
+---
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+ name: pipeline-using-different-subpaths
+spec:
+ workspaces:
+ - name: ws
+ tasks:
+ - name: writer-1
+ taskRef:
+ name: writer
+ workspaces:
+ - name: task-ws
+ workspace: ws
+ subPath: dir-1
+ - name: writer-2
+ taskRef:
+ name: writer
+ workspaces:
+ - name: task-ws
+ workspace: ws
+ subPath: dir-2
+ - name: read-all
+ runAfter:
+ - writer-1
+ - writer-2
+ params:
+ - name: directory1
+ value: dir-1
+ - name: directory2
+ value: dir-2
+ taskRef:
+ name: read-both
+ workspaces:
+ - name: local-ws
+ workspace: ws
+---
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+ generateName: pr-
+spec:
+ pipelineRef:
+ name: pipeline-using-different-subpaths
+ workspaces:
+ - name: ws
+ volumeClaimTemplate:
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi

pkg/apis/pipeline/v1alpha1/pipeline_validation_test.go

@@ -359,7 +359,7 @@ func TestPipeline_Validate(t *testing.T) {
  p: tb.Pipeline("name", tb.PipelineSpec(
  tb.PipelineWorkspaceDeclaration("foo"),
  tb.PipelineTask("taskname", "taskref",
- tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", "pipelineWorkspaceName")),
+ tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", "pipelineWorkspaceName", "")),
  )),
  failureExpected: true,
  }, {

pkg/apis/pipeline/v1beta1/workspace_types.go

@@ -95,4 +95,8 @@ type WorkspacePipelineTaskBinding struct {
  Name string `json:"name"`
  // Workspace is the name of the workspace declared by the pipeline
  Workspace string `json:"workspace"`
+ // SubPath is optionally a directory on the volume which should be used
+ // for this binding (i.e. the volume will be mounted at this sub directory).
+ // +optional
+ SubPath string `json:"subPath,omitempty"`
 }

pkg/reconciler/pipelinerun/pipelinerun.go

@@ -733,9 +733,9 @@ func (c *Reconciler) createTaskRun(rprt *resources.ResolvedPipelineRunTask, pr *
  pipelineRunWorkspaces[binding.Name] = binding
  }
  for _, ws := range rprt.PipelineTask.Workspaces {
- taskWorkspaceName, pipelineWorkspaceName := ws.Name, ws.Workspace
+ taskWorkspaceName, pipelineTaskSubPath, pipelineWorkspaceName := ws.Name, ws.SubPath, ws.Workspace
  if b, hasBinding := pipelineRunWorkspaces[pipelineWorkspaceName]; hasBinding {
- tr.Spec.Workspaces = append(tr.Spec.Workspaces, taskWorkspaceByWorkspaceVolumeSource(b, taskWorkspaceName, pr.GetOwnerReference()))
+ tr.Spec.Workspaces = append(tr.Spec.Workspaces, taskWorkspaceByWorkspaceVolumeSource(b, taskWorkspaceName, pipelineTaskSubPath, pr.GetOwnerReference()))
  } else {
  return nil, fmt.Errorf("expected workspace %q to be provided by pipelinerun for pipeline task %q", pipelineWorkspaceName, rprt.PipelineTask.Name)
  }
@@ -748,16 +748,17 @@ func (c *Reconciler) createTaskRun(rprt *resources.ResolvedPipelineRunTask, pr *
 
 // taskWorkspaceByWorkspaceVolumeSource is returning the WorkspaceBinding with the TaskRun specified name.
 // If the volume source is a volumeClaimTemplate, the template is applied and passed to TaskRun as a persistentVolumeClaim
-func taskWorkspaceByWorkspaceVolumeSource(wb v1alpha1.WorkspaceBinding, taskWorkspaceName string, owner metav1.OwnerReference) v1alpha1.WorkspaceBinding {
+func taskWorkspaceByWorkspaceVolumeSource(wb v1alpha1.WorkspaceBinding, taskWorkspaceName string, pipelineTaskSubPath string, owner metav1.OwnerReference) v1alpha1.WorkspaceBinding {
  if wb.VolumeClaimTemplate == nil {
  binding := *wb.DeepCopy()
  binding.Name = taskWorkspaceName
+ binding.SubPath = combinedSubPath(wb.SubPath, pipelineTaskSubPath)
  return binding
  }
 
  // apply template
  binding := v1alpha1.WorkspaceBinding{
- SubPath: wb.SubPath,
+ SubPath: combinedSubPath(wb.SubPath, pipelineTaskSubPath),
  PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
  ClaimName: volumeclaim.GetPersistentVolumeClaimName(wb.VolumeClaimTemplate, wb, owner),
  },
@@ -766,6 +767,18 @@ func taskWorkspaceByWorkspaceVolumeSource(wb v1alpha1.WorkspaceBinding, taskWork
  return binding
 }
 
+// combinedSubPath returns the combined value of the optional subPath from workspaceBinding and the optional
+// subPath from pipelineTask. If both is set, they are joined with a slash.
+func combinedSubPath(workspaceSubPath string, pipelineTaskSubPath string) string {
+ if workspaceSubPath == "" {
+ return pipelineTaskSubPath
+ } else if pipelineTaskSubPath == "" {
+ return workspaceSubPath
+ } else {
+ return fmt.Sprintf("%s/%s", workspaceSubPath, pipelineTaskSubPath)
+ }
+}
+
 func addRetryHistory(tr *v1alpha1.TaskRun) {
  newStatus := *tr.Status.DeepCopy()
  newStatus.RetriesStatus = nil

pkg/reconciler/pipelinerun/pipelinerun_test.go

@@ -1718,13 +1718,13 @@ func TestReconcileWithVolumeClaimTemplateWorkspace(t *testing.T) {
  claimName := "myclaim"
  pipelineRunName := "test-pipeline-run"
  ps := []*v1alpha1.Pipeline{tb.Pipeline("test-pipeline", tb.PipelineNamespace("foo"), tb.PipelineSpec(
- tb.PipelineTask("hello-world-1", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName)),
+ tb.PipelineTask("hello-world-1", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, "")),
  tb.PipelineTask("hello-world-2", "hello-world"),
  tb.PipelineWorkspaceDeclaration(workspaceName),
  ))}
 
  prs := []*v1alpha1.PipelineRun{tb.PipelineRun(pipelineRunName, tb.PipelineRunNamespace("foo"),
- tb.PipelineRunSpec("test-pipeline", tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceName, claimName))),
+ tb.PipelineRunSpec("test-pipeline", tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceName, claimName, ""))),
  }
  ts := []*v1alpha1.Task{tb.Task("hello-world", tb.TaskNamespace("foo"))}
 
@@ -1792,6 +1792,120 @@ func TestReconcileWithVolumeClaimTemplateWorkspace(t *testing.T) {
  }
 }
 
+// TestReconcileWithVolumeClaimTemplateWorkspaceUsingSubPaths tests that given a pipeline with volumeClaimTemplate workspace and
+// multiple instances of the same task, but using different subPaths in the volume - is seen as taskRuns with expected subPaths.
+func TestReconcileWithVolumeClaimTemplateWorkspaceUsingSubPaths(t *testing.T) {
+ workspaceName := "ws1"
+ workspaceNameWithSubPath := "ws2"
+ subPath1 := "customdirectory"
+ subPath2 := "otherdirecory"
+ pipelineRunWsSubPath := "mypath"
+ ps := []*v1alpha1.Pipeline{tb.Pipeline("test-pipeline", tb.PipelineNamespace("foo"), tb.PipelineSpec(
+ tb.PipelineTask("hello-world-1", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, subPath1)),
+ tb.PipelineTask("hello-world-2", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, subPath2)),
+ tb.PipelineTask("hello-world-3", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, "")),
+ tb.PipelineTask("hello-world-4", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceNameWithSubPath, "")),
+ tb.PipelineTask("hello-world-5", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceNameWithSubPath, subPath1)),
+ tb.PipelineWorkspaceDeclaration(workspaceName, workspaceNameWithSubPath),
+ ))}
+
+ prs := []*v1alpha1.PipelineRun{tb.PipelineRun("test-pipeline-run", tb.PipelineRunNamespace("foo"),
+ tb.PipelineRunSpec("test-pipeline",
+ tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceName, "myclaim", ""),
+ tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceNameWithSubPath, "myclaim", pipelineRunWsSubPath))),
+ }
+ ts := []*v1alpha1.Task{tb.Task("hello-world", tb.TaskNamespace("foo"))}
+
+ d := test.Data{
+ PipelineRuns: prs,
+ Pipelines: ps,
+ Tasks: ts,
+ }
+
+ testAssets, cancel := getPipelineRunController(t, d)
+ defer cancel()
+ c := testAssets.Controller
+ clients := testAssets.Clients
+
+ err := c.Reconciler.Reconcile(context.Background(), "foo/test-pipeline-run")
+ if err != nil {
+ t.Errorf("Did not expect to see error when reconciling PipelineRun but saw %s", err)
+ }
+
+ // Check that the PipelineRun was reconciled correctly
+ reconciledRun, err := clients.Pipeline.TektonV1alpha1().PipelineRuns("foo").Get("test-pipeline-run", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("Somehow had error getting reconciled run out of fake client: %s", err)
+ }
+
+ taskRuns, err := clients.Pipeline.TektonV1alpha1().TaskRuns("foo").List(metav1.ListOptions{})
+ if err != nil {
+ t.Fatalf("unexpected error when listing TaskRuns: %v", err)
+ }
+
+ if len(taskRuns.Items) != 5 {
+ t.Fatalf("unexpected number of taskRuns found, expected 2, but found %d", len(taskRuns.Items))
+ }
+
+ hasSeenWorkspaceWithPipelineTaskSubPath1 := false
+ hasSeenWorkspaceWithPipelineTaskSubPath2 := false
+ hasSeenWorkspaceWithEmptyPipelineTaskSubPath := false
+ hasSeenWorkspaceWithRunSubPathAndEmptyPipelineTaskSubPath := false
+ hasSeenWorkspaceWithRunSubPathAndPipelineTaskSubPath1 := false
+ for _, tr := range taskRuns.Items {
+ for _, ws := range tr.Spec.Workspaces {
+
+ if ws.PersistentVolumeClaim == nil {
+ t.Fatalf("found taskRun workspace that is not PersistentVolumeClaim workspace. Did only expect PersistentVolumeClaims workspaces")
+ }
+
+ if ws.SubPath == subPath1 {
+ hasSeenWorkspaceWithPipelineTaskSubPath1 = true
+ }
+
+ if ws.SubPath == subPath2 {
+ hasSeenWorkspaceWithPipelineTaskSubPath2 = true
+ }
+
+ if ws.SubPath == "" {
+ hasSeenWorkspaceWithEmptyPipelineTaskSubPath = true
+ }
+
+ if ws.SubPath == pipelineRunWsSubPath {
+ hasSeenWorkspaceWithRunSubPathAndEmptyPipelineTaskSubPath = true
+ }
+
+ if ws.SubPath == fmt.Sprintf("%s/%s", pipelineRunWsSubPath, subPath1) {
+ hasSeenWorkspaceWithRunSubPathAndPipelineTaskSubPath1 = true
+ }
+ }
+ }
+
+ if !hasSeenWorkspaceWithPipelineTaskSubPath1 {
+ t.Fatalf("did not see a taskRun with a workspace using pipelineTask subPath1")
+ }
+
+ if !hasSeenWorkspaceWithPipelineTaskSubPath2 {
+ t.Fatalf("did not see a taskRun with a workspace using pipelineTask subPath2")
+ }
+
+ if !hasSeenWorkspaceWithEmptyPipelineTaskSubPath {
+ t.Fatalf("did not see a taskRun with a workspace using empty pipelineTask subPath")
+ }
+
+ if !hasSeenWorkspaceWithRunSubPathAndEmptyPipelineTaskSubPath {
+ t.Fatalf("did not see a taskRun with workspace using empty pipelineTask subPath and a subPath from pipelineRun")
+ }
+
+ if !hasSeenWorkspaceWithRunSubPathAndPipelineTaskSubPath1 {
+ t.Fatalf("did not see a taskRun with workspace using pipelineTaks subPath1 and a subPath from pipelineRun")
+ }
+
+ if !reconciledRun.Status.GetCondition(apis.ConditionSucceeded).IsUnknown() {
+ t.Errorf("Expected PipelineRun to be running, but condition status is %s", reconciledRun.Status.GetCondition(apis.ConditionSucceeded))
+ }
+}
+
 func TestReconcileWithTaskResults(t *testing.T) {
  names.TestingSeed()
  ps := []*v1alpha1.Pipeline{tb.Pipeline("test-pipeline", tb.PipelineNamespace("foo"), tb.PipelineSpec(

test/builder/pipeline.go

@@ -302,11 +302,12 @@ func PipelineTaskConditionResource(name, resource string, from ...string) Pipeli
  }
 }
 
-func PipelineTaskWorkspaceBinding(name, workspace string) PipelineTaskOp {
+func PipelineTaskWorkspaceBinding(name, workspace, subPath string) PipelineTaskOp {
  return func(pt *v1alpha1.PipelineTask) {
  pt.Workspaces = append(pt.Workspaces, v1alpha1.WorkspacePipelineTaskBinding{
  Name: name,
  Workspace: workspace,
+ SubPath: subPath,
  })
  }
 }
@@ -619,10 +620,11 @@ func PipelineRunWorkspaceBindingEmptyDir(name string) PipelineRunSpecOp {
 }
 
 // PipelineRunWorkspaceBindingVolumeClaimTemplate adds an VolumeClaimTemplate Workspace to the workspaces of a pipelineRun spec.
-func PipelineRunWorkspaceBindingVolumeClaimTemplate(name string, claimName string) PipelineRunSpecOp {
+func PipelineRunWorkspaceBindingVolumeClaimTemplate(name string, claimName string, subPath string) PipelineRunSpecOp {
  return func(spec *v1alpha1.PipelineRunSpec) {
  spec.Workspaces = append(spec.Workspaces, v1alpha1.WorkspaceBinding{
- Name: name,
+ Name: name,
+ SubPath: subPath,
  VolumeClaimTemplate: &corev1.PersistentVolumeClaim{
  ObjectMeta: metav1.ObjectMeta{
  Name: claimName,

test/builder/pipeline_test.go

@@ -46,7 +46,7 @@ func TestPipeline(t *testing.T) {
  tb.PipelineTaskConditionParam("param-name", "param-value"),
  tb.PipelineTaskConditionResource("some-resource", "my-only-git-resource", "bar", "never-gonna"),
  ),
- tb.PipelineTaskWorkspaceBinding("task-workspace1", "workspace1"),
+ tb.PipelineTaskWorkspaceBinding("task-workspace1", "workspace1", ""),
  ),
  tb.PipelineTask("bar", "chocolate",
  tb.PipelineTaskRefKind(v1alpha1.ClusterTaskKind),

test/v1alpha1/workspace_test.go

@@ -105,7 +105,7 @@ func TestWorkspacePipelineRunDuplicateWorkspaceEntriesInvalid(t *testing.T) {
 
  pipeline := tb.Pipeline(pipelineName, tb.PipelineSpec(
  tb.PipelineWorkspaceDeclaration("foo"),
- tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo")),
+ tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo", "")),
  ))
  if _, err := c.PipelineClient.Create(pipeline); err != nil {
  t.Fatalf("Failed to create Pipeline: %s", err)
@@ -146,7 +146,7 @@ func TestWorkspacePipelineRunMissingWorkspaceInvalid(t *testing.T) {
 
  pipeline := tb.Pipeline(pipelineName, tb.PipelineSpec(
  tb.PipelineWorkspaceDeclaration("foo"),
- tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo")),
+ tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo", "")),
  ))
  if _, err := c.PipelineClient.Create(pipeline); err != nil {
  t.Fatalf("Failed to create Pipeline: %s", err)