diff --git a/docs/workspaces.md b/docs/workspaces.md index 54b24ee31fa..7f32c548d18 100644 --- a/docs/workspaces.md +++ b/docs/workspaces.md @@ -222,6 +222,10 @@ spec: - use-ws-from-pipeline # important: use-ws-from-pipeline writes to the workspace first ``` +Include a `subPath` in the workspace binding to mount different parts of the same volume for different Tasks. See [a full example of this kind of Pipeline](../examples/v1beta1/pipelineruns/pipelinerun-using-different-subpaths-of-workspace.yaml) which writes data to two adjacent directories on the same Volume. + +The `subPath` specified in a `Pipeline` will be appended to any `subPath` specified as part of the `PipelineRun` workspace declaration. So a `PipelineRun` declaring a Workspace with `subPath` of `/foo` for a `Pipeline` who binds it to a `Task` with `subPath` of `/bar` will end up mounting the `Volume`'s `/foo/bar` directory. + #### Specifying `Workspace` order in a `Pipeline` Sharing a `Workspace` between `Tasks` requires you to define the order in which those `Tasks` diff --git a/examples/v1beta1/pipelineruns/pipelinerun-using-different-subpaths-of-workspace.yaml b/examples/v1beta1/pipelineruns/pipelinerun-using-different-subpaths-of-workspace.yaml new file mode 100644 index 00000000000..4fd1058615b --- /dev/null +++ b/examples/v1beta1/pipelineruns/pipelinerun-using-different-subpaths-of-workspace.yaml @@ -0,0 +1,85 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: writer +spec: + steps: + - name: write + image: ubuntu + script: echo bar > $(workspaces.task-ws.path)/foo + workspaces: + - name: task-ws +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: read-both +spec: + params: + - name: directory1 + type: string + - name: directory2 + type: string + workspaces: + - name: local-ws + steps: + - name: read-1 + image: ubuntu + script: cat $(workspaces.local-ws.path)/$(params.directory1)/foo | grep bar + - name: read-2 + image: ubuntu + script: cat $(workspaces.local-ws.path)/$(params.directory2)/foo | grep bar +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: pipeline-using-different-subpaths +spec: + workspaces: + - name: ws + tasks: + - name: writer-1 + taskRef: + name: writer + workspaces: + - name: task-ws + workspace: ws + subPath: dir-1 + - name: writer-2 + taskRef: + name: writer + workspaces: + - name: task-ws + workspace: ws + subPath: dir-2 + - name: read-all + runAfter: + - writer-1 + - writer-2 + params: + - name: directory1 + value: dir-1 + - name: directory2 + value: dir-2 + taskRef: + name: read-both + workspaces: + - name: local-ws + workspace: ws +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + generateName: pr- +spec: + pipelineRef: + name: pipeline-using-different-subpaths + workspaces: + - name: ws + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/pkg/apis/pipeline/v1alpha1/pipeline_validation_test.go b/pkg/apis/pipeline/v1alpha1/pipeline_validation_test.go index be5ad89ac48..1895af5764c 100644 --- a/pkg/apis/pipeline/v1alpha1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1alpha1/pipeline_validation_test.go @@ -359,7 +359,7 @@ func TestPipeline_Validate(t *testing.T) { p: tb.Pipeline("name", tb.PipelineSpec( tb.PipelineWorkspaceDeclaration("foo"), tb.PipelineTask("taskname", "taskref", - tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", "pipelineWorkspaceName")), + tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", "pipelineWorkspaceName", "")), )), failureExpected: true, }, { diff --git a/pkg/apis/pipeline/v1beta1/workspace_types.go b/pkg/apis/pipeline/v1beta1/workspace_types.go index 94d50b30121..0a47bfaf228 100644 --- a/pkg/apis/pipeline/v1beta1/workspace_types.go +++ b/pkg/apis/pipeline/v1beta1/workspace_types.go @@ -95,4 +95,8 @@ type WorkspacePipelineTaskBinding struct { Name string `json:"name"` // Workspace is the name of the workspace declared by the pipeline Workspace string `json:"workspace"` + // SubPath is optionally a directory on the volume which should be used + // for this binding (i.e. the volume will be mounted at this sub directory). + // +optional + SubPath string `json:"subPath,omitempty"` } diff --git a/pkg/reconciler/pipelinerun/pipelinerun.go b/pkg/reconciler/pipelinerun/pipelinerun.go index eaae85c631a..fc59451d51f 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun.go +++ b/pkg/reconciler/pipelinerun/pipelinerun.go @@ -19,6 +19,7 @@ package pipelinerun import ( "context" "fmt" + "path/filepath" "reflect" "strings" "time" @@ -733,9 +734,9 @@ func (c *Reconciler) createTaskRun(rprt *resources.ResolvedPipelineRunTask, pr * pipelineRunWorkspaces[binding.Name] = binding } for _, ws := range rprt.PipelineTask.Workspaces { - taskWorkspaceName, pipelineWorkspaceName := ws.Name, ws.Workspace + taskWorkspaceName, pipelineTaskSubPath, pipelineWorkspaceName := ws.Name, ws.SubPath, ws.Workspace if b, hasBinding := pipelineRunWorkspaces[pipelineWorkspaceName]; hasBinding { - tr.Spec.Workspaces = append(tr.Spec.Workspaces, taskWorkspaceByWorkspaceVolumeSource(b, taskWorkspaceName, pr.GetOwnerReference())) + tr.Spec.Workspaces = append(tr.Spec.Workspaces, taskWorkspaceByWorkspaceVolumeSource(b, taskWorkspaceName, pipelineTaskSubPath, pr.GetOwnerReference())) } else { return nil, fmt.Errorf("expected workspace %q to be provided by pipelinerun for pipeline task %q", pipelineWorkspaceName, rprt.PipelineTask.Name) } @@ -748,16 +749,17 @@ func (c *Reconciler) createTaskRun(rprt *resources.ResolvedPipelineRunTask, pr * // taskWorkspaceByWorkspaceVolumeSource is returning the WorkspaceBinding with the TaskRun specified name. // If the volume source is a volumeClaimTemplate, the template is applied and passed to TaskRun as a persistentVolumeClaim -func taskWorkspaceByWorkspaceVolumeSource(wb v1alpha1.WorkspaceBinding, taskWorkspaceName string, owner metav1.OwnerReference) v1alpha1.WorkspaceBinding { +func taskWorkspaceByWorkspaceVolumeSource(wb v1alpha1.WorkspaceBinding, taskWorkspaceName string, pipelineTaskSubPath string, owner metav1.OwnerReference) v1alpha1.WorkspaceBinding { if wb.VolumeClaimTemplate == nil { binding := *wb.DeepCopy() binding.Name = taskWorkspaceName + binding.SubPath = combinedSubPath(wb.SubPath, pipelineTaskSubPath) return binding } // apply template binding := v1alpha1.WorkspaceBinding{ - SubPath: wb.SubPath, + SubPath: combinedSubPath(wb.SubPath, pipelineTaskSubPath), PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: volumeclaim.GetPersistentVolumeClaimName(wb.VolumeClaimTemplate, wb, owner), }, @@ -766,6 +768,17 @@ func taskWorkspaceByWorkspaceVolumeSource(wb v1alpha1.WorkspaceBinding, taskWork return binding } +// combinedSubPath returns the combined value of the optional subPath from workspaceBinding and the optional +// subPath from pipelineTask. If both is set, they are joined with a slash. +func combinedSubPath(workspaceSubPath string, pipelineTaskSubPath string) string { + if workspaceSubPath == "" { + return pipelineTaskSubPath + } else if pipelineTaskSubPath == "" { + return workspaceSubPath + } + return filepath.Join(workspaceSubPath, pipelineTaskSubPath) +} + func addRetryHistory(tr *v1alpha1.TaskRun) { newStatus := *tr.Status.DeepCopy() newStatus.RetriesStatus = nil diff --git a/pkg/reconciler/pipelinerun/pipelinerun_test.go b/pkg/reconciler/pipelinerun/pipelinerun_test.go index dac1952ade5..3c21ee48977 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun_test.go +++ b/pkg/reconciler/pipelinerun/pipelinerun_test.go @@ -1718,13 +1718,13 @@ func TestReconcileWithVolumeClaimTemplateWorkspace(t *testing.T) { claimName := "myclaim" pipelineRunName := "test-pipeline-run" ps := []*v1alpha1.Pipeline{tb.Pipeline("test-pipeline", tb.PipelineNamespace("foo"), tb.PipelineSpec( - tb.PipelineTask("hello-world-1", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName)), + tb.PipelineTask("hello-world-1", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, "")), tb.PipelineTask("hello-world-2", "hello-world"), tb.PipelineWorkspaceDeclaration(workspaceName), ))} prs := []*v1alpha1.PipelineRun{tb.PipelineRun(pipelineRunName, tb.PipelineRunNamespace("foo"), - tb.PipelineRunSpec("test-pipeline", tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceName, claimName))), + tb.PipelineRunSpec("test-pipeline", tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceName, claimName, ""))), } ts := []*v1alpha1.Task{tb.Task("hello-world", tb.TaskNamespace("foo"))} @@ -1792,6 +1792,120 @@ func TestReconcileWithVolumeClaimTemplateWorkspace(t *testing.T) { } } +// TestReconcileWithVolumeClaimTemplateWorkspaceUsingSubPaths tests that given a pipeline with volumeClaimTemplate workspace and +// multiple instances of the same task, but using different subPaths in the volume - is seen as taskRuns with expected subPaths. +func TestReconcileWithVolumeClaimTemplateWorkspaceUsingSubPaths(t *testing.T) { + workspaceName := "ws1" + workspaceNameWithSubPath := "ws2" + subPath1 := "customdirectory" + subPath2 := "otherdirecory" + pipelineRunWsSubPath := "mypath" + ps := []*v1alpha1.Pipeline{tb.Pipeline("test-pipeline", tb.PipelineNamespace("foo"), tb.PipelineSpec( + tb.PipelineTask("hello-world-1", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, subPath1)), + tb.PipelineTask("hello-world-2", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, subPath2)), + tb.PipelineTask("hello-world-3", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, "")), + tb.PipelineTask("hello-world-4", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceNameWithSubPath, "")), + tb.PipelineTask("hello-world-5", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceNameWithSubPath, subPath1)), + tb.PipelineWorkspaceDeclaration(workspaceName, workspaceNameWithSubPath), + ))} + + prs := []*v1alpha1.PipelineRun{tb.PipelineRun("test-pipeline-run", tb.PipelineRunNamespace("foo"), + tb.PipelineRunSpec("test-pipeline", + tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceName, "myclaim", ""), + tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceNameWithSubPath, "myclaim", pipelineRunWsSubPath))), + } + ts := []*v1alpha1.Task{tb.Task("hello-world", tb.TaskNamespace("foo"))} + + d := test.Data{ + PipelineRuns: prs, + Pipelines: ps, + Tasks: ts, + } + + testAssets, cancel := getPipelineRunController(t, d) + defer cancel() + c := testAssets.Controller + clients := testAssets.Clients + + err := c.Reconciler.Reconcile(context.Background(), "foo/test-pipeline-run") + if err != nil { + t.Errorf("Did not expect to see error when reconciling PipelineRun but saw %s", err) + } + + // Check that the PipelineRun was reconciled correctly + reconciledRun, err := clients.Pipeline.TektonV1alpha1().PipelineRuns("foo").Get("test-pipeline-run", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Somehow had error getting reconciled run out of fake client: %s", err) + } + + taskRuns, err := clients.Pipeline.TektonV1alpha1().TaskRuns("foo").List(metav1.ListOptions{}) + if err != nil { + t.Fatalf("unexpected error when listing TaskRuns: %v", err) + } + + if len(taskRuns.Items) != 5 { + t.Fatalf("unexpected number of taskRuns found, expected 2, but found %d", len(taskRuns.Items)) + } + + hasSeenWorkspaceWithPipelineTaskSubPath1 := false + hasSeenWorkspaceWithPipelineTaskSubPath2 := false + hasSeenWorkspaceWithEmptyPipelineTaskSubPath := false + hasSeenWorkspaceWithRunSubPathAndEmptyPipelineTaskSubPath := false + hasSeenWorkspaceWithRunSubPathAndPipelineTaskSubPath1 := false + for _, tr := range taskRuns.Items { + for _, ws := range tr.Spec.Workspaces { + + if ws.PersistentVolumeClaim == nil { + t.Fatalf("found taskRun workspace that is not PersistentVolumeClaim workspace. Did only expect PersistentVolumeClaims workspaces") + } + + if ws.SubPath == subPath1 { + hasSeenWorkspaceWithPipelineTaskSubPath1 = true + } + + if ws.SubPath == subPath2 { + hasSeenWorkspaceWithPipelineTaskSubPath2 = true + } + + if ws.SubPath == "" { + hasSeenWorkspaceWithEmptyPipelineTaskSubPath = true + } + + if ws.SubPath == pipelineRunWsSubPath { + hasSeenWorkspaceWithRunSubPathAndEmptyPipelineTaskSubPath = true + } + + if ws.SubPath == fmt.Sprintf("%s/%s", pipelineRunWsSubPath, subPath1) { + hasSeenWorkspaceWithRunSubPathAndPipelineTaskSubPath1 = true + } + } + } + + if !hasSeenWorkspaceWithPipelineTaskSubPath1 { + t.Fatalf("did not see a taskRun with a workspace using pipelineTask subPath1") + } + + if !hasSeenWorkspaceWithPipelineTaskSubPath2 { + t.Fatalf("did not see a taskRun with a workspace using pipelineTask subPath2") + } + + if !hasSeenWorkspaceWithEmptyPipelineTaskSubPath { + t.Fatalf("did not see a taskRun with a workspace using empty pipelineTask subPath") + } + + if !hasSeenWorkspaceWithRunSubPathAndEmptyPipelineTaskSubPath { + t.Fatalf("did not see a taskRun with workspace using empty pipelineTask subPath and a subPath from pipelineRun") + } + + if !hasSeenWorkspaceWithRunSubPathAndPipelineTaskSubPath1 { + t.Fatalf("did not see a taskRun with workspace using pipelineTaks subPath1 and a subPath from pipelineRun") + } + + if !reconciledRun.Status.GetCondition(apis.ConditionSucceeded).IsUnknown() { + t.Errorf("Expected PipelineRun to be running, but condition status is %s", reconciledRun.Status.GetCondition(apis.ConditionSucceeded)) + } +} + func TestReconcileWithTaskResults(t *testing.T) { names.TestingSeed() ps := []*v1alpha1.Pipeline{tb.Pipeline("test-pipeline", tb.PipelineNamespace("foo"), tb.PipelineSpec( diff --git a/test/builder/pipeline.go b/test/builder/pipeline.go index a691b53f1a2..d80b62a7bad 100644 --- a/test/builder/pipeline.go +++ b/test/builder/pipeline.go @@ -302,11 +302,12 @@ func PipelineTaskConditionResource(name, resource string, from ...string) Pipeli } } -func PipelineTaskWorkspaceBinding(name, workspace string) PipelineTaskOp { +func PipelineTaskWorkspaceBinding(name, workspace, subPath string) PipelineTaskOp { return func(pt *v1alpha1.PipelineTask) { pt.Workspaces = append(pt.Workspaces, v1alpha1.WorkspacePipelineTaskBinding{ Name: name, Workspace: workspace, + SubPath: subPath, }) } } @@ -619,10 +620,11 @@ func PipelineRunWorkspaceBindingEmptyDir(name string) PipelineRunSpecOp { } // PipelineRunWorkspaceBindingVolumeClaimTemplate adds an VolumeClaimTemplate Workspace to the workspaces of a pipelineRun spec. -func PipelineRunWorkspaceBindingVolumeClaimTemplate(name string, claimName string) PipelineRunSpecOp { +func PipelineRunWorkspaceBindingVolumeClaimTemplate(name string, claimName string, subPath string) PipelineRunSpecOp { return func(spec *v1alpha1.PipelineRunSpec) { spec.Workspaces = append(spec.Workspaces, v1alpha1.WorkspaceBinding{ - Name: name, + Name: name, + SubPath: subPath, VolumeClaimTemplate: &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: claimName, diff --git a/test/builder/pipeline_test.go b/test/builder/pipeline_test.go index 95c3259e3f0..d776fdd16df 100644 --- a/test/builder/pipeline_test.go +++ b/test/builder/pipeline_test.go @@ -46,7 +46,7 @@ func TestPipeline(t *testing.T) { tb.PipelineTaskConditionParam("param-name", "param-value"), tb.PipelineTaskConditionResource("some-resource", "my-only-git-resource", "bar", "never-gonna"), ), - tb.PipelineTaskWorkspaceBinding("task-workspace1", "workspace1"), + tb.PipelineTaskWorkspaceBinding("task-workspace1", "workspace1", ""), ), tb.PipelineTask("bar", "chocolate", tb.PipelineTaskRefKind(v1alpha1.ClusterTaskKind), diff --git a/test/v1alpha1/workspace_test.go b/test/v1alpha1/workspace_test.go index ce9f887b76f..b5a41256ca9 100644 --- a/test/v1alpha1/workspace_test.go +++ b/test/v1alpha1/workspace_test.go @@ -105,7 +105,7 @@ func TestWorkspacePipelineRunDuplicateWorkspaceEntriesInvalid(t *testing.T) { pipeline := tb.Pipeline(pipelineName, tb.PipelineSpec( tb.PipelineWorkspaceDeclaration("foo"), - tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo")), + tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo", "")), )) if _, err := c.PipelineClient.Create(pipeline); err != nil { t.Fatalf("Failed to create Pipeline: %s", err) @@ -146,7 +146,7 @@ func TestWorkspacePipelineRunMissingWorkspaceInvalid(t *testing.T) { pipeline := tb.Pipeline(pipelineName, tb.PipelineSpec( tb.PipelineWorkspaceDeclaration("foo"), - tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo")), + tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo", "")), )) if _, err := c.PipelineClient.Create(pipeline); err != nil { t.Fatalf("Failed to create Pipeline: %s", err)