diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index f42eaec91ec0..17bf45108f47 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -83,6 +83,10 @@ "description": "name of the artifact. must be unique within a template's inputs/outputs.", "type": "string" }, + "optional": { + "description": "Make Artifacts optional, if Artifacts doesn't generate or exist", + "type": "boolean" + }, "path": { "description": "Path is the container path to the artifact", "type": "string" diff --git a/errors/errors.go b/errors/errors.go index 22177ccaa4f1..1fbe662557a7 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -14,7 +14,7 @@ const ( CodeBadRequest = "ERR_BAD_REQUEST" CodeForbidden = "ERR_FORBIDDEN" CodeNotFound = "ERR_NOT_FOUND" - CodeNotImplemented = "ERR_NOT_INPLEMENTED" + CodeNotImplemented = "ERR_NOT_IMPLEMENTED" CodeTimeout = "ERR_TIMEOUT" CodeInternal = "ERR_INTERNAL" ) diff --git a/pkg/apis/workflow/v1alpha1/openapi_generated.go b/pkg/apis/workflow/v1alpha1/openapi_generated.go index b203430f3f8f..ade406b7685e 100644 --- a/pkg/apis/workflow/v1alpha1/openapi_generated.go +++ b/pkg/apis/workflow/v1alpha1/openapi_generated.go @@ -206,6 +206,13 @@ func schema_pkg_apis_workflow_v1alpha1_Artifact(ref common.ReferenceCallback) co Ref: ref("github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.ArchiveStrategy"), }, }, + "optional": { + SchemaProps: spec.SchemaProps{ + Description: "Make Artifacts optional, if Artifacts doesn't generate or exist", + Type: []string{"boolean"}, + Format: "", + }, + }, }, Required: []string{"name"}, }, diff --git a/pkg/apis/workflow/v1alpha1/types.go b/pkg/apis/workflow/v1alpha1/types.go index 852746236326..3380be49fbc6 100644 --- a/pkg/apis/workflow/v1alpha1/types.go +++ b/pkg/apis/workflow/v1alpha1/types.go @@ -319,6 +319,9 @@ type Artifact struct { // Archive controls how the artifact will be saved to the artifact repository. Archive *ArchiveStrategy `json:"archive,omitempty"` + + // Make Artifacts optional, if Artifacts doesn't generate or exist + Optional bool `json:"optional,omitempty"` } // ArchiveStrategy describes how to archive files/directory when saving artifacts diff --git a/test/e2e/expectedfailures/input-artifact-not-optional.yaml b/test/e2e/expectedfailures/input-artifact-not-optional.yaml new file mode 100644 index 000000000000..e1a3615c71ad --- /dev/null +++ b/test/e2e/expectedfailures/input-artifact-not-optional.yaml @@ -0,0 +1,22 @@ +# This example demonstrates the input artifacts not optionals +# from one step to the next. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: input-artifact-not-optional- +spec: + entrypoint: http-artifact-example + templates: + - name: http-artifact-example + inputs: + artifacts: + - name: kubectl + path: /bin/kubectl + mode: 0755 + optional: false + http: + url: "" + container: + image: debian:9.4 + command: [sh, -c] + args: ["echo NoKubectl"] diff --git a/test/e2e/expectedfailures/output-artifact-not-optional.yaml b/test/e2e/expectedfailures/output-artifact-not-optional.yaml new file mode 100644 index 000000000000..d6fe97da86b6 --- /dev/null +++ b/test/e2e/expectedfailures/output-artifact-not-optional.yaml @@ -0,0 +1,24 @@ +# This example demonstrates the output artifacts not optionals +# from one step to the next. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: output-artifact-not-optional- +spec: + entrypoint: artifact-example + templates: + - name: artifact-example + steps: + - - name: generate-artifact + template: whalesay + + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["cowsay hello world | tee /tmp/hello_world12.txt"] + outputs: + artifacts: + - name: hello-art + optional: false + path: /tmp/hello_world.txt diff --git a/test/e2e/functional/input-artifact-optional.yaml b/test/e2e/functional/input-artifact-optional.yaml new file mode 100644 index 000000000000..9b7a8a051b19 --- /dev/null +++ b/test/e2e/functional/input-artifact-optional.yaml @@ -0,0 +1,22 @@ +# This example demonstrates the input artifacts optionals +# from one step to the next. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: input-artifact-optional- +spec: + entrypoint: http-artifact-example + templates: + - name: http-artifact-example + inputs: + artifacts: + - name: kubectl + path: /bin/kubectl + mode: 0755 + optional: true + http: + url: "" + container: + image: debian:9.4 + command: [sh, -c] + args: ["echo NoKubectl"] diff --git a/test/e2e/functional/output-artifact-optional.yaml b/test/e2e/functional/output-artifact-optional.yaml new file mode 100644 index 000000000000..3713b45450de --- /dev/null +++ b/test/e2e/functional/output-artifact-optional.yaml @@ -0,0 +1,24 @@ +# This example demonstrates the output artifacts optionals +# from one step to the next. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: output-artifact-optional- +spec: + entrypoint: artifact-example + templates: + - name: artifact-example + steps: + - - name: generate-artifact + template: whalesay + + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["cowsay hello world | tee /tmp/hello_world12.txt"] + outputs: + artifacts: + - name: hello-art + optional: true + path: /tmp/hello_world.txt diff --git a/test/e2e/functional/output-input-artifact-optional.yaml b/test/e2e/functional/output-input-artifact-optional.yaml new file mode 100644 index 000000000000..a29959fed04d --- /dev/null +++ b/test/e2e/functional/output-input-artifact-optional.yaml @@ -0,0 +1,40 @@ +# This example demonstrates the output and input artifacts are optionals +# from one step to the next. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: output-input-artifact-optional- +spec: + entrypoint: artifact-example + templates: + - name: artifact-example + steps: + - - name: generate-artifact + template: whalesay + - - name: consume-artifact + template: print-message + arguments: + artifacts: + - name: message + from: "{{steps.generate-artifact.outputs.artifacts.hello-art}}" + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["cowsay hello world | tee /tmp/hello_world123.txt"] + outputs: + artifacts: + - name: hello-art + optional: true + path: /tmp/hello_world.txt + + - name: print-message + inputs: + artifacts: + - name: message + path: /tmp/message + optional: true + container: + image: alpine:latest + command: [sh, -c] + args: ["echo /tmp/message"] diff --git a/workflow/common/util.go b/workflow/common/util.go index 88f3431ecd0e..860c6b47370d 100644 --- a/workflow/common/util.go +++ b/workflow/common/util.go @@ -148,10 +148,10 @@ func ProcessArgs(tmpl *wfv1.Template, args wfv1.Arguments, globalParams, localPa } // artifact must be supplied argArt := args.GetArtifactByName(inArt.Name) - if argArt == nil { + if !inArt.Optional && argArt == nil { return nil, errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s was not supplied", inArt.Name) } - if !argArt.HasLocation() && !validateOnly { + if !inArt.Optional && !argArt.HasLocation() && !validateOnly { return nil, errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s missing location information", inArt.Name) } argArt.Path = inArt.Path diff --git a/workflow/executor/docker/docker.go b/workflow/executor/docker/docker.go index 020ae938d1ea..9b0c7d9266bb 100644 --- a/workflow/executor/docker/docker.go +++ b/workflow/executor/docker/docker.go @@ -65,7 +65,9 @@ func (d *DockerExecutor) CopyFile(containerID string, sourcePath string, destPat return err } if !file.ExistsInTar(sourcePath, tar.NewReader(gzipReader)) { - return errors.InternalErrorf("path %s does not exist (or %s is empty) in archive %s", sourcePath, sourcePath, destPath) + errMsg := fmt.Sprintf("path %s does not exist (or %s is empty) in archive %s", sourcePath, sourcePath, destPath) + log.Warn(errMsg) + return errors.Errorf(errors.CodeNotFound, errMsg) } log.Infof("Archiving completed") return nil diff --git a/workflow/executor/executor.go b/workflow/executor/executor.go index 955c4e7927b3..c361f3f3abc4 100644 --- a/workflow/executor/executor.go +++ b/workflow/executor/executor.go @@ -114,7 +114,17 @@ func (we *WorkflowExecutor) LoadArtifacts() error { log.Infof("Start loading input artifacts...") for _, art := range we.Template.Inputs.Artifacts { + log.Infof("Downloading artifact: %s", art.Name) + + if !art.HasLocation() { + if art.Optional { + log.Warnf("Artifact %s is not supplied. Artifact configured as an optional so, Artifact will be ignored", art.Name) + continue + } else { + return errors.New("required artifact %s not supplied", art.Name) + } + } artDriver, err := we.InitDriver(art) if err != nil { return err @@ -232,6 +242,10 @@ func (we *WorkflowExecutor) saveArtifact(tempOutArtDir string, mainCtrID string, localArtPath := path.Join(tempOutArtDir, fileName) err := we.RuntimeExecutor.CopyFile(mainCtrID, art.Path, localArtPath) if err != nil { + if art.Optional && errors.IsCode(errors.CodeNotFound, err) { + log.Warnf("Error in saving Artifact. Artifact configured as an optional so, Error will be ignored. Error= %v", err) + return nil + } return err } fileName, localArtPath, err = stageArchiveFile(fileName, localArtPath, art) @@ -502,6 +516,7 @@ func (we *WorkflowExecutor) InitDriver(art wfv1.Artifact) (artifact.ArtifactDriv if art.Raw != nil { return &raw.RawArtifactDriver{}, nil } + return nil, errors.Errorf(errors.CodeBadRequest, "Unsupported artifact driver for %s", art.Name) }