Issue#896 Workflow steps with non-existant output artifact path will succeed

api/openapi-spec/swagger.json

@@ -83,6 +83,10 @@
  "description": "name of the artifact. must be unique within a template's inputs/outputs.",
  "type": "string"
  },
+ "optional": {
+ "description": "Make Artifacts optional, if Artifacts doesn't generate or exist",
+ "type": "boolean"
+ },
  "path": {
  "description": "Path is the container path to the artifact",
  "type": "string"

errors/errors.go

@@ -14,7 +14,7 @@ const (
  CodeBadRequest = "ERR_BAD_REQUEST"
  CodeForbidden = "ERR_FORBIDDEN"
  CodeNotFound = "ERR_NOT_FOUND"
- CodeNotImplemented = "ERR_NOT_INPLEMENTED"
+ CodeNotImplemented = "ERR_NOT_IMPLEMENTED"
  CodeTimeout = "ERR_TIMEOUT"
  CodeInternal = "ERR_INTERNAL"
 )

pkg/apis/workflow/v1alpha1/types.go

@@ -319,6 +319,9 @@ type Artifact struct {
 
  // Archive controls how the artifact will be saved to the artifact repository.
  Archive *ArchiveStrategy `json:"archive,omitempty"`
+
+ // Make Artifacts optional, if Artifacts doesn't generate or exist
+ Optional bool `json:"optional,omitempty"`
 }
 
 // ArchiveStrategy describes how to archive files/directory when saving artifacts

test/e2e/expectedfailures/input-artifact-not-optional.yaml

@@ -0,0 +1,22 @@
+# This example demonstrates the input artifacts not optionals
+# from one step to the next.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+ generateName: input-artifact-not-optional-
+spec:
+ entrypoint: http-artifact-example
+ templates:
+ - name: http-artifact-example
+ inputs:
+ artifacts:
+ - name: kubectl
+ path: /bin/kubectl
+ mode: 0755
+ optional: false
+ http:
+ url: ""
+ container:
+ image: debian:9.4
+ command: [sh, -c]
+ args: ["echo NoKubectl"]

test/e2e/expectedfailures/output-artifact-not-optional.yaml

@@ -0,0 +1,24 @@
+# This example demonstrates the output artifacts not optionals
+# from one step to the next.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+ generateName: output-artifact-not-optional-
+spec:
+ entrypoint: artifact-example
+ templates:
+ - name: artifact-example
+ steps:
+ - - name: generate-artifact
+ template: whalesay
+
+ - name: whalesay
+ container:
+ image: docker/whalesay:latest
+ command: [sh, -c]
+ args: ["cowsay hello world | tee /tmp/hello_world12.txt"]
+ outputs:
+ artifacts:
+ - name: hello-art
+ optional: false
+ path: /tmp/hello_world.txt

test/e2e/functional/input-artifact-optional.yaml

@@ -0,0 +1,22 @@
+# This example demonstrates the input artifacts optionals
+# from one step to the next.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+ generateName: input-artifact-optional-
+spec:
+ entrypoint: http-artifact-example
+ templates:
+ - name: http-artifact-example
+ inputs:
+ artifacts:
+ - name: kubectl
+ path: /bin/kubectl
+ mode: 0755
+ optional: true
+ http:
+ url: ""
+ container:
+ image: debian:9.4
+ command: [sh, -c]
+ args: ["echo NoKubectl"]

test/e2e/functional/output-artifact-optional.yaml

@@ -0,0 +1,24 @@
+# This example demonstrates the output artifacts optionals
+# from one step to the next.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+ generateName: output-artifact-optional-
+spec:
+ entrypoint: artifact-example
+ templates:
+ - name: artifact-example
+ steps:
+ - - name: generate-artifact
+ template: whalesay
+
+ - name: whalesay
+ container:
+ image: docker/whalesay:latest
+ command: [sh, -c]
+ args: ["cowsay hello world | tee /tmp/hello_world12.txt"]
+ outputs:
+ artifacts:
+ - name: hello-art
+ optional: true
+ path: /tmp/hello_world.txt

test/e2e/functional/output-input-artifact-optional.yaml

@@ -0,0 +1,40 @@
+# This example demonstrates the output and input artifacts are optionals
+# from one step to the next.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+ generateName: output-input-artifact-optional-
+spec:
+ entrypoint: artifact-example
+ templates:
+ - name: artifact-example
+ steps:
+ - - name: generate-artifact
+ template: whalesay
+ - - name: consume-artifact
+ template: print-message
+ arguments:
+ artifacts:
+ - name: message
+ from: "{{steps.generate-artifact.outputs.artifacts.hello-art}}"
+ - name: whalesay
+ container:
+ image: docker/whalesay:latest
+ command: [sh, -c]
+ args: ["cowsay hello world | tee /tmp/hello_world123.txt"]
+ outputs:
+ artifacts:
+ - name: hello-art
+ optional: true
+ path: /tmp/hello_world.txt
+
+ - name: print-message
+ inputs:
+ artifacts:
+ - name: message
+ path: /tmp/message
+ optional: true
+ container:
+ image: alpine:latest
+ command: [sh, -c]
+ args: ["echo /tmp/message"]

workflow/common/util.go

@@ -148,10 +148,10 @@ func ProcessArgs(tmpl *wfv1.Template, args wfv1.Arguments, globalParams, localPa
  }
  // artifact must be supplied
  argArt := args.GetArtifactByName(inArt.Name)
- if argArt == nil {
+ if !inArt.Optional && argArt == nil {
  return nil, errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s was not supplied", inArt.Name)
  }
- if !argArt.HasLocation() && !validateOnly {
+ if !inArt.Optional && !argArt.HasLocation() && !validateOnly {
  return nil, errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s missing location information", inArt.Name)
  }
  argArt.Path = inArt.Path

workflow/executor/docker/docker.go

@@ -65,7 +65,9 @@ func (d *DockerExecutor) CopyFile(containerID string, sourcePath string, destPat
  return err
  }
  if !file.ExistsInTar(sourcePath, tar.NewReader(gzipReader)) {
- return errors.InternalErrorf("path %s does not exist (or %s is empty) in archive %s", sourcePath, sourcePath, destPath)
+ errMsg := fmt.Sprintf("path %s does not exist (or %s is empty) in archive %s", sourcePath, sourcePath, destPath)
+ log.Warn(errMsg)
+ return errors.Errorf(errors.CodeNotFound, errMsg)
  }
  log.Infof("Archiving completed")
  return nil

workflow/executor/executor.go

@@ -114,7 +114,13 @@ func (we *WorkflowExecutor) LoadArtifacts() error {
  log.Infof("Start loading input artifacts...")
 
  for _, art := range we.Template.Inputs.Artifacts {
+
  log.Infof("Downloading artifact: %s", art.Name)
+
+ if !art.HasLocation() && art.Optional {
+ log.Warnf("Artifact doesn't have location. Artifact configured as an optional so, Artifact will be ignored")
+ continue
+ }
  artDriver, err := we.InitDriver(art)
  if err != nil {
  return err
@@ -232,6 +238,10 @@ func (we *WorkflowExecutor) saveArtifact(tempOutArtDir string, mainCtrID string,
  localArtPath := path.Join(tempOutArtDir, fileName)
  err := we.RuntimeExecutor.CopyFile(mainCtrID, art.Path, localArtPath)
  if err != nil {
+ if art.Optional && errors.IsCode(errors.CodeNotFound, err) {
+ log.Warnf("Error in saving Artifact. Artifact configured as an optional so, Error will be ignored. Error= %v", err)
+ return nil
+ }
  return err
  }
  fileName, localArtPath, err = stageArchiveFile(fileName, localArtPath, art)
@@ -502,6 +512,7 @@ func (we *WorkflowExecutor) InitDriver(art wfv1.Artifact) (artifact.ArtifactDriv
  if art.Raw != nil {
  return &raw.RawArtifactDriver{}, nil
  }
+
  return nil, errors.Errorf(errors.CodeBadRequest, "Unsupported artifact driver for %s", art.Name)
 }