[CSE-13] extended error handling for workflows (#16)

* wip * mixed case imports cause all sorts of problems, switch to lowercase * fixing build issu * fixing error deserialization * fixing error deserialization * unmatched string logic * make workflows fail on error trigger * properly evaluate workflow failures * dev version bump * serialize errors and warnings into wf crd * ugh go types * rewriting error handling to support file sources * temporarily commenting out a test * fixing warning handler * fixing error handling fixing executor fixing executor Fixing executor fixing executor fixing executor fixing executor asdf fixing executor fixing operator operator fixing executor cleanup * cleaning up types * updating codegen * fixing version * updating codegen * add podname and stage name to error result * version 2.5.0->2.4.0 * ErrorCondition->ExceptionCondition * codegen
CyrusBiotechnology · Mar 15, 2019 · b9715f2 · b9715f2
1 parent 2a68399
commit b9715f2
Show file tree

Hide file tree

Showing 102 changed files with 858 additions and 333 deletions.
diff --git a/.argo-ci/ci.yaml b/.argo-ci/ci.yaml
@@ -9,7 +9,7 @@ spec:
  - name: revision
  value: master
  - name: repo
- value: https://github.com/CyrusBiotechnology/argo.git
+ value: https://github.com/cyrusbiotechnology/argo.git
 
  templates:
  - name: argo-ci
@@ -39,31 +39,31 @@ spec:
  - name: cmd
  artifacts:
  - name: code
- path: /go/src/github.com/CyrusBiotechnology/argo
+ path: /go/src/github.com/cyrusbiotechnology/argo
  git:
  repo: "{{workflow.parameters.repo}}"
  revision: "{{workflow.parameters.revision}}"
  container:
  image: argoproj/argo-ci-builder:latest
  command: [sh, -c]
  args: ["{{inputs.parameters.cmd}}"]
- workingDir: /go/src/github.com/CyrusBiotechnology/argo
+ workingDir: /go/src/github.com/cyrusbiotechnology/argo
 
  - name: ci-dind
  inputs:
  parameters:
  - name: cmd
  artifacts:
  - name: code
- path: /go/src/github.com/CyrusBiotechnology/argo
+ path: /go/src/github.com/cyrusbiotechnology/argo
  git:
  repo: "{{workflow.parameters.repo}}"
  revision: "{{workflow.parameters.revision}}"
  container:
  image: argoproj/argo-ci-builder:latest
  command: [sh, -c]
  args: ["until docker ps; do sleep 3; done && {{inputs.parameters.cmd}}"]
- workingDir: /go/src/github.com/CyrusBiotechnology/argo
+ workingDir: /go/src/github.com/cyrusbiotechnology/argo
  env:
  - name: DOCKER_HOST
  value: 127.0.0.1

diff --git a/Branding Assets.md b/Branding Assets.md
@@ -1,3 +1,3 @@
 # Argo Branding Assets
 ## Logo
-![Argo Logo](https://github.com/CyrusBiotechnology/argo/blob/master/argo-logo600.png "Argo Logo")
+![Argo Logo](https://github.com/cyrusbiotechnology/argo/blob/master/argo-logo600.png "Argo Logo")
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -29,8 +29,8 @@ Go to https://groups.google.com/forum/#!forum/argoproj
 
 ### Quickstart
 ```
-$ go get github.com/CyrusBiotechnology/argo
-$ cd $(go env GOPATH)/src/github.com/CyrusBiotechnology/argo
+$ go get github.com/cyrusbiotechnology/argo
+$ cd $(go env GOPATH)/src/github.com/cyrusbiotechnology/argo
 $ dep ensure -vendor-only
 $ make
 ```

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -11,8 +11,8 @@ def runUtilityCommand(buildCommand) {
  // Run an arbitrary command inside the docker builder image
  sh "docker run --rm " +
  "-v ${pwd()}/dist/pkg:/root/go/pkg " +
- "-v ${pwd()}:/root/go/src/github.com/CyrusBiotechnology/argo " +
- "-w /root/go/src/github.com/CyrusBiotechnology/argo argo-builder ${buildCommand}"
+ "-v ${pwd()}:/root/go/src/github.com/cyrusbiotechnology/argo " +
+ "-w /root/go/src/github.com/cyrusbiotechnology/argo argo-builder ${buildCommand}"
 }
 
 pipeline {
@@ -85,8 +85,8 @@ pipeline {
  stage('push CLI to artifactory') {
  steps {
  withCredentials([usernamePassword(credentialsId: 'Artifactory', usernameVariable: 'ARTI_NAME', passwordVariable: 'ARTI_PASS')]) {
- runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /root/go/src/github.com/CyrusBiotechnology/argo/dist/argo-darwin-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-mac-${VERSION}")
- runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /root/go/src/github.com/CyrusBiotechnology/argo/dist/argo-linux-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-linux-${VERSION}")
+ runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /root/go/src/github.com/cyrusbiotechnology/argo/dist/argo-darwin-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-mac-${VERSION}")
+ runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /root/go/src/github.com/cyrusbiotechnology/argo/dist/argo-linux-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-linux-${VERSION}")
  }
  }
  }

diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-PACKAGE=github.com/CyrusBiotechnology/argo
+PACKAGE=github.com/cyrusbiotechnology/argo
 CURRENT_DIR=$(shell pwd)
 DIST_DIR=${CURRENT_DIR}/dist
 ARGO_CLI_NAME=argo

diff --git a/README.md b/README.md
@@ -22,9 +22,9 @@ Argo is an open source container-native workflow engine for getting work done on
 * Argo with Kubernetes puts a cloud-scale supercomputer at your fingertips.
 
 ## Documentation
-* [Get started here](https://github.com/CyrusBiotechnology/argo/blob/master/demo.md)
-* [How to write Argo workflow specs](https://github.com/CyrusBiotechnology/argo/blob/master/examples/README.md)
-* [How to configure your artifact repository](https://github.com/CyrusBiotechnology/argo/blob/master/ARTIFACT_REPO.md)
+* [Get started here](https://github.com/cyrusbiotechnology/argo/blob/master/demo.md)
+* [How to write Argo workflow specs](https://github.com/cyrusbiotechnology/argo/blob/master/examples/README.md)
+* [How to configure your artifact repository](https://github.com/cyrusbiotechnology/argo/blob/master/ARTIFACT_REPO.md)
 
 ## Features
 * DAG or Steps based declaration of workflows

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.3.0
+2.4.0
diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -2,7 +2,7 @@
  "swagger": "2.0",
  "info": {
  "title": "Argo",
- "version": "v2.2.1"
+ "version": "v2.4.0"
  },
  "paths": {},
  "definitions": {
@@ -229,6 +229,52 @@
  }
  }
  },
+ "io.argoproj.workflow.v1alpha1.ExceptionCondition": {
+ "description": "ExceptionCondition is a container for defining an error or warning rule",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "message": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "patternMatched": {
+ "type": "string"
+ },
+ "patternUnmatched": {
+ "type": "string"
+ },
+ "source": {
+ "type": "string"
+ }
+ }
+ },
+ "io.argoproj.workflow.v1alpha1.ExceptionResult": {
+ "description": "ExceptionResult contains the results on an extended error or warning condition evaluation",
+ "required": [
+ "name",
+ "message",
+ "podName",
+ "stepName"
+ ],
+ "properties": {
+ "message": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "podName": {
+ "type": "string"
+ },
+ "stepName": {
+ "type": "string"
+ }
+ }
+ },
  "io.argoproj.workflow.v1alpha1.GCSArtifact": {
  "description": "GCSArtifact is the location of a GCS artifact",
  "required": [
@@ -821,6 +867,12 @@
  "description": "DAG template subtype which runs a DAG",
  "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.DAGTemplate"
  },
+ "errors": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ExceptionCondition"
+ }
+ },
  "inputs": {
  "description": "Inputs describe what inputs parameters and artifacts are supplied to this template",
  "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Inputs"
@@ -888,6 +940,12 @@
  "items": {
  "$ref": "#/definitions/io.k8s.api.core.v1.Toleration"
  }
+ },
+ "warnings": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ExceptionCondition"
+ }
  }
  }
  },

diff --git a/cmd/argo/commands/common.go b/cmd/argo/commands/common.go
@@ -7,9 +7,9 @@ import (
  "strconv"
  "strings"
 
- wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
- wfclientset "github.com/CyrusBiotechnology/argo/pkg/client/clientset/versioned"
- "github.com/CyrusBiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
+ wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
+ wfclientset "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned"
+ "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
  "github.com/spf13/cobra"
  "k8s.io/client-go/kubernetes"
  "k8s.io/client-go/rest"

diff --git a/cmd/argo/commands/delete.go b/cmd/argo/commands/delete.go
@@ -10,7 +10,7 @@ import (
  "github.com/spf13/cobra"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
- "github.com/CyrusBiotechnology/argo/workflow/common"
+ "github.com/cyrusbiotechnology/argo/workflow/common"
 )
 
 var (

diff --git a/cmd/argo/commands/get.go b/cmd/argo/commands/get.go
@@ -13,7 +13,7 @@ import (
  "github.com/spf13/cobra"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
- wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
+ wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
 )
 
 const onExitSuffix = "onExit"

diff --git a/cmd/argo/commands/lint.go b/cmd/argo/commands/lint.go
@@ -7,8 +7,8 @@ import (
  log "github.com/sirupsen/logrus"
  "github.com/spf13/cobra"
 
- cmdutil "github.com/CyrusBiotechnology/argo/util/cmd"
- "github.com/CyrusBiotechnology/argo/workflow/validate"
+ cmdutil "github.com/cyrusbiotechnology/argo/util/cmd"
+ "github.com/cyrusbiotechnology/argo/workflow/validate"
 )
 
 func NewLintCommand() *cobra.Command {

diff --git a/cmd/argo/commands/list.go b/cmd/argo/commands/list.go
@@ -17,10 +17,10 @@ import (
  "k8s.io/apimachinery/pkg/labels"
  "k8s.io/apimachinery/pkg/selection"
 
- wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
- "github.com/CyrusBiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
- "github.com/CyrusBiotechnology/argo/workflow/common"
- "github.com/CyrusBiotechnology/argo/workflow/util"
+ wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
+ "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
+ "github.com/cyrusbiotechnology/argo/workflow/common"
+ "github.com/cyrusbiotechnology/argo/workflow/util"
 )
 
 type listFlags struct {

diff --git a/cmd/argo/commands/logs.go b/cmd/argo/commands/logs.go
@@ -11,10 +11,10 @@ import (
  "sync"
  "time"
 
- "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
- wfclientset "github.com/CyrusBiotechnology/argo/pkg/client/clientset/versioned"
- wfinformers "github.com/CyrusBiotechnology/argo/pkg/client/informers/externalversions"
  "github.com/argoproj/pkg/errors"
+ "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
+ wfclientset "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned"
+ wfinformers "github.com/cyrusbiotechnology/argo/pkg/client/informers/externalversions"
  log "github.com/sirupsen/logrus"
  "github.com/spf13/cobra"
  "k8s.io/api/core/v1"

diff --git a/cmd/argo/commands/resubmit.go b/cmd/argo/commands/resubmit.go
@@ -3,8 +3,8 @@ package commands
 import (
  "os"
 
- "github.com/CyrusBiotechnology/argo/workflow/util"
  "github.com/argoproj/pkg/errors"
+ "github.com/cyrusbiotechnology/argo/workflow/util"
  "github.com/spf13/cobra"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )

diff --git a/cmd/argo/commands/resume.go b/cmd/argo/commands/resume.go
@@ -5,7 +5,7 @@ import (
  "log"
  "os"
 
- "github.com/CyrusBiotechnology/argo/workflow/util"
+ "github.com/cyrusbiotechnology/argo/workflow/util"
  "github.com/spf13/cobra"
 )
 

diff --git a/cmd/argo/commands/retry.go b/cmd/argo/commands/retry.go
@@ -7,7 +7,7 @@ import (
  "github.com/spf13/cobra"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
- "github.com/CyrusBiotechnology/argo/workflow/util"
+ "github.com/cyrusbiotechnology/argo/workflow/util"
 )
 
 func NewRetryCommand() *cobra.Command {

diff --git a/cmd/argo/commands/root.go b/cmd/argo/commands/root.go
@@ -3,7 +3,7 @@ package commands
 import (
  "os"
 
- "github.com/CyrusBiotechnology/argo/util/cmd"
+ "github.com/cyrusbiotechnology/argo/util/cmd"
  "github.com/spf13/cobra"
  "k8s.io/client-go/tools/clientcmd"
 )

diff --git a/cmd/argo/commands/submit.go b/cmd/argo/commands/submit.go
@@ -10,10 +10,10 @@ import (
  "github.com/argoproj/pkg/json"
  "github.com/spf13/cobra"
 
- wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
- cmdutil "github.com/CyrusBiotechnology/argo/util/cmd"
- "github.com/CyrusBiotechnology/argo/workflow/common"
- "github.com/CyrusBiotechnology/argo/workflow/util"
+ wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
+ cmdutil "github.com/cyrusbiotechnology/argo/util/cmd"
+ "github.com/cyrusbiotechnology/argo/workflow/common"
+ "github.com/cyrusbiotechnology/argo/workflow/util"
 )
 
 // cliSubmitOpts holds submition options specific to CLI submission (e.g. controlling output)

diff --git a/cmd/argo/commands/suspend.go b/cmd/argo/commands/suspend.go
@@ -5,7 +5,7 @@ import (
  "log"
  "os"
 
- "github.com/CyrusBiotechnology/argo/workflow/util"
+ "github.com/cyrusbiotechnology/argo/workflow/util"
  "github.com/spf13/cobra"
 )
 

diff --git a/cmd/argo/commands/terminate.go b/cmd/argo/commands/terminate.go
@@ -7,7 +7,7 @@ import (
  "github.com/argoproj/pkg/errors"
  "github.com/spf13/cobra"
 
- "github.com/CyrusBiotechnology/argo/workflow/util"
+ "github.com/cyrusbiotechnology/argo/workflow/util"
 )
 
 func NewTerminateCommand() *cobra.Command {

diff --git a/cmd/argo/commands/wait.go b/cmd/argo/commands/wait.go
@@ -5,8 +5,8 @@ import (
  "os"
  "sync"
 
- wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
  "github.com/argoproj/pkg/errors"
+ wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
  "github.com/spf13/cobra"
  apierr "k8s.io/apimachinery/pkg/api/errors"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

diff --git a/cmd/argo/commands/watch.go b/cmd/argo/commands/watch.go
@@ -10,7 +10,7 @@ import (
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  "k8s.io/apimachinery/pkg/fields"
 
- wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
+ wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
 )
 
 func NewWatchCommand() *cobra.Command {

diff --git a/cmd/argo/main.go b/cmd/argo/main.go
@@ -4,7 +4,7 @@ import (
  "fmt"
  "os"
 
- "github.com/CyrusBiotechnology/argo/cmd/argo/commands"
+ "github.com/cyrusbiotechnology/argo/cmd/argo/commands"
  // load the gcp plugin (required to authenticate against GKE clusters).
  _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
  // load the oidc plugin (required to authenticate with OpenID Connect).

diff --git a/cmd/argoexec/commands/resource.go b/cmd/argoexec/commands/resource.go
@@ -3,7 +3,7 @@ package commands
 import (
  "os"
 
- "github.com/CyrusBiotechnology/argo/workflow/common"
+ "github.com/cyrusbiotechnology/argo/workflow/common"
  log "github.com/sirupsen/logrus"
  "github.com/spf13/cobra"
 )

diff --git a/cmd/argoexec/commands/root.go b/cmd/argoexec/commands/root.go
@@ -10,12 +10,12 @@ import (
  "k8s.io/client-go/kubernetes"
  "k8s.io/client-go/tools/clientcmd"
 
- "github.com/CyrusBiotechnology/argo"
- "github.com/CyrusBiotechnology/argo/util/cmd"
- "github.com/CyrusBiotechnology/argo/workflow/common"
- "github.com/CyrusBiotechnology/argo/workflow/executor"
- "github.com/CyrusBiotechnology/argo/workflow/executor/docker"
- "github.com/CyrusBiotechnology/argo/workflow/executor/kubelet"
+ "github.com/cyrusbiotechnology/argo"
+ "github.com/cyrusbiotechnology/argo/util/cmd"
+ "github.com/cyrusbiotechnology/argo/workflow/common"
+ "github.com/cyrusbiotechnology/argo/workflow/executor"
+ "github.com/cyrusbiotechnology/argo/workflow/executor/docker"
+ "github.com/cyrusbiotechnology/argo/workflow/executor/kubelet"
 )
 
 const (