From f18d2b8a275e263a77a139859a7a33b51d4e5f94 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Tue, 17 Nov 2020 11:38:17 -0800 Subject: [PATCH 01/20] fix(controller): Correct default port logic (#4547) Signed-off-by: Alex Collins Signed-off-by: Paul Brabban --- workflow/controller/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/controller/controller.go b/workflow/controller/controller.go index d947c8577494..e8a820d3cead 100644 --- a/workflow/controller/controller.go +++ b/workflow/controller/controller.go @@ -938,7 +938,7 @@ func (wfc *WorkflowController) getMetricsServerConfig() (metrics.ServerConfig, m path = metrics.DefaultMetricsServerPath } port := wfc.Config.MetricsConfig.Port - if port > 0 { + if port == 0 { port = metrics.DefaultMetricsServerPort } metricsConfig := metrics.ServerConfig{ From 5a76dc04c01cbec7109c6fb14583180189ef6528 Mon Sep 17 00:00:00 2001 From: Zach Date: Tue, 17 Nov 2020 17:01:39 -0500 Subject: [PATCH 02/20] docs: Update cost optimisation to include information about cleaning up workflows and archiving (#4549) Signed-off-by: Zach Brown Signed-off-by: Paul Brabban --- docs/cost-optimisation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cost-optimisation.md b/docs/cost-optimisation.md index 2f5cbe38b23e..6b9b402add3a 100644 --- a/docs/cost-optimisation.md +++ b/docs/cost-optimisation.md @@ -39,9 +39,11 @@ Consider: > Suitable for all. -A workflow (and for that matter, any Kubernetes resource) will incur a cost as long as they exist in your cluster. +A workflow (and for that matter, any Kubernetes resource) will incur a cost as long as they exist in your cluster, even after they are no longer running. -The workflow controller memory and CPU needs increase linearly with the number of pods and workflows you are currently running. +The workflow controller memory and CPU needs increase linearly with the number of pods and workflows you are currently running. + +You should delete workflows once they are no longer needed, or enable a [Workflow Archive](workflow-archive.md) and you can still view them after they are removed from Kubernetes. Limit the total number of workflows using: @@ -65,8 +67,6 @@ spec: You can set these configurations globally using [Default Workflow Spec](default-workflow-specs.md). -If you need to keep records historically, use the [Workflow Archive](workflow-archive.md). - Changing these settings will not delete workflows that have already run. To list old workflows: ``` From a2335eae705da139491ea2b40f00c629a33a2817 Mon Sep 17 00:00:00 2001 From: Lennart Kindermann Date: Wed, 18 Nov 2020 16:45:21 +0100 Subject: [PATCH 03/20] docs: Added CloudSeeds as one of the users for argo (#4553) Signed-off-by: Lennart Kindermann Signed-off-by: Paul Brabban --- USERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/USERS.md b/USERS.md index 980a420d247f..8d785ec6ab7c 100644 --- a/USERS.md +++ b/USERS.md @@ -24,6 +24,7 @@ Currently, the following organizations are **officially** using Argo Workflows: 1. [Capital One](https://www.capitalone.com/tech/) 1. [CarTrack](https://www.cartrack.com/) 1. [CCRi](https://www.ccri.com/) +1. [CloudSeeds](https://www.cloudseeds.de/) 1. [Codec](https://www.codec.ai/) 1. [Commodus Tech](https://www.commodus.tech) 1. [Concierge Render](https://www.conciergerender.com) From 325d5f8c191a1e253a06de730eafb6b2423b75be Mon Sep 17 00:00:00 2001 From: Simon Behar Date: Wed, 18 Nov 2020 12:01:51 -0600 Subject: [PATCH 04/20] fix: Respect continueOn for leaf tasks (#4455) Signed-off-by: Simon Behar Signed-off-by: Paul Brabban --- workflow/controller/dag.go | 9 +++ workflow/controller/dag_test.go | 124 ++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/workflow/controller/dag.go b/workflow/controller/dag.go index e98f0a455d6f..f7090e46da71 100644 --- a/workflow/controller/dag.go +++ b/workflow/controller/dag.go @@ -181,6 +181,15 @@ func (d *dagContext) assessDAGPhase(targetTasks []string, nodes wfv1.Nodes) wfv1 break } } else if branchPhase.FailedOrError() { + // If this target task has continueOn set for its current phase, then don't treat it as failed for the purposes + // of determining DAG status. This is so that target tasks with said continueOn do not fail the overall DAG. + // For non-leaf tasks, this is done by setting all of its dependents to allow for their failure or error in + // their "depends" clause during their respective "dependencies" to "depends" conversion. See "expandDependency" + // in ancestry.go + if task := d.GetTask(depName); task.ContinuesOn(branchPhase) { + continue + } + result = branchPhase // If failFast is enabled, don't check to see if other target tasks are complete and fail now instead if failFast { diff --git a/workflow/controller/dag_test.go b/workflow/controller/dag_test.go index 00efd6b38efe..683b2a9892be 100644 --- a/workflow/controller/dag_test.go +++ b/workflow/controller/dag_test.go @@ -2861,3 +2861,127 @@ func TestFailsWithParamDAG(t *testing.T) { woc.operate() assert.Equal(t, wfv1.NodeFailed, woc.wf.Status.Phase) } + +var testLeafContinueOn = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + name: build-wf-kpxvm +spec: + arguments: {} + entrypoint: test-workflow + templates: + - arguments: {} + dag: + tasks: + - arguments: {} + name: A + template: ok + - arguments: {} + continueOn: + failed: true + dependencies: + - A + name: B + template: fail + inputs: {} + metadata: {} + name: test-workflow + outputs: {} + - arguments: {} + container: + args: + - | + exit 0 + command: + - sh + - -c + image: busybox + name: "" + resources: {} + inputs: {} + metadata: {} + name: ok + outputs: {} + - arguments: {} + container: + args: + - | + exit 1 + command: + - sh + - -c + image: busybox + name: "" + resources: {} + inputs: {} + metadata: {} + name: fail + outputs: {} +status: + finishedAt: "2020-11-04T16:17:59Z" + nodes: + build-wf-kpxvm: + children: + - build-wf-kpxvm-2225940411 + displayName: build-wf-kpxvm + finishedAt: "2020-11-04T16:17:59Z" + id: build-wf-kpxvm + name: build-wf-kpxvm + outboundNodes: + - build-wf-kpxvm-2242718030 + phase: Running + progress: 3/3 + resourcesDuration: + cpu: 13 + memory: 6 + startedAt: "2020-11-04T16:17:43Z" + templateName: test-workflow + templateScope: local/build-wf-kpxvm + type: DAG + build-wf-kpxvm-2225940411: + boundaryID: build-wf-kpxvm + children: + - build-wf-kpxvm-2242718030 + displayName: A + finishedAt: "2020-11-04T16:17:51Z" + hostNodeName: minikube + id: build-wf-kpxvm-2225940411 + name: build-wf-kpxvm.A + phase: Succeeded + startedAt: "2020-11-04T16:17:43Z" + templateName: ok + templateScope: local/build-wf-kpxvm + type: Pod + build-wf-kpxvm-2242718030: + boundaryID: build-wf-kpxvm + displayName: B + finishedAt: "2020-11-04T16:17:57Z" + hostNodeName: minikube + id: build-wf-kpxvm-2242718030 + message: failed with exit code 1 + name: build-wf-kpxvm.B + phase: Failed + startedAt: "2020-11-04T16:17:53Z" + templateName: fail + templateScope: local/build-wf-kpxvm + type: Pod + phase: Running + progress: 3/3 + resourcesDuration: + cpu: 13 + memory: 6 + startedAt: "2020-11-04T16:17:43Z" + +` + +func TestLeafContinueOn(t *testing.T) { + wf := unmarshalWF(testLeafContinueOn) + cancel, controller := newController(wf) + defer cancel() + + woc := newWorkflowOperationCtx(wf, controller) + + woc.operate() + assert.Equal(t, wfv1.NodeSucceeded, woc.wf.Status.Phase) +} From 93da1a0b9bf74b7d59b12e5012b9858d57fd07af Mon Sep 17 00:00:00 2001 From: Lennart Kindermann Date: Thu, 19 Nov 2020 18:13:51 +0100 Subject: [PATCH 05/20] fix: executor/pns containerid prefix fix (#4555) Signed-off-by: Lennart Kindermann Signed-off-by: Paul Brabban --- workflow/executor/pns/pns.go | 12 ++++++++---- workflow/executor/pns/pns_test.go | 6 +++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/workflow/executor/pns/pns.go b/workflow/executor/pns/pns.go index 7feeca49347d..1891e969b9e6 100644 --- a/workflow/executor/pns/pns.go +++ b/workflow/executor/pns/pns.go @@ -418,9 +418,8 @@ func parseContainerIDFromCgroupLine(line string) string { if containerID := parts[len(parts)-1]; containerID != "" { // need to check for empty string because the line may look like: 5:rdma:/ - // for crio we need to get rid of "crio-" prefix and ".scope" suffix - // e.g. crio-7a92a067289f6197148912be1c15f20f0330c7f3c541473d3b9c4043ca137b42.scope - containerID := strings.TrimSuffix(strings.TrimPrefix(containerID, "crio-"), ".scope") + // remove possible ".scope" suffix + containerID := strings.TrimSuffix(containerID, ".scope") // for compatibility with cri-containerd record format when using systemd cgroup path // example record in /proc/{pid}/cgroup: @@ -428,8 +427,13 @@ func parseContainerIDFromCgroupLine(line string) string { if strings.Contains(containerID, "cri-containerd") { strList := strings.Split(containerID, ":") containerID = strList[len(strList)-1] - containerID = strings.TrimPrefix(containerID, "cri-containerd-") } + + // remove possible "*-" prefix + // e.g. crio-7a92a067289f6197148912be1c15f20f0330c7f3c541473d3b9c4043ca137b42.scope + parts := strings.Split(containerID, "-") + containerID = parts[len(parts)-1] + return containerID } } diff --git a/workflow/executor/pns/pns_test.go b/workflow/executor/pns/pns_test.go index 4c9aa37e18a7..b04e1540c2b1 100644 --- a/workflow/executor/pns/pns_test.go +++ b/workflow/executor/pns/pns_test.go @@ -31,10 +31,14 @@ func TestPNSExecutor_parseContainerIDFromCgroupLine(t *testing.T) { line: "8:cpu,cpuacct:/kubepods/besteffort/pod2fad8aad-dcd0-4fef-b45a-151630b9a4b5/crio-7a92a067289f6197148912be1c15f20f0330c7f3c541473d3b9c4043ca137b42.scope", expected: "7a92a067289f6197148912be1c15f20f0330c7f3c541473d3b9c4043ca137b42", }, + { + line: "2:cpuacct,cpu:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod1cd87fe8_8ea0_11ea_8d51_566f300c000a.slice/docker-6b40fc7f75fe3210621a287412ac056e43554b1026a01625b48ba7d136d8a125.scope", + expected: "6b40fc7f75fe3210621a287412ac056e43554b1026a01625b48ba7d136d8a125", + }, } for _, testCase := range testCases { containerID := parseContainerIDFromCgroupLine(testCase.line) - assert.Equal(t, containerID, testCase.expected) + assert.Equal(t, testCase.expected, containerID) } } From 337047b9b74a7ed074893de3836af3cb03bcac5d Mon Sep 17 00:00:00 2001 From: Yuan Tang Date: Thu, 19 Nov 2020 12:53:57 -0500 Subject: [PATCH 06/20] chore: Remove unused image build and push hooks (#4539) Signed-off-by: terrytangyuan Signed-off-by: Paul Brabban --- hooks/README.md | 16 ---------------- hooks/build | 7 ------- hooks/push | 7 ------- 3 files changed, 30 deletions(-) delete mode 100755 hooks/README.md delete mode 100755 hooks/build delete mode 100755 hooks/push diff --git a/hooks/README.md b/hooks/README.md deleted file mode 100755 index f9206a550b82..000000000000 --- a/hooks/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Docker Hub Automated Build Hooks - -This directory contains Docker Hub [Automated Build](https://docs.docker.com/docker-hub/builds/advanced/) hooks. -This is needed since we publish multiple images as part of a single build: -* argoproj/workflow-controller:latest -* argoproj/argoexec:latest -* argoproj/argocli:latest - -It relies the DOCKER_REPO and DOCKER_TAG environment variables that are set by Docker Hub during -the build. - -Hooks can be tested using: -``` -DOCKER_REPO=index.docker.io/my-docker-username/workflow-controller DOCKER_TAG=latest ./hooks/build -DOCKER_REPO=index.docker.io/my-docker-username/workflow-controller DOCKER_TAG=latest ./hooks/push -``` diff --git a/hooks/build b/hooks/build deleted file mode 100755 index 3fa6e5ef98d5..000000000000 --- a/hooks/build +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -e -docker_org=$(echo $DOCKER_REPO | rev | cut -d / -f 2- | rev) -targets="workflow-controller argoexec argocli" -for target in $targets; do - image_name="${docker_org}/${target}:${DOCKER_TAG}" - docker build --target $target -t $image_name . -done diff --git a/hooks/push b/hooks/push deleted file mode 100755 index b5c624c37080..000000000000 --- a/hooks/push +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -e -docker_org=$(echo $DOCKER_REPO | rev | cut -d / -f 2- | rev) -targets="workflow-controller argoexec argocli" -for target in $targets; do - image_name="${docker_org}/${target}:${DOCKER_TAG}" - docker push $image_name -done From 8dc491b94b98f2f9690eb0df9762c02e623464fa Mon Sep 17 00:00:00 2001 From: Espen Finnesand <38684618+secretlifeof@users.noreply.github.com> Date: Thu, 19 Nov 2020 19:20:05 +0100 Subject: [PATCH 07/20] docs: Clean-up examples. Fixes #4124 (#4128) Signed-off-by: github@finnesand.no Signed-off-by: Paul Brabban --- examples/README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/README.md b/examples/README.md index a8fd7f0f1066..9cf11797f7d8 100644 --- a/examples/README.md +++ b/examples/README.md @@ -46,8 +46,7 @@ In case you want to follow along with this walkthrough, here's a quick overview argo submit hello-world.yaml # submit a workflow spec to Kubernetes argo list # list current workflows argo get hello-world-xxx # get info about a specific workflow -argo logs -w hello-world-xxx # get logs from all steps in a workflow -argo logs hello-world-xxx-yyy # get logs from a specific step in a workflow +argo logs hello-world-xxx # print the logs from a workflow argo delete hello-world-xxx # delete workflow ``` @@ -66,7 +65,7 @@ kubectl delete wf hello-world-xxx Let's start by creating a very simple workflow template to echo "hello world" using the docker/whalesay container image from DockerHub. -You can run this directly from your shell with a simple docker command: +You can run this directly from your shell with a simple docker command: ```sh $ docker run docker/whalesay cowsay "hello world" @@ -1521,4 +1520,4 @@ spec: Continuous integration is a popular application for workflows. Currently, Argo does not provide event triggers for automatically kicking off your CI jobs, but we plan to do so in the near future. Until then, you can easily write a cron job that checks for new commits and kicks off the needed workflow, or use your existing Jenkins server to kick off the workflow. -A good example of a CI workflow spec is provided at https://github.com/argoproj/argo/tree/master/examples/influxdb-ci.yaml. Because it just uses the concepts that we've already covered and is somewhat long, we don't go into details here. +A good example of a CI workflow spec is provided at https://github.com/argoproj/argo/tree/master/examples/influxdb-ci.yaml. Because it just uses the concepts that we've already covered and is somewhat long, we don't go into details here. \ No newline at end of file From b48996ab04df941b4c74d7203cbc40d4081548c2 Mon Sep 17 00:00:00 2001 From: tczhao Date: Fri, 20 Nov 2020 05:25:34 +1100 Subject: [PATCH 08/20] feat(ui): Add Template/Cron workflow filter to workflow page. Closes #4532 (#4543) Signed-off-by: Tianchu Zhao Signed-off-by: Paul Brabban --- USERS.md | 1 + ui/src/app/reports/components/reports.tsx | 35 ++++++------- .../components/data-loader-dropdown.tsx | 21 ++++++++ .../workflow-filters/workflow-filters.tsx | 50 ++++++++++++++++++- 4 files changed, 85 insertions(+), 22 deletions(-) create mode 100644 ui/src/app/shared/components/data-loader-dropdown.tsx diff --git a/USERS.md b/USERS.md index 8d785ec6ab7c..b8f4e2567a95 100644 --- a/USERS.md +++ b/USERS.md @@ -104,3 +104,4 @@ Currently, the following organizations are **officially** using Argo Workflows: 1. [UFirstGroup](https://www.ufirstgroup.com) 1. [Wavefront](https://www.wavefront.com/) 1. [Wellcome Trust](https://wellcome.ac.uk/) +1. [Woolworths Group](https://www.woolworthsgroup.com.au/) diff --git a/ui/src/app/reports/components/reports.tsx b/ui/src/app/reports/components/reports.tsx index 70ef43b99899..af431a774d91 100644 --- a/ui/src/app/reports/components/reports.tsx +++ b/ui/src/app/reports/components/reports.tsx @@ -1,4 +1,4 @@ -import {Checkbox, DataLoader, Page, Select} from 'argo-ui/src/index'; +import {Checkbox, Page} from 'argo-ui/src/index'; import {ChartOptions} from 'chart.js'; import 'chartjs-plugin-annotation'; import * as React from 'react'; @@ -7,8 +7,9 @@ import {RouteComponentProps} from 'react-router-dom'; import {getColorForNodePhase, NODE_PHASE, Workflow} from '../../../models'; import {uiUrl} from '../../shared/base'; import {BasePage} from '../../shared/components/base-page'; +import {DataLoaderDropdown} from '../../shared/components/data-loader-dropdown'; import {ErrorNotice} from '../../shared/components/error-notice'; -import {InputFilter} from '../../shared/components/input-filter'; +import {NamespaceFilter} from '../../shared/components/namespace-filter'; import {TagsInput} from '../../shared/components/tags-input/tags-input'; import {ZeroState} from '../../shared/components/zero-state'; import {Consumer, ContextApis} from '../../shared/context'; @@ -43,18 +44,10 @@ export class Reports extends BasePage, State> { this.setLabel(labelKeyPhase, value); } - private get workflowTemplate() { - return this.getLabel(labelKeyWorkflowTemplate); - } - private set workflowTemplate(value: string) { this.setLabel(labelKeyWorkflowTemplate, value); } - private get cronWorkflow() { - return this.getLabel(labelKeyCronWorkflow); - } - private set cronWorkflow(value: string) { this.setLabel(labelKeyCronWorkflow, value); } @@ -257,11 +250,11 @@ export class Reports extends BasePage, State> {

Namespace

- this.fetchReport(namespace, this.state.labels, this.state.archivedWorkflows)} + onChange={namespace => { + this.fetchReport(namespace, this.state.labels, this.state.archivedWorkflows); + }} />
@@ -274,15 +267,17 @@ export class Reports extends BasePage, State> {

Workflow Template

- services.workflowTemplate.list(this.state.namespace).then(list => list.map(x => x.metadata.name))}> - {list => (this.cronWorkflow = x.value)} />} - + list.map(x => x.metadata.name))} + onChange={value => (this.cronWorkflow = value)} + />

Phase

diff --git a/ui/src/app/shared/components/data-loader-dropdown.tsx b/ui/src/app/shared/components/data-loader-dropdown.tsx new file mode 100644 index 000000000000..fb1aa5b0e8cf --- /dev/null +++ b/ui/src/app/shared/components/data-loader-dropdown.tsx @@ -0,0 +1,21 @@ +import {DataLoader, Select, SelectOption} from 'argo-ui'; +import * as React from 'react'; + +export const DataLoaderDropdown = (props: {load: Promise<(string | SelectOption)[]>; onChange: (value: string) => void}) => { + const [selected, setSelected] = React.useState(''); + + return ( + props.load}> + {list => ( +