From f05d00afc8c06e4981258f1cdf1ae737b52b86f5 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Tue, 29 Sep 2020 22:06:30 -0700 Subject: [PATCH 01/10] docs: Add map-reduce example --- examples/map-reduce-workflow.yaml | 138 ++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 examples/map-reduce-workflow.yaml diff --git a/examples/map-reduce-workflow.yaml b/examples/map-reduce-workflow.yaml new file mode 100644 index 000000000000..473d113cbee3 --- /dev/null +++ b/examples/map-reduce-workflow.yaml @@ -0,0 +1,138 @@ +# This workflow demonstrates a basic map-reduce. +# This requries you have a artifact repository configured. +# +# Notes: +# - You'll need to have an user namespaced artifact repository set-up to save intermediate results for this workflow. +# - Only a single reducer. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: map-reduce- +spec: + entrypoint: main + arguments: + parameters: + - name: num_parts + value: "1" + templates: + - name: main + dag: + tasks: + - name: split + template: split + arguments: + parameters: + - name: num_parts + value: "{{workflow.parameters.num_parts}}" + - name: map + template: map + arguments: + parameters: + - name: partId + value: '{{item}}' + artifacts: + - name: parts + from: '{{tasks.split.outputs.artifacts.parts}}' + dependencies: + - split + withParam: '{{tasks.split.outputs.result}}' + - name: reduce + template: reduce + dependencies: + - map + # The `split` task creates a number of "parts". Each part has a unique ID (e.g. part-0, part-1). + # This task writes the part IDs to stdout (so that the `map` task can be expanded to have one task per part). + # And, it writes one "part file" for each of pieces of processing that needs doing, into to single directory + # which is then saved a output artifact. + - name: split + inputs: + parameters: + - name: num_parts + script: + image: 'docker/whalesay:latest' + command: + - bash + - -eu + source: | + mkdir -p /tmp/parts + echo '[' + for i in $(seq 1 {{inputs.parameters.num_parts}}); do + if [ $i -gt 1 ]; then echo , ; fi + echo " \"part-$i\"" + echo "foo=$i" > /tmp/parts/part-$i + done + echo ']' + outputs: + artifacts: + - name: parts + path: /tmp/parts + # One `map` per part ID is started. Finds its own "part file" under `/tmp/parts/${partId}`. + # Each `map` task has an output artifact saved with a unique name for the part into to a common "results directory". + - name: map + inputs: + parameters: + - name: partId + artifacts: + - name: parts + path: /tmp/parts + script: + image: 'docker/whalesay:latest' + command: + - bash + - -eu + source: | + source /tmp/parts/{{inputs.parameters.partId}} + mkdir -p /tmp/results + # map `foo` to `2 + foo` + expr 2 + $foo > /tmp/results/{{inputs.parameters.partId}} + outputs: + artifacts: + - name: result + path: /tmp/results/{{inputs.parameters.partId}} + archive: + none: { } + s3: + bucket: my-bucket + endpoint: minio:9000 + insecure: true + accessKeySecret: + name: my-minio-cred + key: accesskey + secretKeySecret: + name: my-minio-cred + key: secretkey + key: "{{workflow.name}}/results/{{inputs.parameters.partId}}" + # The `reduce` task takes the "results directory" and returns a single result. + - name: reduce + inputs: + artifacts: + - name: result + path: /tmp/results + s3: + bucket: my-bucket + endpoint: minio:9000 + insecure: true + accessKeySecret: + name: my-minio-cred + key: accesskey + secretKeySecret: + name: my-minio-cred + key: secretkey + key: "{{workflow.name}}/results" + script: + image: 'docker/whalesay:latest' + command: + - bash + - -eu + source: | + final_result=0 + for result in $(find /tmp/results -type f -exec cat {} ';'); do + final_result=$(expr $final_result + $result ) + done + echo $final_result > /tmp/final_result + outputs: + parameters: + - name: final_result + globalName: final_result + valueFrom: + path: /tmp/final_result From a6232ee3249b20445baf665460e1cad0845b0498 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Tue, 29 Sep 2020 22:40:23 -0700 Subject: [PATCH 02/10] python --- examples/map-reduce-workflow.yaml | 72 +++++++++++++++++-------------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/examples/map-reduce-workflow.yaml b/examples/map-reduce-workflow.yaml index 473d113cbee3..5ea965504cc7 100644 --- a/examples/map-reduce-workflow.yaml +++ b/examples/map-reduce-workflow.yaml @@ -13,7 +13,7 @@ spec: arguments: parameters: - name: num_parts - value: "1" + value: "2" templates: - name: main dag: @@ -49,19 +49,19 @@ spec: parameters: - name: num_parts script: - image: 'docker/whalesay:latest' + image: python:alpine3.6 command: - - bash - - -eu + - python source: | - mkdir -p /tmp/parts - echo '[' - for i in $(seq 1 {{inputs.parameters.num_parts}}); do - if [ $i -gt 1 ]; then echo , ; fi - echo " \"part-$i\"" - echo "foo=$i" > /tmp/parts/part-$i - done - echo ']' + import json + import os + import sys + os.mkdir("/tmp/parts") + partIds = list(map(lambda x: "part-" + str(x), range({{inputs.parameters.num_parts}}))) + for i, partId in enumerate(partIds, start=1): + with open("/tmp/parts/" + partId + ".json", "w") as out: + json.dump({"foo": i}, out) + json.dump(partIds, sys.stdout) outputs: artifacts: - name: parts @@ -76,19 +76,23 @@ spec: - name: parts path: /tmp/parts script: - image: 'docker/whalesay:latest' + image: python:alpine3.6 command: - - bash - - -eu + - python source: | - source /tmp/parts/{{inputs.parameters.partId}} - mkdir -p /tmp/results - # map `foo` to `2 + foo` - expr 2 + $foo > /tmp/results/{{inputs.parameters.partId}} + import json + import os + import sys + partId = "{{inputs.parameters.partId}}" + os.mkdir("/tmp/results") + with open("/tmp/parts/" + partId + ".json") as f: + part = json.load(f) + with open("/tmp/results/" + partId + ".json", "w") as out: + json.dump({"bar": part["foo"] * 2}, out) outputs: artifacts: - name: result - path: /tmp/results/{{inputs.parameters.partId}} + path: /tmp/results/{{inputs.parameters.partId}}.json archive: none: { } s3: @@ -101,7 +105,7 @@ spec: secretKeySecret: name: my-minio-cred key: secretkey - key: "{{workflow.name}}/results/{{inputs.parameters.partId}}" + key: "{{workflow.name}}/results/{{inputs.parameters.partId}}.json" # The `reduce` task takes the "results directory" and returns a single result. - name: reduce inputs: @@ -120,19 +124,23 @@ spec: key: secretkey key: "{{workflow.name}}/results" script: - image: 'docker/whalesay:latest' + image: python:alpine3.6 command: - - bash - - -eu + - python source: | - final_result=0 - for result in $(find /tmp/results -type f -exec cat {} ';'); do - final_result=$(expr $final_result + $result ) - done - echo $final_result > /tmp/final_result + import json + import os + import sys + total = 0 + for f in list(map(lambda x: open("/tmp/results/" + x), os.listdir("/tmp/results"))): + result = json.load(f) + total = total + result["bar"] + with open("/tmp/total", "w") as f: + f.write(str(total)) + f.close() outputs: parameters: - - name: final_result - globalName: final_result + - name: total + globalName: total valueFrom: - path: /tmp/final_result + path: /tmp/total From 9e28585a878683dd946cf8ad73ef097006f91c31 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Wed, 30 Sep 2020 08:14:52 -0700 Subject: [PATCH 03/10] mr --- docs/fields.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/fields.md b/docs/fields.md index a4e2ef4aa96d..dbef835db0d9 100644 --- a/docs/fields.md +++ b/docs/fields.md @@ -152,6 +152,8 @@ Workflow is the definition of a workflow resource - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -485,6 +487,8 @@ WorkflowSpec is the specification of a Workflow. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -831,6 +835,8 @@ CronWorkflowSpec is the specification of a CronWorkflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1138,6 +1144,8 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1395,6 +1403,8 @@ Arguments to a template - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1735,6 +1745,8 @@ Template is a reusable and composable unit of execution in a workflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2031,6 +2043,8 @@ Outputs hold parameters, artifacts, and results from a step - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2131,6 +2145,8 @@ Artifact indicates an artifact to place at a specified path - [`input-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/input-artifact-s3.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) - [`output-artifact-gcs.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-gcs.yaml) @@ -2244,6 +2260,8 @@ Parameter indicate a passed string parameter to a service template with an optio - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2458,6 +2476,8 @@ DAGTemplate is a template subtype for directed acyclic graph templates - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -2610,6 +2630,8 @@ Inputs are the mechanism for passing parameters, artifacts, volumes from one tem - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2839,6 +2861,8 @@ Pod metdata - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -3030,6 +3054,8 @@ ScriptTemplate is a template subtype to enable scripting through code steps - [`loops-param-result.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-param-result.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) - [`parameter-aggregation-script.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-script.yaml) @@ -3357,6 +3383,8 @@ ArchiveStrategy describes how to archive files/directory when saving artifacts - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -3572,6 +3600,8 @@ ValueFrom describes a location in which to obtain the value to a parameter - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -3715,6 +3745,8 @@ DAGTask represents a node in the graph during DAG execution - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -3877,6 +3909,8 @@ NoneStrategy indicates to skip tar process and upload the files or directory tre - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -4089,6 +4123,8 @@ ObjectMeta is metadata that all persisted resources must have, which includes al - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -5269,6 +5305,8 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -5815,6 +5853,8 @@ EnvVarSource represents a source for the value of an EnvVar. - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -6284,6 +6324,8 @@ ListMeta describes metadata that synthetic resources must have, including lists - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) From 729a50417fb8a03fbaf3d68f6e8edd021cefb736 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Wed, 30 Sep 2020 13:40:25 -0700 Subject: [PATCH 04/10] mr --- docs/fields.md | 44 ++++----- examples/map-reduce-workflow.yaml | 146 ------------------------------ 2 files changed, 23 insertions(+), 167 deletions(-) delete mode 100644 examples/map-reduce-workflow.yaml diff --git a/docs/fields.md b/docs/fields.md index f709be4a8f87..851e6bcc83c0 100644 --- a/docs/fields.md +++ b/docs/fields.md @@ -152,7 +152,7 @@ Workflow is the definition of a workflow resource - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -487,7 +487,7 @@ WorkflowSpec is the specification of a Workflow. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -836,7 +836,7 @@ CronWorkflowSpec is the specification of a CronWorkflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -1145,7 +1145,7 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -1404,7 +1404,7 @@ Arguments to a template - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -1746,7 +1746,7 @@ Template is a reusable and composable unit of execution in a workflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -2045,7 +2045,7 @@ Outputs hold parameters, artifacts, and results from a step - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -2147,7 +2147,7 @@ Artifact indicates an artifact to place at a specified path - [`input-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/input-artifact-s3.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2262,7 +2262,7 @@ Parameter indicate a passed string parameter to a service template with an optio - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -2478,7 +2478,7 @@ DAGTemplate is a template subtype for directed acyclic graph templates - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) @@ -2632,7 +2632,7 @@ Inputs are the mechanism for passing parameters, artifacts, volumes from one tem - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -2863,7 +2863,7 @@ Pod metdata - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -3056,7 +3056,7 @@ ScriptTemplate is a template subtype to enable scripting through code steps - [`loops-param-result.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-param-result.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -3385,7 +3385,7 @@ ArchiveStrategy describes how to archive files/directory when saving artifacts - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -3602,7 +3602,7 @@ ValueFrom describes a location in which to obtain the value to a parameter - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -3747,7 +3747,7 @@ DAGTask represents a node in the graph during DAG execution - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) @@ -3841,6 +3841,8 @@ Item expands a single workflow step into multiple parallel steps The value of It - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`parallelism-limit.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-limit.yaml) - [`parallelism-template-limit.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-template-limit.yaml) @@ -3911,7 +3913,7 @@ NoneStrategy indicates to skip tar process and upload the files or directory tre - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -4125,7 +4127,7 @@ ObjectMeta is metadata that all persisted resources must have, which includes al - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -5307,7 +5309,7 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -5855,7 +5857,7 @@ EnvVarSource represents a source for the value of an EnvVar. - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) @@ -6326,7 +6328,7 @@ ListMeta describes metadata that synthetic resources must have, including lists - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce-workflow.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) diff --git a/examples/map-reduce-workflow.yaml b/examples/map-reduce-workflow.yaml deleted file mode 100644 index 5ea965504cc7..000000000000 --- a/examples/map-reduce-workflow.yaml +++ /dev/null @@ -1,146 +0,0 @@ -# This workflow demonstrates a basic map-reduce. -# This requries you have a artifact repository configured. -# -# Notes: -# - You'll need to have an user namespaced artifact repository set-up to save intermediate results for this workflow. -# - Only a single reducer. -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: map-reduce- -spec: - entrypoint: main - arguments: - parameters: - - name: num_parts - value: "2" - templates: - - name: main - dag: - tasks: - - name: split - template: split - arguments: - parameters: - - name: num_parts - value: "{{workflow.parameters.num_parts}}" - - name: map - template: map - arguments: - parameters: - - name: partId - value: '{{item}}' - artifacts: - - name: parts - from: '{{tasks.split.outputs.artifacts.parts}}' - dependencies: - - split - withParam: '{{tasks.split.outputs.result}}' - - name: reduce - template: reduce - dependencies: - - map - # The `split` task creates a number of "parts". Each part has a unique ID (e.g. part-0, part-1). - # This task writes the part IDs to stdout (so that the `map` task can be expanded to have one task per part). - # And, it writes one "part file" for each of pieces of processing that needs doing, into to single directory - # which is then saved a output artifact. - - name: split - inputs: - parameters: - - name: num_parts - script: - image: python:alpine3.6 - command: - - python - source: | - import json - import os - import sys - os.mkdir("/tmp/parts") - partIds = list(map(lambda x: "part-" + str(x), range({{inputs.parameters.num_parts}}))) - for i, partId in enumerate(partIds, start=1): - with open("/tmp/parts/" + partId + ".json", "w") as out: - json.dump({"foo": i}, out) - json.dump(partIds, sys.stdout) - outputs: - artifacts: - - name: parts - path: /tmp/parts - # One `map` per part ID is started. Finds its own "part file" under `/tmp/parts/${partId}`. - # Each `map` task has an output artifact saved with a unique name for the part into to a common "results directory". - - name: map - inputs: - parameters: - - name: partId - artifacts: - - name: parts - path: /tmp/parts - script: - image: python:alpine3.6 - command: - - python - source: | - import json - import os - import sys - partId = "{{inputs.parameters.partId}}" - os.mkdir("/tmp/results") - with open("/tmp/parts/" + partId + ".json") as f: - part = json.load(f) - with open("/tmp/results/" + partId + ".json", "w") as out: - json.dump({"bar": part["foo"] * 2}, out) - outputs: - artifacts: - - name: result - path: /tmp/results/{{inputs.parameters.partId}}.json - archive: - none: { } - s3: - bucket: my-bucket - endpoint: minio:9000 - insecure: true - accessKeySecret: - name: my-minio-cred - key: accesskey - secretKeySecret: - name: my-minio-cred - key: secretkey - key: "{{workflow.name}}/results/{{inputs.parameters.partId}}.json" - # The `reduce` task takes the "results directory" and returns a single result. - - name: reduce - inputs: - artifacts: - - name: result - path: /tmp/results - s3: - bucket: my-bucket - endpoint: minio:9000 - insecure: true - accessKeySecret: - name: my-minio-cred - key: accesskey - secretKeySecret: - name: my-minio-cred - key: secretkey - key: "{{workflow.name}}/results" - script: - image: python:alpine3.6 - command: - - python - source: | - import json - import os - import sys - total = 0 - for f in list(map(lambda x: open("/tmp/results/" + x), os.listdir("/tmp/results"))): - result = json.load(f) - total = total + result["bar"] - with open("/tmp/total", "w") as f: - f.write(str(total)) - f.close() - outputs: - parameters: - - name: total - globalName: total - valueFrom: - path: /tmp/total From 1ac8da0740276378feda5e27cfa07a2d51aacef3 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Wed, 30 Sep 2020 14:19:46 -0700 Subject: [PATCH 05/10] mr --- docs/fields.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/fields.md b/docs/fields.md index 851e6bcc83c0..50676fe732e0 100644 --- a/docs/fields.md +++ b/docs/fields.md @@ -3841,8 +3841,6 @@ Item expands a single workflow step into multiple parallel steps The value of It - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`parallelism-limit.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-limit.yaml) - [`parallelism-template-limit.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-template-limit.yaml) @@ -3870,6 +3868,8 @@ Sequence expands a workflow step into numeric range - [`loops-sequence.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-sequence.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`work-avoidance.yaml`](https://github.com/argoproj/argo/blob/master/examples/work-avoidance.yaml) From c394daed3ac5984aae5636e8581d390a6c2395c5 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Wed, 30 Sep 2020 17:38:13 -0700 Subject: [PATCH 06/10] mr --- docs/fields.md | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/docs/fields.md b/docs/fields.md index 50676fe732e0..fa166c149060 100644 --- a/docs/fields.md +++ b/docs/fields.md @@ -152,8 +152,6 @@ Workflow is the definition of a workflow resource - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -487,8 +485,6 @@ WorkflowSpec is the specification of a Workflow. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -836,8 +832,6 @@ CronWorkflowSpec is the specification of a CronWorkflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1145,8 +1139,6 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1404,8 +1396,6 @@ Arguments to a template - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1746,8 +1736,6 @@ Template is a reusable and composable unit of execution in a workflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2045,8 +2033,6 @@ Outputs hold parameters, artifacts, and results from a step - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2147,8 +2133,6 @@ Artifact indicates an artifact to place at a specified path - [`input-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/input-artifact-s3.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) - [`output-artifact-gcs.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-gcs.yaml) @@ -2262,8 +2246,6 @@ Parameter indicate a passed string parameter to a service template with an optio - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2478,8 +2460,6 @@ DAGTemplate is a template subtype for directed acyclic graph templates - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -2632,8 +2612,6 @@ Inputs are the mechanism for passing parameters, artifacts, volumes from one tem - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2863,8 +2841,6 @@ Pod metdata - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -3056,8 +3032,6 @@ ScriptTemplate is a template subtype to enable scripting through code steps - [`loops-param-result.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-param-result.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) - [`parameter-aggregation-script.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-script.yaml) @@ -3385,8 +3359,6 @@ ArchiveStrategy describes how to archive files/directory when saving artifacts - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -3602,8 +3574,6 @@ ValueFrom describes a location in which to obtain the value to a parameter - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -3747,8 +3717,6 @@ DAGTask represents a node in the graph during DAG execution - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -3868,8 +3836,6 @@ Sequence expands a workflow step into numeric range - [`loops-sequence.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-sequence.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`work-avoidance.yaml`](https://github.com/argoproj/argo/blob/master/examples/work-avoidance.yaml) @@ -3913,8 +3879,6 @@ NoneStrategy indicates to skip tar process and upload the files or directory tre - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -4127,8 +4091,6 @@ ObjectMeta is metadata that all persisted resources must have, which includes al - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -5309,8 +5271,6 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -5857,8 +5817,6 @@ EnvVarSource represents a source for the value of an EnvVar. - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -6328,8 +6286,6 @@ ListMeta describes metadata that synthetic resources must have, including lists - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) From 4d0638face406ec0ed3fff48fd404ca44f29a88e Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Thu, 1 Oct 2020 11:52:16 -0700 Subject: [PATCH 07/10] mr --- docs/fields.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/fields.md b/docs/fields.md index fa166c149060..50676fe732e0 100644 --- a/docs/fields.md +++ b/docs/fields.md @@ -152,6 +152,8 @@ Workflow is the definition of a workflow resource - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -485,6 +487,8 @@ WorkflowSpec is the specification of a Workflow. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -832,6 +836,8 @@ CronWorkflowSpec is the specification of a CronWorkflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1139,6 +1145,8 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1396,6 +1404,8 @@ Arguments to a template - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1736,6 +1746,8 @@ Template is a reusable and composable unit of execution in a workflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2033,6 +2045,8 @@ Outputs hold parameters, artifacts, and results from a step - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2133,6 +2147,8 @@ Artifact indicates an artifact to place at a specified path - [`input-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/input-artifact-s3.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) - [`output-artifact-gcs.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-gcs.yaml) @@ -2246,6 +2262,8 @@ Parameter indicate a passed string parameter to a service template with an optio - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2460,6 +2478,8 @@ DAGTemplate is a template subtype for directed acyclic graph templates - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -2612,6 +2632,8 @@ Inputs are the mechanism for passing parameters, artifacts, volumes from one tem - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2841,6 +2863,8 @@ Pod metdata - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -3032,6 +3056,8 @@ ScriptTemplate is a template subtype to enable scripting through code steps - [`loops-param-result.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-param-result.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) - [`parameter-aggregation-script.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-script.yaml) @@ -3359,6 +3385,8 @@ ArchiveStrategy describes how to archive files/directory when saving artifacts - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -3574,6 +3602,8 @@ ValueFrom describes a location in which to obtain the value to a parameter - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -3717,6 +3747,8 @@ DAGTask represents a node in the graph during DAG execution - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -3836,6 +3868,8 @@ Sequence expands a workflow step into numeric range - [`loops-sequence.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-sequence.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`work-avoidance.yaml`](https://github.com/argoproj/argo/blob/master/examples/work-avoidance.yaml) @@ -3879,6 +3913,8 @@ NoneStrategy indicates to skip tar process and upload the files or directory tre - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -4091,6 +4127,8 @@ ObjectMeta is metadata that all persisted resources must have, which includes al - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -5271,6 +5309,8 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -5817,6 +5857,8 @@ EnvVarSource represents a source for the value of an EnvVar. - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -6286,6 +6328,8 @@ ListMeta describes metadata that synthetic resources must have, including lists - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) From c166d5895684dee83ebf72f2c53f55b03517e04a Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Thu, 1 Oct 2020 12:02:42 -0700 Subject: [PATCH 08/10] mr --- docs/fields.md | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/docs/fields.md b/docs/fields.md index 50676fe732e0..fa166c149060 100644 --- a/docs/fields.md +++ b/docs/fields.md @@ -152,8 +152,6 @@ Workflow is the definition of a workflow resource - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -487,8 +485,6 @@ WorkflowSpec is the specification of a Workflow. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -836,8 +832,6 @@ CronWorkflowSpec is the specification of a CronWorkflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1145,8 +1139,6 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1404,8 +1396,6 @@ Arguments to a template - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1746,8 +1736,6 @@ Template is a reusable and composable unit of execution in a workflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2045,8 +2033,6 @@ Outputs hold parameters, artifacts, and results from a step - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2147,8 +2133,6 @@ Artifact indicates an artifact to place at a specified path - [`input-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/input-artifact-s3.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) - [`output-artifact-gcs.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-gcs.yaml) @@ -2262,8 +2246,6 @@ Parameter indicate a passed string parameter to a service template with an optio - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2478,8 +2460,6 @@ DAGTemplate is a template subtype for directed acyclic graph templates - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -2632,8 +2612,6 @@ Inputs are the mechanism for passing parameters, artifacts, volumes from one tem - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2863,8 +2841,6 @@ Pod metdata - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -3056,8 +3032,6 @@ ScriptTemplate is a template subtype to enable scripting through code steps - [`loops-param-result.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-param-result.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) - [`parameter-aggregation-script.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-script.yaml) @@ -3385,8 +3359,6 @@ ArchiveStrategy describes how to archive files/directory when saving artifacts - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -3602,8 +3574,6 @@ ValueFrom describes a location in which to obtain the value to a parameter - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -3747,8 +3717,6 @@ DAGTask represents a node in the graph during DAG execution - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -3868,8 +3836,6 @@ Sequence expands a workflow step into numeric range - [`loops-sequence.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-sequence.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`work-avoidance.yaml`](https://github.com/argoproj/argo/blob/master/examples/work-avoidance.yaml) @@ -3913,8 +3879,6 @@ NoneStrategy indicates to skip tar process and upload the files or directory tre - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -4127,8 +4091,6 @@ ObjectMeta is metadata that all persisted resources must have, which includes al - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -5309,8 +5271,6 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -5857,8 +5817,6 @@ EnvVarSource represents a source for the value of an EnvVar. - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -6328,8 +6286,6 @@ ListMeta describes metadata that synthetic resources must have, including lists - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) -- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) - - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) From 8a3ca24223992f8afcf5409c6527eb77756a1aa3 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Thu, 1 Oct 2020 12:03:00 -0700 Subject: [PATCH 09/10] Revert "mr"t This reverts commit c166d5895684dee83ebf72f2c53f55b03517e04a. --- docs/fields.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/fields.md b/docs/fields.md index fa166c149060..50676fe732e0 100644 --- a/docs/fields.md +++ b/docs/fields.md @@ -152,6 +152,8 @@ Workflow is the definition of a workflow resource - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -485,6 +487,8 @@ WorkflowSpec is the specification of a Workflow. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -832,6 +836,8 @@ CronWorkflowSpec is the specification of a CronWorkflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1139,6 +1145,8 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate. - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1396,6 +1404,8 @@ Arguments to a template - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -1736,6 +1746,8 @@ Template is a reusable and composable unit of execution in a workflow - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2033,6 +2045,8 @@ Outputs hold parameters, artifacts, and results from a step - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2133,6 +2147,8 @@ Artifact indicates an artifact to place at a specified path - [`input-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/input-artifact-s3.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) - [`output-artifact-gcs.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-gcs.yaml) @@ -2246,6 +2262,8 @@ Parameter indicate a passed string parameter to a service template with an optio - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2460,6 +2478,8 @@ DAGTemplate is a template subtype for directed acyclic graph templates - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -2612,6 +2632,8 @@ Inputs are the mechanism for passing parameters, artifacts, volumes from one tem - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -2841,6 +2863,8 @@ Pod metdata - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -3032,6 +3056,8 @@ ScriptTemplate is a template subtype to enable scripting through code steps - [`loops-param-result.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-param-result.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) - [`parameter-aggregation-script.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-script.yaml) @@ -3359,6 +3385,8 @@ ArchiveStrategy describes how to archive files/directory when saving artifacts - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -3574,6 +3602,8 @@ ValueFrom describes a location in which to obtain the value to a parameter - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -3717,6 +3747,8 @@ DAGTask represents a node in the graph during DAG execution - [`loops-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-dag.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`parallelism-nested-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parallelism-nested-dag.yaml) - [`parameter-aggregation-dag.yaml`](https://github.com/argoproj/argo/blob/master/examples/parameter-aggregation-dag.yaml) @@ -3836,6 +3868,8 @@ Sequence expands a workflow step into numeric range - [`loops-sequence.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops-sequence.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`work-avoidance.yaml`](https://github.com/argoproj/argo/blob/master/examples/work-avoidance.yaml) @@ -3879,6 +3913,8 @@ NoneStrategy indicates to skip tar process and upload the files or directory tre - [`artifact-passing-subpath.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-passing-subpath.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`output-artifact-s3.yaml`](https://github.com/argoproj/argo/blob/master/examples/output-artifact-s3.yaml) @@ -4091,6 +4127,8 @@ ObjectMeta is metadata that all persisted resources must have, which includes al - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -5271,6 +5309,8 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -5817,6 +5857,8 @@ EnvVarSource represents a source for the value of an EnvVar. - [`k8s-wait-wf.yaml`](https://github.com/argoproj/argo/blob/master/examples/k8s-wait-wf.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) @@ -6286,6 +6328,8 @@ ListMeta describes metadata that synthetic resources must have, including lists - [`loops.yaml`](https://github.com/argoproj/argo/blob/master/examples/loops.yaml) +- [`map-reduce.yaml`](https://github.com/argoproj/argo/blob/master/examples/map-reduce.yaml) + - [`memoize-simple.yaml`](https://github.com/argoproj/argo/blob/master/examples/memoize-simple.yaml) - [`nested-workflow.yaml`](https://github.com/argoproj/argo/blob/master/examples/nested-workflow.yaml) From 44e6601fc3d41a896653fbf700ee8ac2894273b2 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Thu, 1 Oct 2020 12:03:08 -0700 Subject: [PATCH 10/10] ok --- examples/map-reduce.yaml | 162 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 examples/map-reduce.yaml diff --git a/examples/map-reduce.yaml b/examples/map-reduce.yaml new file mode 100644 index 000000000000..8b4c80078a5e --- /dev/null +++ b/examples/map-reduce.yaml @@ -0,0 +1,162 @@ +# This workflow demonstrates a basic map-reduce. +# This requires you have a artifact repository configured. +# +# Notes: +# - You'll need to have an user namespaced artifact repository set-up to save intermediate results for this workflow. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: map-reduce- +spec: + entrypoint: main + arguments: + parameters: + - name: numParts + value: "4" + - name: numGroups + value: "2" + templates: + - name: main + dag: + tasks: + - name: split + template: split + arguments: + parameters: + - name: numParts + value: "{{workflow.parameters.numParts}}" + - name: map + template: map + arguments: + parameters: + - name: partId + value: '{{item}}' + - name: numGroups + value: '{{workflow.parameters.numGroups}}' + artifacts: + - name: parts + from: '{{tasks.split.outputs.artifacts.parts}}' + dependencies: + - split + withParam: '{{tasks.split.outputs.result}}' + - name: reduce + template: reduce + arguments: + parameters: + - name: group + value: '{{item}}' + dependencies: + - map + withSequence: + count: "{{workflow.parameters.numGroups}}" + # The `split` task creates a number of "parts". Each part has a unique ID (e.g. part-0, part-1). + # This task writes the part IDs to stdout (so that the `map` task can be expanded to have one task per part). + # And, it writes one "part file" for each of pieces of processing that needs doing, into to single directory + # which is then saved a output artifact. + - name: split + inputs: + parameters: + - name: numParts + script: + image: python:alpine3.6 + command: + - python + source: | + import json + import os + import sys + os.mkdir("/tmp/parts") + partIds = list(map(lambda x: "part-" + str(x), range({{inputs.parameters.numParts}}))) + for i, partId in enumerate(partIds, start=1): + with open("/tmp/parts/" + partId + ".json", "w") as out: + json.dump({"foo": i}, out) + json.dump(partIds, sys.stdout) + outputs: + artifacts: + - name: parts + path: /tmp/parts + # One `map` per part ID is started. Finds its own "part file" under `/tmp/parts/${partId}`. + # Each `map` task has an output artifact saved with a unique name for the part into to a common "results directory". + - name: map + inputs: + parameters: + - name: partId + - name: numGroups + artifacts: + - name: parts + path: /tmp/parts + script: + image: python:alpine3.6 + command: + - python + source: | + import json + import os + import sys + partId = "{{inputs.parameters.partId}}" + numGroups = {{inputs.parameters.numGroups}} + os.mkdir("/tmp/results") + with open("/tmp/parts/" + partId + ".json") as f: + part = json.load(f) + with open("/tmp/results/" + partId + ".json", "w") as out: + json.dump({"bar": part["foo"] * 2, "group": part["foo"] % numGroups}, out) + outputs: + artifacts: + - name: result + path: /tmp/results/{{inputs.parameters.partId}}.json + archive: + none: { } + s3: + bucket: my-bucket + endpoint: minio:9000 + insecure: true + accessKeySecret: + name: my-minio-cred + key: accesskey + secretKeySecret: + name: my-minio-cred + key: secretkey + key: "{{workflow.name}}/results/{{inputs.parameters.partId}}.json" + # The `reduce` task takes the "results directory" and returns a single result. + - name: reduce + inputs: + parameters: + - name: group + artifacts: + - name: result + path: /tmp/results + s3: + bucket: my-bucket + endpoint: minio:9000 + insecure: true + accessKeySecret: + name: my-minio-cred + key: accesskey + secretKeySecret: + name: my-minio-cred + key: secretkey + key: "{{workflow.name}}/results" + script: + image: python:alpine3.6 + command: + - python + source: | + import json + import os + import sys + total = 0 + group = "{{inputs.parameters.group}}" + os.mkdir("/tmp/totals/") + for f in list(map(lambda x: open("/tmp/results/" + x), os.listdir("/tmp/results"))): + result = json.load(f) + if result["group"] == group: + total = total + result["bar"] + with open("/tmp/totals/" + group, "w") as f: + f.write(str(total)) + f.close() + outputs: + parameters: + - name: total-{{inputs.parameters.group}} + globalName: total-{{inputs.parameters.group}} + valueFrom: + path: /tmp/totals/{{inputs.parameters.group}}