diff --git a/backend/Dockerfile b/backend/Dockerfile index 30b559c3af3..72bf06394ad 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -14,6 +14,15 @@ COPY . . RUN apk add --update gcc musl-dev RUN go build -o /bin/apiserver backend/src/apiserver/*.go +FROM python:3.5.0-slim as compiler + +# This is hard coded to 0.0.26. Once kfp DSK release process is automated, +# we can dynamically refer to the version from same commit SHA. +RUN pip install https://storage.googleapis.com/ml-pipeline/release/0.0.26/kfp-0.0.26.tar.gz --upgrade +WORKDIR /samples +COPY ./samples . +RUN find . -maxdepth 2 -name "*.py" -exec dsl-compile --py {} --output {}.tar.gz \; + FROM alpine ARG COMMIT_SHA=unknown @@ -24,7 +33,8 @@ WORKDIR /bin COPY --from=builder /bin/apiserver /bin/apiserver COPY --from=builder /go/src/github.com/kubeflow/pipelines/third_party/license.txt /bin/license.txt COPY backend/src/apiserver/config/ /config -COPY backend/src/apiserver/samples/ /samples + +COPY --from=compiler /samples/ /samples/ # Adding CA certificate so API server can download pipeline through URL RUN apk add ca-certificates @@ -33,4 +43,4 @@ RUN apk add ca-certificates EXPOSE 8888 # Start the apiserver -CMD apiserver --config=/config --sampleconfig=/samples/sample_config.json +CMD apiserver --config=/config --sampleconfig=/config/sample_config.json diff --git a/backend/src/apiserver/config/sample_config.json b/backend/src/apiserver/config/sample_config.json new file mode 100644 index 00000000000..81e0917bb61 --- /dev/null +++ b/backend/src/apiserver/config/sample_config.json @@ -0,0 +1,37 @@ +[ + { + "name":"[Sample] ML - XGBoost - Training with Confusion Matrix", + "description":"A trainer that does end-to-end distributed training for XGBoost models. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/xgboost-spark", + "file":"/samples/xgboost-spark/xgboost-training-cm.py.tar.gz" + }, + { + "name":"[Sample] ML - TFX - Taxi Tip Prediction Model Trainer", + "description":"Example pipeline that does classification with model analysis based on a public tax cab BigQuery dataset. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/tfx", + "file":"/samples/tfx/taxi-cab-classification-pipeline.py.tar.gz" + }, + { + "name":"[Sample] Basic - Sequential", + "description":"A pipeline with two sequential steps. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/sequential.py", + "file":"/samples/basic/sequential.py.tar.gz" + }, + { + "name":"[Sample] Basic - Parallel Join", + "description":"A pipeline that downloads two messages in parallel and print the concatenated result. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/parallel_join.py", + "file":"/samples/basic/parallel_join.py.tar.gz" + }, + { + "name":"[Sample] Basic - Immediate Value", + "description":"A pipeline with parameter values hard coded. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/immediate_value.py", + "file":"/samples/basic/immediate_value.py.tar.gz" + }, + { + "name":"[Sample] Basic - Exit Handler", + "description":"A pipeline that downloads a message and print it out. Exit Handler will run at the end. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/exit_handler.py", + "file":"/samples/basic/exit_handler.py.tar.gz" + }, + { + "name":"[Sample] Basic - Condition", + "description":"A pipeline shows how to use dsl.Condition. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/condition.py", + "file":"/samples/basic/condition.py.tar.gz" + } +] \ No newline at end of file diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go index 746c9834f90..a0b6562322c 100644 --- a/backend/src/apiserver/main.go +++ b/backend/src/apiserver/main.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "net" "net/http" + "time" "github.com/golang/glog" api "github.com/kubeflow/pipelines/backend/api/go_client" @@ -30,6 +31,9 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/runtime" "google.golang.org/grpc" "google.golang.org/grpc/reflection" + "os" + "github.com/pkg/errors" + "fmt" ) var ( @@ -48,7 +52,10 @@ func main() { initConfig() clientManager := newClientManager() resourceManager := resource.NewResourceManager(&clientManager) - loadSamples(resourceManager) + err:= loadSamples(resourceManager) + if err!=nil{ + glog.Fatalf("Failed to load samples. Err: %v", err.Error()) + } go startRpcServer(resourceManager) startHttpProxy(resourceManager) @@ -119,11 +126,10 @@ func registerHttpHandlerFromEndpoint(handler RegisterHttpHandlerFromEndpoint, se } // Preload a bunch of pipeline samples -func loadSamples(resourceManager *resource.ResourceManager) { +func loadSamples(resourceManager *resource.ResourceManager) error { configBytes, err := ioutil.ReadFile(*sampleConfigPath) if err != nil { - glog.Warningf("Failed to read sample configurations. Err: %v", err.Error()) - return + return errors.New(fmt.Sprintf("Failed to read sample configurations file. Err: %v", err.Error())) } type config struct { Name string @@ -131,17 +137,30 @@ func loadSamples(resourceManager *resource.ResourceManager) { File string } var configs []config - if json.Unmarshal(configBytes, &configs) != nil { - glog.Warningf("Failed to read sample configurations. Err: %v", err.Error()) - return + if err:= json.Unmarshal(configBytes, &configs);err != nil { + return errors.New(fmt.Sprintf("Failed to read sample configurations. Err: %v", err.Error())) } for _, config := range configs { - sampleBytes, err := ioutil.ReadFile(config.File) + reader, err:= os.Open(config.File) if err != nil { - glog.Warningf("Failed to load sample %s. Error: %v", config.Name, err.Error()) + return errors.New(fmt.Sprintf("Failed to load sample %s. Error: %v", config.Name, err.Error())) + } + pipelineFile, err := server.ReadPipelineFile(config.File, reader, server.MaxFileLength) + if err!=nil{ + return errors.New(fmt.Sprintf("Failed to decompress the file %s. Error: %v", config.Name, err.Error())) + } + _, err = resourceManager.CreatePipeline(config.Name, config.Description, pipelineFile) + if err!=nil{ + // Log the error but not fail. The API Server pod can restart and it could potentially cause name collision. + // In the future, we might consider loading samples during deployment, instead of when API server starts. + glog.Warningf(fmt.Sprintf("Failed to create pipeline for %s. Error: %v", config.Name, err.Error())) continue } - resourceManager.CreatePipeline(config.Name, config.Description, sampleBytes) + + // Since the default sorting is by create time, + // Sleep one second makes sure the samples are showing up in the same order as they are added. + time.Sleep(1*time.Second) } glog.Info("All samples are loaded.") + return nil } diff --git a/backend/src/apiserver/samples/condition.yaml b/backend/src/apiserver/samples/condition.yaml deleted file mode 100644 index be071ff203b..00000000000 --- a/backend/src/apiserver/samples/condition.yaml +++ /dev/null @@ -1,269 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: pipeline-flip-coin- -spec: - arguments: - parameters: [] - entrypoint: pipeline-flip-coin - serviceAccountName: pipeline-runner - templates: - - inputs: - parameters: - - name: flip-output - name: condition-1 - steps: - - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: condition-1-child - template: condition-1-child - when: '{{inputs.parameters.flip-output}} == heads' - - dag: - tasks: - - arguments: - parameters: - - name: flip-again-output - value: '{{tasks.flip-again.outputs.parameters.flip-again-output}}' - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - dependencies: - - flip-again - name: condition-2 - template: condition-2 - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: flip-again - template: flip-again - inputs: - parameters: - - name: flip-output - name: condition-1-child - - inputs: - parameters: - - name: flip-again-output - - name: flip-output - name: condition-2 - steps: - - - arguments: - parameters: - - name: flip-again-output - value: '{{inputs.parameters.flip-again-output}}' - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: condition-2-child - template: condition-2-child - when: '{{inputs.parameters.flip-again-output}} == tails' - - dag: - tasks: - - arguments: - parameters: - - name: flip-again-output - value: '{{inputs.parameters.flip-again-output}}' - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: print1 - template: print1 - inputs: - parameters: - - name: flip-again-output - - name: flip-output - name: condition-2-child - - inputs: - parameters: - - name: flip-output - name: condition-3 - steps: - - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: condition-3-child - template: condition-3-child - when: '{{inputs.parameters.flip-output}} == tails' - - dag: - tasks: - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: print2 - template: print2 - inputs: - parameters: - - name: flip-output - name: condition-3-child - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.6 - name: flip - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: flip-output - valueFrom: - path: /tmp/output - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.6 - name: flip-again - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: flip-again-output - valueFrom: - path: /tmp/output - - dag: - tasks: - - arguments: - parameters: - - name: flip-output - value: '{{tasks.flip.outputs.parameters.flip-output}}' - dependencies: - - flip - name: condition-1 - template: condition-1 - - arguments: - parameters: - - name: flip-output - value: '{{tasks.flip.outputs.parameters.flip-output}}' - dependencies: - - flip - name: condition-3 - template: condition-3 - - name: flip - template: flip - name: pipeline-flip-coin - - container: - command: - - echo - - '"it was tail"' - image: alpine:3.6 - name: print1 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - container: - command: - - echo - - '"it was tail"' - image: alpine:3.6 - name: print2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact diff --git a/backend/src/apiserver/samples/default_value.yaml b/backend/src/apiserver/samples/default_value.yaml deleted file mode 100644 index d8ed6d653c2..00000000000 --- a/backend/src/apiserver/samples/default_value.yaml +++ /dev/null @@ -1,114 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: default-value- -spec: - arguments: - parameters: - - name: url - value: gs://ml-pipeline/shakespeare1.txt - entrypoint: default-value - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: echo - template: echo - inputs: - parameters: - - name: url - name: default-value - - container: - args: - - gsutil cat {{inputs.parameters.url}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url - name: download - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo {{inputs.parameters.download-downloaded}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: download-downloaded - name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact diff --git a/backend/src/apiserver/samples/exit_handler.yaml b/backend/src/apiserver/samples/exit_handler.yaml deleted file mode 100644 index 4ef846d5291..00000000000 --- a/backend/src/apiserver/samples/exit_handler.yaml +++ /dev/null @@ -1,162 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: exit-handler- -spec: - arguments: - parameters: - - name: url - entrypoint: exit-handler - onExit: finally - serviceAccountName: pipeline-runner - templates: - - container: - args: - - gsutil cat {{inputs.parameters.url}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url - name: download - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo {{inputs.parameters.download-downloaded}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: download-downloaded - name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: exit-handler-1 - template: exit-handler-1 - - name: finally - template: finally - inputs: - parameters: - - name: url - name: exit-handler - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: echo - template: echo - inputs: - parameters: - - name: url - name: exit-handler-1 - - container: - command: - - echo - - exit! - image: library/bash:4.4.23 - name: finally - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact diff --git a/backend/src/apiserver/samples/parallel_join.yaml b/backend/src/apiserver/samples/parallel_join.yaml deleted file mode 100644 index df445867f91..00000000000 --- a/backend/src/apiserver/samples/parallel_join.yaml +++ /dev/null @@ -1,168 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: parallel-and-join- -spec: - arguments: - parameters: - - name: url1 - - name: url2 - entrypoint: parallel-and-join - serviceAccountName: pipeline-runner - templates: - - container: - args: - - gsutil cat {{inputs.parameters.url1}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url1 - name: download1 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download1-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - gsutil cat {{inputs.parameters.url2}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url2 - name: download2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download2-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo {{inputs.parameters.download1-downloaded}} {{inputs.parameters.download2-downloaded}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: download1-downloaded - - name: download2-downloaded - name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - dag: - tasks: - - arguments: - parameters: - - name: url1 - value: '{{inputs.parameters.url1}}' - name: download1 - template: download1 - - arguments: - parameters: - - name: url2 - value: '{{inputs.parameters.url2}}' - name: download2 - template: download2 - - arguments: - parameters: - - name: download1-downloaded - value: '{{tasks.download1.outputs.parameters.download1-downloaded}}' - - name: download2-downloaded - value: '{{tasks.download2.outputs.parameters.download2-downloaded}}' - dependencies: - - download1 - - download2 - name: echo - template: echo - inputs: - parameters: - - name: url1 - - name: url2 - name: parallel-and-join diff --git a/backend/src/apiserver/samples/resnet_training_pipeline.yaml b/backend/src/apiserver/samples/resnet_training_pipeline.yaml deleted file mode 100644 index 0eecea31bb5..00000000000 --- a/backend/src/apiserver/samples/resnet_training_pipeline.yaml +++ /dev/null @@ -1,305 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: resnet-train-pipeline- -spec: - arguments: - parameters: - - name: project-id - - name: bucket - - name: region - value: us-central1 - - name: model - value: bolts - - name: version - value: beta1 - - name: tf-version - value: '1.8' - - name: train-csv - value: gs://bolts_image_dataset/bolt_images_train.csv - - name: validation-csv - value: gs://bolts_image_dataset/bolt_images_validate.csv - - name: labels - value: gs://bolts_image_dataset/labels.txt - - name: depth - value: '50' - - name: train-batch-size - value: '1024' - - name: eval-batch-size - value: '1024' - - name: steps-per-eval - value: '250' - - name: train-steps - value: '10000' - - name: num-train-images - value: '218593' - - name: num-eval-images - value: '54648' - - name: num-label-classes - value: '10' - entrypoint: resnet-train-pipeline - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --model - - '{{inputs.parameters.model}}' - - --version - - '{{inputs.parameters.version}}' - - --project_id - - '{{inputs.parameters.project-id}}' - - --region - - '{{inputs.parameters.region}}' - - --model_dir - - '{{inputs.parameters.train-trained}}' - - --TFVERSION - - '{{inputs.parameters.tf-version}}' - image: gcr.io/ml-pipeline/resnet-deploy:0.0.42 - inputs: - parameters: - - name: model - - name: project-id - - name: region - - name: tf-version - - name: train-trained - - name: version - name: deploy - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - container: - args: - - --project_id - - '{{inputs.parameters.project-id}}' - - --bucket - - '{{inputs.parameters.bucket}}' - - --train_csv - - '{{inputs.parameters.train-csv}}' - - --validation_csv - - '{{inputs.parameters.validation-csv}}' - - --labels - - '{{inputs.parameters.labels}}' - image: gcr.io/ml-pipeline/resnet-preprocess:0.0.42 - inputs: - parameters: - - name: bucket - - name: labels - - name: project-id - - name: train-csv - - name: validation-csv - name: preprocess - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: preprocess-preprocessed - valueFrom: - path: /output.txt - - dag: - tasks: - - arguments: - parameters: - - name: model - value: '{{inputs.parameters.model}}' - - name: project-id - value: '{{inputs.parameters.project-id}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: tf-version - value: '{{inputs.parameters.tf-version}}' - - name: train-trained - value: '{{tasks.train.outputs.parameters.train-trained}}' - - name: version - value: '{{inputs.parameters.version}}' - dependencies: - - train - name: deploy - template: deploy - - arguments: - parameters: - - name: bucket - value: '{{inputs.parameters.bucket}}' - - name: labels - value: '{{inputs.parameters.labels}}' - - name: project-id - value: '{{inputs.parameters.project-id}}' - - name: train-csv - value: '{{inputs.parameters.train-csv}}' - - name: validation-csv - value: '{{inputs.parameters.validation-csv}}' - name: preprocess - template: preprocess - - arguments: - parameters: - - name: bucket - value: '{{inputs.parameters.bucket}}' - - name: depth - value: '{{inputs.parameters.depth}}' - - name: eval-batch-size - value: '{{inputs.parameters.eval-batch-size}}' - - name: num-eval-images - value: '{{inputs.parameters.num-eval-images}}' - - name: num-label-classes - value: '{{inputs.parameters.num-label-classes}}' - - name: num-train-images - value: '{{inputs.parameters.num-train-images}}' - - name: preprocess-preprocessed - value: '{{tasks.preprocess.outputs.parameters.preprocess-preprocessed}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: steps-per-eval - value: '{{inputs.parameters.steps-per-eval}}' - - name: tf-version - value: '{{inputs.parameters.tf-version}}' - - name: train-batch-size - value: '{{inputs.parameters.train-batch-size}}' - - name: train-steps - value: '{{inputs.parameters.train-steps}}' - dependencies: - - preprocess - name: train - template: train - inputs: - parameters: - - name: bucket - - name: depth - - name: eval-batch-size - - name: labels - - name: model - - name: num-eval-images - - name: num-label-classes - - name: num-train-images - - name: project-id - - name: region - - name: steps-per-eval - - name: tf-version - - name: train-batch-size - - name: train-csv - - name: train-steps - - name: validation-csv - - name: version - name: resnet-train-pipeline - - container: - args: - - --data_dir - - '{{inputs.parameters.preprocess-preprocessed}}' - - --bucket - - '{{inputs.parameters.bucket}}' - - --region - - '{{inputs.parameters.region}}' - - --depth - - '{{inputs.parameters.depth}}' - - --train_batch_size - - '{{inputs.parameters.train-batch-size}}' - - --eval_batch_size - - '{{inputs.parameters.eval-batch-size}}' - - --steps_per_eval - - '{{inputs.parameters.steps-per-eval}}' - - --train_steps - - '{{inputs.parameters.train-steps}}' - - --num_train_images - - '{{inputs.parameters.num-train-images}}' - - --num_eval_images - - '{{inputs.parameters.num-eval-images}}' - - --num_label_classes - - '{{inputs.parameters.num-label-classes}}' - - --TFVERSION - - '{{inputs.parameters.tf-version}}' - image: gcr.io/ml-pipeline/resnet-train:0.0.42 - inputs: - parameters: - - name: bucket - - name: depth - - name: eval-batch-size - - name: num-eval-images - - name: num-label-classes - - name: num-train-images - - name: preprocess-preprocessed - - name: region - - name: steps-per-eval - - name: tf-version - - name: train-batch-size - - name: train-steps - name: train - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: train-trained - valueFrom: - path: /output.txt diff --git a/backend/src/apiserver/samples/sample_config.json b/backend/src/apiserver/samples/sample_config.json deleted file mode 100644 index b97b11c0ff1..00000000000 --- a/backend/src/apiserver/samples/sample_config.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "name":"condition", - "description":"shows how to use dsl.Condition", - "file":"/samples/condition.yaml" - }, - { - "name":"default value", - "description":"A pipeline with parameter and default value", - "file":"/samples/default_value.yaml" - }, - { - "name":"exit handler", - "description":"Download a message and print it out. Exit Handler will run at the end", - "file":"/samples/exit_handler.yaml" - }, - { - "name":"parallel join", - "description":"Download two messages in parallel and print the concatenated result", - "file":"/samples/parallel_join.yaml" - }, - { - "name":"sequential", - "description":"Download a message and print it", - "file":"/samples/sequential.yaml" - }, - { - "name":"taxi cab classification", - "description":"Example pipeline that does classification with model analysis based on a public BigQuery dataset", - "file":"/samples/taxi_cab_classification_pipeline.yaml" - }, - { - "name":"resnet training", - "description":"Demonstrate the ResNet50 predict", - "file":"/samples/resnet_training_pipeline.yaml" - }, - { - "name":"xgboost training - confusion matrix", - "description":"A trainer that does end-to-end distributed training for XGBoost models", - "file":"/samples/xgboost_training_cm.yaml" - } -] \ No newline at end of file diff --git a/backend/src/apiserver/samples/sequential.yaml b/backend/src/apiserver/samples/sequential.yaml deleted file mode 100644 index 2aecc6b6f35..00000000000 --- a/backend/src/apiserver/samples/sequential.yaml +++ /dev/null @@ -1,113 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: sequential- -spec: - arguments: - parameters: - - name: url - entrypoint: sequential - serviceAccountName: pipeline-runner - templates: - - container: - args: - - gsutil cat {{inputs.parameters.url}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url - name: download - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo "{{inputs.parameters.download-downloaded}}" - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: download-downloaded - name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: echo - template: echo - inputs: - parameters: - - name: url - name: sequential diff --git a/backend/src/apiserver/samples/taxi_cab_classification_pipeline.yaml b/backend/src/apiserver/samples/taxi_cab_classification_pipeline.yaml deleted file mode 100644 index e621a9ad0d1..00000000000 --- a/backend/src/apiserver/samples/taxi_cab_classification_pipeline.yaml +++ /dev/null @@ -1,421 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: tfma-taxi-cab-classification-pipeline-example- -spec: - arguments: - parameters: - - name: output - - name: project - - name: schema - value: gs://ml-pipeline-playground/tfma/taxi-cab-classification/schema.json - - name: train - value: gs://ml-pipeline-playground/tfma/taxi-cab-classification/train.csv - - name: evaluation - value: gs://ml-pipeline-playground/tfma/taxi-cab-classification/eval.csv - - name: preprocess-mode - value: local - - name: preprocess-module - value: gs://ml-pipeline-playground/tfma/taxi-cab-classification/preprocessing.py - - name: target - value: tips - - name: learning-rate - value: '0.1' - - name: hidden-layer-size - value: '1500' - - name: steps - value: '3000' - - name: predict-mode - value: local - - name: analyze-mode - value: local - - name: analyze-slice-column - value: trip_start_hour - entrypoint: tfma-taxi-cab-classification-pipeline-example - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --model - - '{{inputs.parameters.training-train}}' - - --eval - - '{{inputs.parameters.evaluation}}' - - --schema - - '{{inputs.parameters.schema}}' - - --project - - '{{inputs.parameters.project}}' - - --mode - - '{{inputs.parameters.analyze-mode}}' - - --slice-columns - - '{{inputs.parameters.analyze-slice-column}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/analysis' - image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:0.0.42 - inputs: - parameters: - - name: analyze-mode - - name: analyze-slice-column - - name: evaluation - - name: output - - name: project - - name: schema - - name: training-train - name: analysis - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: analysis-analysis - valueFrom: - path: /output.txt - - container: - args: - - --model-path - - '{{inputs.parameters.training-train}}' - - --server-name - - taxi-cab-classification-model-{{workflow.name}} - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:0.0.42 - inputs: - parameters: - - name: training-train - name: deploy - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - container: - args: - - --data - - '{{inputs.parameters.evaluation}}' - - --schema - - '{{inputs.parameters.schema}}' - - --target - - '{{inputs.parameters.target}}' - - --model - - '{{inputs.parameters.training-train}}' - - --mode - - '{{inputs.parameters.predict-mode}}' - - --project - - '{{inputs.parameters.project}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/predict' - image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:0.0.42 - inputs: - parameters: - - name: evaluation - - name: output - - name: predict-mode - - name: project - - name: schema - - name: target - - name: training-train - name: prediction - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: prediction-prediction - valueFrom: - path: /output.txt - - container: - args: - - --train - - '{{inputs.parameters.train}}' - - --eval - - '{{inputs.parameters.evaluation}}' - - --schema - - '{{inputs.parameters.schema}}' - - --project - - '{{inputs.parameters.project}}' - - --mode - - '{{inputs.parameters.preprocess-mode}}' - - --preprocessing-module - - '{{inputs.parameters.preprocess-module}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/transformed' - image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:0.0.42 - inputs: - parameters: - - name: evaluation - - name: output - - name: preprocess-mode - - name: preprocess-module - - name: project - - name: schema - - name: train - name: preprocess - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: preprocess-transformed - valueFrom: - path: /output.txt - - dag: - tasks: - - arguments: - parameters: - - name: analyze-mode - value: '{{inputs.parameters.analyze-mode}}' - - name: analyze-slice-column - value: '{{inputs.parameters.analyze-slice-column}}' - - name: evaluation - value: '{{inputs.parameters.evaluation}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: training-train - value: '{{tasks.training.outputs.parameters.training-train}}' - dependencies: - - training - name: analysis - template: analysis - - arguments: - parameters: - - name: training-train - value: '{{tasks.training.outputs.parameters.training-train}}' - dependencies: - - training - name: deploy - template: deploy - - arguments: - parameters: - - name: evaluation - value: '{{inputs.parameters.evaluation}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: predict-mode - value: '{{inputs.parameters.predict-mode}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: target - value: '{{inputs.parameters.target}}' - - name: training-train - value: '{{tasks.training.outputs.parameters.training-train}}' - dependencies: - - training - name: prediction - template: prediction - - arguments: - parameters: - - name: evaluation - value: '{{inputs.parameters.evaluation}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: preprocess-mode - value: '{{inputs.parameters.preprocess-mode}}' - - name: preprocess-module - value: '{{inputs.parameters.preprocess-module}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: train - value: '{{inputs.parameters.train}}' - name: preprocess - template: preprocess - - arguments: - parameters: - - name: hidden-layer-size - value: '{{inputs.parameters.hidden-layer-size}}' - - name: learning-rate - value: '{{inputs.parameters.learning-rate}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: preprocess-module - value: '{{inputs.parameters.preprocess-module}}' - - name: preprocess-transformed - value: '{{tasks.preprocess.outputs.parameters.preprocess-transformed}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: steps - value: '{{inputs.parameters.steps}}' - - name: target - value: '{{inputs.parameters.target}}' - dependencies: - - preprocess - name: training - template: training - inputs: - parameters: - - name: analyze-mode - - name: analyze-slice-column - - name: evaluation - - name: hidden-layer-size - - name: learning-rate - - name: output - - name: predict-mode - - name: preprocess-mode - - name: preprocess-module - - name: project - - name: schema - - name: steps - - name: target - - name: train - name: tfma-taxi-cab-classification-pipeline-example - - container: - args: - - --transformed-data-dir - - '{{inputs.parameters.preprocess-transformed}}' - - --schema - - '{{inputs.parameters.schema}}' - - --learning-rate - - '{{inputs.parameters.learning-rate}}' - - --hidden-layer-size - - '{{inputs.parameters.hidden-layer-size}}' - - --steps - - '{{inputs.parameters.steps}}' - - --target - - '{{inputs.parameters.target}}' - - --preprocessing-module - - '{{inputs.parameters.preprocess-module}}' - - --job-dir - - '{{inputs.parameters.output}}/{{workflow.name}}/train' - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.0.42 - inputs: - parameters: - - name: hidden-layer-size - - name: learning-rate - - name: output - - name: preprocess-module - - name: preprocess-transformed - - name: schema - - name: steps - - name: target - name: training - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: training-train - valueFrom: - path: /output.txt diff --git a/backend/src/apiserver/samples/xgboost_training_cm.yaml b/backend/src/apiserver/samples/xgboost_training_cm.yaml deleted file mode 100644 index 885e7e63bd2..00000000000 --- a/backend/src/apiserver/samples/xgboost_training_cm.yaml +++ /dev/null @@ -1,640 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: xgboost-trainer- -spec: - arguments: - parameters: - - name: output - - name: project - - name: region - value: us-central1 - - name: train-data - value: gs://ml-pipeline-playground/sfpd/train.csv - - name: eval-data - value: gs://ml-pipeline-playground/sfpd/eval.csv - - name: schema - value: gs://ml-pipeline-playground/sfpd/schema.json - - name: target - value: resolution - - name: rounds - value: '200' - - name: workers - value: '2' - - name: true-label - value: ACTION - entrypoint: xgboost-trainer - onExit: delete-cluster - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --cluster - - '{{inputs.parameters.create-cluster-output}}' - - --schema - - '{{inputs.parameters.schema}}' - - --train - - '{{inputs.parameters.train-data}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/analysis' - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:0.0.42 - inputs: - parameters: - - name: create-cluster-output - - name: output - - name: project - - name: region - - name: schema - - name: train-data - name: analyze - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: analyze-output - valueFrom: - path: /output.txt - - container: - args: - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/confusionmatrix' - - --predictions - - '{{inputs.parameters.predict-output}}' - image: gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:0.0.42 - inputs: - parameters: - - name: output - - name: predict-output - name: confusion-matrix - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --name - - xgb-{{workflow.name}} - - --staging - - '{{inputs.parameters.output}}' - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:0.0.42 - inputs: - parameters: - - name: output - - name: project - - name: region - name: create-cluster - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: create-cluster-output - valueFrom: - path: /output.txt - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --name - - xgb-{{workflow.name}} - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:0.0.42 - inputs: - parameters: - - name: project - - name: region - name: delete-cluster - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - dag: - tasks: - - arguments: - parameters: - - name: create-cluster-output - value: '{{tasks.create-cluster.outputs.parameters.create-cluster-output}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: train-data - value: '{{inputs.parameters.train-data}}' - dependencies: - - create-cluster - name: analyze - template: analyze - - arguments: - parameters: - - name: output - value: '{{inputs.parameters.output}}' - - name: predict-output - value: '{{tasks.predict.outputs.parameters.predict-output}}' - dependencies: - - predict - name: confusion-matrix - template: confusion-matrix - - arguments: - parameters: - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - name: create-cluster - template: create-cluster - - arguments: - parameters: - - name: analyze-output - value: '{{tasks.analyze.outputs.parameters.analyze-output}}' - - name: create-cluster-output - value: '{{tasks.create-cluster.outputs.parameters.create-cluster-output}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: target - value: '{{inputs.parameters.target}}' - - name: train-output - value: '{{tasks.train.outputs.parameters.train-output}}' - - name: transform-eval - value: '{{tasks.transform.outputs.parameters.transform-eval}}' - dependencies: - - analyze - - create-cluster - - train - - transform - name: predict - template: predict - - arguments: - parameters: - - name: output - value: '{{inputs.parameters.output}}' - - name: predict-output - value: '{{tasks.predict.outputs.parameters.predict-output}}' - - name: true-label - value: '{{inputs.parameters.true-label}}' - dependencies: - - predict - name: roc - template: roc - - arguments: - parameters: - - name: analyze-output - value: '{{tasks.analyze.outputs.parameters.analyze-output}}' - - name: create-cluster-output - value: '{{tasks.create-cluster.outputs.parameters.create-cluster-output}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: rounds - value: '{{inputs.parameters.rounds}}' - - name: target - value: '{{inputs.parameters.target}}' - - name: transform-eval - value: '{{tasks.transform.outputs.parameters.transform-eval}}' - - name: transform-train - value: '{{tasks.transform.outputs.parameters.transform-train}}' - - name: workers - value: '{{inputs.parameters.workers}}' - dependencies: - - analyze - - create-cluster - - transform - name: train - template: train - - arguments: - parameters: - - name: analyze-output - value: '{{tasks.analyze.outputs.parameters.analyze-output}}' - - name: create-cluster-output - value: '{{tasks.create-cluster.outputs.parameters.create-cluster-output}}' - - name: eval-data - value: '{{inputs.parameters.eval-data}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: target - value: '{{inputs.parameters.target}}' - - name: train-data - value: '{{inputs.parameters.train-data}}' - dependencies: - - analyze - - create-cluster - name: transform - template: transform - inputs: - parameters: - - name: eval-data - - name: output - - name: project - - name: region - - name: rounds - - name: schema - - name: target - - name: train-data - - name: true-label - - name: workers - name: exit-handler-1 - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --cluster - - '{{inputs.parameters.create-cluster-output}}' - - --predict - - '{{inputs.parameters.transform-eval}}' - - --analysis - - '{{inputs.parameters.analyze-output}}' - - --target - - '{{inputs.parameters.target}}' - - --package - - gs://ml-pipeline-playground/xgboost4j-example-0.8-SNAPSHOT-jar-with-dependencies.jar - - --model - - '{{inputs.parameters.train-output}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/predict' - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:0.0.42 - inputs: - parameters: - - name: analyze-output - - name: create-cluster-output - - name: output - - name: project - - name: region - - name: target - - name: train-output - - name: transform-eval - name: predict - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: predict-output - valueFrom: - path: /output.txt - - container: - args: - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/roc' - - --predictions - - '{{inputs.parameters.predict-output}}' - - --trueclass - - '{{inputs.parameters.true-label}}' - image: gcr.io/ml-pipeline/ml-pipeline-local-roc:0.0.42 - inputs: - parameters: - - name: output - - name: predict-output - - name: true-label - name: roc - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --cluster - - '{{inputs.parameters.create-cluster-output}}' - - --train - - '{{inputs.parameters.transform-train}}' - - --eval - - '{{inputs.parameters.transform-eval}}' - - --analysis - - '{{inputs.parameters.analyze-output}}' - - --target - - '{{inputs.parameters.target}}' - - --package - - gs://ml-pipeline-playground/xgboost4j-example-0.8-SNAPSHOT-jar-with-dependencies.jar - - --workers - - '{{inputs.parameters.workers}}' - - --rounds - - '{{inputs.parameters.rounds}}' - - --conf - - gs://ml-pipeline-playground/trainconfcla.json - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/model' - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-train:0.0.42 - inputs: - parameters: - - name: analyze-output - - name: create-cluster-output - - name: output - - name: project - - name: region - - name: rounds - - name: target - - name: transform-eval - - name: transform-train - - name: workers - name: train - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: train-output - valueFrom: - path: /output.txt - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --cluster - - '{{inputs.parameters.create-cluster-output}}' - - --train - - '{{inputs.parameters.train-data}}' - - --eval - - '{{inputs.parameters.eval-data}}' - - --analysis - - '{{inputs.parameters.analyze-output}}' - - --target - - '{{inputs.parameters.target}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/transform' - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:0.0.42 - inputs: - parameters: - - name: analyze-output - - name: create-cluster-output - - name: eval-data - - name: output - - name: project - - name: region - - name: target - - name: train-data - name: transform - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: transform-eval - valueFrom: - path: /output_eval.txt - - name: transform-train - valueFrom: - path: /output_train.txt - - dag: - tasks: - - arguments: - parameters: - - name: eval-data - value: '{{inputs.parameters.eval-data}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: rounds - value: '{{inputs.parameters.rounds}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: target - value: '{{inputs.parameters.target}}' - - name: train-data - value: '{{inputs.parameters.train-data}}' - - name: true-label - value: '{{inputs.parameters.true-label}}' - - name: workers - value: '{{inputs.parameters.workers}}' - name: exit-handler-1 - template: exit-handler-1 - inputs: - parameters: - - name: eval-data - - name: output - - name: project - - name: region - - name: rounds - - name: schema - - name: target - - name: train-data - - name: true-label - - name: workers - name: xgboost-trainer diff --git a/backend/src/apiserver/server/util.go b/backend/src/apiserver/server/util.go index 6c602d2c594..7ab28d346d4 100644 --- a/backend/src/apiserver/server/util.go +++ b/backend/src/apiserver/server/util.go @@ -61,7 +61,7 @@ func isYamlFile(fileName string) bool { return strings.HasSuffix(fileName, ".yaml") || strings.HasSuffix(fileName, ".yml") } -func decompressPipelineTarball(compressedFile []byte) ([]byte, error) { +func DecompressPipelineTarball(compressedFile []byte) ([]byte, error) { gzipReader, err := gzip.NewReader(bytes.NewReader(compressedFile)) if err != nil { return nil, util.NewInvalidInputErrorWithDetails(err, "Error extracting pipeline from the tarball file. Not a valid tarball file.") @@ -98,7 +98,7 @@ func ReadPipelineFile(fileName string, fileReader io.Reader, maxFileLength int) } // Decompress if file is tarball - decompressedFile, err := decompressPipelineTarball(pipelineFileBytes) + decompressedFile, err := DecompressPipelineTarball(pipelineFileBytes) if err != nil { return nil, util.Wrap(err, "Error decompress the pipeline file") } diff --git a/backend/src/apiserver/server/util_test.go b/backend/src/apiserver/server/util_test.go index c3b265d20a9..541e8d58ad0 100644 --- a/backend/src/apiserver/server/util_test.go +++ b/backend/src/apiserver/server/util_test.go @@ -56,7 +56,7 @@ func TestLoadFile_ExceedSizeLimit(t *testing.T) { func TestDecompressPipelineTarball(t *testing.T) { tarballByte, _ := ioutil.ReadFile("test/arguments_tarball/arguments.tar.gz") - pipelineFile, err := decompressPipelineTarball(tarballByte) + pipelineFile, err := DecompressPipelineTarball(tarballByte) assert.Nil(t, err) expectedPipelineFile, _ := ioutil.ReadFile("test/arguments_tarball/arguments-parameters.yaml") @@ -65,21 +65,21 @@ func TestDecompressPipelineTarball(t *testing.T) { func TestDecompressPipelineTarball_MalformattedTarball(t *testing.T) { tarballByte, _ := ioutil.ReadFile("test/malformatted_tarball.tar.gz") - _, err := decompressPipelineTarball(tarballByte) + _, err := DecompressPipelineTarball(tarballByte) assert.NotNil(t, err) assert.Contains(t, err.Error(), "Not a valid tarball file") } func TestDecompressPipelineTarball_NonYamlTarball(t *testing.T) { tarballByte, _ := ioutil.ReadFile("test/non_yaml_tarball/non_yaml_tarball.tar.gz") - _, err := decompressPipelineTarball(tarballByte) + _, err := DecompressPipelineTarball(tarballByte) assert.NotNil(t, err) assert.Contains(t, err.Error(), "Expecting a YAML file inside the tarball") } func TestDecompressPipelineTarball_EmptyTarball(t *testing.T) { tarballByte, _ := ioutil.ReadFile("test/empty_tarball/empty.tar.gz") - _, err := decompressPipelineTarball(tarballByte) + _, err := DecompressPipelineTarball(tarballByte) assert.NotNil(t, err) assert.Contains(t, err.Error(), "Not a valid tarball file") } diff --git a/samples/basic/sequential.yaml b/samples/basic/sequential.yaml deleted file mode 100644 index 195c9981024..00000000000 --- a/samples/basic/sequential.yaml +++ /dev/null @@ -1,125 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: sequential- -spec: - arguments: - parameters: - - name: url - value: gs://ml-pipeline-playground/shakespeare1.txt - entrypoint: sequential - serviceAccountName: pipeline-runner - templates: - - container: - args: - - gsutil cat {{inputs.parameters.url}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url - name: download - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo {{inputs.parameters.download-downloaded}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: download-downloaded - name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: echo - template: echo - inputs: - parameters: - - name: url - name: sequential