From 8df55d3abe1d2d584f2852e5f20a4c6fabdc4a9b Mon Sep 17 00:00:00 2001 From: Yang Pan Date: Mon, 5 Nov 2018 18:00:49 -0800 Subject: [PATCH 01/12] compile samples --- backend/Dockerfile | 14 +- .../src/apiserver/config/sample_config.json | 42 ++ backend/src/apiserver/main.go | 23 +- backend/src/apiserver/samples/condition.yaml | 269 -------- .../src/apiserver/samples/default_value.yaml | 114 ---- .../src/apiserver/samples/exit_handler.yaml | 162 ----- .../src/apiserver/samples/parallel_join.yaml | 168 ----- .../samples/resnet_training_pipeline.yaml | 305 --------- .../src/apiserver/samples/sample_config.json | 42 -- backend/src/apiserver/samples/sequential.yaml | 113 ---- .../taxi_cab_classification_pipeline.yaml | 421 ------------ .../samples/xgboost_training_cm.yaml | 640 ------------------ backend/src/apiserver/server/util.go | 4 +- backend/src/apiserver/server/util_test.go | 8 +- samples/basic/sequential.yaml | 125 ---- 15 files changed, 79 insertions(+), 2371 deletions(-) create mode 100644 backend/src/apiserver/config/sample_config.json delete mode 100644 backend/src/apiserver/samples/condition.yaml delete mode 100644 backend/src/apiserver/samples/default_value.yaml delete mode 100644 backend/src/apiserver/samples/exit_handler.yaml delete mode 100644 backend/src/apiserver/samples/parallel_join.yaml delete mode 100644 backend/src/apiserver/samples/resnet_training_pipeline.yaml delete mode 100644 backend/src/apiserver/samples/sample_config.json delete mode 100644 backend/src/apiserver/samples/sequential.yaml delete mode 100644 backend/src/apiserver/samples/taxi_cab_classification_pipeline.yaml delete mode 100644 backend/src/apiserver/samples/xgboost_training_cm.yaml delete mode 100644 samples/basic/sequential.yaml diff --git a/backend/Dockerfile b/backend/Dockerfile index 30b559c3af3..74d3223e094 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -14,6 +14,15 @@ COPY . . RUN apk add --update gcc musl-dev RUN go build -o /bin/apiserver backend/src/apiserver/*.go +FROM python:3.5.0-slim as compiler + +# This is hard coded to 0.0.26. Once kfp DSK release process is automated, +# we can dynamically refer to the version from same commit SHA. +RUN pip install https://storage.googleapis.com/ml-pipeline/release/0.0.26/kfp-0.0.26.tar.gz --upgrade +WORKDIR /samples +COPY ./samples . +RUN find . -maxdepth 2 -name "*.py" -exec python3 {} \; + FROM alpine ARG COMMIT_SHA=unknown @@ -24,7 +33,8 @@ WORKDIR /bin COPY --from=builder /bin/apiserver /bin/apiserver COPY --from=builder /go/src/github.com/kubeflow/pipelines/third_party/license.txt /bin/license.txt COPY backend/src/apiserver/config/ /config -COPY backend/src/apiserver/samples/ /samples + +COPY --from=compiler /samples/ /samples/ # Adding CA certificate so API server can download pipeline through URL RUN apk add ca-certificates @@ -33,4 +43,4 @@ RUN apk add ca-certificates EXPOSE 8888 # Start the apiserver -CMD apiserver --config=/config --sampleconfig=/samples/sample_config.json +CMD apiserver --config=/config --sampleconfig=/config/sample_config.json diff --git a/backend/src/apiserver/config/sample_config.json b/backend/src/apiserver/config/sample_config.json new file mode 100644 index 00000000000..06f61e08fe6 --- /dev/null +++ b/backend/src/apiserver/config/sample_config.json @@ -0,0 +1,42 @@ +[ + { + "name":"Sample-Basic-Condition", + "description":"shows how to use dsl.Condition", + "file":"/samples/basic/condition.py.tar.gz" + }, + { + "name":"Sample-Basic-Default Value", + "description":"A pipeline with parameter and default value", + "file":"/samples/basic/default_value.py.tar.gz" + }, + { + "name":"Sample-Basic-Exit Handler", + "description":"Download a message and print it out. Exit Handler will run at the end", + "file":"/samples/basic/exit_handler.py.tar.gz" + }, + { + "name":"Sample-Basic-Parallel Join", + "description":"Download two messages in parallel and print the concatenated result", + "file":"/samples/basic/parallel_join.py.tar.gz" + }, + { + "name":"Sample-Basic-Sequential", + "description":"Download a message and print it", + "file":"/samples/basic/sequential.py.tar.gz" + }, + { + "name":"Sample-TFX-Taxi Cab Classification", + "description":"Example pipeline that does classification with model analysis based on a public BigQuery dataset", + "file":"/samples/tfx/taxi_cab_classification_pipeline.py.tar.gz" + }, + { + "name":"Sample-CMLE-Resnet Training", + "description":"Demonstrate the ResNet50 predict", + "file":"/samples/resnet-cmle/resnet_training_pipeline.py.tar.gz" + }, + { + "name":"Sample-Xgboost-Training with Confusion Matrix", + "description":"A trainer that does end-to-end distributed training for XGBoost models", + "file":"/samples/xgboost-spark/xgboost_training_cm.py.tar.gz" + } +] \ No newline at end of file diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go index 746c9834f90..2d4bf31c2be 100644 --- a/backend/src/apiserver/main.go +++ b/backend/src/apiserver/main.go @@ -30,6 +30,7 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/runtime" "google.golang.org/grpc" "google.golang.org/grpc/reflection" + "time" ) var ( @@ -122,7 +123,7 @@ func registerHttpHandlerFromEndpoint(handler RegisterHttpHandlerFromEndpoint, se func loadSamples(resourceManager *resource.ResourceManager) { configBytes, err := ioutil.ReadFile(*sampleConfigPath) if err != nil { - glog.Warningf("Failed to read sample configurations. Err: %v", err.Error()) + glog.Fatalf("Failed to read sample configurations. Err: %v", err.Error()) return } type config struct { @@ -132,16 +133,30 @@ func loadSamples(resourceManager *resource.ResourceManager) { } var configs []config if json.Unmarshal(configBytes, &configs) != nil { - glog.Warningf("Failed to read sample configurations. Err: %v", err.Error()) + glog.Fatalf("Failed to read sample configurations. Err: %v", err.Error()) return } for _, config := range configs { sampleBytes, err := ioutil.ReadFile(config.File) if err != nil { - glog.Warningf("Failed to load sample %s. Error: %v", config.Name, err.Error()) + glog.Fatalf("Failed to load sample %s. Error: %v", config.Name, err.Error()) continue } - resourceManager.CreatePipeline(config.Name, config.Description, sampleBytes) + // Decompress if file is tarball + decompressedFile, err := server.DecompressPipelineTarball(sampleBytes) + if err!=nil{ + glog.Fatalf("Failed to decompress the file %s. Error: %v", config.Name, err.Error()) + continue + } + _, err = resourceManager.CreatePipeline(config.Name, config.Description, decompressedFile) + if err!=nil{ + glog.Fatalf("Failed to create pipeline for %s. Error: %v", config.Name, err.Error()) + continue + } + + // Since the default sorting is by create time, + // Sleep one second makes sure the samples are showing up in the same order as they are added. + time.Sleep(1*time.Second) } glog.Info("All samples are loaded.") } diff --git a/backend/src/apiserver/samples/condition.yaml b/backend/src/apiserver/samples/condition.yaml deleted file mode 100644 index be071ff203b..00000000000 --- a/backend/src/apiserver/samples/condition.yaml +++ /dev/null @@ -1,269 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: pipeline-flip-coin- -spec: - arguments: - parameters: [] - entrypoint: pipeline-flip-coin - serviceAccountName: pipeline-runner - templates: - - inputs: - parameters: - - name: flip-output - name: condition-1 - steps: - - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: condition-1-child - template: condition-1-child - when: '{{inputs.parameters.flip-output}} == heads' - - dag: - tasks: - - arguments: - parameters: - - name: flip-again-output - value: '{{tasks.flip-again.outputs.parameters.flip-again-output}}' - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - dependencies: - - flip-again - name: condition-2 - template: condition-2 - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: flip-again - template: flip-again - inputs: - parameters: - - name: flip-output - name: condition-1-child - - inputs: - parameters: - - name: flip-again-output - - name: flip-output - name: condition-2 - steps: - - - arguments: - parameters: - - name: flip-again-output - value: '{{inputs.parameters.flip-again-output}}' - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: condition-2-child - template: condition-2-child - when: '{{inputs.parameters.flip-again-output}} == tails' - - dag: - tasks: - - arguments: - parameters: - - name: flip-again-output - value: '{{inputs.parameters.flip-again-output}}' - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: print1 - template: print1 - inputs: - parameters: - - name: flip-again-output - - name: flip-output - name: condition-2-child - - inputs: - parameters: - - name: flip-output - name: condition-3 - steps: - - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: condition-3-child - template: condition-3-child - when: '{{inputs.parameters.flip-output}} == tails' - - dag: - tasks: - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: print2 - template: print2 - inputs: - parameters: - - name: flip-output - name: condition-3-child - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.6 - name: flip - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: flip-output - valueFrom: - path: /tmp/output - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.6 - name: flip-again - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: flip-again-output - valueFrom: - path: /tmp/output - - dag: - tasks: - - arguments: - parameters: - - name: flip-output - value: '{{tasks.flip.outputs.parameters.flip-output}}' - dependencies: - - flip - name: condition-1 - template: condition-1 - - arguments: - parameters: - - name: flip-output - value: '{{tasks.flip.outputs.parameters.flip-output}}' - dependencies: - - flip - name: condition-3 - template: condition-3 - - name: flip - template: flip - name: pipeline-flip-coin - - container: - command: - - echo - - '"it was tail"' - image: alpine:3.6 - name: print1 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - container: - command: - - echo - - '"it was tail"' - image: alpine:3.6 - name: print2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact diff --git a/backend/src/apiserver/samples/default_value.yaml b/backend/src/apiserver/samples/default_value.yaml deleted file mode 100644 index d8ed6d653c2..00000000000 --- a/backend/src/apiserver/samples/default_value.yaml +++ /dev/null @@ -1,114 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: default-value- -spec: - arguments: - parameters: - - name: url - value: gs://ml-pipeline/shakespeare1.txt - entrypoint: default-value - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: echo - template: echo - inputs: - parameters: - - name: url - name: default-value - - container: - args: - - gsutil cat {{inputs.parameters.url}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url - name: download - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo {{inputs.parameters.download-downloaded}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: download-downloaded - name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact diff --git a/backend/src/apiserver/samples/exit_handler.yaml b/backend/src/apiserver/samples/exit_handler.yaml deleted file mode 100644 index 4ef846d5291..00000000000 --- a/backend/src/apiserver/samples/exit_handler.yaml +++ /dev/null @@ -1,162 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: exit-handler- -spec: - arguments: - parameters: - - name: url - entrypoint: exit-handler - onExit: finally - serviceAccountName: pipeline-runner - templates: - - container: - args: - - gsutil cat {{inputs.parameters.url}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url - name: download - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo {{inputs.parameters.download-downloaded}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: download-downloaded - name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: exit-handler-1 - template: exit-handler-1 - - name: finally - template: finally - inputs: - parameters: - - name: url - name: exit-handler - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: echo - template: echo - inputs: - parameters: - - name: url - name: exit-handler-1 - - container: - command: - - echo - - exit! - image: library/bash:4.4.23 - name: finally - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact diff --git a/backend/src/apiserver/samples/parallel_join.yaml b/backend/src/apiserver/samples/parallel_join.yaml deleted file mode 100644 index df445867f91..00000000000 --- a/backend/src/apiserver/samples/parallel_join.yaml +++ /dev/null @@ -1,168 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: parallel-and-join- -spec: - arguments: - parameters: - - name: url1 - - name: url2 - entrypoint: parallel-and-join - serviceAccountName: pipeline-runner - templates: - - container: - args: - - gsutil cat {{inputs.parameters.url1}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url1 - name: download1 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download1-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - gsutil cat {{inputs.parameters.url2}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url2 - name: download2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download2-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo {{inputs.parameters.download1-downloaded}} {{inputs.parameters.download2-downloaded}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: download1-downloaded - - name: download2-downloaded - name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - dag: - tasks: - - arguments: - parameters: - - name: url1 - value: '{{inputs.parameters.url1}}' - name: download1 - template: download1 - - arguments: - parameters: - - name: url2 - value: '{{inputs.parameters.url2}}' - name: download2 - template: download2 - - arguments: - parameters: - - name: download1-downloaded - value: '{{tasks.download1.outputs.parameters.download1-downloaded}}' - - name: download2-downloaded - value: '{{tasks.download2.outputs.parameters.download2-downloaded}}' - dependencies: - - download1 - - download2 - name: echo - template: echo - inputs: - parameters: - - name: url1 - - name: url2 - name: parallel-and-join diff --git a/backend/src/apiserver/samples/resnet_training_pipeline.yaml b/backend/src/apiserver/samples/resnet_training_pipeline.yaml deleted file mode 100644 index 0eecea31bb5..00000000000 --- a/backend/src/apiserver/samples/resnet_training_pipeline.yaml +++ /dev/null @@ -1,305 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: resnet-train-pipeline- -spec: - arguments: - parameters: - - name: project-id - - name: bucket - - name: region - value: us-central1 - - name: model - value: bolts - - name: version - value: beta1 - - name: tf-version - value: '1.8' - - name: train-csv - value: gs://bolts_image_dataset/bolt_images_train.csv - - name: validation-csv - value: gs://bolts_image_dataset/bolt_images_validate.csv - - name: labels - value: gs://bolts_image_dataset/labels.txt - - name: depth - value: '50' - - name: train-batch-size - value: '1024' - - name: eval-batch-size - value: '1024' - - name: steps-per-eval - value: '250' - - name: train-steps - value: '10000' - - name: num-train-images - value: '218593' - - name: num-eval-images - value: '54648' - - name: num-label-classes - value: '10' - entrypoint: resnet-train-pipeline - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --model - - '{{inputs.parameters.model}}' - - --version - - '{{inputs.parameters.version}}' - - --project_id - - '{{inputs.parameters.project-id}}' - - --region - - '{{inputs.parameters.region}}' - - --model_dir - - '{{inputs.parameters.train-trained}}' - - --TFVERSION - - '{{inputs.parameters.tf-version}}' - image: gcr.io/ml-pipeline/resnet-deploy:0.0.42 - inputs: - parameters: - - name: model - - name: project-id - - name: region - - name: tf-version - - name: train-trained - - name: version - name: deploy - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - container: - args: - - --project_id - - '{{inputs.parameters.project-id}}' - - --bucket - - '{{inputs.parameters.bucket}}' - - --train_csv - - '{{inputs.parameters.train-csv}}' - - --validation_csv - - '{{inputs.parameters.validation-csv}}' - - --labels - - '{{inputs.parameters.labels}}' - image: gcr.io/ml-pipeline/resnet-preprocess:0.0.42 - inputs: - parameters: - - name: bucket - - name: labels - - name: project-id - - name: train-csv - - name: validation-csv - name: preprocess - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: preprocess-preprocessed - valueFrom: - path: /output.txt - - dag: - tasks: - - arguments: - parameters: - - name: model - value: '{{inputs.parameters.model}}' - - name: project-id - value: '{{inputs.parameters.project-id}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: tf-version - value: '{{inputs.parameters.tf-version}}' - - name: train-trained - value: '{{tasks.train.outputs.parameters.train-trained}}' - - name: version - value: '{{inputs.parameters.version}}' - dependencies: - - train - name: deploy - template: deploy - - arguments: - parameters: - - name: bucket - value: '{{inputs.parameters.bucket}}' - - name: labels - value: '{{inputs.parameters.labels}}' - - name: project-id - value: '{{inputs.parameters.project-id}}' - - name: train-csv - value: '{{inputs.parameters.train-csv}}' - - name: validation-csv - value: '{{inputs.parameters.validation-csv}}' - name: preprocess - template: preprocess - - arguments: - parameters: - - name: bucket - value: '{{inputs.parameters.bucket}}' - - name: depth - value: '{{inputs.parameters.depth}}' - - name: eval-batch-size - value: '{{inputs.parameters.eval-batch-size}}' - - name: num-eval-images - value: '{{inputs.parameters.num-eval-images}}' - - name: num-label-classes - value: '{{inputs.parameters.num-label-classes}}' - - name: num-train-images - value: '{{inputs.parameters.num-train-images}}' - - name: preprocess-preprocessed - value: '{{tasks.preprocess.outputs.parameters.preprocess-preprocessed}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: steps-per-eval - value: '{{inputs.parameters.steps-per-eval}}' - - name: tf-version - value: '{{inputs.parameters.tf-version}}' - - name: train-batch-size - value: '{{inputs.parameters.train-batch-size}}' - - name: train-steps - value: '{{inputs.parameters.train-steps}}' - dependencies: - - preprocess - name: train - template: train - inputs: - parameters: - - name: bucket - - name: depth - - name: eval-batch-size - - name: labels - - name: model - - name: num-eval-images - - name: num-label-classes - - name: num-train-images - - name: project-id - - name: region - - name: steps-per-eval - - name: tf-version - - name: train-batch-size - - name: train-csv - - name: train-steps - - name: validation-csv - - name: version - name: resnet-train-pipeline - - container: - args: - - --data_dir - - '{{inputs.parameters.preprocess-preprocessed}}' - - --bucket - - '{{inputs.parameters.bucket}}' - - --region - - '{{inputs.parameters.region}}' - - --depth - - '{{inputs.parameters.depth}}' - - --train_batch_size - - '{{inputs.parameters.train-batch-size}}' - - --eval_batch_size - - '{{inputs.parameters.eval-batch-size}}' - - --steps_per_eval - - '{{inputs.parameters.steps-per-eval}}' - - --train_steps - - '{{inputs.parameters.train-steps}}' - - --num_train_images - - '{{inputs.parameters.num-train-images}}' - - --num_eval_images - - '{{inputs.parameters.num-eval-images}}' - - --num_label_classes - - '{{inputs.parameters.num-label-classes}}' - - --TFVERSION - - '{{inputs.parameters.tf-version}}' - image: gcr.io/ml-pipeline/resnet-train:0.0.42 - inputs: - parameters: - - name: bucket - - name: depth - - name: eval-batch-size - - name: num-eval-images - - name: num-label-classes - - name: num-train-images - - name: preprocess-preprocessed - - name: region - - name: steps-per-eval - - name: tf-version - - name: train-batch-size - - name: train-steps - name: train - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: train-trained - valueFrom: - path: /output.txt diff --git a/backend/src/apiserver/samples/sample_config.json b/backend/src/apiserver/samples/sample_config.json deleted file mode 100644 index b97b11c0ff1..00000000000 --- a/backend/src/apiserver/samples/sample_config.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "name":"condition", - "description":"shows how to use dsl.Condition", - "file":"/samples/condition.yaml" - }, - { - "name":"default value", - "description":"A pipeline with parameter and default value", - "file":"/samples/default_value.yaml" - }, - { - "name":"exit handler", - "description":"Download a message and print it out. Exit Handler will run at the end", - "file":"/samples/exit_handler.yaml" - }, - { - "name":"parallel join", - "description":"Download two messages in parallel and print the concatenated result", - "file":"/samples/parallel_join.yaml" - }, - { - "name":"sequential", - "description":"Download a message and print it", - "file":"/samples/sequential.yaml" - }, - { - "name":"taxi cab classification", - "description":"Example pipeline that does classification with model analysis based on a public BigQuery dataset", - "file":"/samples/taxi_cab_classification_pipeline.yaml" - }, - { - "name":"resnet training", - "description":"Demonstrate the ResNet50 predict", - "file":"/samples/resnet_training_pipeline.yaml" - }, - { - "name":"xgboost training - confusion matrix", - "description":"A trainer that does end-to-end distributed training for XGBoost models", - "file":"/samples/xgboost_training_cm.yaml" - } -] \ No newline at end of file diff --git a/backend/src/apiserver/samples/sequential.yaml b/backend/src/apiserver/samples/sequential.yaml deleted file mode 100644 index 2aecc6b6f35..00000000000 --- a/backend/src/apiserver/samples/sequential.yaml +++ /dev/null @@ -1,113 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: sequential- -spec: - arguments: - parameters: - - name: url - entrypoint: sequential - serviceAccountName: pipeline-runner - templates: - - container: - args: - - gsutil cat {{inputs.parameters.url}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url - name: download - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo "{{inputs.parameters.download-downloaded}}" - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: download-downloaded - name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: echo - template: echo - inputs: - parameters: - - name: url - name: sequential diff --git a/backend/src/apiserver/samples/taxi_cab_classification_pipeline.yaml b/backend/src/apiserver/samples/taxi_cab_classification_pipeline.yaml deleted file mode 100644 index e621a9ad0d1..00000000000 --- a/backend/src/apiserver/samples/taxi_cab_classification_pipeline.yaml +++ /dev/null @@ -1,421 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: tfma-taxi-cab-classification-pipeline-example- -spec: - arguments: - parameters: - - name: output - - name: project - - name: schema - value: gs://ml-pipeline-playground/tfma/taxi-cab-classification/schema.json - - name: train - value: gs://ml-pipeline-playground/tfma/taxi-cab-classification/train.csv - - name: evaluation - value: gs://ml-pipeline-playground/tfma/taxi-cab-classification/eval.csv - - name: preprocess-mode - value: local - - name: preprocess-module - value: gs://ml-pipeline-playground/tfma/taxi-cab-classification/preprocessing.py - - name: target - value: tips - - name: learning-rate - value: '0.1' - - name: hidden-layer-size - value: '1500' - - name: steps - value: '3000' - - name: predict-mode - value: local - - name: analyze-mode - value: local - - name: analyze-slice-column - value: trip_start_hour - entrypoint: tfma-taxi-cab-classification-pipeline-example - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --model - - '{{inputs.parameters.training-train}}' - - --eval - - '{{inputs.parameters.evaluation}}' - - --schema - - '{{inputs.parameters.schema}}' - - --project - - '{{inputs.parameters.project}}' - - --mode - - '{{inputs.parameters.analyze-mode}}' - - --slice-columns - - '{{inputs.parameters.analyze-slice-column}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/analysis' - image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:0.0.42 - inputs: - parameters: - - name: analyze-mode - - name: analyze-slice-column - - name: evaluation - - name: output - - name: project - - name: schema - - name: training-train - name: analysis - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: analysis-analysis - valueFrom: - path: /output.txt - - container: - args: - - --model-path - - '{{inputs.parameters.training-train}}' - - --server-name - - taxi-cab-classification-model-{{workflow.name}} - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:0.0.42 - inputs: - parameters: - - name: training-train - name: deploy - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - container: - args: - - --data - - '{{inputs.parameters.evaluation}}' - - --schema - - '{{inputs.parameters.schema}}' - - --target - - '{{inputs.parameters.target}}' - - --model - - '{{inputs.parameters.training-train}}' - - --mode - - '{{inputs.parameters.predict-mode}}' - - --project - - '{{inputs.parameters.project}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/predict' - image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:0.0.42 - inputs: - parameters: - - name: evaluation - - name: output - - name: predict-mode - - name: project - - name: schema - - name: target - - name: training-train - name: prediction - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: prediction-prediction - valueFrom: - path: /output.txt - - container: - args: - - --train - - '{{inputs.parameters.train}}' - - --eval - - '{{inputs.parameters.evaluation}}' - - --schema - - '{{inputs.parameters.schema}}' - - --project - - '{{inputs.parameters.project}}' - - --mode - - '{{inputs.parameters.preprocess-mode}}' - - --preprocessing-module - - '{{inputs.parameters.preprocess-module}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/transformed' - image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:0.0.42 - inputs: - parameters: - - name: evaluation - - name: output - - name: preprocess-mode - - name: preprocess-module - - name: project - - name: schema - - name: train - name: preprocess - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: preprocess-transformed - valueFrom: - path: /output.txt - - dag: - tasks: - - arguments: - parameters: - - name: analyze-mode - value: '{{inputs.parameters.analyze-mode}}' - - name: analyze-slice-column - value: '{{inputs.parameters.analyze-slice-column}}' - - name: evaluation - value: '{{inputs.parameters.evaluation}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: training-train - value: '{{tasks.training.outputs.parameters.training-train}}' - dependencies: - - training - name: analysis - template: analysis - - arguments: - parameters: - - name: training-train - value: '{{tasks.training.outputs.parameters.training-train}}' - dependencies: - - training - name: deploy - template: deploy - - arguments: - parameters: - - name: evaluation - value: '{{inputs.parameters.evaluation}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: predict-mode - value: '{{inputs.parameters.predict-mode}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: target - value: '{{inputs.parameters.target}}' - - name: training-train - value: '{{tasks.training.outputs.parameters.training-train}}' - dependencies: - - training - name: prediction - template: prediction - - arguments: - parameters: - - name: evaluation - value: '{{inputs.parameters.evaluation}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: preprocess-mode - value: '{{inputs.parameters.preprocess-mode}}' - - name: preprocess-module - value: '{{inputs.parameters.preprocess-module}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: train - value: '{{inputs.parameters.train}}' - name: preprocess - template: preprocess - - arguments: - parameters: - - name: hidden-layer-size - value: '{{inputs.parameters.hidden-layer-size}}' - - name: learning-rate - value: '{{inputs.parameters.learning-rate}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: preprocess-module - value: '{{inputs.parameters.preprocess-module}}' - - name: preprocess-transformed - value: '{{tasks.preprocess.outputs.parameters.preprocess-transformed}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: steps - value: '{{inputs.parameters.steps}}' - - name: target - value: '{{inputs.parameters.target}}' - dependencies: - - preprocess - name: training - template: training - inputs: - parameters: - - name: analyze-mode - - name: analyze-slice-column - - name: evaluation - - name: hidden-layer-size - - name: learning-rate - - name: output - - name: predict-mode - - name: preprocess-mode - - name: preprocess-module - - name: project - - name: schema - - name: steps - - name: target - - name: train - name: tfma-taxi-cab-classification-pipeline-example - - container: - args: - - --transformed-data-dir - - '{{inputs.parameters.preprocess-transformed}}' - - --schema - - '{{inputs.parameters.schema}}' - - --learning-rate - - '{{inputs.parameters.learning-rate}}' - - --hidden-layer-size - - '{{inputs.parameters.hidden-layer-size}}' - - --steps - - '{{inputs.parameters.steps}}' - - --target - - '{{inputs.parameters.target}}' - - --preprocessing-module - - '{{inputs.parameters.preprocess-module}}' - - --job-dir - - '{{inputs.parameters.output}}/{{workflow.name}}/train' - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.0.42 - inputs: - parameters: - - name: hidden-layer-size - - name: learning-rate - - name: output - - name: preprocess-module - - name: preprocess-transformed - - name: schema - - name: steps - - name: target - name: training - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: training-train - valueFrom: - path: /output.txt diff --git a/backend/src/apiserver/samples/xgboost_training_cm.yaml b/backend/src/apiserver/samples/xgboost_training_cm.yaml deleted file mode 100644 index 885e7e63bd2..00000000000 --- a/backend/src/apiserver/samples/xgboost_training_cm.yaml +++ /dev/null @@ -1,640 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: xgboost-trainer- -spec: - arguments: - parameters: - - name: output - - name: project - - name: region - value: us-central1 - - name: train-data - value: gs://ml-pipeline-playground/sfpd/train.csv - - name: eval-data - value: gs://ml-pipeline-playground/sfpd/eval.csv - - name: schema - value: gs://ml-pipeline-playground/sfpd/schema.json - - name: target - value: resolution - - name: rounds - value: '200' - - name: workers - value: '2' - - name: true-label - value: ACTION - entrypoint: xgboost-trainer - onExit: delete-cluster - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --cluster - - '{{inputs.parameters.create-cluster-output}}' - - --schema - - '{{inputs.parameters.schema}}' - - --train - - '{{inputs.parameters.train-data}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/analysis' - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:0.0.42 - inputs: - parameters: - - name: create-cluster-output - - name: output - - name: project - - name: region - - name: schema - - name: train-data - name: analyze - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: analyze-output - valueFrom: - path: /output.txt - - container: - args: - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/confusionmatrix' - - --predictions - - '{{inputs.parameters.predict-output}}' - image: gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:0.0.42 - inputs: - parameters: - - name: output - - name: predict-output - name: confusion-matrix - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --name - - xgb-{{workflow.name}} - - --staging - - '{{inputs.parameters.output}}' - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:0.0.42 - inputs: - parameters: - - name: output - - name: project - - name: region - name: create-cluster - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: create-cluster-output - valueFrom: - path: /output.txt - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --name - - xgb-{{workflow.name}} - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:0.0.42 - inputs: - parameters: - - name: project - - name: region - name: delete-cluster - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - dag: - tasks: - - arguments: - parameters: - - name: create-cluster-output - value: '{{tasks.create-cluster.outputs.parameters.create-cluster-output}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: train-data - value: '{{inputs.parameters.train-data}}' - dependencies: - - create-cluster - name: analyze - template: analyze - - arguments: - parameters: - - name: output - value: '{{inputs.parameters.output}}' - - name: predict-output - value: '{{tasks.predict.outputs.parameters.predict-output}}' - dependencies: - - predict - name: confusion-matrix - template: confusion-matrix - - arguments: - parameters: - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - name: create-cluster - template: create-cluster - - arguments: - parameters: - - name: analyze-output - value: '{{tasks.analyze.outputs.parameters.analyze-output}}' - - name: create-cluster-output - value: '{{tasks.create-cluster.outputs.parameters.create-cluster-output}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: target - value: '{{inputs.parameters.target}}' - - name: train-output - value: '{{tasks.train.outputs.parameters.train-output}}' - - name: transform-eval - value: '{{tasks.transform.outputs.parameters.transform-eval}}' - dependencies: - - analyze - - create-cluster - - train - - transform - name: predict - template: predict - - arguments: - parameters: - - name: output - value: '{{inputs.parameters.output}}' - - name: predict-output - value: '{{tasks.predict.outputs.parameters.predict-output}}' - - name: true-label - value: '{{inputs.parameters.true-label}}' - dependencies: - - predict - name: roc - template: roc - - arguments: - parameters: - - name: analyze-output - value: '{{tasks.analyze.outputs.parameters.analyze-output}}' - - name: create-cluster-output - value: '{{tasks.create-cluster.outputs.parameters.create-cluster-output}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: rounds - value: '{{inputs.parameters.rounds}}' - - name: target - value: '{{inputs.parameters.target}}' - - name: transform-eval - value: '{{tasks.transform.outputs.parameters.transform-eval}}' - - name: transform-train - value: '{{tasks.transform.outputs.parameters.transform-train}}' - - name: workers - value: '{{inputs.parameters.workers}}' - dependencies: - - analyze - - create-cluster - - transform - name: train - template: train - - arguments: - parameters: - - name: analyze-output - value: '{{tasks.analyze.outputs.parameters.analyze-output}}' - - name: create-cluster-output - value: '{{tasks.create-cluster.outputs.parameters.create-cluster-output}}' - - name: eval-data - value: '{{inputs.parameters.eval-data}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: target - value: '{{inputs.parameters.target}}' - - name: train-data - value: '{{inputs.parameters.train-data}}' - dependencies: - - analyze - - create-cluster - name: transform - template: transform - inputs: - parameters: - - name: eval-data - - name: output - - name: project - - name: region - - name: rounds - - name: schema - - name: target - - name: train-data - - name: true-label - - name: workers - name: exit-handler-1 - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --cluster - - '{{inputs.parameters.create-cluster-output}}' - - --predict - - '{{inputs.parameters.transform-eval}}' - - --analysis - - '{{inputs.parameters.analyze-output}}' - - --target - - '{{inputs.parameters.target}}' - - --package - - gs://ml-pipeline-playground/xgboost4j-example-0.8-SNAPSHOT-jar-with-dependencies.jar - - --model - - '{{inputs.parameters.train-output}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/predict' - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:0.0.42 - inputs: - parameters: - - name: analyze-output - - name: create-cluster-output - - name: output - - name: project - - name: region - - name: target - - name: train-output - - name: transform-eval - name: predict - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: predict-output - valueFrom: - path: /output.txt - - container: - args: - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/roc' - - --predictions - - '{{inputs.parameters.predict-output}}' - - --trueclass - - '{{inputs.parameters.true-label}}' - image: gcr.io/ml-pipeline/ml-pipeline-local-roc:0.0.42 - inputs: - parameters: - - name: output - - name: predict-output - - name: true-label - name: roc - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --cluster - - '{{inputs.parameters.create-cluster-output}}' - - --train - - '{{inputs.parameters.transform-train}}' - - --eval - - '{{inputs.parameters.transform-eval}}' - - --analysis - - '{{inputs.parameters.analyze-output}}' - - --target - - '{{inputs.parameters.target}}' - - --package - - gs://ml-pipeline-playground/xgboost4j-example-0.8-SNAPSHOT-jar-with-dependencies.jar - - --workers - - '{{inputs.parameters.workers}}' - - --rounds - - '{{inputs.parameters.rounds}}' - - --conf - - gs://ml-pipeline-playground/trainconfcla.json - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/model' - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-train:0.0.42 - inputs: - parameters: - - name: analyze-output - - name: create-cluster-output - - name: output - - name: project - - name: region - - name: rounds - - name: target - - name: transform-eval - - name: transform-train - - name: workers - name: train - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: train-output - valueFrom: - path: /output.txt - - container: - args: - - --project - - '{{inputs.parameters.project}}' - - --region - - '{{inputs.parameters.region}}' - - --cluster - - '{{inputs.parameters.create-cluster-output}}' - - --train - - '{{inputs.parameters.train-data}}' - - --eval - - '{{inputs.parameters.eval-data}}' - - --analysis - - '{{inputs.parameters.analyze-output}}' - - --target - - '{{inputs.parameters.target}}' - - --output - - '{{inputs.parameters.output}}/{{workflow.name}}/transform' - image: gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:0.0.42 - inputs: - parameters: - - name: analyze-output - - name: create-cluster-output - - name: eval-data - - name: output - - name: project - - name: region - - name: target - - name: train-data - name: transform - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: transform-eval - valueFrom: - path: /output_eval.txt - - name: transform-train - valueFrom: - path: /output_train.txt - - dag: - tasks: - - arguments: - parameters: - - name: eval-data - value: '{{inputs.parameters.eval-data}}' - - name: output - value: '{{inputs.parameters.output}}' - - name: project - value: '{{inputs.parameters.project}}' - - name: region - value: '{{inputs.parameters.region}}' - - name: rounds - value: '{{inputs.parameters.rounds}}' - - name: schema - value: '{{inputs.parameters.schema}}' - - name: target - value: '{{inputs.parameters.target}}' - - name: train-data - value: '{{inputs.parameters.train-data}}' - - name: true-label - value: '{{inputs.parameters.true-label}}' - - name: workers - value: '{{inputs.parameters.workers}}' - name: exit-handler-1 - template: exit-handler-1 - inputs: - parameters: - - name: eval-data - - name: output - - name: project - - name: region - - name: rounds - - name: schema - - name: target - - name: train-data - - name: true-label - - name: workers - name: xgboost-trainer diff --git a/backend/src/apiserver/server/util.go b/backend/src/apiserver/server/util.go index 6c602d2c594..7ab28d346d4 100644 --- a/backend/src/apiserver/server/util.go +++ b/backend/src/apiserver/server/util.go @@ -61,7 +61,7 @@ func isYamlFile(fileName string) bool { return strings.HasSuffix(fileName, ".yaml") || strings.HasSuffix(fileName, ".yml") } -func decompressPipelineTarball(compressedFile []byte) ([]byte, error) { +func DecompressPipelineTarball(compressedFile []byte) ([]byte, error) { gzipReader, err := gzip.NewReader(bytes.NewReader(compressedFile)) if err != nil { return nil, util.NewInvalidInputErrorWithDetails(err, "Error extracting pipeline from the tarball file. Not a valid tarball file.") @@ -98,7 +98,7 @@ func ReadPipelineFile(fileName string, fileReader io.Reader, maxFileLength int) } // Decompress if file is tarball - decompressedFile, err := decompressPipelineTarball(pipelineFileBytes) + decompressedFile, err := DecompressPipelineTarball(pipelineFileBytes) if err != nil { return nil, util.Wrap(err, "Error decompress the pipeline file") } diff --git a/backend/src/apiserver/server/util_test.go b/backend/src/apiserver/server/util_test.go index c3b265d20a9..541e8d58ad0 100644 --- a/backend/src/apiserver/server/util_test.go +++ b/backend/src/apiserver/server/util_test.go @@ -56,7 +56,7 @@ func TestLoadFile_ExceedSizeLimit(t *testing.T) { func TestDecompressPipelineTarball(t *testing.T) { tarballByte, _ := ioutil.ReadFile("test/arguments_tarball/arguments.tar.gz") - pipelineFile, err := decompressPipelineTarball(tarballByte) + pipelineFile, err := DecompressPipelineTarball(tarballByte) assert.Nil(t, err) expectedPipelineFile, _ := ioutil.ReadFile("test/arguments_tarball/arguments-parameters.yaml") @@ -65,21 +65,21 @@ func TestDecompressPipelineTarball(t *testing.T) { func TestDecompressPipelineTarball_MalformattedTarball(t *testing.T) { tarballByte, _ := ioutil.ReadFile("test/malformatted_tarball.tar.gz") - _, err := decompressPipelineTarball(tarballByte) + _, err := DecompressPipelineTarball(tarballByte) assert.NotNil(t, err) assert.Contains(t, err.Error(), "Not a valid tarball file") } func TestDecompressPipelineTarball_NonYamlTarball(t *testing.T) { tarballByte, _ := ioutil.ReadFile("test/non_yaml_tarball/non_yaml_tarball.tar.gz") - _, err := decompressPipelineTarball(tarballByte) + _, err := DecompressPipelineTarball(tarballByte) assert.NotNil(t, err) assert.Contains(t, err.Error(), "Expecting a YAML file inside the tarball") } func TestDecompressPipelineTarball_EmptyTarball(t *testing.T) { tarballByte, _ := ioutil.ReadFile("test/empty_tarball/empty.tar.gz") - _, err := decompressPipelineTarball(tarballByte) + _, err := DecompressPipelineTarball(tarballByte) assert.NotNil(t, err) assert.Contains(t, err.Error(), "Not a valid tarball file") } diff --git a/samples/basic/sequential.yaml b/samples/basic/sequential.yaml deleted file mode 100644 index 195c9981024..00000000000 --- a/samples/basic/sequential.yaml +++ /dev/null @@ -1,125 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: sequential- -spec: - arguments: - parameters: - - name: url - value: gs://ml-pipeline-playground/shakespeare1.txt - entrypoint: sequential - serviceAccountName: pipeline-runner - templates: - - container: - args: - - gsutil cat {{inputs.parameters.url}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk:216.0.0 - inputs: - parameters: - - name: url - name: download - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo {{inputs.parameters.download-downloaded}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: download-downloaded - name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: accesskey - name: mlpipeline-minio-artifact - bucket: mlpipeline - endpoint: minio-service.kubeflow:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz - secretKeySecret: - key: secretkey - name: mlpipeline-minio-artifact - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: echo - template: echo - inputs: - parameters: - - name: url - name: sequential From 0ba9eee3c949e1bef19061cf8bb9e6897a97b361 Mon Sep 17 00:00:00 2001 From: IronPan Date: Tue, 6 Nov 2018 00:15:17 -0800 Subject: [PATCH 02/12] update logging --- backend/src/apiserver/main.go | 8 +++----- test/presubmit-tests.sh | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go index 2d4bf31c2be..a39d2b418b9 100644 --- a/backend/src/apiserver/main.go +++ b/backend/src/apiserver/main.go @@ -124,7 +124,6 @@ func loadSamples(resourceManager *resource.ResourceManager) { configBytes, err := ioutil.ReadFile(*sampleConfigPath) if err != nil { glog.Fatalf("Failed to read sample configurations. Err: %v", err.Error()) - return } type config struct { Name string @@ -134,23 +133,22 @@ func loadSamples(resourceManager *resource.ResourceManager) { var configs []config if json.Unmarshal(configBytes, &configs) != nil { glog.Fatalf("Failed to read sample configurations. Err: %v", err.Error()) - return } for _, config := range configs { sampleBytes, err := ioutil.ReadFile(config.File) if err != nil { - glog.Fatalf("Failed to load sample %s. Error: %v", config.Name, err.Error()) + glog.Warningf("Failed to load sample %s. Error: %v", config.Name, err.Error()) continue } // Decompress if file is tarball decompressedFile, err := server.DecompressPipelineTarball(sampleBytes) if err!=nil{ - glog.Fatalf("Failed to decompress the file %s. Error: %v", config.Name, err.Error()) + glog.Warningf("Failed to decompress the file %s. Error: %v", config.Name, err.Error()) continue } _, err = resourceManager.CreatePipeline(config.Name, config.Description, decompressedFile) if err!=nil{ - glog.Fatalf("Failed to create pipeline for %s. Error: %v", config.Name, err.Error()) + glog.Warningf("Failed to create pipeline for %s. Error: %v", config.Name, err.Error()) continue } diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh index 667f8681804..80ab6f310b8 100755 --- a/test/presubmit-tests.sh +++ b/test/presubmit-tests.sh @@ -72,7 +72,7 @@ if [ "$CLUSTER_TYPE" == "create-gke" ]; then echo "Delete cluster..." gcloud container clusters delete ${TEST_CLUSTER} --async } - trap delete_cluster EXIT + # trap delete_cluster EXIT gcloud config set project ml-pipeline-test gcloud config set compute/zone us-central1-a From 3cd0d309f7e912b8da6b1b3e5358b1d16e5b34b0 Mon Sep 17 00:00:00 2001 From: IronPan Date: Tue, 6 Nov 2018 00:39:21 -0800 Subject: [PATCH 03/12] update description --- .../src/apiserver/config/sample_config.json | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/backend/src/apiserver/config/sample_config.json b/backend/src/apiserver/config/sample_config.json index 06f61e08fe6..9a60ea9034a 100644 --- a/backend/src/apiserver/config/sample_config.json +++ b/backend/src/apiserver/config/sample_config.json @@ -1,42 +1,37 @@ [ { - "name":"Sample-Basic-Condition", - "description":"shows how to use dsl.Condition", + "name":"[Sample] Basic - Condition", + "description":"A pipeline shows how to use dsl.Condition. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/condition.py", "file":"/samples/basic/condition.py.tar.gz" }, { - "name":"Sample-Basic-Default Value", - "description":"A pipeline with parameter and default value", - "file":"/samples/basic/default_value.py.tar.gz" - }, - { - "name":"Sample-Basic-Exit Handler", - "description":"Download a message and print it out. Exit Handler will run at the end", + "name":"[Sample] Basic - Exit Handler", + "description":"A pipeline that downloads a message and print it out. Exit Handler will run at the end. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/exit_handler.py", "file":"/samples/basic/exit_handler.py.tar.gz" }, { - "name":"Sample-Basic-Parallel Join", - "description":"Download two messages in parallel and print the concatenated result", + "name":"[Sample] Basic - Parallel Join", + "description":"A pipeline that downloads two messages in parallel and print the concatenated result. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/parallel_join.py", "file":"/samples/basic/parallel_join.py.tar.gz" }, { - "name":"Sample-Basic-Sequential", - "description":"Download a message and print it", + "name":"[Sample] Basic - Sequential", + "description":"A pipeline with two sequential steps. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/sequential.py", "file":"/samples/basic/sequential.py.tar.gz" }, { - "name":"Sample-TFX-Taxi Cab Classification", - "description":"Example pipeline that does classification with model analysis based on a public BigQuery dataset", + "name":"[Sample] ML - TFX - Taxi Cab Classification", + "description":"Example pipeline that does classification with model analysis based on a public tax cab BigQuery dataset. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/tfx", "file":"/samples/tfx/taxi_cab_classification_pipeline.py.tar.gz" }, { - "name":"Sample-CMLE-Resnet Training", - "description":"Demonstrate the ResNet50 predict", + "name":"[Sample] ML - CMLE - Resnet Training", + "description":"Demonstrate the ResNet50 predict. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/resnet-cmle", "file":"/samples/resnet-cmle/resnet_training_pipeline.py.tar.gz" }, { - "name":"Sample-Xgboost-Training with Confusion Matrix", - "description":"A trainer that does end-to-end distributed training for XGBoost models", + "name":"[Sample] ML - XGBoost - Training with Confusion Matrix", + "description":"A trainer that does end-to-end distributed training for XGBoost models. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/xgboost-spark", "file":"/samples/xgboost-spark/xgboost_training_cm.py.tar.gz" } ] \ No newline at end of file From 4c599ba01de8120c87452d7bc4a2cd9616d0a346 Mon Sep 17 00:00:00 2001 From: IronPan Date: Tue, 6 Nov 2018 02:09:23 -0800 Subject: [PATCH 04/12] update sample --- backend/src/apiserver/config/sample_config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/apiserver/config/sample_config.json b/backend/src/apiserver/config/sample_config.json index 9a60ea9034a..bd7918ffa89 100644 --- a/backend/src/apiserver/config/sample_config.json +++ b/backend/src/apiserver/config/sample_config.json @@ -22,16 +22,16 @@ { "name":"[Sample] ML - TFX - Taxi Cab Classification", "description":"Example pipeline that does classification with model analysis based on a public tax cab BigQuery dataset. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/tfx", - "file":"/samples/tfx/taxi_cab_classification_pipeline.py.tar.gz" + "file":"/samples/tfx/taxi-cab-classification-pipeline.py.tar.gz" }, { "name":"[Sample] ML - CMLE - Resnet Training", "description":"Demonstrate the ResNet50 predict. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/resnet-cmle", - "file":"/samples/resnet-cmle/resnet_training_pipeline.py.tar.gz" + "file":"/samples/resnet-cmle/resnet-train-pipeline.py.tar.gz" }, { "name":"[Sample] ML - XGBoost - Training with Confusion Matrix", "description":"A trainer that does end-to-end distributed training for XGBoost models. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/xgboost-spark", - "file":"/samples/xgboost-spark/xgboost_training_cm.py.tar.gz" + "file":"/samples/xgboost-spark/xgboost-training-cm.py.tar.gz" } ] \ No newline at end of file From 204d7bfabfab1594eddc7fdbaeee64bf761fd89d Mon Sep 17 00:00:00 2001 From: Yang Pan Date: Tue, 6 Nov 2018 10:09:55 -0800 Subject: [PATCH 05/12] add immediate value sample --- backend/src/apiserver/config/sample_config.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/apiserver/config/sample_config.json b/backend/src/apiserver/config/sample_config.json index bd7918ffa89..6c68581c7b1 100644 --- a/backend/src/apiserver/config/sample_config.json +++ b/backend/src/apiserver/config/sample_config.json @@ -9,6 +9,11 @@ "description":"A pipeline that downloads a message and print it out. Exit Handler will run at the end. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/exit_handler.py", "file":"/samples/basic/exit_handler.py.tar.gz" }, + { + "name":"[Sample] Basic - Immediate Value", + "description":"A pipeline with parameter values hard coded. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/immediate_value.py", + "file":"/samples/basic/immediate_value.py.tar.gz" + }, { "name":"[Sample] Basic - Parallel Join", "description":"A pipeline that downloads two messages in parallel and print the concatenated result. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/parallel_join.py", From 8a1ca012e4fc883771f45405395aea178cf8fb11 Mon Sep 17 00:00:00 2001 From: Yang Pan Date: Tue, 6 Nov 2018 10:10:23 -0800 Subject: [PATCH 06/12] revert --- test/presubmit-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh index 80ab6f310b8..667f8681804 100755 --- a/test/presubmit-tests.sh +++ b/test/presubmit-tests.sh @@ -72,7 +72,7 @@ if [ "$CLUSTER_TYPE" == "create-gke" ]; then echo "Delete cluster..." gcloud container clusters delete ${TEST_CLUSTER} --async } - # trap delete_cluster EXIT + trap delete_cluster EXIT gcloud config set project ml-pipeline-test gcloud config set compute/zone us-central1-a From 1b305742289741d596a2f096ca8e2ecb040c4988 Mon Sep 17 00:00:00 2001 From: Yang Pan Date: Tue, 6 Nov 2018 10:52:53 -0800 Subject: [PATCH 07/12] fail fast if the samples are failed to load --- backend/src/apiserver/main.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go index a39d2b418b9..1c6ba3c7fab 100644 --- a/backend/src/apiserver/main.go +++ b/backend/src/apiserver/main.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "net" "net/http" + "time" "github.com/golang/glog" api "github.com/kubeflow/pipelines/backend/api/go_client" @@ -30,7 +31,6 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/runtime" "google.golang.org/grpc" "google.golang.org/grpc/reflection" - "time" ) var ( @@ -137,19 +137,16 @@ func loadSamples(resourceManager *resource.ResourceManager) { for _, config := range configs { sampleBytes, err := ioutil.ReadFile(config.File) if err != nil { - glog.Warningf("Failed to load sample %s. Error: %v", config.Name, err.Error()) - continue + glog.Fatalf("Failed to load sample %s. Error: %v", config.Name, err.Error()) } // Decompress if file is tarball decompressedFile, err := server.DecompressPipelineTarball(sampleBytes) if err!=nil{ - glog.Warningf("Failed to decompress the file %s. Error: %v", config.Name, err.Error()) - continue + glog.Fatalf("Failed to decompress the file %s. Error: %v", config.Name, err.Error()) } _, err = resourceManager.CreatePipeline(config.Name, config.Description, decompressedFile) if err!=nil{ - glog.Warningf("Failed to create pipeline for %s. Error: %v", config.Name, err.Error()) - continue + glog.Fatalf("Failed to create pipeline for %s. Error: %v", config.Name, err.Error()) } // Since the default sorting is by create time, From 1998970f722f0bdc7173ff38733b14af3693974a Mon Sep 17 00:00:00 2001 From: Yang Pan Date: Tue, 6 Nov 2018 11:28:49 -0800 Subject: [PATCH 08/12] comment --- backend/Dockerfile | 2 +- backend/src/apiserver/config/sample_config.json | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 74d3223e094..192866374e3 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -21,7 +21,7 @@ FROM python:3.5.0-slim as compiler RUN pip install https://storage.googleapis.com/ml-pipeline/release/0.0.26/kfp-0.0.26.tar.gz --upgrade WORKDIR /samples COPY ./samples . -RUN find . -maxdepth 2 -name "*.py" -exec python3 {} \; +RUN find . -maxdepth 2 -name "*.py" -exec dsk-compile --py {} --output {}.tar.gz \; FROM alpine diff --git a/backend/src/apiserver/config/sample_config.json b/backend/src/apiserver/config/sample_config.json index 6c68581c7b1..ac6a71c6d51 100644 --- a/backend/src/apiserver/config/sample_config.json +++ b/backend/src/apiserver/config/sample_config.json @@ -25,15 +25,10 @@ "file":"/samples/basic/sequential.py.tar.gz" }, { - "name":"[Sample] ML - TFX - Taxi Cab Classification", + "name":"[Sample] ML - TFX - Taxi Tip Prediction Model Trainer", "description":"Example pipeline that does classification with model analysis based on a public tax cab BigQuery dataset. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/tfx", "file":"/samples/tfx/taxi-cab-classification-pipeline.py.tar.gz" }, - { - "name":"[Sample] ML - CMLE - Resnet Training", - "description":"Demonstrate the ResNet50 predict. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/resnet-cmle", - "file":"/samples/resnet-cmle/resnet-train-pipeline.py.tar.gz" - }, { "name":"[Sample] ML - XGBoost - Training with Confusion Matrix", "description":"A trainer that does end-to-end distributed training for XGBoost models. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/xgboost-spark", From 5060b2575652eaa0b7d3edd045992ee60166b296 Mon Sep 17 00:00:00 2001 From: Yang Pan Date: Tue, 6 Nov 2018 12:40:36 -0800 Subject: [PATCH 09/12] address comments --- backend/src/apiserver/main.go | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go index 1c6ba3c7fab..252cf0ff4ef 100644 --- a/backend/src/apiserver/main.go +++ b/backend/src/apiserver/main.go @@ -31,6 +31,9 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/runtime" "google.golang.org/grpc" "google.golang.org/grpc/reflection" + "os" + "github.com/pkg/errors" + "fmt" ) var ( @@ -49,7 +52,10 @@ func main() { initConfig() clientManager := newClientManager() resourceManager := resource.NewResourceManager(&clientManager) - loadSamples(resourceManager) + err:= loadSamples(resourceManager) + if err!=nil{ + glog.Fatalf("Failed to load samples. Err: %v", err.Error()) + } go startRpcServer(resourceManager) startHttpProxy(resourceManager) @@ -120,10 +126,10 @@ func registerHttpHandlerFromEndpoint(handler RegisterHttpHandlerFromEndpoint, se } // Preload a bunch of pipeline samples -func loadSamples(resourceManager *resource.ResourceManager) { +func loadSamples(resourceManager *resource.ResourceManager) error { configBytes, err := ioutil.ReadFile(*sampleConfigPath) if err != nil { - glog.Fatalf("Failed to read sample configurations. Err: %v", err.Error()) + return errors.New(fmt.Sprintf("Failed to read sample configurations file. Err: %v", err.Error())) } type config struct { Name string @@ -132,21 +138,20 @@ func loadSamples(resourceManager *resource.ResourceManager) { } var configs []config if json.Unmarshal(configBytes, &configs) != nil { - glog.Fatalf("Failed to read sample configurations. Err: %v", err.Error()) + return errors.New(fmt.Sprintf("Failed to parse sample configurations.")) } for _, config := range configs { - sampleBytes, err := ioutil.ReadFile(config.File) + reader, err:= os.Open(config.File) if err != nil { - glog.Fatalf("Failed to load sample %s. Error: %v", config.Name, err.Error()) + return errors.New(fmt.Sprintf("Failed to load sample %s. Error: %v", config.Name, err.Error())) } - // Decompress if file is tarball - decompressedFile, err := server.DecompressPipelineTarball(sampleBytes) + pipelineFile, err := server.ReadPipelineFile(config.File, reader, server.MaxFileLength) if err!=nil{ - glog.Fatalf("Failed to decompress the file %s. Error: %v", config.Name, err.Error()) + return errors.New(fmt.Sprintf("Failed to decompress the file %s. Error: %v", config.Name, err.Error())) } - _, err = resourceManager.CreatePipeline(config.Name, config.Description, decompressedFile) + _, err = resourceManager.CreatePipeline(config.Name, config.Description, pipelineFile) if err!=nil{ - glog.Fatalf("Failed to create pipeline for %s. Error: %v", config.Name, err.Error()) + return errors.New(fmt.Sprintf("Failed to create pipeline for %s. Error: %v", config.Name, err.Error())) } // Since the default sorting is by create time, @@ -154,4 +159,5 @@ func loadSamples(resourceManager *resource.ResourceManager) { time.Sleep(1*time.Second) } glog.Info("All samples are loaded.") + return nil } From 0389e640f57e1adef9327d36edd9f20756319deb Mon Sep 17 00:00:00 2001 From: Yang Pan Date: Tue, 6 Nov 2018 12:41:04 -0800 Subject: [PATCH 10/12] comment out --- test/presubmit-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh index 667f8681804..80ab6f310b8 100755 --- a/test/presubmit-tests.sh +++ b/test/presubmit-tests.sh @@ -72,7 +72,7 @@ if [ "$CLUSTER_TYPE" == "create-gke" ]; then echo "Delete cluster..." gcloud container clusters delete ${TEST_CLUSTER} --async } - trap delete_cluster EXIT + # trap delete_cluster EXIT gcloud config set project ml-pipeline-test gcloud config set compute/zone us-central1-a From 05b5b21658f66f5267625364bfd86c9ec8d3b310 Mon Sep 17 00:00:00 2001 From: Yang Pan Date: Tue, 6 Nov 2018 13:36:00 -0800 Subject: [PATCH 11/12] update command --- backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 192866374e3..72bf06394ad 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -21,7 +21,7 @@ FROM python:3.5.0-slim as compiler RUN pip install https://storage.googleapis.com/ml-pipeline/release/0.0.26/kfp-0.0.26.tar.gz --upgrade WORKDIR /samples COPY ./samples . -RUN find . -maxdepth 2 -name "*.py" -exec dsk-compile --py {} --output {}.tar.gz \; +RUN find . -maxdepth 2 -name "*.py" -exec dsl-compile --py {} --output {}.tar.gz \; FROM alpine From ee8911c502e3ac89cf20d2a363c3e85027c0f521 Mon Sep 17 00:00:00 2001 From: Yang Pan Date: Tue, 6 Nov 2018 14:27:57 -0800 Subject: [PATCH 12/12] comments --- .../src/apiserver/config/sample_config.json | 36 +++++++++---------- backend/src/apiserver/main.go | 9 +++-- test/presubmit-tests.sh | 2 +- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/backend/src/apiserver/config/sample_config.json b/backend/src/apiserver/config/sample_config.json index ac6a71c6d51..81e0917bb61 100644 --- a/backend/src/apiserver/config/sample_config.json +++ b/backend/src/apiserver/config/sample_config.json @@ -1,18 +1,18 @@ [ { - "name":"[Sample] Basic - Condition", - "description":"A pipeline shows how to use dsl.Condition. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/condition.py", - "file":"/samples/basic/condition.py.tar.gz" + "name":"[Sample] ML - XGBoost - Training with Confusion Matrix", + "description":"A trainer that does end-to-end distributed training for XGBoost models. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/xgboost-spark", + "file":"/samples/xgboost-spark/xgboost-training-cm.py.tar.gz" }, { - "name":"[Sample] Basic - Exit Handler", - "description":"A pipeline that downloads a message and print it out. Exit Handler will run at the end. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/exit_handler.py", - "file":"/samples/basic/exit_handler.py.tar.gz" + "name":"[Sample] ML - TFX - Taxi Tip Prediction Model Trainer", + "description":"Example pipeline that does classification with model analysis based on a public tax cab BigQuery dataset. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/tfx", + "file":"/samples/tfx/taxi-cab-classification-pipeline.py.tar.gz" }, { - "name":"[Sample] Basic - Immediate Value", - "description":"A pipeline with parameter values hard coded. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/immediate_value.py", - "file":"/samples/basic/immediate_value.py.tar.gz" + "name":"[Sample] Basic - Sequential", + "description":"A pipeline with two sequential steps. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/sequential.py", + "file":"/samples/basic/sequential.py.tar.gz" }, { "name":"[Sample] Basic - Parallel Join", @@ -20,18 +20,18 @@ "file":"/samples/basic/parallel_join.py.tar.gz" }, { - "name":"[Sample] Basic - Sequential", - "description":"A pipeline with two sequential steps. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/sequential.py", - "file":"/samples/basic/sequential.py.tar.gz" + "name":"[Sample] Basic - Immediate Value", + "description":"A pipeline with parameter values hard coded. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/immediate_value.py", + "file":"/samples/basic/immediate_value.py.tar.gz" }, { - "name":"[Sample] ML - TFX - Taxi Tip Prediction Model Trainer", - "description":"Example pipeline that does classification with model analysis based on a public tax cab BigQuery dataset. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/tfx", - "file":"/samples/tfx/taxi-cab-classification-pipeline.py.tar.gz" + "name":"[Sample] Basic - Exit Handler", + "description":"A pipeline that downloads a message and print it out. Exit Handler will run at the end. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/exit_handler.py", + "file":"/samples/basic/exit_handler.py.tar.gz" }, { - "name":"[Sample] ML - XGBoost - Training with Confusion Matrix", - "description":"A trainer that does end-to-end distributed training for XGBoost models. For source code, refer to https://github.com/kubeflow/pipelines/tree/master/samples/xgboost-spark", - "file":"/samples/xgboost-spark/xgboost-training-cm.py.tar.gz" + "name":"[Sample] Basic - Condition", + "description":"A pipeline shows how to use dsl.Condition. For source code, refer to https://github.com/kubeflow/pipelines/blob/master/samples/basic/condition.py", + "file":"/samples/basic/condition.py.tar.gz" } ] \ No newline at end of file diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go index 252cf0ff4ef..a0b6562322c 100644 --- a/backend/src/apiserver/main.go +++ b/backend/src/apiserver/main.go @@ -137,8 +137,8 @@ func loadSamples(resourceManager *resource.ResourceManager) error { File string } var configs []config - if json.Unmarshal(configBytes, &configs) != nil { - return errors.New(fmt.Sprintf("Failed to parse sample configurations.")) + if err:= json.Unmarshal(configBytes, &configs);err != nil { + return errors.New(fmt.Sprintf("Failed to read sample configurations. Err: %v", err.Error())) } for _, config := range configs { reader, err:= os.Open(config.File) @@ -151,7 +151,10 @@ func loadSamples(resourceManager *resource.ResourceManager) error { } _, err = resourceManager.CreatePipeline(config.Name, config.Description, pipelineFile) if err!=nil{ - return errors.New(fmt.Sprintf("Failed to create pipeline for %s. Error: %v", config.Name, err.Error())) + // Log the error but not fail. The API Server pod can restart and it could potentially cause name collision. + // In the future, we might consider loading samples during deployment, instead of when API server starts. + glog.Warningf(fmt.Sprintf("Failed to create pipeline for %s. Error: %v", config.Name, err.Error())) + continue } // Since the default sorting is by create time, diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh index 80ab6f310b8..667f8681804 100755 --- a/test/presubmit-tests.sh +++ b/test/presubmit-tests.sh @@ -72,7 +72,7 @@ if [ "$CLUSTER_TYPE" == "create-gke" ]; then echo "Delete cluster..." gcloud container clusters delete ${TEST_CLUSTER} --async } - # trap delete_cluster EXIT + trap delete_cluster EXIT gcloud config set project ml-pipeline-test gcloud config set compute/zone us-central1-a