.cloudbuild.yaml

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Test before submit:
# gcloud builds submit --config=.cloudbuild.yaml --substitutions=COMMIT_SHA="$(git rev-parse HEAD)" --project=ml-pipeline-test

steps:

# Build the Python SDK
- name: 'python:3-alpine'
  entrypoint: '/bin/sh'
  args: ['-c', 'cd /workspace/sdk/python/; python3 setup.py sdist --format=gztar; cp dist/*.tar.gz /workspace/kfp.tar.gz']
  id:   'preparePythonSDK'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz']
  id:   'copyPythonSDK'
  waitFor: ['preparePythonSDK']
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp.tar.gz']
  id:   'copyPythonSDKToLatest'
  waitFor: ['preparePythonSDK']

# Build the Python Component SDK
- name: 'python:2-alpine'
  entrypoint: '/bin/sh'
  args: ['-c', 'cd /workspace/components/gcp/container/component_sdk/python;python setup.py sdist --format=gztar; cp dist/*.tar.gz /workspace/kfp-component.tar.gz']
  id:   'preparePythonComponentSDK'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp-component.tar.gz']
  id:   'copyPythonComponentSDK'
  waitFor: ['preparePythonComponentSDK']
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp-component.tar.gz']
  id:   'copyPythonComponentSDKToLatest'
  waitFor: ['preparePythonComponentSDK']

# Build the pipeline system images
- name: 'debian'
  entrypoint: '/bin/bash'
  args: ['-c', 'sed -i -e "s/ARG DATE/ENV DATE \"$(date -u)\"/" /workspace/frontend/Dockerfile']
  id:   'prepareFrontend'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA',
         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
         '/workspace/frontend/Dockerfile', '/workspace']
  id:   'buildFrontend'
  waitFor: ['prepareFrontend']
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA',
         '--build-arg', 'COMMIT_SHA=$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile', '/workspace']
  id:   'buildApiServer'
  waitFor: ['copyPythonSDK']

- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.scheduledworkflow', '/workspace']
  id:   'buildScheduledWorkflow'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.viewercontroller', '/workspace']
  id:   'buildViewerCrdController'
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.persistenceagent', '/workspace']
  id:   'buildPersistenceAgent'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA', '-f',
         '/workspace/proxy/Dockerfile', '/workspace/proxy']
  id:   'buildInverseProxyAgent'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA',
         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.visualization', '/workspace']
  id: 'buildVisualizationServer'
  waitFor: ["-"]
- id: 'buildMetadataWriter'
  name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA',
         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
         '/workspace/backend/metadata_writer/Dockerfile', '/workspace']
  waitFor: ["-"]
- id: 'buildCacheServer'
  name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA',
          '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
          '/workspace/backend/Dockerfile.cacheserver', '/workspace']
  waitFor: ["-"]
- id: 'buildCacheDeployer'
  name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA',
          '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
          '/workspace/backend/src/cache/deployer/Dockerfile', '/workspace']
  waitFor: ["-"]

# Build marketplace deployer
- id: 'buildMarketplaceDeployer'
  name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA',
         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
         '/workspace/manifests/gcp_marketplace/deployer/Dockerfile', '/workspace/manifests/gcp_marketplace']
  waitFor: ["-"]

# Build the Kubeflow-based pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA',
         '/workspace/components/kubeflow/deployer']
  id: 'buildDeployer'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/kubeflow/launcher && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildLauncher'
  waitFor: ["-"]
- id: 'buildCpuTrainer'
  name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/kubeflow/dnntrainer && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA -l ml-pipeline-kubeflow-tf-trainer -b 1.6.0']
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/kubeflow/dnntrainer && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA -l ml-pipeline-kubeflow-tf-trainer-gpu -b 1.6.0-gpu']
  id: 'buildGpuTrainer'
  waitFor: ["-"]

# Build the Dataproc-based pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/analyze && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocAnalyze'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/create_cluster && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocCreateCluster'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/delete_cluster && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocDeleteCluster'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/predict && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocPredict'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/transform && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocTransform'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/train && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocTrain'
  waitFor: ["-"]

# Build the Generic GCP component image
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/gcp/container/ && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildGcpGenericComponent'
  waitFor: ["-"]

# Build the local pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/local/confusion_matrix && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildConfusionMatrix'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/local/roc && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildROC'
  waitFor: ["-"]

# Build third_party images
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA', '-f',
         '/workspace/third_party/metadata_envoy/Dockerfile', '/workspace']
  id:   'buildMetadataEnvoy'

# Pull third_party images
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:0.21.1']
  id:   'pullMetadataServer'
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance']
  id:   'pullMinio'
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/ml-pipeline/mysql:5.6']
  id:   'pullMysql'
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.14']
  id:   'pullCloudsqlProxy'
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance']
  id:   'pullArgoExecutor'
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance']
  id:   'pullArgoWorkflowController'

# Tag for Hosted - SemVersion to Major.Minor parsing
- id: "parseMajorMinorVersion"
  waitFor: ["-"]
  name: gcr.io/cloud-builders/docker
  entrypoint: /bin/bash
  args:
    - -ceux
    - |
      # Parse major minor version and save to a file for reusing in other steps.
      cat /workspace/VERSION | sed -e "s#[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)#\1.\2#" > /workspace/mm.ver

# Tag for Hosted - Tag to hosted folder with MKP friendly name
- id:   'tagForHosted'
  waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildScheduledWorkflow',            
            'buildViewerCrdController', 'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
            'buildMetadataWriter', 'buildCacheServer', 'buildCacheDeployer', 'buildMetadataEnvoy',
            'buildMarketplaceDeployer', 'pullMetadataServer', 'pullMinio', 'pullMysql', 'pullCloudsqlProxy',
            'pullArgoExecutor', 'pullArgoWorkflowController']
  name: 'gcr.io/cloud-builders/docker'
  entrypoint: bash
  args:
  - -ceux
  - |
    /workspace/test/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)

images:
# Images for the pipeline system itself
- 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA'

# Images for Marketplace
- 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA'

# Images for the Kubeflow-based pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer-gpu:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tfjob:$COMMIT_SHA'

# Images for the Dataproc-based pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-analyze:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-create-cluster:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-delete-cluster:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-predict:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-transform:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-train:$COMMIT_SHA'

# Images for the GCP generic pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-gcp:$COMMIT_SHA'

# Images for the local components
- 'gcr.io/$PROJECT_ID/ml-pipeline-local-confusion-matrix:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-local-roc:$COMMIT_SHA'

# Images for the third_party components
- 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA'

timeout: '3600s'
options:
 diskSizeGb: 300
 machineType: 'N1_HIGHCPU_8'