build_tests.yaml

# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

name: Build Tests

on:
  workflow_dispatch:
    inputs:
      tpuType:
        description: 'TPU Type'
        required: true
        default: 'v4-8'
        type: choice
        options:
          - v4-8
  push:
    branches: ["main"]
  pull_request: # By default this runs for types assigned, opened and synchronize.

env:
  # Names must be unique in parallel running tests.
  TPU_TYPE: ${{ inputs.tpuType || 'v4-8' }}
  TPU_CLUSTER_NAME: build-xpk-2-v4-8-nodepools
  TPU_FILESTORE_CLUSTER_NAME: build-xpk-filestore
  WORKLOAD_NAME: xpktest-build-${{ github.run_attempt }}
  PATHWAYS_WORKLOAD_NAME: xpkpw-build-${{ github.run_attempt }}
  RUN_ID: "pr-${{ github.event.number }}"
  PROJECT_ID: ${{secrets.PROJECT_NAME}}
  A3_MEGA_TEST_CLUSTER_NAME: "xpk-mega-ctk-int"
  A3_ULTRA_TEST_CLUSTER_NAME: "xpk-ultra-ctk-int"
  GKE_ML_TEST_CLUSTER_NAME: "xpk-gke-ml"
  ZONE: us-central2-a
  REGION: us-central2
  STORAGE_NAME: test-storage
  CLUSTER_ARGUMENTS: "--network=${{secrets.NETWORK_NAME}} --subnetwork=${{secrets.SUBNETWORK_NAME}} --maintenance-window=23:50"
  FS_STORAGE_NAME: ${{secrets.INSTANCE_NAME}}-test-storage
  FS_DELETE_WORKLOAD: "fs-delete-workload"
  FS_READ_WORKLOAD: "fs-read-workload"
  FS_WRITE_WORKLOAD: "fs-write-workload"
  PW_CLUSTER_ARGUMENTS: "--network=${{secrets.NETWORK_NAME}} --subnetwork=${{secrets.SUBNETWORK_NAME}} --maintenance-window=23:50"

jobs:
  # run-unit-tests:
  #   runs-on: [ubuntu-22.04]
  #   concurrency: # We support one build or nightly test to run at a time currently.
  #     group: build-test-cluster-group
  #     cancel-in-progress: false
  #   steps:
  #   - uses: actions/checkout@v4
  #   - uses: actions/setup-python@v5
  #     with:
  #       python-version: '3.10'
  #   - uses: google-github-actions/setup-gcloud@v2
  #     with:
  #       version: '>= 363.0.0'
  #       install_components: 'beta, gke-gcloud-auth-plugin'
  #   - name: Install dependencies
  #     run : make install-dev
  #   - name: Run unit tests
  #     run: make run-unittests

  # run-integration-tests:
  #   runs-on: [ubuntu-22.04]
  #   needs: [run-unit-tests]
  #   concurrency: # We support one build or nightly test to run at a time currently.
  #     group: build-test-cluster-group
  #     cancel-in-progress: false
  #   steps:
  #   - uses: actions/checkout@v4
  #   - uses: actions/setup-python@v5
  #     with:
  #       python-version: '3.10'
  #   - uses: 'google-github-actions/auth@v2'
  #     with:
  #       credentials_json: '${{ secrets.GCP_SA_KEY }}'
  #   - uses: google-github-actions/setup-gcloud@v2
  #     with:
  #       version: '>= 363.0.0'
  #       install_components: 'beta,gke-gcloud-auth-plugin, gke-gcloud-auth-plugin'
  #   - name: Verify gcp setup
  #     run: gcloud info
  #   - name: Install dependencies
  #     run : make install-dev
  #   - name: "Set auth cidr"
  #     run: echo "AUTH_CIDR=$(curl api.ipify.org)/32" >> $GITHUB_ENV
  #   - name: "Set GCLOUD_CFG_PATH"
  #     run: echo "GCLOUD_CFG_PATH=/home/runner/work/xpk/xpk/" >> $GITHUB_ENV
  #   - name: "Copy credentials"
  #     run: cp $GOOGLE_APPLICATION_CREDENTIALS $GCLOUD_CFG_PATH/application_default_credentials.json
  #   - name: "Set DEPLOYMENT_DIR"
  #     run: echo "DEPLOYMENT_DIR=$HOME/deployment" >> $GITHUB_ENV
  #   - name: Create deployment dir
  #     run: mkdir -p $DEPLOYMENT_DIR
  #   - name: Run integration tests
  #     run: make run-integrationtests

  cluster-create-and-delete:
    runs-on: [ubuntu-22.04]
    # needs: [run-integration-tests]
    concurrency: # We support one nightly test and one build test for each branch to run at a time currently.
      group: build-test-cluster-group-${{ github.ref }}
      cancel-in-progress: false
    steps:
    - name: Change RUN_ID env var if merge to main
      run: echo "RUN_ID=main" >> $GITHUB_ENV
      if: ${{ github.ref == 'refs/heads/main' }}
    - name: Update cluster name with TPU_TYPE and RUN_ID
      run: echo "TPU_CLUSTER_NAME=$TPU_CLUSTER_NAME-$TPU_TYPE-$RUN_ID" >> $GITHUB_ENV
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v5
      with:
        python-version: '3.10'
    - name: Install expect package
      run: sudo apt-get install expect
    - uses: 'google-github-actions/auth@v2'
      with:
        credentials_json: '${{ secrets.GCP_SA_KEY }}'
    - uses: google-github-actions/setup-gcloud@v2
      with:
        version: '>= 363.0.0'
        install_components: 'beta,gke-gcloud-auth-plugin'
    - name: Generate random seed
      run: |
        RANDOM_SEED=$((RANDOM % 10000))  # Generate a random number between 0 and 9999
        echo "RANDOM_SEED=$RANDOM_SEED" >> $GITHUB_ENV
    - name: Verify gcp setup
      run: gcloud info
    - name: Set Google Cloud CLI properties to a unused zone to verify --zone arg is passed properly in commands.
      run: |
        gcloud config set compute/zone us-east4-a
        gcloud config get compute/zone
    - name: Install xpk dependencies
      run: |
        make install 
        echo $PWD/bin >> "$GITHUB_PATH"
    - name: Check xpk installation
      run: xpk --help
    # - name: Create a private Pathways-enabled XPK Cluster with 2x $TPU_TYPE nodepools. Larger num-nodes to avoid master resizing.
    #   run: python xpk.py cluster create-pathways --cluster $TPU_CLUSTER_NAME --private --tpu-type=$TPU_TYPE  --num-slices=2 --zone=us-central2-b --default-pool-cpu-machine-type=n1-standard-16 --default-pool-cpu-num-nodes=16 --reservation='${{ secrets.GCP_TPU_V4_RESERVATION }}' --custom-cluster-arguments="${CLUSTER_ARGUMENTS}"
    # - name: Verify the created cluster is private
    #   run: gcloud container clusters describe $TPU_CLUSTER_NAME --region=us-central2 --format="value(privateClusterConfig.enablePrivateNodes)" | grep 'True' || (echo 'The created cluster is not private.' && exit 1)
    - name: Create a Pathways-enabled XPK Cluster with 2x v4-8 nodepools. Larger num-nodes to avoid master resizing.
      run: |
        python3 xpk.py cluster create-pathways --cluster $TPU_CLUSTER_NAME --tpu-type=v4-8  --num-slices=2 \
        --zone=us-central2-b --default-pool-cpu-machine-type=n1-standard-16 --default-pool-cpu-num-nodes=16 \
        --reservation='${{ secrets.GCP_TPU_V4_RESERVATION }}' --enable-workload-identity --enable-gcsfuse-csi-driver --enable-gcpfilestore-csi-driver --custom-cluster-arguments="${PW_CLUSTER_ARGUMENTS}"
    - name: Authenticate Docker
      run: gcloud auth configure-docker --quiet
    - name: Create auto-mount Storage instance
      run: |
        python3 xpk.py storage create $STORAGE_NAME --cluster=$TPU_CLUSTER_NAME --zone=us-central2-b --type=gcsfuse \
        --auto-mount=true \
        --mount-point='/test-mount-point' --readonly=false --manifest='./tests/data/pv-pvc-templates.yaml'
    - name: List and verify existing Storages
      run: python3 xpk.py storage list --cluster $TPU_CLUSTER_NAME --zone=us-central2-b | tee output.txt | grep 'test-storage' || (echo 'No storage found' && cat output.txt && exit 1)
    - name: Create test script to execute in workloads
      run: |
        echo -e \
        '#!/bin/bash \n
        echo "Hello world from a test script!"
        cd ~/../test-mount-point && echo "Hello world from a Github Action CI/CD test script!" > '$RANDOM_SEED'.txt' \
        > test.sh
    - name: Run a base-docker-image workload
      run: |
        python3 xpk.py workload create --cluster $TPU_CLUSTER_NAME --workload $WORKLOAD_NAME  --command "bash test.sh"  \
        --tpu-type=v4-8 --num-slices=2 --zone=us-central2-b
    - name: Run xpk inspector with the workload created above
      run: python3 xpk.py inspector --cluster $TPU_CLUSTER_NAME --zone=us-central2-b  --workload $WORKLOAD_NAME
    - name: Wait for workload completion and confirm it succeeded
      run: python3 xpk.py workload list --cluster $TPU_CLUSTER_NAME --zone=us-central2-b --wait-for-job-completion $WORKLOAD_NAME --timeout 300
    - name: Run a Pathways workload on Ubuntu base image
      run: |
        python3 xpk.py workload create-pathways --cluster $TPU_CLUSTER_NAME --workload $PATHWAYS_WORKLOAD_NAME \
        --docker-image='marketplace.gcr.io/google/ubuntu2004' --tpu-type=v4-8 --num-slices=2 --zone=us-central2-b \
        --command "echo \"Hello world from a test script! \""
    - name: Verify if the file was created in the GCS bucket
      run: gsutil cp gs://xpk-ci-cd-tests/$RANDOM_SEED.txt .
    - name: Check if the file contains desired content
      run: grep 'Hello world from a Github Action CI/CD test script!' $RANDOM_SEED.txt
    - name: Wait for Pathways workload completion and confirm it succeeded
      run: python3 xpk.py workload list --cluster $TPU_CLUSTER_NAME --zone=us-central2-b --wait-for-job-completion $PATHWAYS_WORKLOAD_NAME --timeout 300
    - name: List out the workloads on the cluster
      run: python3 xpk.py workload list --cluster $TPU_CLUSTER_NAME --zone=us-central2-b
    - name: Run xpk info
      run: python3 xpk.py info --cluster $TPU_CLUSTER_NAME --zone=us-central2-b | tee output.txt | grep -P "^(?=.*QUEUE)(?=.*PENDING_WORKLOADS)(?=.*ADMITTED_WORKLOADS)(?=.*2x$TPU_TYPE:google.com/tpu)(?=.*cpu-rm:cpu)(?=.*cpu-rm:memory)(?=.*cpu-proxy:cpu)(?=.*cpu-proxy:memory)(?=.*cpu-user:cpu)(?=.*cpu-user:memory)" || (echo 'Invalid command output' && cat output.txt && exit 1)
    - name: Delete the workload on the cluster
      run: python3 xpk.py workload delete --workload $WORKLOAD_NAME --cluster $TPU_CLUSTER_NAME --zone=us-central2-b
    - name: Delete the Pathways workload on the cluster
      run: python3 xpk.py workload delete --workload $PATHWAYS_WORKLOAD_NAME --cluster $TPU_CLUSTER_NAME --zone=us-central2-b
    - name: Delete created GCS file
      run: gsutil rm gs://xpk-ci-cd-tests/$RANDOM_SEED.txt
    - name: Delete existing Storage
      run: python3 xpk.py storage delete $STORAGE_NAME  --cluster $TPU_CLUSTER_NAME --zone=us-central2-b
    - name: Create test script to execute in batch
      run: echo -e '#!/bin/bash \n#SBATCH --unknown-flag=value\n echo "Hello world from a test script!"' > batch.sh
    - name: Run a batch job on the cluster
      run: python3 xpk.py batch --cluster $TPU_CLUSTER_NAME --zone=us-central2-b batch.sh --ignore-unknown-flags --array 1-5 --nodes 2 --ntasks 3 --time 60
    - name: List out the jobs on the cluster
      run: python3 xpk.py job ls --cluster $TPU_CLUSTER_NAME --zone=us-central2-b | grep 'xpk-def-app-profile-slurm-'
    - name: Get created job name
      run: |
        JOB_NAME=$(python3 xpk.py job ls --cluster $TPU_CLUSTER_NAME --zone=us-central2-b | grep 'xpk-def-app-profile-slurm-' | head -1 | awk '{print $1}')
        echo "JOB_NAME=${JOB_NAME}" >> $GITHUB_ENV
    - name: Check created job
      run: |
        kubectl get job ${JOB_NAME} -o jsonpath='{.metadata.labels}' | grep '"kueue.x-k8s.io/max-exec-time-seconds":"3600"'
        job_spec=$(kubectl get job ${JOB_NAME} -o jsonpath='{.spec}')
        echo "$job_spec" | grep '"completions":2'
        echo "$job_spec" | grep '"parallelism":2'
        echo "$job_spec" | jq '.template.spec.containers | length' | grep 3
    - name: Get job info for the last job created on the cluster
      run: python3 xpk.py job info ${JOB_NAME} | grep -e "Entrypoint environment variables template:" -e "Job name:" -e "Labels:" -e "Mounts:" -e "Pods:" -e "Profile:" -e "Script name:" | wc -l | grep "7"
    - name: Cancel the batch job on the cluster
      run: |
        python3 xpk.py job cancel ${JOB_NAME} --cluster $TPU_CLUSTER_NAME --zone=us-central2-b | grep "job.batch/${JOB_NAME} deleted"
    - name: Create shell and exit it immediately
      run: |
        cat <<'EOF' >> create-shell.exp
        ##!/usr/bin/expect
        spawn python3 ./xpk.py shell
        expect "/ # "
        send "exit\n"
        EOF
        chmod +x ./create-shell.exp
        expect ./create-shell.exp
    # TODO - deflake tests
    # - name: Check if shell exists and is running
    #   run: kubectl get pods | grep xpk-def-app-profile-interactive- | grep Running
    - name: Stop the shell
      run: python3 xpk.py shell stop
    - name: Delete the cluster created
      if: always()
      run: echo 'y' | python xpk.py cluster delete --cluster $TPU_CLUSTER_NAME --zone=us-central2-b
    - name: Delete the cluster created
      if: always()
      run: python3 xpk.py cluster delete --cluster $TPU_CLUSTER_NAME --zone=us-central2-b

  run-filestore-workload:
    runs-on: [ubuntu-20.04]
    concurrency: # We support one build or nightly test to run at a time currently.
      group: filestore-test-group
      cancel-in-progress: false
    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v5
      with:
        python-version: '3.10'
    - uses: 'google-github-actions/auth@v2'
      with:
        credentials_json: '${{ secrets.GCP_SA_KEY }}'
    - uses: google-github-actions/setup-gcloud@v2
      with:
        version: '>= 363.0.0'
        install_components: 'beta,gke-gcloud-auth-plugin'
    - name: Generate random seed
      run: |
        RANDOM_SEED=$((RANDOM % 10000))  # Generate a random number between 0 and 9999
        echo "RANDOM_SEED=$RANDOM_SEED" >> $GITHUB_ENV
    - name: Verify gcp setup
      run: gcloud info
    - name: Set Google Cloud CLI properties to a unused zone to verify --zone arg is passed properly in commands.
      run: |
        gcloud config set compute/zone us-east4-a
        gcloud config get compute/zone
    - name: Install xpk with pip and verify it executes corretly
      run: |
        pip install .
        xpk --help
    - name: Authenticate Docker
      run: gcloud auth configure-docker --quiet
    - name: Create a XPK Cluster with 2x v4-8 nodepools. Larger num-nodes to avoid master resizing.
      run: |
        python3 xpk.py cluster create --cluster $TPU_FILESTORE_CLUSTER_NAME --tpu-type=v4-8  --num-slices=2 \
        --zone=us-central2-b --default-pool-cpu-machine-type=n1-standard-16 --default-pool-cpu-num-nodes=16 \
        --reservation='${{ secrets.GCP_TPU_V4_RESERVATION }}' --enable-workload-identity --enable-gcpfilestore-csi-driver --custom-cluster-arguments="${CLUSTER_ARGUMENTS}"
    - name: Fill Filestore manifest file
      run: |
        sed -i 's/PROJECT_NAME/${{secrets.PROJECT_NAME}}/g;  s/ZONE/us-central2-a/g; s/INSTANCE_NAME/${{secrets.INSTANCE_NAME}}/g; s/VOL_NAME/${{secrets.VOL_NAME}}/g; s/IP_ADDRESS/${{secrets.IP_ADDRESS}}/g' ./tests/data/fs-manifest.yaml 
    - name: Create auto-mount GCP Filestore Storage instance
      run: |
        python3 xpk.py storage create $FS_STORAGE_NAME --cluster=$TPU_FILESTORE_CLUSTER_NAME --zone=us-central2-b --type=gcpfilestore \
        --auto-mount=true \
        --mount-point='/fs-test-mount-point' --readonly=false --manifest='./tests/data/fs-manifest.yaml'
    - name: List and verify existing Storages
      run: python3 xpk.py storage list --cluster $TPU_FILESTORE_CLUSTER_NAME --zone=us-central2-b | tee output.txt | grep $FS_STORAGE_NAME || (echo 'No storage found' && exit 143)
    - name: Run workload to write file on filestore
      run : python3 xpk.py workload create --workload $FS_WRITE_WORKLOAD --command "mkdir /fs-test-mount-point/$RANDOM_SEED/ && echo 'Test text message' > /fs-test-mount-point/$RANDOM_SEED/test.txt || (echo 'Writing to filestore failed' && exit 143)" --cluster $TPU_FILESTORE_CLUSTER_NAME --tpu-type=v4-8 --zone us-central2
    - name: Wait for writer workload completion and confirm it succeeded
      run: python3 xpk.py workload list --cluster $TPU_FILESTORE_CLUSTER_NAME --zone=us-central2-b --wait-for-job-completion $FS_WRITE_WORKLOAD --timeout 300
    - name: Run workload to read file on filestore
      run : python3 xpk.py workload create --workload $FS_READ_WORKLOAD --command "cat /fs-test-mount-point/$RANDOM_SEED/test.txt | grep 'Test text message' || (echo 'Reading from filestore failed' && exit 143)" --cluster $TPU_FILESTORE_CLUSTER_NAME --tpu-type=v4-8 --zone us-central2
    - name: Wait for reader workload completion and confirm it succeeded
      run: python3 xpk.py workload list --cluster $TPU_FILESTORE_CLUSTER_NAME --zone=us-central2-b --wait-for-job-completion $FS_READ_WORKLOAD --timeout 300
    - name: Run workload to delete file on filestore
      run : python3 xpk.py workload create --workload $FS_DELETE_WORKLOAD --command "rm -rf /fs-test-mount-point/$RANDOM_SEED/test.txt || exit 143" --cluster $TPU_FILESTORE_CLUSTER_NAME --tpu-type=v4-8 --zone us-central2
    - name: Wait for delete workload completion and confirm it succeeded
      run: python3 xpk.py workload list --cluster $TPU_FILESTORE_CLUSTER_NAME --zone=us-central2-b --wait-for-job-completion $FS_DELETE_WORKLOAD --timeout 300
    - name: Delete the writer workload on the cluster
      if: always()
      run: python3 xpk.py workload delete --workload $FS_WRITE_WORKLOAD --cluster $TPU_FILESTORE_CLUSTER_NAME --zone=us-central2-b
    - name: Delete the reader workload on the cluster
      if: always()
      run: python3 xpk.py workload delete --workload $FS_READ_WORKLOAD --cluster $TPU_FILESTORE_CLUSTER_NAME --zone=us-central2-b
    - name: Delete the delete workload on the cluster
      if: always()
      run: python3 xpk.py workload delete --workload $FS_DELETE_WORKLOAD --cluster $TPU_FILESTORE_CLUSTER_NAME --zone=us-central2-b
    - name: Delete the cluster created
      if: always()
      run: python3 xpk.py cluster delete --cluster $TPU_FILESTORE_CLUSTER_NAME --zone=us-central2-b