ci-benchmark.yaml

name: Run benchmarks
on:
  pull_request:
    types:
      - labeled
  schedule:
    - cron: '0 0 * * 1'

# This allows a subsequently queued workflow run to interrupt and cancel previous runs
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  Run-Benchmark:
    if: >-
      (github.event_name == 'schedule' && github.event.schedule != '0 0 * * *') ||
      (
        github.event.label.name == 'run_benchmark' &&
        github.event.name != 'pull_request_target'
      )
    runs-on: ubuntu-latest
    env:
      SETUPTOOLS_USE_DISTUTILS: stdlib
      GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
      GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
      POSTGRES_HOST: 0.0.0.0
      POSTGRES_PORT: 5432
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AIRFLOW__ASTRO_SDK__SQL_SCHEMA: astroflow_ci
      REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN }}
      REDSHIFT_DATABASE: dev
      REDSHIFT_HOST: utkarsh-cluster.cdru7mxqmtyx.us-east-2.redshift.amazonaws.com
      REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }}
      REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }}
      SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }}
      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
      SNOWFLAKE_SCHEMA: ASTROFLOW_CI
      SNOWFLAKE_DATABASE: SANDBOX
      SNOWFLAKE_WAREHOUSE: DEMO
      SNOWFLAKE_HOST: https://gp21411.us-east-1.snowflakecomputing.com
      SNOWFLAKE_ACCOUNT: gp21411
      SNOWFLAKE_REGION: us-east-1
      SNOWFLAKE_ROLE: AIRFLOW_TEST_USER
      AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS: True
      AIRFLOW__CORE__ENABLE_XCOM_PICKLING: True
      AIRFLOW_VAR_FOO: templated_file_name
      AIRFLOW__ASTRO_SDK__DATAFRAME_ALLOW_UNSAFE_STORAGE: True
      FORCE_COLOR: "true"
      ASTRO_PUBLISH_BENCHMARK_DATA: True
      USE_GKE_GCLOUD_AUTH_PLUGIN: True
    steps:
      - uses: actions/checkout@v3
      - name: Install google-cloud-sdk-auth-plugin
        run: |
          echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
          curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
          sudo apt update
          sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin kubectl
      - name: Run benchmark
        run: |
          rm -f python-sdk/test-connections.yaml
          ( echo "cat <<EOF >python-sdk/test-connections.yaml"; cat .github/ci-test-connections.yaml; ) >python-sdk/test-connections.yaml && . python-sdk/test-connections.yaml
          python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
          cd python-sdk/tests/benchmark && make