parse_catalog.yaml

name: Parse Catalog

on:
  push:
    branches: ["main"]
  workflow_dispatch:
  schedule:
    - cron: "0 */12 * * *"

jobs:
  parse-catalog:
    name: dump-catalog-csvs-from-bq
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: "Authenticate to Google Cloud"
        id: "auth"
        uses: "google-github-actions/auth@v2"
        with:
          credentials_json: "${{ secrets.GCP_DATAFLOW_SERVICE_KEY }}"
      - uses: actions/setup-python@v5
        with:
          python-version: "3.9"
      - name: Install dependencies
        # TODO: base on an actual release
        run: python -m pip install gcsfs "leap-data-management-utils[bigquery]==0.0.14"
      - name: "Parse catalog"
        shell: bash
        run: |
          python scripts/dump_bigquery_to_csv.py
        env:
          GOOGLE_APPLICATION_CREDENTIALS: "${{ steps.auth.outputs.credentials_file_path }}"
      - name: Save csv file as artifacts
        uses: actions/upload-artifact@v4
        with:
          name: csv-files
          path: "*.csv"