ci_lint_package.yml

name: CI lint package

on:
  workflow_dispatch:
  # all PRs, important to note that `pull_request_target` workflows
  # will run in the context of the target branch of a PR
  pull_request_target:

env:
  # These are configured in GitHub secrets
  DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project
  GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger
  DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
  DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
  DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
  DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
  DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
  DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
  DBT_ENV_SECRET_DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
  DBT_ENV_SECRET_DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
  DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
  DBT_ENV_SECRET_GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
  # Env vars to test invocations model
  DBT_CLOUD_PROJECT_ID: 123
  DBT_CLOUD_JOB_ID: ABC
  DBT_CLOUD_RUN_REASON: "String with 'quotes' !"
  TEST_ENV_VAR_1: TEST_VALUE
  TEST_ENV_VAR_NUMBER: 3
  TEST_ENV_VAR_EMPTY: ""
  TEST_ENV_VAR_WITH_QUOTE: "Triggered via Apache Airflow by task 'trigger_dbt_cloud_job_run' in the airtable_ingest DAG."
  DBT_ENV_CUSTOM_ENV_FAVOURITE_DBT_PACKAGE: dbt_artifacts

jobs:
  sqlfluff-lint-models:
    name: Lint dbt models using SQLFluff
    runs-on: ubuntu-latest
    environment:
      name: Approve Integration Tests

    steps:
      - name: Checkout Branch
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9.x"
          architecture: "x64"

      - name: Install Python packages
        run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.3.2

      - name: Test database connection
        run: dbt debug

      - name: Install dbt packages
        run: dbt deps

      - name: Get changed files
        id: get_file_changes
        uses: trilom/file-changes-action@v1.2.4
        with:
          output: " "

      - name: Get new and changed .sql files in /models to lint
        id: get_files_to_lint
        shell: bash -l {0}
        run: |
          # Set the command in the $() brackets as an output to use in later steps
          echo "lintees=$(
          # Issue where grep regular expressions don't work as expected on the
          # Github Actions shell, check dbt/models/ folder
          echo \
          $(echo ${{ steps.get_file_changes.outputs.files_modified }} |
          tr -s ' ' '\n' |
          grep -E '^models.*[.]sql$' |
          tr -s '\n' ' ') \
          $(echo ${{ steps.get_file_changes.outputs.files_added }} |
          tr -s ' ' '\n' |
          grep -E '^models.*[.]sql$' |
          tr -s '\n' ' ')
          )" >> $GITHUB_OUTPUT

      - name: Lint dbt models
        if: steps.get_files_to_lint.outputs.lintees != ''
        shell: bash -l {0}
        run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} --write-output annotations.json

      - name: Annotate
        uses: yuzutech/annotations-action@v0.4.0
        with:
          repo-token: "${{ secrets.GITHUB_TOKEN }}"
          title: "SQLFluff Lint"
          input: "./annotations.json"