Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert tox tests to central workflow #947

Merged
merged 4 commits into from
Sep 26, 2024
Merged

Convert tox tests to central workflow #947

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c8c6806
use central workflow
emmyoop Sep 9, 2024
aaadd84
Update .github/workflows/ci.yml
emmyoop Sep 9, 2024
48d855e
use tag
emmyoop Sep 26, 2024
7073836
Merge branch 'er/use-shared-workflow' of https://github.com/dbt-labs/…
emmyoop Sep 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 24 additions & 119 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,125 +15,30 @@ on:
- main
pull_request:
workflow_dispatch:
inputs:
adapter:
description: The adapter to test against. Defaults to all supported adapters when blank.
type: string
required: false

env:
PYTHON_VERSION: "3.11"

jobs:
determine-supported-adapters:
runs-on: ubuntu-latest
outputs:
adapters: ${{ steps.supported-adapters.outputs.adapters }}
steps:
- name: "Checkout ${{ github.event.repository }}"
uses: actions/checkout@v4

- name: "Set up Python ${{ env.PYTHON_VERSION }}"
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: "Install tox"
run: |
python -m pip install --upgrade pip
pip install tox

- name: "Get list of supported adapters or use input adapter only"
id: list-adapters
run: |
if [ -z "${{ inputs.adapter }}" ]; then
# github adds a pip freeze and a new line we need to strip out
source supported_adapters.env
echo $SUPPORTED_ADAPTERS
echo "test_adapters=$SUPPORTED_ADAPTERS" >> $GITHUB_OUTPUT
else
echo "test_adapters=${{ inputs.adapter }}" >> $GITHUB_OUTPUT
fi

- name: "Format adapter list for use as the matrix"
id: supported-adapters
run: |
# Convert to JSON array and output
supported_adapters=$(echo "${{ steps.list-adapters.outputs.test_adapters }}" | jq -Rc 'split(",")')
echo $supported_adapters
echo "adapters=$supported_adapters" >> $GITHUB_OUTPUT

- name: "[ANNOTATION] ${{ github.event.repository.name }} - Testing ${{ steps.supported-adapters.outputs.adapters }}"
run: |
title="${{ github.event.repository.name }} - adapters to test"
message="The workflow will run tests for the following adapters: ${{ steps.supported-adapters.outputs.adapters }}"
echo "::notice $title::$message"

run-tests:
runs-on: ubuntu-latest
needs: [determine-supported-adapters]
services:
postgres:
image: postgres
env:
POSTGRES_USER: ${{ vars.POSTGRES_USER }}
POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASS }}
POSTGRES_DB: ${{ vars.POSTGRES_DATABASE }}
POSTGRES_HOST: ${{ vars.POSTGRES_HOST }}
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
strategy:
fail-fast: false
matrix:
adapter: ${{fromJson(needs.determine-supported-adapters.outputs.adapters)}}

steps:
- name: "Checkout ${{ github.event.repository }} "
uses: actions/checkout@v4

- name: "Set up Python ${{ env.PYTHON_VERSION }}"
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: "Install ${{ matrix.adapter }}"
run: |
python -m pip install --upgrade pip
pip install dbt-${{ matrix.adapter }}

- name: "Install tox"
run: |
python -m pip install --upgrade pip
pip install tox

- name: "Run integration tests with tox on ${{ matrix.adapter }}"
run: |
tox -e dbt_integration_${{ matrix.adapter }}
env:
POSTGRES_HOST: ${{ vars.POSTGRES_HOST }}
POSTGRES_USER: ${{ vars.POSTGRES_USER }}
DBT_ENV_SECRET_POSTGRES_PASS: ${{ secrets.POSTGRES_PASS }}
POSTGRES_PORT: 5432
POSTGRES_DATABASE: ${{ vars.POSTGRES_DATABASE }}
POSTGRES_SCHEMA: "dbt_utils_integration_tests_postgres_${{ github.run_number }}"
SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}"
REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}"
REDSHIFT_PORT: 5439
BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}"
run-tests:
uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
with:
# no need to pass postgres vars in. We can just use the defaults in the local container
# redshift
REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}"
REDSHIFT_PORT: 5439
# bigquery
BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}"
# snowflake
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}"
secrets:
DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
Loading