From 88139703aaa5f9f0891d4397a5493595a443d3ba Mon Sep 17 00:00:00 2001 From: David Zajac Date: Thu, 16 Nov 2023 23:40:25 -0300 Subject: [PATCH] Makefile and CI upgrades --- .github/workflows/generate-docs.yml | 3 +- .github/workflows/run-db-tests.yml | 81 +++++++++++++++-------------- Makefile | 34 ++++++++++++ README.md | 12 +++-- integration_tests/.gitignore | 2 + 5 files changed, 87 insertions(+), 45 deletions(-) create mode 100644 Makefile diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 4795135f..319b1961 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -5,6 +5,7 @@ on: [push] env: DBT_PROFILES_DIR: ./ DBT_VERSION: 1.7 + PYTHON_VERSION: "3.8.x" jobs: generate-docs: @@ -30,7 +31,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.8.x" + python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies and run run: | diff --git a/.github/workflows/run-db-tests.yml b/.github/workflows/run-db-tests.yml index 4b244902..874ba138 100644 --- a/.github/workflows/run-db-tests.yml +++ b/.github/workflows/run-db-tests.yml @@ -19,11 +19,48 @@ env: BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL: ${{ secrets.BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL }} BIGQUERY_TESTING_CLIENT_X509_CERT_URL: ${{ secrets.BIGQUERY_TESTING_CLIENT_X509_CERT_URL }} DBT_VERSION: 1.7 + PYTHON_VERSION: "3.8.x" jobs: + test-postgres: + runs-on: ubuntu-latest + if: github.event_name == 'push' + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + steps: + - name: Check out + uses: actions/checkout@v2 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + working-directory: ./integration_tests + run: | + pip install -r requirements.txt + pip install dbt-postgres==$DBT_VERSION + dbt deps + + - name: Test DB + working-directory: ./integration_tests/python_tests + run: pytest --db postgres --source_schema dq + test-other-dbs: runs-on: ubuntu-latest - if: github.event_name == 'push' && github.repository == 're-data/dbt-re-data' + if: github.event_name == 'push' && github.repository == 're-data/dbt-re-data' && github.ref == 'refs/heads/master' strategy: fail-fast: false matrix: @@ -34,7 +71,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.8.x" + python-version: ${{ env.PYTHON_VERSION }} - name: Inject slug/short variables uses: rlespinasse/github-slug-action@v3.x @@ -71,45 +108,9 @@ jobs: run: | pytest --db ${{ matrix.database }} --source_schema ${{ env.DQ_SCHEMA }} - test-postgres: - runs-on: ubuntu-latest - if: github.event_name == 'push' - services: - postgres: - image: postgres - env: - POSTGRES_PASSWORD: postgres - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 - steps: - - name: Check out - uses: actions/checkout@v2 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8.x" - - - name: Install dependencies - working-directory: ./integration_tests - run: | - pip install -r requirements.txt - pip install dbt-postgres==$DBT_VERSION - dbt deps - - - name: Test DB - working-directory: ./integration_tests/python_tests - run: pytest --db postgres --source_schema dq - clean-up-schemas: runs-on: ubuntu-latest - if: github.event_name == 'delete' && github.repository == 're-data/dbt-re-data' + if: github.event_name == 'delete' && github.repository == 're-data/dbt-re-data' && github.ref == 'refs/heads/master' strategy: fail-fast: false matrix: @@ -120,7 +121,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.8.x" + python-version: ${{ env.PYTHON_VERSION }} - name: Inject slug/short variables uses: rlespinasse/github-slug-action@v3.x diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..e3341dee --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ + +# Maybe change this if you are not running on a Mac +CONTAINER_ARCH = linux/amd64 + +.PHONY: help run-all-ci test-postgres generate-docs + +help: + $(info ${HELP_MESSAGE}) + @exit 0 + + +# Run GitHub Actions CI jobs locally +run-all-ci: test-postgres generate-docs + @echo "All CI steps completed." + +test-postgres: + @echo "Running test-postgres job..." + act -j test-postgres --container-architecture $(CONTAINER_ARCH) + +generate-docs: + @echo "Running generate-docs job..." + act -j generate-docs --container-architecture $(CONTAINER_ARCH) + + +define HELP_MESSAGE +Usage: $ make [TARGETS] + +TARGETS + help Shows this help message + run-all-ci Runs all CI steps + test-postgres Runs test-postgres job + generate-docs Generates documentation + +endef diff --git a/README.md b/README.md index 8cbc53d3..5711f9d4 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,19 @@ re_data is an open-source data reliability framework for the modern data stack. 😊 -Currently, re_data focuses on observing the dbt project (together with underlaying data warehouse - Postgres, BigQuery, Snowflake, Redshift). +Currently, re_data focuses on observing the dbt project (together with underlying data warehouse - Postgres, BigQuery, Snowflake, Redshift). Data transformations in re_data are implemented and exposed as models & macros in this dbt package. # Live demo -Check out our **[live demo](https://getre.io/demo)** of what re_data can do for you 😊 +Check out our **[live demo](https://docs.getre.io/ui-latest)** of what re_data can do for you 😊 # Getting started [Check our docs!](https://docs.getre.io/) 🙂 -[Join re_data community on Slack](https://www.getre.io/slack) (we are very responsive there) +[Join re_data community on Slack](https://join.slack.com/t/re-data/shared_invite/zt-vkauq1y8-tL4R4_H5nZoVvyXyy0hdug) (we are very responsive there) [Check out more info, issues, etc. in master repo](https://github.com/re-data/re-data) @@ -22,6 +22,10 @@ Check out our **[live demo](https://getre.io/demo)** of what re_data can do for Say, hi to us on! 🙂 -- [Slack](https://www.getre.io/slack) +- [Slack](https://join.slack.com/t/re-data/shared_invite/zt-vkauq1y8-tL4R4_H5nZoVvyXyy0hdug) - [Twitter](https://twitter.com/re_data_labs) - [LinkedIn](https://www.linkedin.com/company/74608627/) + +# Contributing + +Any contributions are greatly appreciated! Most of our documentation and GitHub issues are managed in the primary [re-data](https://github.com/re-data/re-data) repo. See the Contributing section in [re-data](https://github.com/re-data/re-data) for details. diff --git a/integration_tests/.gitignore b/integration_tests/.gitignore index bd457446..eb35edc1 100644 --- a/integration_tests/.gitignore +++ b/integration_tests/.gitignore @@ -3,3 +3,5 @@ target/ dbt_modules/ dbt_packages/ logs/ +.user.yml +package-lock.yml