Makefile and CI upgrades (#120)

.github/workflows/generate-docs.yml

@@ -5,6 +5,7 @@ on: [push]
 env:
   DBT_PROFILES_DIR: ./
   DBT_VERSION: 1.7
+  PYTHON_VERSION: "3.8.x"
 
 jobs:
   generate-docs:
@@ -30,7 +31,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8.x"
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install dependencies and run
         run: |

.github/workflows/run-db-tests.yml

@@ -19,11 +19,48 @@ env:
   BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL: ${{ secrets.BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL }}
   BIGQUERY_TESTING_CLIENT_X509_CERT_URL: ${{ secrets.BIGQUERY_TESTING_CLIENT_X509_CERT_URL }}
   DBT_VERSION: 1.7
+  PYTHON_VERSION: "3.8.x"
 
 jobs:
+  test-postgres:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:        
+      - name: Check out
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        working-directory: ./integration_tests
+        run: |
+          pip install -r requirements.txt
+          pip install dbt-postgres==$DBT_VERSION
+          dbt deps
+
+      - name: Test DB
+        working-directory: ./integration_tests/python_tests
+        run: pytest --db postgres --source_schema dq
+
   test-other-dbs:
     runs-on: ubuntu-latest
-    if: github.event_name == 'push' && github.repository == 're-data/dbt-re-data'
+    if: github.event_name == 'push' && github.repository == 're-data/dbt-re-data' && github.ref == 'refs/heads/master'
     strategy:
       fail-fast: false
       matrix:
@@ -34,7 +71,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8.x"
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v3.x
@@ -71,45 +108,9 @@ jobs:
         run: |
           pytest --db ${{ matrix.database }} --source_schema ${{ env.DQ_SCHEMA }}
 
-  test-postgres:
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push'
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: postgres
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-    steps:        
-      - name: Check out
-        uses: actions/checkout@v2
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.8.x"
-
-      - name: Install dependencies
-        working-directory: ./integration_tests
-        run: |
-          pip install -r requirements.txt
-          pip install dbt-postgres==$DBT_VERSION
-          dbt deps
-
-      - name: Test DB
-        working-directory: ./integration_tests/python_tests
-        run: pytest --db postgres --source_schema dq
-
   clean-up-schemas:
     runs-on: ubuntu-latest
-    if: github.event_name == 'delete' && github.repository == 're-data/dbt-re-data'
+    if: github.event_name == 'delete' && github.repository == 're-data/dbt-re-data' && github.ref == 'refs/heads/master'
     strategy:
       fail-fast: false
       matrix:
@@ -120,7 +121,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8.x"
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v3.x

Makefile

@@ -0,0 +1,34 @@
+
+# Maybe change this if you are not running on a Mac
+CONTAINER_ARCH = linux/amd64
+
+.PHONY: help run-all-ci test-postgres generate-docs
+
+help:
+	$(info ${HELP_MESSAGE})
+	@exit 0
+
+
+# Run GitHub Actions CI jobs locally
+run-all-ci: test-postgres generate-docs
+	@echo "All CI steps completed."
+
+test-postgres:
+	@echo "Running test-postgres job..."
+	act -j test-postgres --container-architecture $(CONTAINER_ARCH)
+
+generate-docs:
+	@echo "Running generate-docs job..."
+	act -j generate-docs --container-architecture $(CONTAINER_ARCH)
+
+
+define HELP_MESSAGE
+Usage: $ make [TARGETS]
+
+TARGETS
+  help                   Shows this help message
+  run-all-ci             Runs all CI steps
+  test-postgres          Runs test-postgres job
+  generate-docs          Generates documentation
+
+endef

README.md

@@ -2,26 +2,30 @@
 
 re_data is an open-source data reliability framework for the modern data stack. 😊
 
-Currently, re_data focuses on observing the dbt project (together with underlaying data warehouse - Postgres, BigQuery, Snowflake, Redshift).
+Currently, re_data focuses on observing the dbt project (together with underlying data warehouse - Postgres, BigQuery, Snowflake, Redshift).
 
 Data transformations in re_data are implemented and exposed as models & macros in this dbt package.
 
 # Live demo
 
-Check out our **[live demo](https://getre.io/demo)** of what re_data can do for you 😊
+Check out our **[live demo](https://docs.getre.io/ui-latest)** of what re_data can do for you 😊
 
 # Getting started
 
 [Check our docs!](https://docs.getre.io/) 🙂
 
-[Join re_data community on Slack](https://www.getre.io/slack) (we are very responsive there)
+[Join re_data community on Slack](https://join.slack.com/t/re-data/shared_invite/zt-vkauq1y8-tL4R4_H5nZoVvyXyy0hdug) (we are very responsive there)
 
 [Check out more info, issues, etc. in master repo](https://github.com/re-data/re-data)
 
 # Community
 
 Say, hi to us on! 🙂
 
-- [Slack](https://www.getre.io/slack)
+- [Slack](https://join.slack.com/t/re-data/shared_invite/zt-vkauq1y8-tL4R4_H5nZoVvyXyy0hdug)
 - [Twitter](https://twitter.com/re_data_labs)
 - [LinkedIn](https://www.linkedin.com/company/74608627/)
+
+# Contributing
+
+Any contributions are greatly appreciated! Most of our documentation and GitHub issues are managed in the primary [re-data](https://github.com/re-data/re-data) repo. See the Contributing section in [re-data](https://github.com/re-data/re-data) for details.

integration_tests/.gitignore

@@ -3,3 +3,5 @@ target/
 dbt_modules/
 dbt_packages/
 logs/
+.user.yml
+package-lock.yml