Upgrade/2.6.2

.github/pull_request_template.md

@@ -13,6 +13,7 @@
 - [ ] New features (breaking change)
 - [ ] Other (non-breaking change)
 - [ ] Other (breaking change)
+- [ ] Release preparation
 
 ## What does this solve?
 

.github/workflows/ci_lint_package.yml

@@ -43,8 +43,14 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR
 
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9.x"
+          architecture: "x64"
+
       - name: Install Python packages
-        run: python -m pip install dbt-snowflake~=1.5.0 sqlfluff-templater-dbt
+        run: python -m pip install dbt-snowflake~=1.7.0 sqlfluff-templater-dbt~=2.3.2
 
       - name: Test database connection
         run: dbt debug

.github/workflows/ci_test_package.yml

@@ -21,7 +21,7 @@ env:
   DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
   DBT_ENV_SECRET_GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
   # Env var to test version
-  LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_4_0 # A dbt version supported by both the last release and this one
+  LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_7_0 # A dbt version supported by both the last release and this one
   # Env vars to test invocations model
   DBT_CLOUD_PROJECT_ID: 123
   DBT_CLOUD_JOB_ID: ABC
@@ -37,14 +37,27 @@ jobs:
     strategy:
       fail-fast: false # Don't fail one DWH if the others fail
       matrix:
-        warehouse: ["snowflake", "bigquery"]
+        warehouse: ["snowflake", "bigquery", "postgres"]
     runs-on: ubuntu-latest
     environment:
       name: Approve Integration Tests
     permissions:
       contents: "read"
       id-token: "write"
 
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: Get latest release
         uses: rez0n/actions-github-release@main
@@ -100,16 +113,29 @@ jobs:
     strategy:
       fail-fast: false # Don't fail one DWH if the others fail
       matrix:
-        warehouse: ["snowflake", "bigquery"]
+        warehouse: ["snowflake", "bigquery", "postgres"]
         # When supporting a new version, update the list here
-        version: ["1_3_0", "1_4_0", "1_5_0"]
+        version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0"]
     runs-on: ubuntu-latest
     environment:
       name: Approve Integration Tests
     permissions:
       contents: "read"
       id-token: "write"
 
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - uses: actions/setup-python@v4
         with:

.github/workflows/main_lint_package.yml

@@ -39,8 +39,14 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@v3
 
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9.x"
+          architecture: "x64"
+
       - name: Install Python packages
-        run: python -m pip install dbt-snowflake~=1.5.0 sqlfluff-templater-dbt
+        run: python -m pip install dbt-snowflake~=1.7.0 sqlfluff-templater-dbt~=2.3.2
 
       - name: Test database connection
         run: dbt debug

.github/workflows/main_test_package.yml

@@ -34,13 +34,26 @@ jobs:
   integration:
     strategy:
       matrix:
-        warehouse: ["snowflake", "bigquery"]
-        version: ["1_3_0", "1_4_0", "1_5_0"]
+        warehouse: ["snowflake", "bigquery", "postgres"]
+        version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0"]
     runs-on: ubuntu-latest
     permissions:
       contents: "read"
       id-token: "write"
 
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.github/workflows/publish_docs_on_release.yml

@@ -39,7 +39,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install Python packages
-        run: python -m pip install dbt-snowflake~=1.3.0
+        run: python -m pip install dbt-snowflake~=1.7.0
 
       - name: Test database connection
         run: dbt debug

README.md

@@ -18,6 +18,7 @@ The package currently supports
 - Spark :white_check_mark:
 - Snowflake :white_check_mark:
 - Google BigQuery :white_check_mark:
+- Postgres :white_check_mark:
 
 Models included:
 
@@ -45,7 +46,7 @@ See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/
 ```
 packages:
   - package: brooklyn-data/dbt_artifacts
-    version: 2.4.2
+    version: 2.6.2
 ```
 
 :construction_worker: Make sure to fix at least the **minor** version, to avoid issues when a new release is open. See the notes on upgrading below for more detail.
@@ -55,15 +56,15 @@ packages:
 3. Add an on-run-end hook to your `dbt_project.yml`
 
     ```yml
-    `on-run-end:
-      - "{{ dbt_artifacts.upload_results(results) }}"`
+    on-run-end:
+      - "{{ dbt_artifacts.upload_results(results) }}"
     ```
 
     We recommend adding a conditional here so that the upload only occurs in your production environment, such as:
 
     ```yml
     on-run-end:
-      - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}"`)
+      - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}"
     ```
 
 4. Run the tables!
@@ -141,6 +142,37 @@ vars:
   ]
 ```
 
+## Creating custom marts tables
+
+Multiple modelled `dim` and `fct` models have been provided for ease of use, but we recognise that some use cases may require custom ones. To this end, you can disable all but the raw sources tables using the following in your `dbt_project.yml` file:
+
+```yml
+# dbt_project.yml
+
+models:
+  dbt_artifacts:
+    +enabled: false
+    sources:
+      +enabled: true
+```
+
+In these sources tables, you will find a JSON column `all_results` which contains a JSON blob of the results object used, which you can use in your own analysis:
+
+- exposures
+- models
+- seeds
+- snapshots
+- sources
+- tests
+
+This column can cause queries to become too long - particularly in BigQuery. Therefore, if you want to disable this column, you can make use of the `dbt_artifacts_exclude_all_results` variable, and set this to `true` in your `dbt_project.yml` file.
+
+```
+# dbt_project.yml
+vars:
+  dbt_artifacts_exclude_all_results: true
+```
+
 ## Upgrading from 1.x to >=2.0.0
 
 If you were using the following variables:
@@ -189,29 +221,6 @@ An example operation is as follows:
 dbt run-operation migrate_from_v0_to_v1 --args '{old_database: analytics, old_schema: dbt_artifacts, new_database: analytics, new_schema: artifact_sources}'
 ```
 
-## Creating custom marts tables
-
-Multiple modelled `dim` and `fct` models have been provided for ease of use, but we recognise that some use cases may require custom ones. To this end, you can disable all but the raw sources tables using the following in your `dbt_project.yml` file:
-
-```yml
-# dbt_project.yml
-
-models:
-  dbt_artifacts:
-    +enabled: false
-    sources:
-      +enabled: true
-```
-
-In these sources tables, you will find a JSON column `all_results` which contains a JSON blob of the results object used, which you can use in your own analysis:
-
-- exposures
-- models
-- seeds
-- snapshots
-- sources
-- tests
-
 ## Acknowledgements
 
 Thank you to [Tails.com](https://tails.com/gb/careers/) for initial development and maintenance of this package. On 2021/12/20, the repository was transferred from the Tails.com GitHub organization to Brooklyn Data Co.

dbt_project.yml

@@ -1,7 +1,7 @@
 name: "dbt_artifacts"
-version: "2.4.2"
+version: "2.6.2"
 config-version: 2
-require-dbt-version: [">=1.3.0", "<1.6.0"]
+require-dbt-version: [">=1.3.0", "<1.8.0"]
 profile: "dbt_artifacts"
 
 clean-targets: # folders to be removed by `dbt clean`

integration_test_project/dbt_project.yml

@@ -21,6 +21,7 @@ vars:
   env_vars:
     ["TEST_ENV_VAR_NUMBER", "TEST_ENV_VAR_EMPTY", "TEST_ENV_VAR_WITH_QUOTE"]
   dbt_vars: ["test_dbt_vars_1", "test_dbt_vars_2", "test_dbt_vars_3"]
+  dbt_artifacts_exclude_all_results: true
 
 models:
   +persist_docs:

integration_test_project/example-env.sh

@@ -18,6 +18,7 @@ export DBT_ENV_SPARK_DRIVER_PATH= # /Library/simba/spark/lib/libsparkodbc_sbu.dy
 export DBT_ENV_SPARK_ENDPOINT= # The endpoint ID from the Databricks HTTP path
 
 # dbt environment variables, change these
+export DBT_VERSION="1_5_0"
 export DBT_CLOUD_PROJECT_ID=
 export DBT_CLOUD_JOB_ID=
 export DBT_CLOUD_RUN_ID=

integration_test_project/profiles.yml

@@ -2,8 +2,8 @@
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
 config:
-    send_anonymous_usage_stats: False
-    use_colors: True
+  send_anonymous_usage_stats: False
+  use_colors: True
 
 dbt_artifacts:
   target: snowflake
@@ -43,3 +43,12 @@ dbt_artifacts:
       timeout_seconds: 300
       priority: interactive
       retries: 1
+    postgres:
+      type: postgres
+      host: localhost
+      user: postgres
+      password: postgres
+      port: 5432
+      dbname: postgres
+      schema: public
+      threads: 8

integration_test_project/tests/singular_test.sql

@@ -1 +1 @@
-select 1 as failures from (select 2) where 1 = 2
+select 1 as failures from (select 2) as foo where 1 = 2