From b444bf836132d316efa0c1286eb366f30c9b941f Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:42:38 -0700 Subject: [PATCH 1/7] Fix quoted identifiers in the `generate_base_model` macro for BigQuery (#199) * Use `adapter.quote` to create a case-sensitive quoted identifier for column names * Force a failure for all adapters to help troubleshoot * Revert "Force a failure for all adapters to help troubleshoot" This reverts commit d707832d3e13d12e3619b248cc87f1c72447bafb. * Use `adapter.quote` to create a case-sensitive quoted identifier for column names in `generate_base_model` macro --- integration_tests/macros/operations/create_source_table.sql | 4 ++-- .../tests/test_generate_base_models_all_args.sql | 4 ++-- .../tests/test_generate_base_models_case_sensitive.sql | 4 ++-- macros/generate_base_model.sql | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/integration_tests/macros/operations/create_source_table.sql b/integration_tests/macros/operations/create_source_table.sql index 6794e51..8123f33 100644 --- a/integration_tests/macros/operations/create_source_table.sql +++ b/integration_tests/macros/operations/create_source_table.sql @@ -48,8 +48,8 @@ set enable_case_sensitive_identifier to true; {% set create_table_sql_case_sensitive %} create table {{ target_schema }}.codegen_integration_tests__data_source_table_case_sensitive as ( select - 1 as {% if target.type == "bigquery" %}My_Integer_Col{% else %}"My_Integer_Col"{% endif %}, - true as {% if target.type == "bigquery" %}My_Bool_Col{% else %}"My_Bool_Col"{% endif %} + 1 as {{ adapter.quote("My_Integer_Col") }}, + true as {{ adapter.quote("My_Bool_Col") }} ) {% endset %} diff --git a/integration_tests/tests/test_generate_base_models_all_args.sql b/integration_tests/tests/test_generate_base_models_all_args.sql index 5a0b21e..89497de 100644 --- a/integration_tests/tests/test_generate_base_models_all_args.sql +++ b/integration_tests/tests/test_generate_base_models_all_args.sql @@ -20,8 +20,8 @@ with source as ( renamed as ( select - {% if target.type == "bigquery" %}My_Integer_Col{% else %}"My_Integer_Col"{% endif %} - , {% if target.type == "bigquery" %}My_Bool_Col{% else %}"My_Bool_Col"{% endif %} + {{ adapter.quote("My_Integer_Col") }} + , {{ adapter.quote("My_Bool_Col") }} from source diff --git a/integration_tests/tests/test_generate_base_models_case_sensitive.sql b/integration_tests/tests/test_generate_base_models_case_sensitive.sql index 2fd3123..1f18a1c 100644 --- a/integration_tests/tests/test_generate_base_models_case_sensitive.sql +++ b/integration_tests/tests/test_generate_base_models_case_sensitive.sql @@ -16,8 +16,8 @@ with source as ( renamed as ( select - {% if target.type == "bigquery" %}My_Integer_Col{% else %}"My_Integer_Col"{% endif %}, - {% if target.type == "bigquery" %}My_Bool_Col{% else %}"My_Bool_Col"{% endif %} + {{ adapter.quote("My_Integer_Col") }}, + {{ adapter.quote("My_Bool_Col") }} from source diff --git a/macros/generate_base_model.sql b/macros/generate_base_model.sql index 0a58784..aa74145 100644 --- a/macros/generate_base_model.sql +++ b/macros/generate_base_model.sql @@ -25,11 +25,11 @@ renamed as ( select {%- if leading_commas -%} {%- for column in column_names %} - {{", " if not loop.first}}{% if not case_sensitive_cols %}{{ column | lower }}{% elif target.type == "bigquery" %}{{ column }}{% else %}{{ "\"" ~ column ~ "\"" }}{% endif %} + {{", " if not loop.first}}{% if not case_sensitive_cols %}{{ column | lower }}{% else %}{{ adapter.quote(column) }}{% endif %} {%- endfor %} {%- else -%} {%- for column in column_names %} - {% if not case_sensitive_cols %}{{ column | lower }}{% elif target.type == "bigquery" %}{{ column }}{% else %}{{ "\"" ~ column ~ "\"" }}{% endif %}{{"," if not loop.last}} + {% if not case_sensitive_cols %}{{ column | lower }}{% else %}{{ adapter.quote(column) }}{% endif %}{{"," if not loop.last}} {%- endfor -%} {%- endif %} From 696c9f01758d727f9c1a360fcbe5af7b5ab7fa1e Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:04:23 -0700 Subject: [PATCH 2/7] Try removing Redshift-specific logic (#208) --- .../macros/operations/create_source_table.sql | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/integration_tests/macros/operations/create_source_table.sql b/integration_tests/macros/operations/create_source_table.sql index 8123f33..61d0814 100644 --- a/integration_tests/macros/operations/create_source_table.sql +++ b/integration_tests/macros/operations/create_source_table.sql @@ -1,12 +1,5 @@ {% macro create_source_table() %} -{% if target.type == "redshift" %} -{% set disable_case_sensitive %} -reset enable_case_sensitive_identifier; -{% endset %} -{{ run_query(disable_case_sensitive) }} -{% endif %} - {% set target_schema=api.Relation.create( database=target.database, schema="codegen_integration_tests__data_source_schema" @@ -38,13 +31,6 @@ drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_ {{ run_query(drop_table_sql_case_sensitive) }} -{% if target.type == "redshift" %} -{% set enable_case_sensitive %} -set enable_case_sensitive_identifier to true; -{% endset %} -{{ run_query(enable_case_sensitive) }} -{% endif %} - {% set create_table_sql_case_sensitive %} create table {{ target_schema }}.codegen_integration_tests__data_source_table_case_sensitive as ( select From 5caf2e39bbc171a620049ada7336d125d8993bbf Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:31:03 -0700 Subject: [PATCH 3/7] =?UTF-8?q?Use=20the=20`cimg/postgres`=20Docker=20imag?= =?UTF-8?q?e=20created=20by=20CircleCI=20with=20continu=E2=80=A6=20(#214)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use the `cimg/postgres` Docker image created by CircleCI with continuous integration builds in mind * Add the root Postgres user to the environment --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6746196..cf6f1a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,8 +4,9 @@ jobs: build: docker: - image: cimg/python:3.9.9 - - image: circleci/postgres:9.6.5-alpine-ram - + - image: cimg/postgres:9.6 + environment: + POSTGRES_USER: root steps: - checkout From 6388e32c4629d7ff8a31dc0a1b90419937c208cf Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:01:17 -0700 Subject: [PATCH 4/7] Independent workflow job for dbt-postgres (#215) * Independent workflow job for dbt-postgres * Remove activation of virtual environment * Try without `python -m` * Independent workflow job for dbt-redshift * Independent workflow job for dbt-snowflake * Independent workflow job for dbt-snowflake * Independent workflow job for dbt-bigquery * Independent workflow job for dbt-bigquery * Independent workflow job for dbt-bigquery * Independent workflow job for dbt-bigquery * Independent workflow job for dbt-bigquery * Setup environment variables for dbt-bigquery --- .circleci/config.yml | 123 ++++++++++++++++++++++++++----------------- 1 file changed, 76 insertions(+), 47 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf6f1a3..acc10fd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,100 +1,129 @@ version: 2 jobs: - build: + + integration-postgres: docker: - - image: cimg/python:3.9.9 + - image: cimg/python:3.9 - image: cimg/postgres:9.6 environment: POSTGRES_USER: root + environment: + POSTGRES_HOST: localhost + POSTGRES_USER: root + DBT_ENV_SECRET_POSTGRES_PASS: '' + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: circle_test + POSTGRES_SCHEMA: codegen_integration_tests_postgres + steps: - checkout - - - run: - name: setup_creds - command: | - echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json - - - restore_cache: - key: deps1-{{ .Branch }} - - - run: - name: "Setup dbt" - command: | - python3 -m venv dbt_venv - . dbt_venv/bin/activate - - python -m pip install --upgrade pip setuptools - python -m pip install --pre dbt-core dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery - + - run: pip install --pre dbt-core dbt-postgres - run: name: "Run Tests - Postgres" - environment: - POSTGRES_HOST: localhost - POSTGRES_USER: root - DBT_ENV_SECRET_POSTGRES_PASS: "" - POSTGRES_PORT: 5432 - POSTGRES_DATABASE: circle_test - POSTGRES_SCHEMA: codegen_integration_tests_postgres command: | - . dbt_venv/bin/activate cd integration_tests dbt --warn-error deps --target postgres dbt --warn-error run-operation create_source_table --target postgres dbt --warn-error seed --target postgres --full-refresh dbt --warn-error run --target postgres dbt --warn-error test --target postgres + - store_artifacts: + path: integration_tests/logs + - store_artifacts: + path: integration_tests/target + + # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass + resource_class: large + integration-redshift: + docker: + - image: cimg/python:3.9 + steps: + - checkout + - run: pip install --pre dbt-core dbt-redshift - run: name: "Run Tests - Redshift" command: | - . dbt_venv/bin/activate - echo `pwd` cd integration_tests dbt --warn-error deps --target redshift dbt --warn-error run-operation create_source_table --target redshift dbt --warn-error seed --target redshift --full-refresh dbt --warn-error run --target redshift dbt --warn-error test --target redshift + - store_artifacts: + path: integration_tests/logs + - store_artifacts: + path: integration_tests/target + # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass + resource_class: large + integration-snowflake: + docker: + - image: cimg/python:3.9 + steps: + - checkout + - run: pip install --pre dbt-core dbt-snowflake - run: name: "Run Tests - Snowflake" command: | - . dbt_venv/bin/activate - echo `pwd` cd integration_tests dbt --warn-error deps --target snowflake dbt --warn-error run-operation create_source_table --target snowflake dbt --warn-error seed --target snowflake --full-refresh dbt --warn-error run --target snowflake dbt --warn-error test --target snowflake + - store_artifacts: + path: integration_tests/logs + - store_artifacts: + path: integration_tests/target + # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass + resource_class: large + integration-bigquery: + environment: + BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" + docker: + - image: cimg/python:3.9 + steps: + - checkout + - run: pip install --pre dbt-core dbt-bigquery + - run: + name: Setup Environment Variables + command: | + echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json + echo 'export BIGQUERY_KEYFILE_JSON="$BIGQUERY_SERVICE_ACCOUNT_JSON"' >> "$BASH_ENV" - run: name: "Run Tests - BigQuery" - environment: - BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" - command: | - . dbt_venv/bin/activate - echo `pwd` cd integration_tests dbt --warn-error deps --target bigquery dbt --warn-error run-operation create_source_table --target bigquery dbt --warn-error seed --target bigquery --full-refresh dbt --warn-error run --target bigquery dbt --warn-error test --target bigquery - - - save_cache: - key: deps1-{{ .Branch }} - paths: - - "dbt_venv" + - store_artifacts: + path: integration_tests/logs + - store_artifacts: + path: integration_tests/target + # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass + resource_class: large workflows: version: 2 test-all: jobs: - - build: - context: - - profile-redshift - - profile-snowflake - - profile-bigquery + - integration-postgres: + context: profile-postgres + - integration-redshift: + context: profile-redshift + requires: + - integration-postgres + - integration-snowflake: + context: profile-snowflake + requires: + - integration-postgres + - integration-bigquery: + context: profile-bigquery + requires: + - integration-postgres From f41d1b9c23f9d9dad4ab05de74a3f0ed0a78a766 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:22:00 -0700 Subject: [PATCH 5/7] Simplify environment variables for BigQuery in CircleCI (#216) * Simplify environment variables for BigQuery in CircleCI * Fix YAML parsing error * Fix reference to environment variable * Fix reference to environment variable --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index acc10fd..e6ccb79 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,7 +91,7 @@ jobs: - run: name: Setup Environment Variables command: | - echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json + echo $BIGQUERY_SERVICE_ACCOUNT_JSON > $BIGQUERY_SERVICE_KEY_PATH echo 'export BIGQUERY_KEYFILE_JSON="$BIGQUERY_SERVICE_ACCOUNT_JSON"' >> "$BASH_ENV" - run: name: "Run Tests - BigQuery" From 1cc3090cb75629274f1d4d80f3bbb9aca030e962 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Fri, 22 Nov 2024 08:08:31 -0700 Subject: [PATCH 6/7] Stop installing prereleases from PyPI in favor of stable releases only (#220) --- .circleci/config.yml | 8 ++++---- Makefile | 2 +- integration_tests/README.md | 7 ++----- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e6ccb79..d873aa4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: steps: - checkout - - run: pip install --pre dbt-core dbt-postgres + - run: pip install dbt-core dbt-postgres - run: name: "Run Tests - Postgres" command: | @@ -41,7 +41,7 @@ jobs: - image: cimg/python:3.9 steps: - checkout - - run: pip install --pre dbt-core dbt-redshift + - run: pip install dbt-core dbt-redshift - run: name: "Run Tests - Redshift" command: | @@ -63,7 +63,7 @@ jobs: - image: cimg/python:3.9 steps: - checkout - - run: pip install --pre dbt-core dbt-snowflake + - run: pip install dbt-core dbt-snowflake - run: name: "Run Tests - Snowflake" command: | @@ -87,7 +87,7 @@ jobs: - image: cimg/python:3.9 steps: - checkout - - run: pip install --pre dbt-core dbt-bigquery + - run: pip install dbt-core dbt-bigquery - run: name: Setup Environment Variables command: | diff --git a/Makefile b/Makefile index e3545fc..d842cb9 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ dev: ## Installs dbt-* packages in develop mode along with development dependenc @\ echo "Install dbt-$(target)..."; \ python -m pip install --upgrade pip setuptools; \ - python -m pip install --pre dbt-core "dbt-$(target)"; + python -m pip install dbt-core "dbt-$(target)"; .PHONY: setup-db setup-db: ## Setup Postgres database with docker-compose for system testing. diff --git a/integration_tests/README.md b/integration_tests/README.md index ed9c863..91f9ef0 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -89,7 +89,7 @@ Next, install `dbt-core` (and its dependencies) with: ```shell make dev target=[postgres|redshift|...] # or -python3 -m pip install --pre dbt-core dbt-[postgres|redshift|...] +python3 -m pip install dbt-core dbt-[postgres|redshift|...] ``` Or more specific: @@ -97,12 +97,9 @@ Or more specific: ```shell make dev target=postgres # or -python3 -m pip install --pre dbt-core dbt-postgres +python3 -m pip install dbt-core dbt-postgres ``` -> [!NOTE] -> The `--pre` flag tells pip to install the latest pre-release version of whatever you pass to install. This ensures you're always using the latest version of dbt, so if your code interacts with dbt in a way that causes issues or test failures, we'll know about it ahead of a release. - Make sure to reload your virtual environment after installing the dependencies: ```shell From fdc998c8d1bda197bcc3be9f85272a6bdc3f2622 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Fri, 22 Nov 2024 08:38:52 -0700 Subject: [PATCH 7/7] Upgrade to Python 3.11 in CircleCI (#222) --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d873aa4..348ba3f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ jobs: integration-postgres: docker: - - image: cimg/python:3.9 + - image: cimg/python:3.11 - image: cimg/postgres:9.6 environment: POSTGRES_USER: root @@ -38,7 +38,7 @@ jobs: integration-redshift: docker: - - image: cimg/python:3.9 + - image: cimg/python:3.11 steps: - checkout - run: pip install dbt-core dbt-redshift @@ -60,7 +60,7 @@ jobs: integration-snowflake: docker: - - image: cimg/python:3.9 + - image: cimg/python:3.11 steps: - checkout - run: pip install dbt-core dbt-snowflake @@ -84,7 +84,7 @@ jobs: environment: BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" docker: - - image: cimg/python:3.9 + - image: cimg/python:3.11 steps: - checkout - run: pip install dbt-core dbt-bigquery