From 5738b7cbe01991c8c26489ea75137114e300a855 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 20 Nov 2024 14:35:23 -0600 Subject: [PATCH 01/12] add support for bigquery testing --- .github/workflows/ci.yml | 4 ++++ integration_tests/profiles.yml | 9 +++++---- supported_adapters.env | 2 +- tox.ini | 18 ++++++++++++++++++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7497bc..bcd3004 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,10 @@ jobs: SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} SNOWFLAKE_SCHEMA: "integration_tests_snowflake_${{ github.run_number }}" + # bigquery + BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} + BIGQUERY_SCHEMA: "integration_tests_bigquery_${{ github.run_number }}" secrets: SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} + BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index f2ddf18..fde62f5 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -28,10 +28,11 @@ integration_tests: bigquery: type: bigquery - method: service-account - keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" - project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" - schema: codegen_integration_tests_bigquery + method: service-account-json + keyfile_json: + "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}" + project: "{{ env_var('BIGQUERY_PROJECT') }}" + dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" threads: 1 snowflake: diff --git a/supported_adapters.env b/supported_adapters.env index 8d18434..158d7eb 100644 --- a/supported_adapters.env +++ b/supported_adapters.env @@ -1 +1 @@ -SUPPORTED_ADAPTERS=postgres,snowflake +SUPPORTED_ADAPTERS=postgres,snowflake,bigquery diff --git a/tox.ini b/tox.ini index d10286b..65616d4 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,9 @@ passenv = SNOWFLAKE_DATABASE SNOWFLAKE_WAREHOUSE SNOWFLAKE_SCHEMA + # bigquery env vars + BIGQUERY_PROJECT + BIGQUERY_SCHEMA # Postgres integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment @@ -49,3 +52,18 @@ commands = dbt --warn-error seed --target snowflake --full-refresh dbt --warn-error run --target snowflake dbt --warn-error test --target snowflake + +# bigquery integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_bigquery] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt --warn-error deps --target bigquery + dbt --warn-error run-operation create_source_table --target bigquery + dbt --warn-error seed --target bigquery --full-refresh + dbt --warn-error run --target bigquery + dbt --warn-error test --target bigquery From 12f836968a5abf7435fb32cfea7b7a7144e5ff84 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 20 Nov 2024 14:44:00 -0600 Subject: [PATCH 02/12] add missing var in tox file --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 65616d4..b5540dc 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,7 @@ passenv = # bigquery env vars BIGQUERY_PROJECT BIGQUERY_SCHEMA + BIGQUERY_KEYFILE_JSON # Postgres integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment From 72b18799f6ffbd5b26f7c7b31262518342c38b3e Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Thu, 21 Nov 2024 08:58:53 -0700 Subject: [PATCH 03/12] Temporarily only run CI tests for BigQuery --- .circleci/config.yml | 78 ++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6746196..b3a6aab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,47 +26,47 @@ jobs: python -m pip install --upgrade pip setuptools python -m pip install --pre dbt-core dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery - - run: - name: "Run Tests - Postgres" - environment: - POSTGRES_HOST: localhost - POSTGRES_USER: root - DBT_ENV_SECRET_POSTGRES_PASS: "" - POSTGRES_PORT: 5432 - POSTGRES_DATABASE: circle_test - POSTGRES_SCHEMA: codegen_integration_tests_postgres - command: | - . dbt_venv/bin/activate - cd integration_tests - dbt --warn-error deps --target postgres - dbt --warn-error run-operation create_source_table --target postgres - dbt --warn-error seed --target postgres --full-refresh - dbt --warn-error run --target postgres - dbt --warn-error test --target postgres + # - run: + # name: "Run Tests - Postgres" + # environment: + # POSTGRES_HOST: localhost + # POSTGRES_USER: root + # DBT_ENV_SECRET_POSTGRES_PASS: "" + # POSTGRES_PORT: 5432 + # POSTGRES_DATABASE: circle_test + # POSTGRES_SCHEMA: codegen_integration_tests_postgres + # command: | + # . dbt_venv/bin/activate + # cd integration_tests + # dbt --warn-error deps --target postgres + # dbt --warn-error run-operation create_source_table --target postgres + # dbt --warn-error seed --target postgres --full-refresh + # dbt --warn-error run --target postgres + # dbt --warn-error test --target postgres - - run: - name: "Run Tests - Redshift" - command: | - . dbt_venv/bin/activate - echo `pwd` - cd integration_tests - dbt --warn-error deps --target redshift - dbt --warn-error run-operation create_source_table --target redshift - dbt --warn-error seed --target redshift --full-refresh - dbt --warn-error run --target redshift - dbt --warn-error test --target redshift + # - run: + # name: "Run Tests - Redshift" + # command: | + # . dbt_venv/bin/activate + # echo `pwd` + # cd integration_tests + # dbt --warn-error deps --target redshift + # dbt --warn-error run-operation create_source_table --target redshift + # dbt --warn-error seed --target redshift --full-refresh + # dbt --warn-error run --target redshift + # dbt --warn-error test --target redshift - - run: - name: "Run Tests - Snowflake" - command: | - . dbt_venv/bin/activate - echo `pwd` - cd integration_tests - dbt --warn-error deps --target snowflake - dbt --warn-error run-operation create_source_table --target snowflake - dbt --warn-error seed --target snowflake --full-refresh - dbt --warn-error run --target snowflake - dbt --warn-error test --target snowflake + # - run: + # name: "Run Tests - Snowflake" + # command: | + # . dbt_venv/bin/activate + # echo `pwd` + # cd integration_tests + # dbt --warn-error deps --target snowflake + # dbt --warn-error run-operation create_source_table --target snowflake + # dbt --warn-error seed --target snowflake --full-refresh + # dbt --warn-error run --target snowflake + # dbt --warn-error test --target snowflake - run: name: "Run Tests - BigQuery" From b3002ee71ef6a52f99382d1882d47e31517f2ae3 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Thu, 21 Nov 2024 09:19:48 -0700 Subject: [PATCH 04/12] Prefix the schema for the data source in CI with the name of the target schema --- integration_tests/macros/operations/create_source_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/macros/operations/create_source_table.sql b/integration_tests/macros/operations/create_source_table.sql index 6794e51..5584742 100644 --- a/integration_tests/macros/operations/create_source_table.sql +++ b/integration_tests/macros/operations/create_source_table.sql @@ -9,7 +9,7 @@ reset enable_case_sensitive_identifier; {% set target_schema=api.Relation.create( database=target.database, - schema="codegen_integration_tests__data_source_schema" + schema=target.schema ~ "__data_source_schema" ) %} From 6ec5f88e375edd29da33a7b9182e54d2716af511 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Thu, 21 Nov 2024 10:25:07 -0700 Subject: [PATCH 05/12] Store artifacts for logs and target directories for BigQuery --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3a6aab..e5652fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,6 +88,12 @@ jobs: paths: - "dbt_venv" + - store_artifacts: + path: integration_tests/logs + + - store_artifacts: + path: integration_tests/target + workflows: version: 2 test-all: From bea1d53b5a0dd4a971086ade34dc87a1608bd540 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Thu, 21 Nov 2024 10:37:07 -0700 Subject: [PATCH 06/12] Set up environment variable for BigQuery credentials (keyfile for service account JSON) --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e5652fc..9333941 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,9 +10,8 @@ jobs: - checkout - run: - name: setup_creds - command: | - echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json + name: Setup Environment Variables + command: echo 'export BIGQUERY_KEYFILE_JSON="$BIGQUERY_SERVICE_ACCOUNT_JSON"' >> "$BASH_ENV" - restore_cache: key: deps1-{{ .Branch }} From 16f51fc4c3c6144790a64a6354ec37e142e08f70 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Thu, 21 Nov 2024 11:00:00 -0700 Subject: [PATCH 07/12] Set the custom schema in the source definition --- integration_tests/models/source.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/models/source.yml b/integration_tests/models/source.yml index 528139c..13a8d4c 100644 --- a/integration_tests/models/source.yml +++ b/integration_tests/models/source.yml @@ -2,6 +2,7 @@ version: 2 sources: - name: codegen_integration_tests__data_source_schema + schema: "{{ target.schema ~ '__data_source_schema' }}" tables: - name: codegen_integration_tests__data_source_table columns: From edc2c3ae69ec6bd8bbf4b45eba8fa7dd94c78554 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Thu, 21 Nov 2024 11:08:53 -0700 Subject: [PATCH 08/12] Use the target schema --- integration_tests/models/model_without_import_ctes.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/models/model_without_import_ctes.sql b/integration_tests/models/model_without_import_ctes.sql index 94ace74..68a552f 100644 --- a/integration_tests/models/model_without_import_ctes.sql +++ b/integration_tests/models/model_without_import_ctes.sql @@ -20,7 +20,7 @@ with my_first_cte as ( my_second_cte as ( select 1 as id - from codegen_integration_tests__data_source_schema.codegen_integration_tests__data_source_table + from {{ target.schema }}__data_source_schema.codegen_integration_tests__data_source_table union all select 2 as id From d542ce1ed1a44840b00c02c3c38b82944e742705 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Thu, 21 Nov 2024 11:23:53 -0700 Subject: [PATCH 09/12] Try to align actual vs. expected when the schema name is variable --- integration_tests/tests/test_generate_model_import_ctes.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/tests/test_generate_model_import_ctes.sql b/integration_tests/tests/test_generate_model_import_ctes.sql index 8975550..7ee58bf 100644 --- a/integration_tests/tests/test_generate_model_import_ctes.sql +++ b/integration_tests/tests/test_generate_model_import_ctes.sql @@ -12,9 +12,9 @@ materialized='table', ) }}{% endraw %} -with codegen_integration_tests__data_source_schema_codegen_integration_tests__data_source_table as ( +with {{ target.schema.replace(".", "_") }}__data_source_schema_codegen_integration_tests__data_source_table as ( - select * from codegen_integration_tests__data_source_schema.codegen_integration_tests__data_source_table + select * from {{ target.schema }}__data_source_schema.codegen_integration_tests__data_source_table -- CAUTION: It's best practice to use the ref or source function instead of a direct reference ), From 332b139b1d000a8bfb4e16d3698aed11434702d9 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 22 Nov 2024 08:58:21 -0700 Subject: [PATCH 10/12] Remove extraneous storage of artifacts --- .circleci/config.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 94b7ea5..348ba3f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -109,12 +109,6 @@ jobs: # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass resource_class: large - - store_artifacts: - path: integration_tests/logs - - - store_artifacts: - path: integration_tests/target - workflows: version: 2 test-all: From d70d776be66e3ee7085434a461b90a86f527645f Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 22 Nov 2024 09:15:43 -0700 Subject: [PATCH 11/12] Temporarily disable two failing CI tests --- ...odel_import_ctes.sql => test_generate_model_import_ctes.sql.x} | 0 ..._leading.sql => test_generate_model_import_ctes_leading.sql.x} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename integration_tests/tests/{test_generate_model_import_ctes.sql => test_generate_model_import_ctes.sql.x} (100%) rename integration_tests/tests/{test_generate_model_import_ctes_leading.sql => test_generate_model_import_ctes_leading.sql.x} (100%) diff --git a/integration_tests/tests/test_generate_model_import_ctes.sql b/integration_tests/tests/test_generate_model_import_ctes.sql.x similarity index 100% rename from integration_tests/tests/test_generate_model_import_ctes.sql rename to integration_tests/tests/test_generate_model_import_ctes.sql.x diff --git a/integration_tests/tests/test_generate_model_import_ctes_leading.sql b/integration_tests/tests/test_generate_model_import_ctes_leading.sql.x similarity index 100% rename from integration_tests/tests/test_generate_model_import_ctes_leading.sql rename to integration_tests/tests/test_generate_model_import_ctes_leading.sql.x From b329d821231a42b255587c624c5e63a265630712 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 22 Nov 2024 15:04:41 -0700 Subject: [PATCH 12/12] Revert "Temporarily disable two failing CI tests" This reverts commit d70d776be66e3ee7085434a461b90a86f527645f. --- ...odel_import_ctes.sql.x => test_generate_model_import_ctes.sql} | 0 ..._leading.sql.x => test_generate_model_import_ctes_leading.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename integration_tests/tests/{test_generate_model_import_ctes.sql.x => test_generate_model_import_ctes.sql} (100%) rename integration_tests/tests/{test_generate_model_import_ctes_leading.sql.x => test_generate_model_import_ctes_leading.sql} (100%) diff --git a/integration_tests/tests/test_generate_model_import_ctes.sql.x b/integration_tests/tests/test_generate_model_import_ctes.sql similarity index 100% rename from integration_tests/tests/test_generate_model_import_ctes.sql.x rename to integration_tests/tests/test_generate_model_import_ctes.sql diff --git a/integration_tests/tests/test_generate_model_import_ctes_leading.sql.x b/integration_tests/tests/test_generate_model_import_ctes_leading.sql similarity index 100% rename from integration_tests/tests/test_generate_model_import_ctes_leading.sql.x rename to integration_tests/tests/test_generate_model_import_ctes_leading.sql