From 9abd7151c59cbde26629b79d6e94baf86d487262 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 10 Oct 2024 13:27:51 -0400 Subject: [PATCH 1/9] Standardize integration tests. --- .github/workflows/ci.yml | 33 +++++++++++++++++++++++++++++ integration_tests/.env/postgres.env | 6 ++++++ profiles.yml | 12 +++++++++++ supported_adapters.env | 1 + tox.ini | 24 +++++++++++++++++++++ 5 files changed, 76 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 integration_tests/.env/postgres.env create mode 100644 profiles.yml create mode 100644 supported_adapters.env create mode 100644 tox.ini diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..26f7c7ae --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +# **what?** +# Run tests for dbt-codegen against supported adapters + +# **why?** +# To ensure that dbt-codegen works as expected with all supported adapters + +# **when?** +# On every PR, and every push to main and when manually triggered + +name: Package Integration Tests + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + run-tests: + uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1 + # this just tests with postgres so no variables need to be passed through. + # When it's time to add more adapters you will need to pass through inputs for + # the other adapters as shown in the below example for redshift + # with: + # # redshift + # REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + # REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + # REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} + # REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}" + # REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }} + # secrets: + # DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }} \ No newline at end of file diff --git a/integration_tests/.env/postgres.env b/integration_tests/.env/postgres.env new file mode 100644 index 00000000..4298beb6 --- /dev/null +++ b/integration_tests/.env/postgres.env @@ -0,0 +1,6 @@ +POSTGRES_HOST=localhost +POSTGRES_USER=root +DBT_ENV_SECRET_POSTGRES_PASS=password +POSTGRES_PORT=5432 +POSTGRES_DATABASE=audit_helper_test +POSTGRES_SCHEMA=audit_helper_integration_tests_postgres \ No newline at end of file diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 00000000..11e0a5f2 --- /dev/null +++ b/profiles.yml @@ -0,0 +1,12 @@ +integration_tests: + target: postgres + outputs: + postgres: + type: "postgres" + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" + port: "{{ env_var('POSTGRES_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + schema: "{{ env_var('POSTGRES_SCHEMA') }}" + threads: 5 diff --git a/supported_adapters.env b/supported_adapters.env new file mode 100644 index 00000000..14c965c9 --- /dev/null +++ b/supported_adapters.env @@ -0,0 +1 @@ +SUPPORTED_ADAPTERS=postgres \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..c93ca725 --- /dev/null +++ b/tox.ini @@ -0,0 +1,24 @@ +[tox] +skipsdist = True +envlist = lint_all, testenv + +[testenv] +passenv = + # postgres env vars + POSTGRES_HOST + POSTGRES_USER + DBT_ENV_SECRET_POSTGRES_PASS + POSTGRES_PORT + POSTGRES_DATABASE + POSTGRES_SCHEMA + +# Postgres integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_postgres] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target postgres \ No newline at end of file From a292ed4ff202c1377f7cb98238b358e136f7d1a2 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 10 Oct 2024 13:35:49 -0400 Subject: [PATCH 2/9] Move profiles file. --- integration_tests/profiles.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 integration_tests/profiles.yml diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml new file mode 100644 index 00000000..11e0a5f2 --- /dev/null +++ b/integration_tests/profiles.yml @@ -0,0 +1,12 @@ +integration_tests: + target: postgres + outputs: + postgres: + type: "postgres" + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" + port: "{{ env_var('POSTGRES_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + schema: "{{ env_var('POSTGRES_SCHEMA') }}" + threads: 5 From e68fec274c712ca96f9d06e4a6e23e5b578c6071 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 10 Oct 2024 13:56:13 -0400 Subject: [PATCH 3/9] Update config, remove extra profiles file. --- .circleci/config.yml | 10 +++++----- profiles.yml | 12 ------------ 2 files changed, 5 insertions(+), 17 deletions(-) delete mode 100644 profiles.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 950a6cfd..2a4f5dff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,11 +41,11 @@ jobs: - run: name: "Run Tests - Postgres" environment: - POSTGRES_TEST_HOST: localhost - POSTGRES_TEST_USER: root - POSTGRES_TEST_PASS: '' - POSTGRES_TEST_PORT: 5432 - POSTGRES_TEST_DBNAME: circle_test + POSTGRES_HOST: localhost + POSTGRES_USER: root + POSTGRES_PASS: '' + POSTGRES_PORT: 5432 + POSTGRES_DBNAME: circle_test command: | . dbt_venv/bin/activate cd integration_tests diff --git a/profiles.yml b/profiles.yml deleted file mode 100644 index 11e0a5f2..00000000 --- a/profiles.yml +++ /dev/null @@ -1,12 +0,0 @@ -integration_tests: - target: postgres - outputs: - postgres: - type: "postgres" - host: "{{ env_var('POSTGRES_HOST') }}" - user: "{{ env_var('POSTGRES_USER') }}" - pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" - port: "{{ env_var('POSTGRES_PORT') | as_number }}" - dbname: "{{ env_var('POSTGRES_DATABASE') }}" - schema: "{{ env_var('POSTGRES_SCHEMA') }}" - threads: 5 From 7e35e73f8f932fd2787f0e076b777ca8e6cf7c2e Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 10 Oct 2024 13:59:46 -0400 Subject: [PATCH 4/9] Remove old sample profiles YAML. Modify conifg accordingly. --- .circleci/config.yml | 3 -- integration_tests/ci/sample.profiles.yml | 53 ------------------------ 2 files changed, 56 deletions(-) delete mode 100644 integration_tests/ci/sample.profiles.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a4f5dff..61172b4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,9 +35,6 @@ jobs: python -m pip install --upgrade pip setuptools python -m pip install --pre dbt-core dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery dbt-databricks - mkdir -p ~/.dbt - cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - - run: name: "Run Tests - Postgres" environment: diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml deleted file mode 100644 index ea8effc1..00000000 --- a/integration_tests/ci/sample.profiles.yml +++ /dev/null @@ -1,53 +0,0 @@ - -# HEY! This file is used in the dbt-audit-helper integrations tests with CircleCI. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -integration_tests: - target: postgres - outputs: - postgres: - type: postgres - host: "{{ env_var('POSTGRES_TEST_HOST') }}" - user: "{{ env_var('POSTGRES_TEST_USER') }}" - pass: "{{ env_var('POSTGRES_TEST_PASS') }}" - port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" - dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" - schema: audit_helper_integration_tests_postgres - threads: 1 - - redshift: - type: redshift - host: "{{ env_var('REDSHIFT_TEST_HOST') }}" - user: "{{ env_var('REDSHIFT_TEST_USER') }}" - pass: "{{ env_var('REDSHIFT_TEST_PASS') }}" - dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" - port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" - schema: audit_helper_integration_tests_redshift - threads: 8 - - bigquery: - type: bigquery - method: service-account - keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" - project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" - schema: audit_helper_integration_tests_bigquery - threads: 8 - - snowflake: - type: snowflake - account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" - user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" - password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" - role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" - database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" - warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" - schema: audit_helper_integration_tests_snowflake - threads: 8 - - databricks: - type: databricks - schema: dbt_project_evaluator_integration_tests_databricks - host: "{{ env_var('DATABRICKS_TEST_HOST') }}" - http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}" - token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}" - threads: 10 From 61e586de1d711d3d4ab43687816c1e86801e13c3 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 10 Oct 2024 14:22:16 -0400 Subject: [PATCH 5/9] Tweak CircleCI config again. --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 61172b4e..8cc21627 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,8 +10,12 @@ jobs: username: dbt-labs password: '' environment: + POSTGRES_HOST: localhost POSTGRES_USER: root - POSTGRES_DB: circle_test + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: circle_test + POSTGRES_SCHEMA: dbt_utils_integration_tests_postgres + DBT_ENV_SECRET_POSTGRES_PASS: '' steps: - checkout From 985689659b6a54c2a4bed79005bb5de2905055d1 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 10 Oct 2024 14:27:31 -0400 Subject: [PATCH 6/9] One more try tweaking config. --- .circleci/config.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8cc21627..00a6a201 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,11 +42,12 @@ jobs: - run: name: "Run Tests - Postgres" environment: - POSTGRES_HOST: localhost - POSTGRES_USER: root - POSTGRES_PASS: '' - POSTGRES_PORT: 5432 - POSTGRES_DBNAME: circle_test + POSTGRES_HOST: localhost + POSTGRES_USER: root + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: circle_test + POSTGRES_SCHEMA: dbt_utils_integration_tests_postgres + DBT_ENV_SECRET_POSTGRES_PASS: '' command: | . dbt_venv/bin/activate cd integration_tests From e2a8d2d746e4e4ade54e532a180d334a24e18f10 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 10 Oct 2024 14:32:49 -0400 Subject: [PATCH 7/9] Another tweak. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 00a6a201..0fe4c3db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,12 +42,12 @@ jobs: - run: name: "Run Tests - Postgres" environment: - POSTGRES_HOST: localhost - POSTGRES_USER: root - POSTGRES_PORT: 5432 - POSTGRES_DATABASE: circle_test - POSTGRES_SCHEMA: dbt_utils_integration_tests_postgres - DBT_ENV_SECRET_POSTGRES_PASS: '' + POSTGRES_HOST: localhost + POSTGRES_USER: root + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: circle_test + POSTGRES_SCHEMA: dbt_utils_integration_tests_postgres + DBT_ENV_SECRET_POSTGRES_PASS: '' command: | . dbt_venv/bin/activate cd integration_tests From 5c830fd51cd3ac8f2b86d3f1b70e5b74ee0db2f7 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 10 Oct 2024 15:25:10 -0400 Subject: [PATCH 8/9] Add back other adapter profiles. --- integration_tests/profiles.yml | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 11e0a5f2..5ee348da 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -10,3 +10,41 @@ integration_tests: dbname: "{{ env_var('POSTGRES_DATABASE') }}" schema: "{{ env_var('POSTGRES_SCHEMA') }}" threads: 5 + + redshift: + type: redshift + host: "{{ env_var('REDSHIFT_TEST_HOST') }}" + user: "{{ env_var('REDSHIFT_TEST_USER') }}" + pass: "{{ env_var('REDSHIFT_TEST_PASS') }}" + dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" + port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" + schema: audit_helper_integration_tests_redshift + threads: 8 + + bigquery: + type: bigquery + method: service-account + keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" + project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" + schema: audit_helper_integration_tests_bigquery + threads: 8 + + snowflake: + type: snowflake + account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" + password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" + role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" + schema: audit_helper_integration_tests_snowflake + threads: 8 + + databricks: + type: databricks + schema: dbt_project_evaluator_integration_tests_databricks + host: "{{ env_var('DATABRICKS_TEST_HOST') }}" + http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}" + token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}" + threads: 10 + From 9b2b2103d607b20c112461f81a3a601090f65ec7 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 15 Oct 2024 07:28:45 -0500 Subject: [PATCH 9/9] Apply suggestions from code review --- .github/workflows/ci.yml | 2 +- integration_tests/.env/postgres.env | 2 +- supported_adapters.env | 2 +- tox.ini | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26f7c7ae..6a8d2886 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,4 +30,4 @@ jobs: # REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}" # REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }} # secrets: - # DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }} \ No newline at end of file + # DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }} diff --git a/integration_tests/.env/postgres.env b/integration_tests/.env/postgres.env index 4298beb6..f2ee721d 100644 --- a/integration_tests/.env/postgres.env +++ b/integration_tests/.env/postgres.env @@ -3,4 +3,4 @@ POSTGRES_USER=root DBT_ENV_SECRET_POSTGRES_PASS=password POSTGRES_PORT=5432 POSTGRES_DATABASE=audit_helper_test -POSTGRES_SCHEMA=audit_helper_integration_tests_postgres \ No newline at end of file +POSTGRES_SCHEMA=audit_helper_integration_tests_postgres diff --git a/supported_adapters.env b/supported_adapters.env index 14c965c9..79c9a199 100644 --- a/supported_adapters.env +++ b/supported_adapters.env @@ -1 +1 @@ -SUPPORTED_ADAPTERS=postgres \ No newline at end of file +SUPPORTED_ADAPTERS=postgres diff --git a/tox.ini b/tox.ini index c93ca725..bd23fa9b 100644 --- a/tox.ini +++ b/tox.ini @@ -21,4 +21,4 @@ allowlist_externals = skip_install = true commands = dbt --version - dbt debug --target postgres \ No newline at end of file + dbt debug --target postgres