From ecf59ff3c487ba1d679265e28c2d1ecfb80535c8 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 4 Feb 2022 16:39:43 -0600 Subject: [PATCH 01/18] added unique_key to the test_docs_generate places relevant to changes made in core, TODO: double check that any other references to unique_key in bigquery there are two more on that file need to be changed to None value as well, and design an integration test --- tests/integration/docs_generate_tests/test_docs_generate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/docs_generate_tests/test_docs_generate.py b/tests/integration/docs_generate_tests/test_docs_generate.py index 8621d1e15..b354dfd8f 100644 --- a/tests/integration/docs_generate_tests/test_docs_generate.py +++ b/tests/integration/docs_generate_tests/test_docs_generate.py @@ -532,6 +532,7 @@ def rendered_model_config(self, **updates): 'full_refresh': None, 'on_schema_change': 'ignore', 'meta': {}, + 'unique_key': None, } result.update(updates) return result @@ -556,6 +557,7 @@ def rendered_seed_config(self, **updates): 'schema': None, 'alias': None, 'meta': {}, + 'unique_key': None } result.update(updates) return result From 25ca18fbf9160bfdc3919577f781b5c4789fb089 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Mon, 7 Feb 2022 17:35:42 -0600 Subject: [PATCH 02/18] init attempt at test, for unique key, getting type error for hashable currently --- .../unique_id_test/model/expected.sql | 7 +++++ .../unique_id_test/model/result.sql | 6 ++++ .../integration/unique_id_test/seeds/seed.csv | 3 ++ tests/integration/unique_id_test/unique_id.py | 31 +++++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 tests/integration/unique_id_test/model/expected.sql create mode 100644 tests/integration/unique_id_test/model/result.sql create mode 100644 tests/integration/unique_id_test/seeds/seed.csv create mode 100644 tests/integration/unique_id_test/unique_id.py diff --git a/tests/integration/unique_id_test/model/expected.sql b/tests/integration/unique_id_test/model/expected.sql new file mode 100644 index 000000000..5ce6e1300 --- /dev/null +++ b/tests/integration/unique_id_test/model/expected.sql @@ -0,0 +1,7 @@ +select + 'Texas' as state, + 'Austin' as city +union all +select + 'Texas', + 'Dallas' \ No newline at end of file diff --git a/tests/integration/unique_id_test/model/result.sql b/tests/integration/unique_id_test/model/result.sql new file mode 100644 index 000000000..8888d40c5 --- /dev/null +++ b/tests/integration/unique_id_test/model/result.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'incremental', + unique_key = ['state', 'city'] +)}} + +select * from {{ ref('seed') }}) \ No newline at end of file diff --git a/tests/integration/unique_id_test/seeds/seed.csv b/tests/integration/unique_id_test/seeds/seed.csv new file mode 100644 index 000000000..0819e8f5a --- /dev/null +++ b/tests/integration/unique_id_test/seeds/seed.csv @@ -0,0 +1,3 @@ +state,city +Texas,Austin +Texas,Dallas \ No newline at end of file diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py new file mode 100644 index 000000000..eee3efac1 --- /dev/null +++ b/tests/integration/unique_id_test/unique_id.py @@ -0,0 +1,31 @@ +from tests.integration.base import DBTIntegrationTest, use_profile +import hashlib + + +class TestUniqueKey(DBTIntegrationTest): + + def setUp(self): + super().setUp() + + @property + def models(self): + return 'models' + + + @property + def schema(self): + return 'unique_id' + + @property + def project_config(self): + return { + 'config-version': 2, + 'seed-paths': ['seeds'], + } + + @use_profile('bigquery') + def test_bigquery_unique_key(self): + seed = hashlib.sha256(self.run_dbt(['seed'])) + expected = hashlib.sha256(self.run_dbt(['expected'])) + self.assertCountEqual(list(seed), list(expected)) + \ No newline at end of file From 6a035011cde47382daca76932422921fac2a48e1 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 8 Feb 2022 01:18:01 -0600 Subject: [PATCH 03/18] moved to using basic hash method, and changing up calls and saving of values, failing now due to being unequal --- tests/integration/unique_id_test/unique_id.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py index eee3efac1..a0849b1a7 100644 --- a/tests/integration/unique_id_test/unique_id.py +++ b/tests/integration/unique_id_test/unique_id.py @@ -22,10 +22,13 @@ def project_config(self): 'config-version': 2, 'seed-paths': ['seeds'], } + @use_profile('bigquery') def test_bigquery_unique_key(self): - seed = hashlib.sha256(self.run_dbt(['seed'])) - expected = hashlib.sha256(self.run_dbt(['expected'])) - self.assertCountEqual(list(seed), list(expected)) + self.run_dbt(['seed']) + self.run_dbt() + hashed_result = hash('result') + hashed_expected = hash('expected') + self.assertEqual(hashed_result, hashed_expected) \ No newline at end of file From f984df2a9ddee03a09e7946ab17e66de5571198e Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 8 Feb 2022 11:50:33 -0600 Subject: [PATCH 04/18] fixed minor errors have all models building properly including incremental model now only have to fine tune till assertions are correct --- .../unique_id_test/{model => models}/expected.sql | 0 .../integration/unique_id_test/{model => models}/result.sql | 5 +++-- tests/integration/unique_id_test/unique_id.py | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) rename tests/integration/unique_id_test/{model => models}/expected.sql (100%) rename tests/integration/unique_id_test/{model => models}/result.sql (67%) diff --git a/tests/integration/unique_id_test/model/expected.sql b/tests/integration/unique_id_test/models/expected.sql similarity index 100% rename from tests/integration/unique_id_test/model/expected.sql rename to tests/integration/unique_id_test/models/expected.sql diff --git a/tests/integration/unique_id_test/model/result.sql b/tests/integration/unique_id_test/models/result.sql similarity index 67% rename from tests/integration/unique_id_test/model/result.sql rename to tests/integration/unique_id_test/models/result.sql index 8888d40c5..7410f6b0a 100644 --- a/tests/integration/unique_id_test/model/result.sql +++ b/tests/integration/unique_id_test/models/result.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = ['state', 'city'] -)}} +) +}} -select * from {{ ref('seed') }}) \ No newline at end of file +select * from {{ ref('seed') }} diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py index a0849b1a7..48d403c7e 100644 --- a/tests/integration/unique_id_test/unique_id.py +++ b/tests/integration/unique_id_test/unique_id.py @@ -23,11 +23,10 @@ def project_config(self): 'seed-paths': ['seeds'], } - @use_profile('bigquery') def test_bigquery_unique_key(self): self.run_dbt(['seed']) - self.run_dbt() + self.run_dbt(['run']) hashed_result = hash('result') hashed_expected = hash('expected') self.assertEqual(hashed_result, hashed_expected) From bc838662bb9de9f39d07e3fdd89e481f9789874d Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 9 Feb 2022 14:06:02 -0600 Subject: [PATCH 05/18] remove items from docs_generate.py objects to move to secondary pr --- tests/integration/docs_generate_tests/test_docs_generate.py | 4 +--- tests/integration/unique_id_test/unique_id.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/integration/docs_generate_tests/test_docs_generate.py b/tests/integration/docs_generate_tests/test_docs_generate.py index b354dfd8f..51df84485 100644 --- a/tests/integration/docs_generate_tests/test_docs_generate.py +++ b/tests/integration/docs_generate_tests/test_docs_generate.py @@ -532,7 +532,6 @@ def rendered_model_config(self, **updates): 'full_refresh': None, 'on_schema_change': 'ignore', 'meta': {}, - 'unique_key': None, } result.update(updates) return result @@ -556,8 +555,7 @@ def rendered_seed_config(self, **updates): 'database': None, 'schema': None, 'alias': None, - 'meta': {}, - 'unique_key': None + 'meta': {}, } result.update(updates) return result diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py index 48d403c7e..4d4205f43 100644 --- a/tests/integration/unique_id_test/unique_id.py +++ b/tests/integration/unique_id_test/unique_id.py @@ -27,7 +27,7 @@ def project_config(self): def test_bigquery_unique_key(self): self.run_dbt(['seed']) self.run_dbt(['run']) - hashed_result = hash('result') - hashed_expected = hash('expected') + hashed_result = hashlib.md5('result') + hashed_expected = hashlib.md5('expected') self.assertEqual(hashed_result, hashed_expected) \ No newline at end of file From df08072a9d088d058c5d65f9af371901f10cd683 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 9 Feb 2022 15:39:59 -0600 Subject: [PATCH 06/18] using assertTableEqual --- tests/integration/unique_id_test/unique_id.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py index 4d4205f43..8dceb9f90 100644 --- a/tests/integration/unique_id_test/unique_id.py +++ b/tests/integration/unique_id_test/unique_id.py @@ -27,7 +27,5 @@ def project_config(self): def test_bigquery_unique_key(self): self.run_dbt(['seed']) self.run_dbt(['run']) - hashed_result = hashlib.md5('result') - hashed_expected = hashlib.md5('expected') - self.assertEqual(hashed_result, hashed_expected) + self.assertTablesEqual("result", "expected") \ No newline at end of file From 2de84a29f32c63a6a85ad0c2608e79cb8989a88a Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 9 Feb 2022 16:12:50 -0600 Subject: [PATCH 07/18] added new models and test for single unique key --- .../models/{expected.sql => list_expected.sql} | 0 .../unique_id_test/models/{result.sql => list_result.sql} | 0 .../integration/unique_id_test/models/single_expected.sql | 7 +++++++ tests/integration/unique_id_test/models/single_result.sql | 7 +++++++ tests/integration/unique_id_test/unique_id.py | 3 ++- 5 files changed, 16 insertions(+), 1 deletion(-) rename tests/integration/unique_id_test/models/{expected.sql => list_expected.sql} (100%) rename tests/integration/unique_id_test/models/{result.sql => list_result.sql} (100%) create mode 100644 tests/integration/unique_id_test/models/single_expected.sql create mode 100644 tests/integration/unique_id_test/models/single_result.sql diff --git a/tests/integration/unique_id_test/models/expected.sql b/tests/integration/unique_id_test/models/list_expected.sql similarity index 100% rename from tests/integration/unique_id_test/models/expected.sql rename to tests/integration/unique_id_test/models/list_expected.sql diff --git a/tests/integration/unique_id_test/models/result.sql b/tests/integration/unique_id_test/models/list_result.sql similarity index 100% rename from tests/integration/unique_id_test/models/result.sql rename to tests/integration/unique_id_test/models/list_result.sql diff --git a/tests/integration/unique_id_test/models/single_expected.sql b/tests/integration/unique_id_test/models/single_expected.sql new file mode 100644 index 000000000..5ce6e1300 --- /dev/null +++ b/tests/integration/unique_id_test/models/single_expected.sql @@ -0,0 +1,7 @@ +select + 'Texas' as state, + 'Austin' as city +union all +select + 'Texas', + 'Dallas' \ No newline at end of file diff --git a/tests/integration/unique_id_test/models/single_result.sql b/tests/integration/unique_id_test/models/single_result.sql new file mode 100644 index 000000000..66fae899d --- /dev/null +++ b/tests/integration/unique_id_test/models/single_result.sql @@ -0,0 +1,7 @@ +{{ config( + materialized = 'incremental', + unique_key = 'state' +) +}} + +select * from {{ ref('seed') }} \ No newline at end of file diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py index 8dceb9f90..d3f6f0456 100644 --- a/tests/integration/unique_id_test/unique_id.py +++ b/tests/integration/unique_id_test/unique_id.py @@ -27,5 +27,6 @@ def project_config(self): def test_bigquery_unique_key(self): self.run_dbt(['seed']) self.run_dbt(['run']) - self.assertTablesEqual("result", "expected") + self.assertTablesEqual("list_result", "list_expected") + self.assertTablesEqual('single_result', 'single_expected') \ No newline at end of file From 57657db094405c50ab9686de0ac9d6d88f49587d Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 11 Feb 2022 09:03:31 -0600 Subject: [PATCH 08/18] adding tests for single unique key, and removing unused import --- tests/integration/unique_id_test/models/single_expected.sql | 4 ++-- tests/integration/unique_id_test/models/single_result.sql | 2 +- tests/integration/unique_id_test/seeds/single_seed.csv | 3 +++ tests/integration/unique_id_test/unique_id.py | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 tests/integration/unique_id_test/seeds/single_seed.csv diff --git a/tests/integration/unique_id_test/models/single_expected.sql b/tests/integration/unique_id_test/models/single_expected.sql index 5ce6e1300..9a0ab7b53 100644 --- a/tests/integration/unique_id_test/models/single_expected.sql +++ b/tests/integration/unique_id_test/models/single_expected.sql @@ -3,5 +3,5 @@ select 'Austin' as city union all select - 'Texas', - 'Dallas' \ No newline at end of file + 'California', + 'San Diego' \ No newline at end of file diff --git a/tests/integration/unique_id_test/models/single_result.sql b/tests/integration/unique_id_test/models/single_result.sql index 66fae899d..7f04d3848 100644 --- a/tests/integration/unique_id_test/models/single_result.sql +++ b/tests/integration/unique_id_test/models/single_result.sql @@ -4,4 +4,4 @@ ) }} -select * from {{ ref('seed') }} \ No newline at end of file +select * from {{ ref('single_seed') }} \ No newline at end of file diff --git a/tests/integration/unique_id_test/seeds/single_seed.csv b/tests/integration/unique_id_test/seeds/single_seed.csv new file mode 100644 index 000000000..0fa289685 --- /dev/null +++ b/tests/integration/unique_id_test/seeds/single_seed.csv @@ -0,0 +1,3 @@ +state,city +Texas,Austin +California,San Diego \ No newline at end of file diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py index d3f6f0456..6dcb1268f 100644 --- a/tests/integration/unique_id_test/unique_id.py +++ b/tests/integration/unique_id_test/unique_id.py @@ -11,7 +11,6 @@ def setUp(self): def models(self): return 'models' - @property def schema(self): return 'unique_id' @@ -29,4 +28,6 @@ def test_bigquery_unique_key(self): self.run_dbt(['run']) self.assertTablesEqual("list_result", "list_expected") self.assertTablesEqual('single_result', 'single_expected') + # used to test against mismatch (might be unneeded, first fail condition we produce) + # self.assertTablesEqual('single_result','list_expected') \ No newline at end of file From 0b9492fae65f04ca1bdd700f6fb3528e1b8c01af Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 11 Feb 2022 09:04:20 -0600 Subject: [PATCH 09/18] removing import --- tests/integration/unique_id_test/unique_id.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py index 6dcb1268f..8d0f56e48 100644 --- a/tests/integration/unique_id_test/unique_id.py +++ b/tests/integration/unique_id_test/unique_id.py @@ -1,5 +1,5 @@ from tests.integration.base import DBTIntegrationTest, use_profile -import hashlib + class TestUniqueKey(DBTIntegrationTest): From f4625f6f169b5461b97ce7bf55527f140b170fad Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 11 Feb 2022 11:32:13 -0600 Subject: [PATCH 10/18] add first draft for changelog which links to bigquery issue which points back to core pr to initial contributor issue/pr --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7049c2456..bf416cce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## dbt-bigquery 1.1.0 (Release TBD) ### Features - Provide a fine-grained control of the timeout and retry of BigQuery query with four new dbt profile configs: `job_creation_timeout_seconds`, `job_execution_timeout_seconds`, `job_retry_deadline_seconds`, and `job_retries` ([#45](https://github.com/dbt-labs/dbt-bigquery/issues/45), [#50](https://github.com/dbt-labs/dbt-bigquery/pull/50)) +- Adds new integration test to check against new ability to allow unique_key to be a list. [#112](https://github.com/dbt-labs/dbt-bigquery/issues/112) ### Fixes - Fix test related to preventing coercion of boolean values (True, False) to numeric values (0, 1) in query results ([#93](https://github.com/dbt-labs/dbt-bigquery/issues/93)) From 1cef0c636e79adab90efc505428e431700fc658c Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 15 Feb 2022 12:25:20 -0600 Subject: [PATCH 11/18] working on new version of unique_key as list test in incremental_schema_tests after pair coding with kyle, similiar test in vain to my previous version which is still in code but is using modified prexisting test functionality which seems a much cleaner simple test --- .../models/incremental_unique_key_list.sql | 18 +++++++++++++ .../incremental_unique_key_list_target.sql | 15 +++++++++++ .../models/model_b.sql | 17 ++++++++++++ .../test_incremental_schema.py | 27 +++++++++++++++++++ .../unique_id_test/models/single_expected.sql | 4 +++ .../unique_id_test/seeds/single_seed.csv | 1 + tests/integration/unique_id_test/unique_id.py | 1 - 7 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 tests/integration/incremental_schema_tests/models/incremental_unique_key_list.sql create mode 100644 tests/integration/incremental_schema_tests/models/incremental_unique_key_list_target.sql create mode 100644 tests/integration/incremental_schema_tests/models/model_b.sql diff --git a/tests/integration/incremental_schema_tests/models/incremental_unique_key_list.sql b/tests/integration/incremental_schema_tests/models/incremental_unique_key_list.sql new file mode 100644 index 000000000..7f12a5c9b --- /dev/null +++ b/tests/integration/incremental_schema_tests/models/incremental_unique_key_list.sql @@ -0,0 +1,18 @@ +{{ + config( + materialized='incremental', + unique_key=['field1', 'field2'] + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_b') }} ) + +{% if is_incremental() %} + +SELECT id, field1, field2 FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +SELECT id, field1, field2 FROM source_data LIMIT 2 + +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_schema_tests/models/incremental_unique_key_list_target.sql b/tests/integration/incremental_schema_tests/models/incremental_unique_key_list_target.sql new file mode 100644 index 000000000..858dcd41f --- /dev/null +++ b/tests/integration/incremental_schema_tests/models/incremental_unique_key_list_target.sql @@ -0,0 +1,15 @@ +{{ + config(materialized='table') +}} + +with source_data as ( + + select * from {{ ref('model_b') }} + +) + +select id + ,field1 + ,field2 + +from source_data \ No newline at end of file diff --git a/tests/integration/incremental_schema_tests/models/model_b.sql b/tests/integration/incremental_schema_tests/models/model_b.sql new file mode 100644 index 000000000..fe680df1e --- /dev/null +++ b/tests/integration/incremental_schema_tests/models/model_b.sql @@ -0,0 +1,17 @@ +{{ + config(materialized='table') +}} + +with source_data as ( + + select 1 as id, 'Texas' as field1, 'Austin' as field2 + union all select 2 as id, 'California' as field1, 'San Diego' as field2 + union all select 3 as id, 'Texas' as field1, 'Dallas' as field2 + +) + +select id + ,field1 + ,field2 + +from source_data \ No newline at end of file diff --git a/tests/integration/incremental_schema_tests/test_incremental_schema.py b/tests/integration/incremental_schema_tests/test_incremental_schema.py index 113a53585..99388660e 100644 --- a/tests/integration/incremental_schema_tests/test_incremental_schema.py +++ b/tests/integration/incremental_schema_tests/test_incremental_schema.py @@ -54,6 +54,22 @@ def run_tests_and_assert( assert sorted(tests_run) == sorted(expected_tests) self.assertTablesEqual(compare_source, compare_target) + + def run_dbt_and_assert( + self, include, compare_source, compare_target + ): + + run_args = ['run'] + if include: + run_args.extend(('--models', include)) + results_one = self.run_dbt(run_args) + results_two = self.run_dbt(run_args) + + self.assertEqual(len(results_one), 3) + self.assertEqual(len(results_two), 3) + + self.assertTablesEqual(compare_source, compare_target) + def run_incremental_ignore(self): select = 'model_a incremental_ignore incremental_ignore_target' compare_source = 'incremental_ignore' @@ -140,3 +156,14 @@ def test__bigquery__run_incremental_sync_all_columns(self): @use_profile('bigquery') def test__bigquery__run_incremental_fail_on_schema_change(self): self.run_incremental_fail_on_schema_change() + + @use_profile('bigquery') + def test__bigquery__run_incremental_unique_key_list(self): + + select = 'model_b incremental_unique_key_list incremental_unique_key_list_target' + compare_source = 'incremental_unique_key_list' + compare_target = 'incremental_unique_key_list_target' + + self.run_dbt_and_assert(select, compare_source, compare_target) + + \ No newline at end of file diff --git a/tests/integration/unique_id_test/models/single_expected.sql b/tests/integration/unique_id_test/models/single_expected.sql index 9a0ab7b53..d70920c3b 100644 --- a/tests/integration/unique_id_test/models/single_expected.sql +++ b/tests/integration/unique_id_test/models/single_expected.sql @@ -2,6 +2,10 @@ select 'Texas' as state, 'Austin' as city union all +select + 'Texas', + 'Dallas' +union all select 'California', 'San Diego' \ No newline at end of file diff --git a/tests/integration/unique_id_test/seeds/single_seed.csv b/tests/integration/unique_id_test/seeds/single_seed.csv index 0fa289685..c93496dc4 100644 --- a/tests/integration/unique_id_test/seeds/single_seed.csv +++ b/tests/integration/unique_id_test/seeds/single_seed.csv @@ -1,3 +1,4 @@ state,city Texas,Austin +Texas,Dallas California,San Diego \ No newline at end of file diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py index 8d0f56e48..7c5e70818 100644 --- a/tests/integration/unique_id_test/unique_id.py +++ b/tests/integration/unique_id_test/unique_id.py @@ -1,7 +1,6 @@ from tests.integration.base import DBTIntegrationTest, use_profile - class TestUniqueKey(DBTIntegrationTest): def setUp(self): From 4688b90c374411fd8bb11a42450246ede26726a0 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 16 Feb 2022 13:34:48 -0600 Subject: [PATCH 12/18] comments and docstring addition --- .../incremental_schema_tests/test_incremental_schema.py | 9 ++++++--- tests/integration/unique_id_test/unique_id.py | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/integration/incremental_schema_tests/test_incremental_schema.py b/tests/integration/incremental_schema_tests/test_incremental_schema.py index 99388660e..a118d756a 100644 --- a/tests/integration/incremental_schema_tests/test_incremental_schema.py +++ b/tests/integration/incremental_schema_tests/test_incremental_schema.py @@ -58,6 +58,7 @@ def run_tests_and_assert( def run_dbt_and_assert( self, include, compare_source, compare_target ): + """runs dbt twice to test incremental models.""" run_args = ['run'] if include: @@ -67,7 +68,7 @@ def run_dbt_and_assert( self.assertEqual(len(results_one), 3) self.assertEqual(len(results_two), 3) - + # Asserts that tables after second dbt run are the same. self.assertTablesEqual(compare_source, compare_target) def run_incremental_ignore(self): @@ -159,11 +160,13 @@ def test__bigquery__run_incremental_fail_on_schema_change(self): @use_profile('bigquery') def test__bigquery__run_incremental_unique_key_list(self): - + """tests that lists are a accepted values for unique_key. see model_b for source of test.""" + # picks which models to run for the tests from models folder select = 'model_b incremental_unique_key_list incremental_unique_key_list_target' + # assigns correct model to variable as source comparision compare_source = 'incremental_unique_key_list' + # assigns incremental model as an expected value to be in a end result. compare_target = 'incremental_unique_key_list_target' - self.run_dbt_and_assert(select, compare_source, compare_target) \ No newline at end of file diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py index 7c5e70818..203bd9c41 100644 --- a/tests/integration/unique_id_test/unique_id.py +++ b/tests/integration/unique_id_test/unique_id.py @@ -23,10 +23,10 @@ def project_config(self): @use_profile('bigquery') def test_bigquery_unique_key(self): + """""" self.run_dbt(['seed']) self.run_dbt(['run']) + # self.run_dbt(['run']) self.assertTablesEqual("list_result", "list_expected") - self.assertTablesEqual('single_result', 'single_expected') - # used to test against mismatch (might be unneeded, first fail condition we produce) - # self.assertTablesEqual('single_result','list_expected') + # self.assertTablesEqual('single_result', 'single_expected') \ No newline at end of file From e3c5bcb02a7f1629da1ada3fc914a542c5ad188c Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Thu, 17 Feb 2022 17:01:38 -0600 Subject: [PATCH 13/18] beginning transition to unify tests between adapters, only issue seems to be around models using ::date will fix tomorrow --- .../models/incremental_unique_key_list.sql | 18 -- .../incremental_unique_key_list_target.sql | 15 -- .../models/model_b.sql | 17 -- .../test_incremental_schema.py | 31 +--- .../duplicated_unary_unique_key_list.sql | 17 ++ .../models/empty_str_unique_key.sql | 14 ++ .../models/empty_unique_key_list.sql | 12 ++ .../models/expected/one_str__overwrite.sql | 21 +++ .../unique_key_list__inplace_overwrite.sql | 21 +++ .../models/no_unique_key.sql | 13 ++ .../nontyped_trinary_unique_key_list.sql | 19 ++ .../models/not_found_unique_key.sql | 14 ++ .../models/not_found_unique_key_list.sql | 8 + .../models/str_unique_key.sql | 17 ++ .../models/trinary_unique_key_list.sql | 19 ++ .../models/unary_unique_key_list.sql | 17 ++ .../models/unique_key_list__no_update.sql | 21 +++ .../seeds/add_new_rows.sql | 9 + .../seeds/duplicate_insert.sql | 5 + .../incremental_unique_id_test/seeds/seed.csv | 7 + .../incremental_unique_id_test/unique_id.py | 164 ++++++++++++++++++ .../unique_id_test/models/list_expected.sql | 7 - .../unique_id_test/models/list_result.sql | 7 - .../unique_id_test/models/single_expected.sql | 11 -- .../unique_id_test/models/single_result.sql | 7 - .../integration/unique_id_test/seeds/seed.csv | 3 - .../unique_id_test/seeds/single_seed.csv | 4 - tests/integration/unique_id_test/unique_id.py | 32 ---- 28 files changed, 399 insertions(+), 151 deletions(-) delete mode 100644 tests/integration/incremental_schema_tests/models/incremental_unique_key_list.sql delete mode 100644 tests/integration/incremental_schema_tests/models/incremental_unique_key_list_target.sql delete mode 100644 tests/integration/incremental_schema_tests/models/model_b.sql create mode 100644 tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql create mode 100644 tests/integration/incremental_unique_id_test/models/empty_str_unique_key.sql create mode 100644 tests/integration/incremental_unique_id_test/models/empty_unique_key_list.sql create mode 100644 tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql create mode 100644 tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql create mode 100644 tests/integration/incremental_unique_id_test/models/no_unique_key.sql create mode 100644 tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql create mode 100644 tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql create mode 100644 tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql create mode 100644 tests/integration/incremental_unique_id_test/models/str_unique_key.sql create mode 100644 tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql create mode 100644 tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql create mode 100644 tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql create mode 100644 tests/integration/incremental_unique_id_test/seeds/add_new_rows.sql create mode 100644 tests/integration/incremental_unique_id_test/seeds/duplicate_insert.sql create mode 100644 tests/integration/incremental_unique_id_test/seeds/seed.csv create mode 100644 tests/integration/incremental_unique_id_test/unique_id.py delete mode 100644 tests/integration/unique_id_test/models/list_expected.sql delete mode 100644 tests/integration/unique_id_test/models/list_result.sql delete mode 100644 tests/integration/unique_id_test/models/single_expected.sql delete mode 100644 tests/integration/unique_id_test/models/single_result.sql delete mode 100644 tests/integration/unique_id_test/seeds/seed.csv delete mode 100644 tests/integration/unique_id_test/seeds/single_seed.csv delete mode 100644 tests/integration/unique_id_test/unique_id.py diff --git a/tests/integration/incremental_schema_tests/models/incremental_unique_key_list.sql b/tests/integration/incremental_schema_tests/models/incremental_unique_key_list.sql deleted file mode 100644 index 7f12a5c9b..000000000 --- a/tests/integration/incremental_schema_tests/models/incremental_unique_key_list.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key=['field1', 'field2'] - ) -}} - -WITH source_data AS (SELECT * FROM {{ ref('model_b') }} ) - -{% if is_incremental() %} - -SELECT id, field1, field2 FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -SELECT id, field1, field2 FROM source_data LIMIT 2 - -{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_schema_tests/models/incremental_unique_key_list_target.sql b/tests/integration/incremental_schema_tests/models/incremental_unique_key_list_target.sql deleted file mode 100644 index 858dcd41f..000000000 --- a/tests/integration/incremental_schema_tests/models/incremental_unique_key_list_target.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ - config(materialized='table') -}} - -with source_data as ( - - select * from {{ ref('model_b') }} - -) - -select id - ,field1 - ,field2 - -from source_data \ No newline at end of file diff --git a/tests/integration/incremental_schema_tests/models/model_b.sql b/tests/integration/incremental_schema_tests/models/model_b.sql deleted file mode 100644 index fe680df1e..000000000 --- a/tests/integration/incremental_schema_tests/models/model_b.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ - config(materialized='table') -}} - -with source_data as ( - - select 1 as id, 'Texas' as field1, 'Austin' as field2 - union all select 2 as id, 'California' as field1, 'San Diego' as field2 - union all select 3 as id, 'Texas' as field1, 'Dallas' as field2 - -) - -select id - ,field1 - ,field2 - -from source_data \ No newline at end of file diff --git a/tests/integration/incremental_schema_tests/test_incremental_schema.py b/tests/integration/incremental_schema_tests/test_incremental_schema.py index a118d756a..d9655dd35 100644 --- a/tests/integration/incremental_schema_tests/test_incremental_schema.py +++ b/tests/integration/incremental_schema_tests/test_incremental_schema.py @@ -54,23 +54,6 @@ def run_tests_and_assert( assert sorted(tests_run) == sorted(expected_tests) self.assertTablesEqual(compare_source, compare_target) - - def run_dbt_and_assert( - self, include, compare_source, compare_target - ): - """runs dbt twice to test incremental models.""" - - run_args = ['run'] - if include: - run_args.extend(('--models', include)) - results_one = self.run_dbt(run_args) - results_two = self.run_dbt(run_args) - - self.assertEqual(len(results_one), 3) - self.assertEqual(len(results_two), 3) - # Asserts that tables after second dbt run are the same. - self.assertTablesEqual(compare_source, compare_target) - def run_incremental_ignore(self): select = 'model_a incremental_ignore incremental_ignore_target' compare_source = 'incremental_ignore' @@ -157,16 +140,4 @@ def test__bigquery__run_incremental_sync_all_columns(self): @use_profile('bigquery') def test__bigquery__run_incremental_fail_on_schema_change(self): self.run_incremental_fail_on_schema_change() - - @use_profile('bigquery') - def test__bigquery__run_incremental_unique_key_list(self): - """tests that lists are a accepted values for unique_key. see model_b for source of test.""" - # picks which models to run for the tests from models folder - select = 'model_b incremental_unique_key_list incremental_unique_key_list_target' - # assigns correct model to variable as source comparision - compare_source = 'incremental_unique_key_list' - # assigns incremental model as an expected value to be in a end result. - compare_target = 'incremental_unique_key_list_target' - self.run_dbt_and_assert(select, compare_source, compare_target) - - \ No newline at end of file + \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql new file mode 100644 index 000000000..7292d666c --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql @@ -0,0 +1,17 @@ +{{ + config( + materialized='incremental', + unique_key=['state', 'state'] + ) +}} + +select + state::varchar(2) as state, + county::varchar(12) as county, + city::varchar(12) as city, + last_visit_date::date as last_visit_date +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/empty_str_unique_key.sql b/tests/integration/incremental_unique_id_test/models/empty_str_unique_key.sql new file mode 100644 index 000000000..5260e177c --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/empty_str_unique_key.sql @@ -0,0 +1,14 @@ +{{ + config( + materialized='incremental', + unique_key='' + ) +}} + +select + * +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/empty_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/empty_unique_key_list.sql new file mode 100644 index 000000000..c582d532c --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/empty_unique_key_list.sql @@ -0,0 +1,12 @@ +{{ + config( + materialized='incremental', + unique_key=[] + ) +}} + +select * from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql b/tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql new file mode 100644 index 000000000..7f03b2dd4 --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql @@ -0,0 +1,21 @@ +{{ + config( + materialized='table' + ) +}} + +select + 'CT'::varchar(2) as state, + 'Hartford'::varchar(12) as county, + 'Hartford'::varchar(12) as city, + '2022-02-14'::date as last_visit_date +union all +select 'MA','Suffolk','Boston','2020-02-12' +union all +select 'NJ','Mercer','Trenton','2022-01-01' +union all +select 'NY','Kings','Brooklyn','2021-04-02' +union all +select 'NY','New York','Manhattan','2021-04-01' +union all +select 'PA','Philadelphia','Philadelphia','2021-05-21' \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql b/tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql new file mode 100644 index 000000000..7f03b2dd4 --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql @@ -0,0 +1,21 @@ +{{ + config( + materialized='table' + ) +}} + +select + 'CT'::varchar(2) as state, + 'Hartford'::varchar(12) as county, + 'Hartford'::varchar(12) as city, + '2022-02-14'::date as last_visit_date +union all +select 'MA','Suffolk','Boston','2020-02-12' +union all +select 'NJ','Mercer','Trenton','2022-01-01' +union all +select 'NY','Kings','Brooklyn','2021-04-02' +union all +select 'NY','New York','Manhattan','2021-04-01' +union all +select 'PA','Philadelphia','Philadelphia','2021-05-21' \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/no_unique_key.sql b/tests/integration/incremental_unique_id_test/models/no_unique_key.sql new file mode 100644 index 000000000..44a63e75c --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/no_unique_key.sql @@ -0,0 +1,13 @@ +{{ + config( + materialized='incremental' + ) +}} + +select + * +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql new file mode 100644 index 000000000..52b4509f0 --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql @@ -0,0 +1,19 @@ +-- for comparing against auto-typed seeds + +{{ + config( + materialized='incremental', + unique_key=['state', 'county', 'city'] + ) +}} + +select + state as state, + county as county, + city as city, + last_visit_date as last_visit_date +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql b/tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql new file mode 100644 index 000000000..026f26994 --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql @@ -0,0 +1,14 @@ +{{ + config( + materialized='incremental', + unique_key='asdf' + ) +}} + +select + * +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql new file mode 100644 index 000000000..58d0eba59 --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql @@ -0,0 +1,8 @@ +{{ + config( + materialized='incremental', + unique_key=['state', 'asdf'] + ) +}} + +select * from {{ ref('seed') }} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/str_unique_key.sql b/tests/integration/incremental_unique_id_test/models/str_unique_key.sql new file mode 100644 index 000000000..bb7c5deb9 --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/str_unique_key.sql @@ -0,0 +1,17 @@ +{{ + config( + materialized='incremental', + unique_key='state' + ) +}} + +select + state::varchar(2) as state, + county::varchar(12) as county, + city::varchar(12) as city, + last_visit_date::date as last_visit_date +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql new file mode 100644 index 000000000..2e9d51d21 --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql @@ -0,0 +1,19 @@ +-- types needed to compare against expected model reliably + +{{ + config( + materialized='incremental', + unique_key=['state', 'county', 'city'] + ) +}} + +select + state::varchar(2) as state, + county::varchar(12) as county, + city::varchar(12) as city, + last_visit_date::date as last_visit_date +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql new file mode 100644 index 000000000..f86537b5b --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql @@ -0,0 +1,17 @@ +{{ + config( + materialized='incremental', + unique_key=['state'] + ) +}} + +select + state::varchar(2) as state, + county::varchar(12) as county, + city::varchar(12) as city, + last_visit_date::date as last_visit_date +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql b/tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql new file mode 100644 index 000000000..7f03b2dd4 --- /dev/null +++ b/tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql @@ -0,0 +1,21 @@ +{{ + config( + materialized='table' + ) +}} + +select + 'CT'::varchar(2) as state, + 'Hartford'::varchar(12) as county, + 'Hartford'::varchar(12) as city, + '2022-02-14'::date as last_visit_date +union all +select 'MA','Suffolk','Boston','2020-02-12' +union all +select 'NJ','Mercer','Trenton','2022-01-01' +union all +select 'NY','Kings','Brooklyn','2021-04-02' +union all +select 'NY','New York','Manhattan','2021-04-01' +union all +select 'PA','Philadelphia','Philadelphia','2021-05-21' \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/seeds/add_new_rows.sql b/tests/integration/incremental_unique_id_test/seeds/add_new_rows.sql new file mode 100644 index 000000000..6d515ca3a --- /dev/null +++ b/tests/integration/incremental_unique_id_test/seeds/add_new_rows.sql @@ -0,0 +1,9 @@ +-- insert two new rows, both of which should be in incremental model +-- with any unique columns +insert into {schema}.seed + (state, county, city, last_visit_date) +values ('WA','King','Seattle','2022-02-01'); + +insert into {schema}.seed + (state, county, city, last_visit_date) +values ('CA','Los Angeles','Los Angeles','2022-02-01'); \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/seeds/duplicate_insert.sql b/tests/integration/incremental_unique_id_test/seeds/duplicate_insert.sql new file mode 100644 index 000000000..bd73cee86 --- /dev/null +++ b/tests/integration/incremental_unique_id_test/seeds/duplicate_insert.sql @@ -0,0 +1,5 @@ +-- insert new row, which should not be in incremental model +-- with primary or first three columns unique +insert into {schema}.seed + (state, county, city, last_visit_date) +values ('CT','Hartford','Hartford','2022-02-14'); \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/seeds/seed.csv b/tests/integration/incremental_unique_id_test/seeds/seed.csv new file mode 100644 index 000000000..b988827fb --- /dev/null +++ b/tests/integration/incremental_unique_id_test/seeds/seed.csv @@ -0,0 +1,7 @@ +state,county,city,last_visit_date +CT,Hartford,Hartford,2020-09-23 +MA,Suffolk,Boston,2020-02-12 +NJ,Mercer,Trenton,2022-01-01 +NY,Kings,Brooklyn,2021-04-02 +NY,New York,Manhattan,2021-04-01 +PA,Philadelphia,Philadelphia,2021-05-21 \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/unique_id.py b/tests/integration/incremental_unique_id_test/unique_id.py new file mode 100644 index 000000000..6335aed5a --- /dev/null +++ b/tests/integration/incremental_unique_id_test/unique_id.py @@ -0,0 +1,164 @@ +from tests.integration.base import DBTIntegrationTest, use_profile +from pathlib import Path + + +class TestIncrementalUniqueKey(DBTIntegrationTest): + + # def setUp(self): + # super().setUp() + + @property + def models(self): + return "models" + + @property + def schema(self): + return "incremental_unique_key" + + # @property + # def project_config(self): + # return { + # 'config-version': 2, + # 'seed-paths': ['seeds'], + # } + + def build_test_case( self, seed, incremental_model, update_sql_file, seed_expected_row_count): + """Make incremental model from seed, then change seed""" + seeds = self.run_dbt(["seed", "--full-refresh"]) + self.assertEqual(len(seeds), 1) + + models = self.run_dbt(["run", "--select", incremental_model, "--full-refresh"]) + self.assertEqual(len(models), 1) + + if update_sql_file: + self.run_sql_file(Path('seeds') / Path(update_sql_file + '.sql')) + + get_row_count = "select * from {}.{}".format(self.unique_schema(), seed) + self.assertEqual(seed_expected_row_count, len(self.run_sql(get_row_count, fetch="all"))) + + def run_incremental_update(self, incremental_model): + """update incremental model after the seed table has been updated""" + models = self.run_dbt(["run", "--select", incremental_model]) + self.assertEqual(len(models), 1) + + def run_incremental_mirror_seed_test(self, incremental_model, seed, seed_expected_row_count): + """invoke idempotent seed and model build with no model overwrite""" + self.build_test_case( + seed=seed, + incremental_model=incremental_model, + update_sql_file="add_new_rows", + seed_expected_row_count=seed_expected_row_count + ) + self.run_incremental_update(incremental_model=incremental_model) + + self.assertTablesEqual(seed, incremental_model) + + def run_incremental_match_test( + self, incremental_model, update_sql_file, expected_model, + seed_expected_row_count + ): + """invoke idempotent model seed and build with model overwrite""" + self.build_test_case( + seed='seed', + incremental_model=incremental_model, + update_sql_file=update_sql_file, + seed_expected_row_count=seed_expected_row_count + ) + self.run_incremental_update(incremental_model=incremental_model) + + models = self.run_dbt(['run', '--select', expected_model]) + self.assertEqual(len(models), 1) + + self.assertTablesEqual(expected_model, incremental_model) + + @use_profile('bigquery') + def test__bigquery_no_unique_keys(self): + """with no unique keys, seed and model should match""" + self.run_incremental_mirror_seed_test( + incremental_model='no_unique_key', + seed='seed', + seed_expected_row_count=8 + ) + + +class TestIncrementalStrUniqueKey(TestIncrementalUniqueKey): + @use_profile('bigquery') + def test__bigquery_empty_str_unique_key(self): + """with empty string for unique key, seed and model should match""" + self.run_incremental_mirror_seed_test( + incremental_model='empty_str_unique_key', + seed='seed', + seed_expected_row_count=8 + ) + + @use_profile('bigquery') + def test__bigquery_one_unique_key(self): + """with one unique key, model will overwrite existing row""" + self.run_incremental_match_test( + incremental_model='str_unique_key', + update_sql_file='duplicate_insert', + expected_model='one_str__overwrite', + seed_expected_row_count=7 + ) + + @use_profile('bigquery') + def test__bigquery_bad_unique_key(self): + ""'''""using a unique key not in seed or derived CTEs leads to an error""'''"" + with self.assertRaises(AssertionError) as exc: + self.run_dbt(['run', '--select', 'not_found_unique_key']) + + +class TestIncrementalListUniqueKey(TestIncrementalUniqueKey): + @use_profile('bigquery') + def test__bigquery_empty_unique_key_list(self): + ""'''""with no unique keys, seed and model should match""'''"" + self.run_incremental_mirror_seed_test( + incremental_model='empty_unique_key_list', + seed='seed', + seed_expected_row_count=8 + ) + + @use_profile('bigquery') + def test__bigquery_unary_unique_key_list(self): + '''with one unique key, model will overwrite existing row''' + self.run_incremental_match_test( + incremental_model='unary_unique_key_list', + update_sql_file='duplicate_insert', + expected_model='unique_key_list__inplace_overwrite', + seed_expected_row_count=7 + ) + + @use_profile('bigquery') + def test__bigquery_duplicated_unary_unique_key_list(self): + ""'''""with two of the same unique key, model will overwrite existing row""'''"" + self.run_incremental_match_test( + incremental_model='duplicated_unary_unique_key_list', + update_sql_file='duplicate_insert', + expected_model='unique_key_list__inplace_overwrite', + seed_expected_row_count=7 + ) + + @use_profile('bigquery') + def test__bigquery_trinary_unique_key_list(self): + ""'''""with three unique keys, model will overwrite existing row""'''"" + self.run_incremental_match_test( + incremental_model='trinary_unique_key_list', + update_sql_file='duplicate_insert', + expected_model='unique_key_list__inplace_overwrite', + seed_expected_row_count=7 + ) + + @use_profile('bigquery') + def test__bigquery_unique_key_list_no_update(self): + ""'''""with a fitting unique key, model will not overwrite existing row""'''"" + self.run_incremental_mirror_seed_test( + incremental_model='nontyped_trinary_unique_key_list', + seed='seed', + seed_expected_row_count=8 + ) + + @use_profile('bigquery') + def test__bigquery_bad_unique_key_list(self): + ""'''""using a unique key not in seed or derived CTEs leads to an error""'''"" + with self.assertRaises(AssertionError) as exc: + self.run_dbt(['run', '--select', 'not_found_unique_key_list']) \ No newline at end of file diff --git a/tests/integration/unique_id_test/models/list_expected.sql b/tests/integration/unique_id_test/models/list_expected.sql deleted file mode 100644 index 5ce6e1300..000000000 --- a/tests/integration/unique_id_test/models/list_expected.sql +++ /dev/null @@ -1,7 +0,0 @@ -select - 'Texas' as state, - 'Austin' as city -union all -select - 'Texas', - 'Dallas' \ No newline at end of file diff --git a/tests/integration/unique_id_test/models/list_result.sql b/tests/integration/unique_id_test/models/list_result.sql deleted file mode 100644 index 7410f6b0a..000000000 --- a/tests/integration/unique_id_test/models/list_result.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = ['state', 'city'] -) -}} - -select * from {{ ref('seed') }} diff --git a/tests/integration/unique_id_test/models/single_expected.sql b/tests/integration/unique_id_test/models/single_expected.sql deleted file mode 100644 index d70920c3b..000000000 --- a/tests/integration/unique_id_test/models/single_expected.sql +++ /dev/null @@ -1,11 +0,0 @@ -select - 'Texas' as state, - 'Austin' as city -union all -select - 'Texas', - 'Dallas' -union all -select - 'California', - 'San Diego' \ No newline at end of file diff --git a/tests/integration/unique_id_test/models/single_result.sql b/tests/integration/unique_id_test/models/single_result.sql deleted file mode 100644 index 7f04d3848..000000000 --- a/tests/integration/unique_id_test/models/single_result.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = 'state' -) -}} - -select * from {{ ref('single_seed') }} \ No newline at end of file diff --git a/tests/integration/unique_id_test/seeds/seed.csv b/tests/integration/unique_id_test/seeds/seed.csv deleted file mode 100644 index 0819e8f5a..000000000 --- a/tests/integration/unique_id_test/seeds/seed.csv +++ /dev/null @@ -1,3 +0,0 @@ -state,city -Texas,Austin -Texas,Dallas \ No newline at end of file diff --git a/tests/integration/unique_id_test/seeds/single_seed.csv b/tests/integration/unique_id_test/seeds/single_seed.csv deleted file mode 100644 index c93496dc4..000000000 --- a/tests/integration/unique_id_test/seeds/single_seed.csv +++ /dev/null @@ -1,4 +0,0 @@ -state,city -Texas,Austin -Texas,Dallas -California,San Diego \ No newline at end of file diff --git a/tests/integration/unique_id_test/unique_id.py b/tests/integration/unique_id_test/unique_id.py deleted file mode 100644 index 203bd9c41..000000000 --- a/tests/integration/unique_id_test/unique_id.py +++ /dev/null @@ -1,32 +0,0 @@ -from tests.integration.base import DBTIntegrationTest, use_profile - - -class TestUniqueKey(DBTIntegrationTest): - - def setUp(self): - super().setUp() - - @property - def models(self): - return 'models' - - @property - def schema(self): - return 'unique_id' - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - } - - @use_profile('bigquery') - def test_bigquery_unique_key(self): - """""" - self.run_dbt(['seed']) - self.run_dbt(['run']) - # self.run_dbt(['run']) - self.assertTablesEqual("list_result", "list_expected") - # self.assertTablesEqual('single_result', 'single_expected') - \ No newline at end of file From 86c93734168a41a865d20f03dfd016a127b16e59 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 18 Feb 2022 15:45:45 -0600 Subject: [PATCH 14/18] applied castings to expected tables as bigquery auto detects certain patterns from seeds, created a new class to cut down on runs of a test that was being inherited --- .../duplicated_unary_unique_key_list.sql | 8 +- .../models/expected/one_str__overwrite.sql | 8 +- .../unique_key_list__inplace_overwrite.sql | 8 +- .../nontyped_trinary_unique_key_list.sql | 2 +- .../models/str_unique_key.sql | 8 +- .../models/trinary_unique_key_list.sql | 8 +- .../models/unary_unique_key_list.sql | 8 +- .../models/unique_key_list__no_update.sql | 8 +- .../incremental_unique_id_test/unique_id.py | 319 ++++++++++++------ 9 files changed, 242 insertions(+), 135 deletions(-) diff --git a/tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql index 7292d666c..7290b6c43 100644 --- a/tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql +++ b/tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql @@ -6,10 +6,10 @@ }} select - state::varchar(2) as state, - county::varchar(12) as county, - city::varchar(12) as city, - last_visit_date::date as last_visit_date + state as state, + county as county, + city as city, + last_visit_date as last_visit_date from {{ ref('seed') }} {% if is_incremental() %} diff --git a/tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql b/tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql index 7f03b2dd4..baa37c601 100644 --- a/tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql +++ b/tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql @@ -5,10 +5,10 @@ }} select - 'CT'::varchar(2) as state, - 'Hartford'::varchar(12) as county, - 'Hartford'::varchar(12) as city, - '2022-02-14'::date as last_visit_date + 'CT' as state, + 'Hartford' as county, + 'Hartford' as city, + cast('2022-02-14' as date) as last_visit_date union all select 'MA','Suffolk','Boston','2020-02-12' union all diff --git a/tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql b/tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql index 7f03b2dd4..baa37c601 100644 --- a/tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql +++ b/tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql @@ -5,10 +5,10 @@ }} select - 'CT'::varchar(2) as state, - 'Hartford'::varchar(12) as county, - 'Hartford'::varchar(12) as city, - '2022-02-14'::date as last_visit_date + 'CT' as state, + 'Hartford' as county, + 'Hartford' as city, + cast('2022-02-14' as date) as last_visit_date union all select 'MA','Suffolk','Boston','2020-02-12' union all diff --git a/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql index 52b4509f0..b70185c71 100644 --- a/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql +++ b/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql @@ -11,7 +11,7 @@ select state as state, county as county, city as city, - last_visit_date as last_visit_date + cast(last_visit_date as date) as last_visit_date from {{ ref('seed') }} {% if is_incremental() %} diff --git a/tests/integration/incremental_unique_id_test/models/str_unique_key.sql b/tests/integration/incremental_unique_id_test/models/str_unique_key.sql index bb7c5deb9..f0ed50bb2 100644 --- a/tests/integration/incremental_unique_id_test/models/str_unique_key.sql +++ b/tests/integration/incremental_unique_id_test/models/str_unique_key.sql @@ -6,10 +6,10 @@ }} select - state::varchar(2) as state, - county::varchar(12) as county, - city::varchar(12) as city, - last_visit_date::date as last_visit_date + state as state, + county as county, + city as city, + cast(last_visit_date as date) as last_visit_date from {{ ref('seed') }} {% if is_incremental() %} diff --git a/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql index 2e9d51d21..97e5c6f2f 100644 --- a/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql +++ b/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql @@ -8,10 +8,10 @@ }} select - state::varchar(2) as state, - county::varchar(12) as county, - city::varchar(12) as city, - last_visit_date::date as last_visit_date + state as state, + county as county, + city as city, + cast(last_visit_date as date) as last_visit_date from {{ ref('seed') }} {% if is_incremental() %} diff --git a/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql index f86537b5b..1496824d9 100644 --- a/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql +++ b/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql @@ -6,10 +6,10 @@ }} select - state::varchar(2) as state, - county::varchar(12) as county, - city::varchar(12) as city, - last_visit_date::date as last_visit_date + state as state, + county as county, + city as city, + cast(last_visit_date as date) as last_visit_date from {{ ref('seed') }} {% if is_incremental() %} diff --git a/tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql b/tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql index 7f03b2dd4..d4d2e285c 100644 --- a/tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql +++ b/tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql @@ -5,10 +5,10 @@ }} select - 'CT'::varchar(2) as state, - 'Hartford'::varchar(12) as county, - 'Hartford'::varchar(12) as city, - '2022-02-14'::date as last_visit_date + 'CT' as state, + 'Hartford' as county, + 'Hartford' as city, + '2022-02-14' as last_visit_date union all select 'MA','Suffolk','Boston','2020-02-12' union all diff --git a/tests/integration/incremental_unique_id_test/unique_id.py b/tests/integration/incremental_unique_id_test/unique_id.py index 6335aed5a..27646610e 100644 --- a/tests/integration/incremental_unique_id_test/unique_id.py +++ b/tests/integration/incremental_unique_id_test/unique_id.py @@ -1,109 +1,167 @@ from tests.integration.base import DBTIntegrationTest, use_profile +from collections import namedtuple from pathlib import Path -class TestIncrementalUniqueKey(DBTIntegrationTest): +TestResults = namedtuple( + 'TestResults', + ['seed_count', 'model_count', 'seed_rows', 'inc_test_model_count', + 'opt_model_count', 'relation'], +) - # def setUp(self): - # super().setUp() +class TestIncrementalUniqueKey(DBTIntegrationTest): @property - def models(self): - return "models" + def schema(self): + return 'incremental_unique_key' @property - def schema(self): - return "incremental_unique_key" - - # @property - # def project_config(self): - # return { - # 'config-version': 2, - # 'seed-paths': ['seeds'], - # } - - def build_test_case( self, seed, incremental_model, update_sql_file, seed_expected_row_count): - """Make incremental model from seed, then change seed""" - seeds = self.run_dbt(["seed", "--full-refresh"]) - self.assertEqual(len(seeds), 1) - - models = self.run_dbt(["run", "--select", incremental_model, "--full-refresh"]) - self.assertEqual(len(models), 1) - - if update_sql_file: - self.run_sql_file(Path('seeds') / Path(update_sql_file + '.sql')) - - get_row_count = "select * from {}.{}".format(self.unique_schema(), seed) - self.assertEqual(seed_expected_row_count, len(self.run_sql(get_row_count, fetch="all"))) - - def run_incremental_update(self, incremental_model): - """update incremental model after the seed table has been updated""" - models = self.run_dbt(["run", "--select", incremental_model]) - self.assertEqual(len(models), 1) - - def run_incremental_mirror_seed_test(self, incremental_model, seed, seed_expected_row_count): - """invoke idempotent seed and model build with no model overwrite""" - self.build_test_case( - seed=seed, - incremental_model=incremental_model, - update_sql_file="add_new_rows", - seed_expected_row_count=seed_expected_row_count - ) - self.run_incremental_update(incremental_model=incremental_model) + def models(self): + return 'models' - self.assertTablesEqual(seed, incremental_model) + def count_test_relations(self, seed, incremental_model): + '''Idempotently create some number of seeds and incremental models''' + seed_result_set = self.run_dbt( + ['seed', '--select', seed, '--full-refresh'] + ) + model_result_set = self.run_dbt( + ['run', '--select', incremental_model, '--full-refresh'] + ) + return len(seed_result_set), len(model_result_set) - def run_incremental_match_test( - self, incremental_model, update_sql_file, expected_model, - seed_expected_row_count - ): - """invoke idempotent model seed and build with model overwrite""" - self.build_test_case( - seed='seed', - incremental_model=incremental_model, - update_sql_file=update_sql_file, - seed_expected_row_count=seed_expected_row_count + def get_seed_rows_after_update(self, seed, update_sql_file): + '''Upate seed and return new row count''' + row_count_query = 'select * from {}.{}'.format( + self.unique_schema(), + seed + ) + + self.run_sql_file(Path('seeds') / Path(update_sql_file + '.sql')) + return len(self.run_sql(row_count_query, fetch='all')) + + def update_incremental_model(self, incremental_model): + '''update incremental model after the seed table has been updated''' + model_result_set = self.run_dbt(['run', '--select', incremental_model]) + return len(model_result_set) + + def setup_test(self, seed, incremental_model, update_sql_file): + '''build a test case and return values for assertions''' + (seed_count, model_count) = self.count_test_relations( + seed=seed, incremental_model=incremental_model + ) + seed_rows = self.get_seed_rows_after_update( + seed=seed, update_sql_file=update_sql_file ) - self.run_incremental_update(incremental_model=incremental_model) + inc_test_model_count = self.update_incremental_model( + incremental_model=incremental_model + ) + + return (seed_count, model_count, seed_rows, inc_test_model_count) - models = self.run_dbt(['run', '--select', expected_model]) - self.assertEqual(len(models), 1) + def run_test(self, expected_fields, test_case_fields): + '''Invoke assertions to verify correct build functionality''' + # 1. test seed(s) should build afresh + self.assertEqual( + expected_fields.seed_count, test_case_fields.seed_count + ) + # 2. test model(s) should build afresh + self.assertEqual( + expected_fields.model_count, test_case_fields.model_count + ) + # 3. seeds should have intended row counts post update + self.assertEqual( + expected_fields.seed_rows, test_case_fields.seed_rows + ) + # 4. incremental test model(s) should be updated + self.assertEqual( + expected_fields.inc_test_model_count, + test_case_fields.inc_test_model_count + ) + # 5. extra incremental model(s) should be built; optional since + # comparison may be between an incremental model and seed + if (expected_fields.opt_model_count and + test_case_fields.opt_model_count): + self.assertEqual( + expected_fields.opt_model_count, + test_case_fields.opt_model_count + ) + # 6. result table should match intended result set (itself a relation) + self.assertTablesEqual( + expected_fields.relation, test_case_fields.relation + ) + + def stub_expected_fields( + self, relation, seed_rows, opt_model_count=None + ): + return TestResults( + seed_count=1, model_count=1, seed_rows=seed_rows, + inc_test_model_count=1, opt_model_count=opt_model_count, + relation=relation + ) - self.assertTablesEqual(expected_model, incremental_model) +class TestNoIncrementalUniqueKey(TestIncrementalUniqueKey): @use_profile('bigquery') def test__bigquery_no_unique_keys(self): - """with no unique keys, seed and model should match""" - self.run_incremental_mirror_seed_test( - incremental_model='no_unique_key', - seed='seed', - seed_expected_row_count=8 + '''with no unique keys, seed and model should match''' + seed='seed' + seed_rows=8 + incremental_model='no_unique_key' + update_sql_file='add_new_rows' + + expected_fields = self.stub_expected_fields( + relation=seed, seed_rows=seed_rows + ) + test_case_fields = TestResults( + *self.setup_test(seed, incremental_model, update_sql_file), + opt_model_count=None, relation=incremental_model ) - + + self.run_test(expected_fields, test_case_fields) + class TestIncrementalStrUniqueKey(TestIncrementalUniqueKey): @use_profile('bigquery') def test__bigquery_empty_str_unique_key(self): - """with empty string for unique key, seed and model should match""" - self.run_incremental_mirror_seed_test( - incremental_model='empty_str_unique_key', - seed='seed', - seed_expected_row_count=8 + '''with empty string for unique key, seed and model should match''' + seed='seed' + seed_rows=8 + incremental_model='empty_str_unique_key' + update_sql_file='add_new_rows' + + expected_fields = self.stub_expected_fields( + relation=seed, seed_rows=seed_rows + ) + test_case_fields = TestResults( + *self.setup_test(seed, incremental_model, update_sql_file), + opt_model_count=None, relation=incremental_model ) + self.run_test(expected_fields, test_case_fields) + @use_profile('bigquery') def test__bigquery_one_unique_key(self): - """with one unique key, model will overwrite existing row""" - self.run_incremental_match_test( - incremental_model='str_unique_key', - update_sql_file='duplicate_insert', - expected_model='one_str__overwrite', - seed_expected_row_count=7 + '''with one unique key, model will overwrite existing row''' + seed='seed' + seed_rows=7 + incremental_model='str_unique_key' + update_sql_file='duplicate_insert' + expected_model='one_str__overwrite' + + expected_fields = self.stub_expected_fields( + relation=expected_model, seed_rows=seed_rows, opt_model_count=1 + ) + test_case_fields = TestResults( + *self.setup_test(seed, incremental_model, update_sql_file), + opt_model_count=self.update_incremental_model(expected_model), + relation=incremental_model ) + self.run_test(expected_fields, test_case_fields) + @use_profile('bigquery') def test__bigquery_bad_unique_key(self): - ""'''""using a unique key not in seed or derived CTEs leads to an error""'''"" + '''using a unique key not in seed or derived CTEs leads to an error''' with self.assertRaises(AssertionError) as exc: self.run_dbt(['run', '--select', 'not_found_unique_key']) @@ -111,54 +169,103 @@ def test__bigquery_bad_unique_key(self): class TestIncrementalListUniqueKey(TestIncrementalUniqueKey): @use_profile('bigquery') def test__bigquery_empty_unique_key_list(self): - ""'''""with no unique keys, seed and model should match""'''"" - self.run_incremental_mirror_seed_test( - incremental_model='empty_unique_key_list', - seed='seed', - seed_expected_row_count=8 + '''with no unique keys, seed and model should match''' + seed='seed' + seed_rows=8 + incremental_model='empty_unique_key_list' + update_sql_file='add_new_rows' + + expected_fields = self.stub_expected_fields( + relation=seed, seed_rows=seed_rows + ) + test_case_fields = TestResults( + *self.setup_test(seed, incremental_model, update_sql_file), + opt_model_count=None, relation=incremental_model ) + self.run_test(expected_fields, test_case_fields) + @use_profile('bigquery') def test__bigquery_unary_unique_key_list(self): '''with one unique key, model will overwrite existing row''' - self.run_incremental_match_test( - incremental_model='unary_unique_key_list', - update_sql_file='duplicate_insert', - expected_model='unique_key_list__inplace_overwrite', - seed_expected_row_count=7 + seed='seed' + seed_rows=7 + incremental_model='unary_unique_key_list' + update_sql_file='duplicate_insert' + expected_model='unique_key_list__inplace_overwrite' + + expected_fields = self.stub_expected_fields( + relation=expected_model, seed_rows=seed_rows, opt_model_count=1 ) + test_case_fields = TestResults( + *self.setup_test(seed, incremental_model, update_sql_file), + opt_model_count=self.update_incremental_model(expected_model), + relation=incremental_model + ) + + self.run_test(expected_fields, test_case_fields) @use_profile('bigquery') def test__bigquery_duplicated_unary_unique_key_list(self): - ""'''""with two of the same unique key, model will overwrite existing row""'''"" - self.run_incremental_match_test( - incremental_model='duplicated_unary_unique_key_list', - update_sql_file='duplicate_insert', - expected_model='unique_key_list__inplace_overwrite', - seed_expected_row_count=7 + '''with two of the same unique key, model will overwrite existing row''' + seed='seed' + seed_rows=7 + incremental_model='duplicated_unary_unique_key_list' + update_sql_file='duplicate_insert' + expected_model='unique_key_list__inplace_overwrite' + + expected_fields = self.stub_expected_fields( + relation=expected_model, seed_rows=seed_rows, opt_model_count=1 + ) + test_case_fields = TestResults( + *self.setup_test(seed, incremental_model, update_sql_file), + opt_model_count=self.update_incremental_model(expected_model), + relation=incremental_model ) + self.run_test(expected_fields, test_case_fields) + @use_profile('bigquery') def test__bigquery_trinary_unique_key_list(self): - ""'''""with three unique keys, model will overwrite existing row""'''"" - self.run_incremental_match_test( - incremental_model='trinary_unique_key_list', - update_sql_file='duplicate_insert', - expected_model='unique_key_list__inplace_overwrite', - seed_expected_row_count=7 + '''with three unique keys, model will overwrite existing row''' + seed='seed' + seed_rows=7 + incremental_model='trinary_unique_key_list' + update_sql_file='duplicate_insert' + expected_model='unique_key_list__inplace_overwrite' + + expected_fields = self.stub_expected_fields( + relation=expected_model, seed_rows=seed_rows, opt_model_count=1 + ) + test_case_fields = TestResults( + *self.setup_test(seed, incremental_model, update_sql_file), + opt_model_count=self.update_incremental_model(expected_model), + relation=incremental_model ) + self.run_test(expected_fields, test_case_fields) + @use_profile('bigquery') - def test__bigquery_unique_key_list_no_update(self): - ""'''""with a fitting unique key, model will not overwrite existing row""'''"" - self.run_incremental_mirror_seed_test( - incremental_model='nontyped_trinary_unique_key_list', - seed='seed', - seed_expected_row_count=8 + def test__bigquery_trinary_unique_key_list_no_update(self): + '''even with three unique keys, adding distinct rows to seed does not + cause seed and model to diverge''' + seed='seed' + seed_rows=8 + incremental_model='nontyped_trinary_unique_key_list' + update_sql_file='add_new_rows' + + expected_fields = self.stub_expected_fields( + relation=seed, seed_rows=seed_rows + ) + test_case_fields = TestResults( + *self.setup_test(seed, incremental_model, update_sql_file), + opt_model_count=None, relation=incremental_model ) + self.run_test(expected_fields, test_case_fields) + @use_profile('bigquery') def test__bigquery_bad_unique_key_list(self): - ""'''""using a unique key not in seed or derived CTEs leads to an error""'''"" + '''using a unique key not in seed or derived CTEs leads to an error''' with self.assertRaises(AssertionError) as exc: - self.run_dbt(['run', '--select', 'not_found_unique_key_list']) \ No newline at end of file + self.run_dbt(['run', '--select', 'not_found_unique_key_list']) From 81cb8ca731ce377d5607340179792e9200af0eea Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 18 Feb 2022 16:03:01 -0600 Subject: [PATCH 15/18] minor fixes --- .../models/nontyped_trinary_unique_key_list.sql | 2 +- .../incremental_unique_id_test/models/str_unique_key.sql | 2 +- .../models/trinary_unique_key_list.sql | 2 +- .../incremental_unique_id_test/models/unary_unique_key_list.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql index b70185c71..52b4509f0 100644 --- a/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql +++ b/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql @@ -11,7 +11,7 @@ select state as state, county as county, city as city, - cast(last_visit_date as date) as last_visit_date + last_visit_date as last_visit_date from {{ ref('seed') }} {% if is_incremental() %} diff --git a/tests/integration/incremental_unique_id_test/models/str_unique_key.sql b/tests/integration/incremental_unique_id_test/models/str_unique_key.sql index f0ed50bb2..2f9fc2987 100644 --- a/tests/integration/incremental_unique_id_test/models/str_unique_key.sql +++ b/tests/integration/incremental_unique_id_test/models/str_unique_key.sql @@ -9,7 +9,7 @@ select state as state, county as county, city as city, - cast(last_visit_date as date) as last_visit_date + last_visit_date as last_visit_date from {{ ref('seed') }} {% if is_incremental() %} diff --git a/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql index 97e5c6f2f..0359546bf 100644 --- a/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql +++ b/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql @@ -11,7 +11,7 @@ select state as state, county as county, city as city, - cast(last_visit_date as date) as last_visit_date + last_visit_date as last_visit_date from {{ ref('seed') }} {% if is_incremental() %} diff --git a/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql index 1496824d9..7f5875f85 100644 --- a/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql +++ b/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql @@ -9,7 +9,7 @@ select state as state, county as county, city as city, - cast(last_visit_date as date) as last_visit_date + last_visit_date as last_visit_date from {{ ref('seed') }} {% if is_incremental() %} From feddfda92b97ce7a572418a146ab339dc10ab682 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 22 Feb 2022 10:09:57 -0600 Subject: [PATCH 16/18] minor naming changes and removal of file from older tests --- .../models/unique_key_list__no_update.sql | 21 ------------------- ...ue_id.py => test_incremental_unique_id.py} | 0 2 files changed, 21 deletions(-) delete mode 100644 tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql rename tests/integration/incremental_unique_id_test/{unique_id.py => test_incremental_unique_id.py} (100%) diff --git a/tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql b/tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql deleted file mode 100644 index d4d2e285c..000000000 --- a/tests/integration/incremental_unique_id_test/models/unique_key_list__no_update.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - -select - 'CT' as state, - 'Hartford' as county, - 'Hartford' as city, - '2022-02-14' as last_visit_date -union all -select 'MA','Suffolk','Boston','2020-02-12' -union all -select 'NJ','Mercer','Trenton','2022-01-01' -union all -select 'NY','Kings','Brooklyn','2021-04-02' -union all -select 'NY','New York','Manhattan','2021-04-01' -union all -select 'PA','Philadelphia','Philadelphia','2021-05-21' \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/unique_id.py b/tests/integration/incremental_unique_id_test/test_incremental_unique_id.py similarity index 100% rename from tests/integration/incremental_unique_id_test/unique_id.py rename to tests/integration/incremental_unique_id_test/test_incremental_unique_id.py From 5561b359e64bd796ac0ba3edbe0322a5d6dfa145 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 22 Feb 2022 15:43:43 -0600 Subject: [PATCH 17/18] updates implementing mila final versions to remain similar --- .../models/not_found_unique_key.sql | 2 +- .../models/not_found_unique_key_list.sql | 2 +- .../test_incremental_unique_id.py | 105 +++++++++++------- 3 files changed, 67 insertions(+), 42 deletions(-) diff --git a/tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql b/tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql index 026f26994..d247aa341 100644 --- a/tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql +++ b/tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql @@ -1,7 +1,7 @@ {{ config( materialized='incremental', - unique_key='asdf' + unique_key='thisisnotacolumn' ) }} diff --git a/tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql index 58d0eba59..f1462a48f 100644 --- a/tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql +++ b/tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql @@ -1,7 +1,7 @@ {{ config( materialized='incremental', - unique_key=['state', 'asdf'] + unique_key=['state', 'thisisnotacolumn'] ) }} diff --git a/tests/integration/incremental_unique_id_test/test_incremental_unique_id.py b/tests/integration/incremental_unique_id_test/test_incremental_unique_id.py index 27646610e..a21d84327 100644 --- a/tests/integration/incremental_unique_id_test/test_incremental_unique_id.py +++ b/tests/integration/incremental_unique_id_test/test_incremental_unique_id.py @@ -1,4 +1,5 @@ from tests.integration.base import DBTIntegrationTest, use_profile +from dbt.contracts.results import RunStatus from collections import namedtuple from pathlib import Path @@ -19,26 +20,6 @@ def schema(self): def models(self): return 'models' - def count_test_relations(self, seed, incremental_model): - '''Idempotently create some number of seeds and incremental models''' - seed_result_set = self.run_dbt( - ['seed', '--select', seed, '--full-refresh'] - ) - model_result_set = self.run_dbt( - ['run', '--select', incremental_model, '--full-refresh'] - ) - return len(seed_result_set), len(model_result_set) - - def get_seed_rows_after_update(self, seed, update_sql_file): - '''Upate seed and return new row count''' - row_count_query = 'select * from {}.{}'.format( - self.unique_schema(), - seed - ) - - self.run_sql_file(Path('seeds') / Path(update_sql_file + '.sql')) - return len(self.run_sql(row_count_query, fetch='all')) - def update_incremental_model(self, incremental_model): '''update incremental model after the seed table has been updated''' model_result_set = self.run_dbt(['run', '--select', incremental_model]) @@ -46,19 +27,30 @@ def update_incremental_model(self, incremental_model): def setup_test(self, seed, incremental_model, update_sql_file): '''build a test case and return values for assertions''' - (seed_count, model_count) = self.count_test_relations( - seed=seed, incremental_model=incremental_model - ) - seed_rows = self.get_seed_rows_after_update( - seed=seed, update_sql_file=update_sql_file + + # Idempotently create some number of seeds and incremental models + seed_count = len(self.run_dbt( + ['seed', '--select', seed, '--full-refresh'] + )) + model_count = len(self.run_dbt( + ['run', '--select', incremental_model, '--full-refresh'] + )) + + # Upate seed and return new row count + row_count_query = 'select * from {}.{}'.format( + self.unique_schema(), + seed ) + self.run_sql_file(Path('seeds') / Path(update_sql_file + '.sql')) + seed_rows = len(self.run_sql(row_count_query, fetch='all')) + inc_test_model_count = self.update_incremental_model( incremental_model=incremental_model ) return (seed_count, model_count, seed_rows, inc_test_model_count) - def run_test(self, expected_fields, test_case_fields): + def test_scenario_correctness(self, expected_fields, test_case_fields): '''Invoke assertions to verify correct build functionality''' # 1. test seed(s) should build afresh self.assertEqual( @@ -99,6 +91,25 @@ def stub_expected_fields( relation=relation ) + def fail_to_build_inc_missing_unique_key_column(self, incremental_model_name): + '''should pass back error state when trying build an incremental + model whose unique key or keylist includes a column missing + from the incremental model''' + seed_count = len(self.run_dbt( + ['seed', '--select', 'seed', '--full-refresh'] + )) + # unique keys are not applied on first run, so two are needed + self.run_dbt( + ['run', '--select', incremental_model_name, '--full-refresh'], + expect_pass=True + ) + run_result = self.run_dbt( + ['run', '--select', incremental_model_name], + expect_pass=False + ).results[0] + + return run_result.status, run_result.message + class TestNoIncrementalUniqueKey(TestIncrementalUniqueKey): @use_profile('bigquery') @@ -117,7 +128,7 @@ def test__bigquery_no_unique_keys(self): opt_model_count=None, relation=incremental_model ) - self.run_test(expected_fields, test_case_fields) + self.test_scenario_correctness(expected_fields, test_case_fields) class TestIncrementalStrUniqueKey(TestIncrementalUniqueKey): @@ -137,7 +148,7 @@ def test__bigquery_empty_str_unique_key(self): opt_model_count=None, relation=incremental_model ) - self.run_test(expected_fields, test_case_fields) + self.test_scenario_correctness(expected_fields, test_case_fields) @use_profile('bigquery') def test__bigquery_one_unique_key(self): @@ -157,13 +168,20 @@ def test__bigquery_one_unique_key(self): relation=incremental_model ) - self.run_test(expected_fields, test_case_fields) + self.test_scenario_correctness(expected_fields, test_case_fields) @use_profile('bigquery') def test__bigquery_bad_unique_key(self): - '''using a unique key not in seed or derived CTEs leads to an error''' - with self.assertRaises(AssertionError) as exc: - self.run_dbt(['run', '--select', 'not_found_unique_key']) + '''expect compilation error from unique key not being a column''' + + err_msg = "Name thisisnotacolumn not found inside DBT_INTERNAL_SOURCE" + + (status, exc) = self.fail_to_build_inc_missing_unique_key_column( + incremental_model_name='not_found_unique_key' + ) + + self.assertEqual(status, RunStatus.Error) + self.assertTrue(err_msg in exc) class TestIncrementalListUniqueKey(TestIncrementalUniqueKey): @@ -183,7 +201,7 @@ def test__bigquery_empty_unique_key_list(self): opt_model_count=None, relation=incremental_model ) - self.run_test(expected_fields, test_case_fields) + self.test_scenario_correctness(expected_fields, test_case_fields) @use_profile('bigquery') def test__bigquery_unary_unique_key_list(self): @@ -203,7 +221,7 @@ def test__bigquery_unary_unique_key_list(self): relation=incremental_model ) - self.run_test(expected_fields, test_case_fields) + self.test_scenario_correctness(expected_fields, test_case_fields) @use_profile('bigquery') def test__bigquery_duplicated_unary_unique_key_list(self): @@ -223,7 +241,7 @@ def test__bigquery_duplicated_unary_unique_key_list(self): relation=incremental_model ) - self.run_test(expected_fields, test_case_fields) + self.test_scenario_correctness(expected_fields, test_case_fields) @use_profile('bigquery') def test__bigquery_trinary_unique_key_list(self): @@ -243,7 +261,7 @@ def test__bigquery_trinary_unique_key_list(self): relation=incremental_model ) - self.run_test(expected_fields, test_case_fields) + self.test_scenario_correctness(expected_fields, test_case_fields) @use_profile('bigquery') def test__bigquery_trinary_unique_key_list_no_update(self): @@ -262,10 +280,17 @@ def test__bigquery_trinary_unique_key_list_no_update(self): opt_model_count=None, relation=incremental_model ) - self.run_test(expected_fields, test_case_fields) + self.test_scenario_correctness(expected_fields, test_case_fields) @use_profile('bigquery') def test__bigquery_bad_unique_key_list(self): - '''using a unique key not in seed or derived CTEs leads to an error''' - with self.assertRaises(AssertionError) as exc: - self.run_dbt(['run', '--select', 'not_found_unique_key_list']) + '''expect compilation error from unique key not being a column''' + + err_msg = "Name thisisnotacolumn not found inside DBT_INTERNAL_SOURCE" + + (status, exc) = self.fail_to_build_inc_missing_unique_key_column( + incremental_model_name='not_found_unique_key_list' + ) + + self.assertEqual(status, RunStatus.Error) + self.assertTrue(err_msg in exc) From ac33d1bdc3c4446b556eb5df937fe5f08afdcff0 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 22 Feb 2022 15:46:45 -0600 Subject: [PATCH 18/18] removing unnecessary line --- .../incremental_schema_tests/test_incremental_schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/incremental_schema_tests/test_incremental_schema.py b/tests/integration/incremental_schema_tests/test_incremental_schema.py index d9655dd35..113a53585 100644 --- a/tests/integration/incremental_schema_tests/test_incremental_schema.py +++ b/tests/integration/incremental_schema_tests/test_incremental_schema.py @@ -140,4 +140,3 @@ def test__bigquery__run_incremental_sync_all_columns(self): @use_profile('bigquery') def test__bigquery__run_incremental_fail_on_schema_change(self): self.run_incremental_fail_on_schema_change() - \ No newline at end of file