From e12c51863bf5b6584bb098f25c40c53a0838234f Mon Sep 17 00:00:00 2001 From: Randy Caddell Date: Thu, 10 Mar 2022 09:35:00 -0500 Subject: [PATCH 1/6] add test for column not to exist --- ...xpect_table_columns_not_to_contain_set.sql | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql diff --git a/macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql b/macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql new file mode 100644 index 0000000..dc4737b --- /dev/null +++ b/macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql @@ -0,0 +1,29 @@ +{%- test expect_table_columns_to_contain_set(model, column_list, transform="upper") -%} +{%- if execute -%} + {%- set column_list = column_list | map(transform) | list -%} + {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} + {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%} + with relation_columns as ( + + {% for col_name in relation_column_names %} + select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as relation_column + {% if not loop.last %}union all{% endif %} + {% endfor %} + ), + input_columns as ( + + {% for col_name in column_list %} + select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as input_column + {% if not loop.last %}union all{% endif %} + {% endfor %} + ) + select * + from + input_columns i + left join + relation_columns r on r.relation_column = i.input_column + where + -- catch any column in input list that is not in the list of table columns + r.relation_column is not null +{%- endif -%} +{%- endtest -%} From 852e99983dde970cbbab71e151f7a746ec8dff30 Mon Sep 17 00:00:00 2001 From: Randy Caddell <64412616+rcaddell@users.noreply.github.com> Date: Thu, 10 Mar 2022 09:37:52 -0500 Subject: [PATCH 2/6] Update expect_table_columns_not_to_contain_set.sql --- .../table_shape/expect_table_columns_not_to_contain_set.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql b/macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql index dc4737b..d7292e0 100644 --- a/macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql +++ b/macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql @@ -1,4 +1,4 @@ -{%- test expect_table_columns_to_contain_set(model, column_list, transform="upper") -%} +{%- test expect_table_columns_not_to_contain_set(model, column_list, transform="upper") -%} {%- if execute -%} {%- set column_list = column_list | map(transform) | list -%} {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} From e6f27c9bd5fe6cb355e67717812da396095e2507 Mon Sep 17 00:00:00 2001 From: Randy Caddell <64412616+rcaddell@users.noreply.github.com> Date: Thu, 10 Mar 2022 09:59:01 -0500 Subject: [PATCH 3/6] Update README.md --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index d227677..26f977c 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ To run the tests: - [expect_table_column_count_to_be_between](#expect_table_column_count_to_be_between) - [expect_table_column_count_to_equal_other_table](#expect_table_column_count_to_equal_other_table) - [expect_table_column_count_to_equal](#expect_table_column_count_to_equal) +- [expect_table_columns_not_to_contain_set](#expect_table_columns_not_to_contain_set) - [expect_table_columns_to_contain_set](#expect_table_columns_to_contain_set) - [expect_table_columns_to_match_ordered_list](#expect_table_columns_to_match_ordered_list) - [expect_table_columns_to_match_set](#expect_table_columns_to_match_set) @@ -225,6 +226,20 @@ models: # or seeds: compare_model: ref("other_model") ``` +### [expect_table_columns_not_to_contain_set](macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql) + +Expect the columns in a model not to contain a given list. + +*Applies to:* Model, Seed, Source + +```yaml +models: # or seeds: + - name: my_model + tests: + - dbt_expectations.expect_table_columns_not_to_contain_set: + column_list: ["col_a", "col_b"] + transform: uppper # (Optional) +``` ### [expect_table_columns_to_contain_set](macros/schema_tests/table_shape/expect_table_columns_to_contain_set.sql) Expect the columns in a model to contain a given list. From f5432f103fc5ccea7f22a089c25cd30280289e1f Mon Sep 17 00:00:00 2001 From: Randy Caddell Date: Thu, 10 Mar 2022 11:02:04 -0500 Subject: [PATCH 4/6] name change and inner joined --- integration_tests/models/schema_tests/schema.yml | 2 ++ ...t.sql => expect_table_columns_to_not_contain_set.sql} | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) rename macros/schema_tests/table_shape/{expect_table_columns_not_to_contain_set.sql => expect_table_columns_to_not_contain_set.sql} (81%) diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index 8a1d6bb..21300d1 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -166,6 +166,8 @@ models: column_list: ["idx", "date_col", "col_numeric_a", "col_numeric_b", "col_string_a", "col_string_b", "col_null"] - dbt_expectations.expect_table_column_count_to_equal_other_table: compare_model: ref("data_test") + - dbt_expectations.expect_table_columns_to_not_contain_set: + column_list: ["col_numeric_b", "col_string_a"] - dbt_expectations.expression_is_true: expression: (col_numeric_a + col_numeric_b = 1) - dbt_expectations.equal_expression: diff --git a/macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql b/macros/schema_tests/table_shape/expect_table_columns_to_not_contain_set.sql similarity index 81% rename from macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql rename to macros/schema_tests/table_shape/expect_table_columns_to_not_contain_set.sql index d7292e0..a6f3bea 100644 --- a/macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql +++ b/macros/schema_tests/table_shape/expect_table_columns_to_not_contain_set.sql @@ -1,4 +1,4 @@ -{%- test expect_table_columns_not_to_contain_set(model, column_list, transform="upper") -%} +{%- test expect_table_columns_to_not_contain_set(model, column_list, transform="upper") -%} {%- if execute -%} {%- set column_list = column_list | map(transform) | list -%} {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} @@ -17,13 +17,12 @@ {% if not loop.last %}union all{% endif %} {% endfor %} ) + -- catch any column in input list that is in the list of table columns select * from input_columns i - left join + inner join relation_columns r on r.relation_column = i.input_column - where - -- catch any column in input list that is not in the list of table columns - r.relation_column is not null + {%- endif -%} {%- endtest -%} From 489491edd6fbd36a913a43b0f48325706d5fb940 Mon Sep 17 00:00:00 2001 From: Randy Caddell Date: Thu, 10 Mar 2022 11:24:45 -0500 Subject: [PATCH 5/6] update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 26f977c..be06bdd 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ To run the tests: - [expect_table_column_count_to_be_between](#expect_table_column_count_to_be_between) - [expect_table_column_count_to_equal_other_table](#expect_table_column_count_to_equal_other_table) - [expect_table_column_count_to_equal](#expect_table_column_count_to_equal) -- [expect_table_columns_not_to_contain_set](#expect_table_columns_not_to_contain_set) +- [expect_table_columns_to_not_contain_set](#expect_table_columns_to_not_contain_set) - [expect_table_columns_to_contain_set](#expect_table_columns_to_contain_set) - [expect_table_columns_to_match_ordered_list](#expect_table_columns_to_match_ordered_list) - [expect_table_columns_to_match_set](#expect_table_columns_to_match_set) @@ -226,7 +226,7 @@ models: # or seeds: compare_model: ref("other_model") ``` -### [expect_table_columns_not_to_contain_set](macros/schema_tests/table_shape/expect_table_columns_not_to_contain_set.sql) +### [expect_table_columns_to_not_contain_set](macros/schema_tests/table_shape/expect_table_columns_to_not_contain_set.sql) Expect the columns in a model not to contain a given list. @@ -236,7 +236,7 @@ Expect the columns in a model not to contain a given list. models: # or seeds: - name: my_model tests: - - dbt_expectations.expect_table_columns_not_to_contain_set: + - dbt_expectations.expect_table_columns_to_not_contain_set: column_list: ["col_a", "col_b"] transform: uppper # (Optional) ``` From b51880202f476f230d8befe8f155b37b62ad6d8a Mon Sep 17 00:00:00 2001 From: Randy Caddell Date: Wed, 16 Mar 2022 13:17:14 -0400 Subject: [PATCH 6/6] update test --- integration_tests/models/schema_tests/schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index 21300d1..125302f 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -167,7 +167,7 @@ models: - dbt_expectations.expect_table_column_count_to_equal_other_table: compare_model: ref("data_test") - dbt_expectations.expect_table_columns_to_not_contain_set: - column_list: ["col_numeric_b", "col_string_a"] + column_list: ["col_numeric_c", "col_string_d"] - dbt_expectations.expression_is_true: expression: (col_numeric_a + col_numeric_b = 1) - dbt_expectations.equal_expression: