From 40ab8ff6df66de64ee40dd9f3b31084630d9607b Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 15 Oct 2021 11:06:32 +0200 Subject: [PATCH 1/5] Fix --store-failures --- dbt/include/spark/macros/adapters.sql | 2 +- test/custom/store_failures/models/schema.yml | 9 +++++ .../store_failures/models/view_model.sql | 5 +++ .../store_failures/test_store_failures.py | 36 +++++++++++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 test/custom/store_failures/models/schema.yml create mode 100644 test/custom/store_failures/models/view_model.sql create mode 100644 test/custom/store_failures/test_store_failures.py diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index fcdc46c6d..b966e9aa6 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -32,7 +32,7 @@ {%- if raw_relation -%} comment '{{ model.description | replace("'", "\\'") }}' {% endif %} - {%- else -%} + {%- elif raw_persist_docs -%} {{ exceptions.raise_compiler_error("Invalid value provided for 'persist_docs'. Expected dict but got value: " ~ raw_persist_docs) }} {% endif %} {%- endmacro -%} diff --git a/test/custom/store_failures/models/schema.yml b/test/custom/store_failures/models/schema.yml new file mode 100644 index 000000000..be559b206 --- /dev/null +++ b/test/custom/store_failures/models/schema.yml @@ -0,0 +1,9 @@ +version: 2 + +models: + - name: view_model + columns: + - name: id + tests: + - unique + - not_null diff --git a/test/custom/store_failures/models/view_model.sql b/test/custom/store_failures/models/view_model.sql new file mode 100644 index 000000000..2ff36b4e2 --- /dev/null +++ b/test/custom/store_failures/models/view_model.sql @@ -0,0 +1,5 @@ +select 1 as id +union all +select 1 as id +union all +select null as id diff --git a/test/custom/store_failures/test_store_failures.py b/test/custom/store_failures/test_store_failures.py new file mode 100644 index 000000000..6306ddb4f --- /dev/null +++ b/test/custom/store_failures/test_store_failures.py @@ -0,0 +1,36 @@ +from test.custom.base import DBTSparkIntegrationTest, use_profile + +class TestStoreFailures(DBTSparkIntegrationTest): + @property + def schema(self): + return "store_failures" + + @property + def models(self): + return "models" + + @property + def project_config(self): + return { + 'config-version': 2, + 'tests': { + '+store_failures': True, + '+severity': 'warn' + } + } + + def test_store_failures(self): + self.run_dbt(['run']) + results = self.run_dbt(['test', '--store-failures'], strict = False) + + @use_profile("apache_spark") + def test_store_failures_apache_spark(self): + self.test_store_failures() + + @use_profile("databricks_cluster") + def test_store_failures_databricks_cluster(self): + self.test_store_failures() + + @use_profile("databricks_sql_endpoint") + def test_store_failures_databricks_sql_endpoint(self): + self.test_store_failures() From de96a8ab9a7218a86caf1a62c31bb8ee4179bd3a Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 15 Oct 2021 15:38:54 +0200 Subject: [PATCH 2/5] Add chnagelog entry --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7405e9e0e..240c6810f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ -## dbt-spark 0.21.0 (Release TBD) +## dbt-spark 0.21.1 (Release TBD) + +### Fixes +- Fix `--store-failures` for tests, by suppressing irrelevant error in `comment_clause()` macro ([#232](https://github.com/dbt-labs/dbt-spark/issues/232), [#233](https://github.com/dbt-labs/dbt-spark/pull/233)) + +## dbt-spark 0.21.0 (October 4, 2021) ### Fixes - Enhanced get_columns_in_relation method to handle a bug in open source deltalake which doesnt return schema details in `show table extended in databasename like '*'` query output. This impacts dbt snapshots if file format is open source deltalake ([#207](https://github.com/dbt-labs/dbt-spark/pull/207)) From 6fcf378b575ced65880be574ad79d0eee9274078 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 15 Oct 2021 16:01:42 +0200 Subject: [PATCH 3/5] Maybe this? --- .../store_failures/test_store_failures.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/test/custom/store_failures/test_store_failures.py b/test/custom/store_failures/test_store_failures.py index 6306ddb4f..0170e3450 100644 --- a/test/custom/store_failures/test_store_failures.py +++ b/test/custom/store_failures/test_store_failures.py @@ -15,7 +15,7 @@ def project_config(self): 'config-version': 2, 'tests': { '+store_failures': True, - '+severity': 'warn' + '+severity': 'warn', } } @@ -26,11 +26,24 @@ def test_store_failures(self): @use_profile("apache_spark") def test_store_failures_apache_spark(self): self.test_store_failures() + +class TestStoreFailuresDelta(TestStoreFailures): + + @property + def project_config(self): + return { + 'config-version': 2, + 'tests': { + '+store_failures': True, + '+severity': 'warn', + '+file_format': 'delta', + } + } @use_profile("databricks_cluster") def test_store_failures_databricks_cluster(self): - self.test_store_failures() - + self.test_store_failures() + @use_profile("databricks_sql_endpoint") def test_store_failures_databricks_sql_endpoint(self): self.test_store_failures() From 112b6ac83a537e176b1d84d955261986f71f528e Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 15 Oct 2021 16:19:54 +0200 Subject: [PATCH 4/5] Fix indentation --- test/custom/store_failures/test_store_failures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/custom/store_failures/test_store_failures.py b/test/custom/store_failures/test_store_failures.py index 0170e3450..84744bd34 100644 --- a/test/custom/store_failures/test_store_failures.py +++ b/test/custom/store_failures/test_store_failures.py @@ -42,7 +42,7 @@ def project_config(self): @use_profile("databricks_cluster") def test_store_failures_databricks_cluster(self): - self.test_store_failures() + self.test_store_failures() @use_profile("databricks_sql_endpoint") def test_store_failures_databricks_sql_endpoint(self): From 30fc3d5676f451b68cb8a7a933fa14f9b7ae3813 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 15 Oct 2021 16:42:48 +0200 Subject: [PATCH 5/5] Class inheritance, silly --- test/custom/store_failures/test_store_failures.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/custom/store_failures/test_store_failures.py b/test/custom/store_failures/test_store_failures.py index 84744bd34..7a4aae7d8 100644 --- a/test/custom/store_failures/test_store_failures.py +++ b/test/custom/store_failures/test_store_failures.py @@ -23,6 +23,8 @@ def test_store_failures(self): self.run_dbt(['run']) results = self.run_dbt(['test', '--store-failures'], strict = False) +class TestStoreFailuresApacheSpark(TestStoreFailures): + @use_profile("apache_spark") def test_store_failures_apache_spark(self): self.test_store_failures()