Fix persist_docs for columns

dbt/include/spark/macros/adapters.sql

@@ -174,15 +174,23 @@
   {% do return(None) %}
 {%- endmacro %}
 
+{% macro spark__persist_docs(relation, model, for_relation, for_columns) -%}
+  {% if for_columns and config.persist_column_docs() and model.columns %}
+    {% do alter_column_comment(relation, model.columns) %}
+  {% endif %}
+{% endmacro %}
+
 {% macro spark__alter_column_comment(relation, column_dict) %}
   {% if config.get('file_format', validator=validation.any[basestring]) == 'delta' %}
     {% for column_name in column_dict %}
       {% set comment = column_dict[column_name]['description'] %}
+      {% set escaped_comment = comment | replace('\'', '\\\'') %}
       {% set comment_query %}
-        alter table {{ relation }} change column {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} comment '{{ comment }}';
+        alter table {{ relation }} change column 
+            {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}
+            comment '{{ escaped_comment }}';
       {% endset %}
       {% do run_query(comment_query) %}
     {% endfor %}
   {% endif %}
 {% endmacro %}
-

dbt/include/spark/macros/materializations/seed.sql

@@ -81,10 +81,7 @@
   {%- set agate_table = load_agate_table() -%}
   {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}
 
-  {{ run_hooks(pre_hooks, inside_transaction=False) }}
-
-  -- `BEGIN` happens here:
-  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+  {{ run_hooks(pre_hooks) }}
 
   -- build model
   {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}
@@ -98,10 +95,9 @@
     {{ sql }}
   {% endcall %}
 
-  {{ run_hooks(post_hooks, inside_transaction=True) }}
-  -- `COMMIT` happens here
-  {{ adapter.commit() }}
-  {{ run_hooks(post_hooks, inside_transaction=False) }}
+  {% do persist_docs(target_relation, model) %}
+
+  {{ run_hooks(post_hooks) }}
 
   {{ return({'relations': [target_relation]}) }}
 

dbt/include/spark/macros/materializations/table.sql

@@ -21,6 +21,8 @@
   {% call statement('main') -%}
     {{ create_table_as(False, target_relation, sql) }}
   {%- endcall %}
+
+  {% do persist_docs(target_relation, model) %}
 
   {{ run_hooks(post_hooks) }}
 

test/custom/persist_docs/data/seed.csv

@@ -0,0 +1,3 @@
+id,name
+1,Alice
+2,Bob

test/custom/persist_docs/data/seeds.yml

@@ -0,0 +1,26 @@
+version: 2
+
+seeds:
+  - name: seed
+    description: |
+      Seed model description "with double quotes"
+      and with 'single  quotes' as welll as other;
+      '''abc123'''
+      reserved -- characters
+      --
+      /* comment */
+      Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+    columns:
+      - name: id
+        description: |
+          id Column description "with double quotes"
+          and with 'single  quotes' as welll as other;
+          '''abc123'''
+          reserved -- characters
+          --
+          /* comment */
+          Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+      - name: name
+        description: |
+          Some stuff here and then a call to
+          {{ doc('my_fun_doc')}}

test/custom/persist_docs/models/my_fun_docs.md

@@ -0,0 +1,10 @@
+{% docs my_fun_doc %}
+name Column description "with double quotes"
+and with 'single  quotes' as welll as other;
+'''abc123'''
+reserved -- characters
+--
+/* comment */
+Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+
+{% enddocs %}

test/custom/persist_docs/models/no_docs_model.sql

@@ -0,0 +1 @@
+select 1 as id, 'Alice' as name

test/custom/persist_docs/models/schema.yml

@@ -0,0 +1,71 @@
+version: 2
+
+models:
+
+  - name: table_parquet_model
+    description: |
+      Table model description "with double quotes"
+      and with 'single  quotes' as welll as other;
+      '''abc123'''
+      reserved -- characters
+      --
+      /* comment */
+      Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+    columns:
+      - name: id
+        description: |
+          id Column description "with double quotes"
+          and with 'single  quotes' as welll as other;
+          '''abc123'''
+          reserved -- characters
+          --
+          /* comment */
+          Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+      - name: name
+        description: |
+          Some stuff here and then a call to
+          {{ doc('my_fun_doc')}}
+
+  - name: table_delta_model
+    description: |
+      Table model description "with double quotes"
+      and with 'single  quotes' as welll as other;
+      '''abc123'''
+      reserved -- characters
+      --
+      /* comment */
+      Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+    columns:
+      - name: id
+        description: |
+          id Column description "with double quotes"
+          and with 'single  quotes' as welll as other;
+          '''abc123'''
+          reserved -- characters
+          --
+          /* comment */
+          Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+      - name: name
+        description: |
+          Some stuff here and then a call to
+          {{ doc('my_fun_doc')}}
+
+  - name: view_model
+    description: |
+      View model description "with double quotes"
+      and with 'single  quotes' as welll as other;
+      '''abc123'''
+      reserved -- characters
+      --
+      /* comment */
+      Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+    columns:
+      - name: id
+        description: |
+          id Column description "with double quotes"
+          and with 'single  quotes' as welll as other;
+          '''abc123'''
+          reserved -- characters
+          --
+          /* comment */
+          Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting

test/custom/persist_docs/models/table_delta_model.sql

@@ -0,0 +1,2 @@
+{{ config(materialized='table', file_format='delta') }}
+select 1 as id, 'Joe' as name

test/custom/persist_docs/models/table_parquet_model.sql

@@ -0,0 +1,2 @@
+{{ config(materialized='table', file_format='parquet') }}
+select 1 as id, 'Joe' as name

test/custom/persist_docs/models/view_model.sql

@@ -0,0 +1,2 @@
+{{ config(materialized='view') }}
+select 2 as id, 'Bob' as name

test/custom/persist_docs/test_persist_docs.py

@@ -0,0 +1,72 @@
+from cProfile import run
+from test.custom.base import DBTSparkIntegrationTest, use_profile
+import dbt.exceptions
+
+import json
+
+
+class TestPersistDocsDelta(DBTSparkIntegrationTest):
+    @property
+    def schema(self):
+        return "persist_docs_columns"
+
+    @property
+    def models(self):
+        return "models"
+
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            'models': {
+                'test': {
+                    '+persist_docs': {
+                        "relation": True,
+                        "columns": True,
+                    },
+                }
+            },
+            'seeds': {
+                'test': {
+                    '+persist_docs': {
+                        "relation": True,
+                        "columns": True,
+                    },
+                    '+file_format': 'delta',
+                    '+quote_columns': True
+                }
+            },
+        }
+
+    def test_delta_comments(self):
+        self.run_dbt(['seed'])
+        self.run_dbt(['run'])
+
+        for table in ['table_parquet_model', 'table_delta_model', 'seed']:
+            results = self.run_sql(
+                'describe extended {schema}.{table}'.format(schema=self.unique_schema(), table=table),
+                fetch='all'
+            )
+
+            for result in results:
+                # parquet model will have table-level comment only
+                if result[0] == 'Comment':
+                    whatis = 'Seed' if table == 'seed' else 'Table'
+                    assert result[1].startswith(f'{whatis} model description')
+
+                # delta tables will have table-level and column-level comments
+                if table in ['table_delta_model', 'seed']:
+                    if result[0] == 'id':
+                        assert result[2].startswith('id Column description')
+                    if result[0] == 'name':
+                        assert result[2].startswith('Some stuff here and then a call to')
+
+    # runs on Spark v3.0
+    @use_profile("databricks_cluster")
+    def test_delta_comments_databricks_cluster(self):
+        self.test_delta_comments()
+
+    # runs on Spark v3.0
+    @use_profile("databricks_sql_endpoint")
+    def test_delta_comments_databricks_sql_endpoint(self):
+        self.test_delta_comments()