Snapshot hard-delete opt-in during config

dbt-labs · Sep 15, 2020 · 2581e98 · 2581e98
1 parent afc7136
commit 2581e98
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 4 deletions.
diff --git a/core/dbt/include/global_project/macros/materializations/snapshot/snapshot.sql b/core/dbt/include/global_project/macros/materializations/snapshot/snapshot.sql
@@ -65,13 +65,15 @@
         from snapshot_query
     ),
 
+    {%- if strategy.invalidate_hard_deletes %}
     deletes_source_data as (
 
         select 
             *,
             {{ strategy.unique_key }} as dbt_unique_key
         from snapshot_query
     ),
+    {% endif %}
 
     insertions as (
 
@@ -105,7 +107,10 @@
         and (
             {{ strategy.row_changed }}
         )
-    ),
+    )
+
+    {%- if strategy.invalidate_hard_deletes -%}
+    ,
 
     deletes as (
 
@@ -122,12 +127,15 @@
         where snapshotted_data.dbt_valid_to is null
         and source_data.dbt_unique_key is null
     )
+    {%- endif %}
 
     select * from insertions
     union all
     select * from updates
+    {%- if strategy.invalidate_hard_deletes %}
     union all
     select * from deletes
+    {%- endif %}
 
 {%- endmacro %}
 

diff --git a/core/dbt/include/global_project/macros/materializations/snapshot/strategies.sql b/core/dbt/include/global_project/macros/materializations/snapshot/strategies.sql
@@ -66,6 +66,7 @@
 {% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}
     {% set primary_key = config['unique_key'] %}
     {% set updated_at = config['updated_at'] %}
+    {% set invalidate_hard_deletes = config['invalidate_hard_deletes'] %}
 
     {#/*
         The snapshot relation might not have an {{ updated_at }} value if the
@@ -86,7 +87,8 @@
         "unique_key": primary_key,
         "updated_at": updated_at,
         "row_changed": row_changed_expr,
-        "scd_id": scd_id_expr
+        "scd_id": scd_id_expr,
+        "invalidate_hard_deletes": invalidate_hard_deletes
     }) %}
 {% endmacro %}
 
@@ -131,6 +133,8 @@
 {% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}
     {% set check_cols_config = config['check_cols'] %}
     {% set primary_key = config['unique_key'] %}
+    {% set invalidate_hard_deletes = config['invalidate_hard_deletes'] %}
+
     {% set select_current_time -%}
         select {{ snapshot_get_time() }} as snapshot_start
     {%- endset %}
@@ -173,6 +177,7 @@
         "unique_key": primary_key,
         "updated_at": updated_at,
         "row_changed": row_changed_expr,
-        "scd_id": scd_id_expr
+        "scd_id": scd_id_expr,
+        "invalidate_hard_deletes": invalidate_hard_deletes
     }) %}
 {% endmacro %}
diff --git a/test/integration/004_simple_snapshot_test/test-snapshots-pg/snapshot.sql b/test/integration/004_simple_snapshot_test/test-snapshots-pg/snapshot.sql
@@ -9,6 +9,11 @@
             updated_at='updated_at',
         )
     }}
+
+    {% if var('invalidate_hard_deletes', 'false') | as_bool %}
+        {{ config(invalidate_hard_deletes=True) }}
+    {% endif %}
+
     select * from {{target.database}}.{{target.schema}}.seed
 
 {% endsnapshot %}
diff --git a/test/integration/004_simple_snapshot_test/test_simple_snapshot.py b/test/integration/004_simple_snapshot_test/test_simple_snapshot.py
@@ -783,7 +783,7 @@ def test__postgres__snapshot_hard_delete(self):
 
         begin_snapshot_datetime = datetime.utcnow()
 
-        results = self.run_snapshot()
+        results = self.run_dbt(['snapshot', '--vars', '{invalidate_hard_deletes: true}'])
         self.assertEqual(len(results), self.NUM_SNAPSHOT_MODELS)
 
         results = self.run_sql(
@@ -794,4 +794,5 @@ def test__postgres__snapshot_hard_delete(self):
         self.assertEqual(len(results), 20)
         for result in results[10:]:
             # result is a tuple, the dbt_valid_to column is the latest
+            self.assertIsInstance(result[-1], datetime)
             self.assertGreaterEqual(result[-1], begin_snapshot_datetime)