changes to add support for snapshots data masking (#45)

* Snapshots (#1) * added in ability to process snapshots * added unapply resource type for snapshots * added in snapshot test
entechlog · Oct 9, 2022 · 307c8ab · 307c8ab
1 parent 6a7abcd
commit 307c8ab
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -164,35 +164,52 @@ vars:
 | ------------- | ------------------------------------------------------------------------------ |
 | sources       | `dbt run-operation apply_masking_policy --args '{"resource_type": "sources"}'` |
 | models        | `dbt run -- model <model-name>`                                                |
+| snapshots     | `dbt snapshot --select <snapshot-name> --target <target-name>`                 |
 
 - Alternatively, you can also apply the masking policies by specifying below `post-hook` OR `on-run-end` to `dbt_project.yml`
 
   **Example** : dbt_project.yml
 
+(For models)
   ```yaml
   models:
     post-hook: 
       - "{{ dbt_snow_mask.apply_masking_policy('models') }}"
   ```
+
+  (For snapshots)  
+  ```yaml
+  snapshots:
+    post-hook: 
+      - "{{ dbt_snow_mask.apply_masking_policy('snapshots') }}"
+  ```
 
 # How to remove masking policy ?
 
 - Remove the masking policy applied by this package by running below commands  
 
-| Resource Type | Command                                                                          |
-| ------------- | -------------------------------------------------------------------------------- |
-| sources       | `dbt run-operation unapply_masking_policy --args '{"resource_type": "sources"}'` |
-| models        | `dbt run-operation unapply_masking_policy --args '{"resource_type": "models"}'`  |
+| Resource Type | Command                                                                             |
+| ------------- | ----------------------------------------------------------------------------------- |
+| sources       | `dbt run-operation unapply_masking_policy --args '{"resource_type": "sources"}'`    |
+| models        | `dbt run-operation unapply_masking_policy --args '{"resource_type": "models"}'`     |
+| snapshots     | `dbt run-operation unapply_masking_policy --args '{"resource_type": "snapshots "}'` |
 
 - Alternatively, you can also apply the unmasking policies by specifying below `post-hook` OR `on-run-end` to `dbt_project.yml`
 
   **Example** : dbt_project.yml
 
+(For models)
   ```yaml
   models:
     post-hook: 
       - "{{ dbt_snow_mask.unapply_masking_policy('models') }}"
   ```
+(For snapshots)
+```yaml
+  snapshots:
+    post-hook: 
+      - "{{ dbt_snow_mask.unapply_masking_policy('snapshots') }}"
+  ```
 
 # How to validate masking policy ?
 

diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -9,6 +9,7 @@ analysis-paths: ["analyses"]
 test-paths: ["tests"]
 seed-paths: ["seeds"]
 macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
 
 target-path: "target"
 clean-targets:
@@ -35,6 +36,10 @@ models:
   post-hook: 
     - "{{ dbt_snow_mask.apply_masking_policy('models') }}"
 #    - "{{ dbt_snow_mask.unapply_masking_policy('models') }}"
+
+snapshots:
+  post-hook: 
+    - "{{ dbt_snow_mask.apply_masking_policy('snapshots') }}"
 
   dbt_snow_mask_integration_tests:
     staging:

diff --git a/integration_tests/snapshots/pii/ods_customers.sql b/integration_tests/snapshots/pii/ods_customers.sql
@@ -0,0 +1,26 @@
+{% snapshot ods_customers %}
+
+{{
+    config(
+        target_schema='pii',
+        strategy='timestamp',
+        unique_key='customer_id',
+        updated_at='last_update',
+        invalidate_hard_deletes=true
+    )
+}}
+
+SELECT
+    customer_id,
+    store_id,
+    first_name,
+    last_name,
+    email,
+    address_id,
+    active,
+    create_date,
+    last_update
+FROM {{ source('seeds', 'customer') }}
+
+
+{% endsnapshot %}
diff --git a/integration_tests/snapshots/pii/ods_customers.yml b/integration_tests/snapshots/pii/ods_customers.yml
@@ -0,0 +1,8 @@
+version: 2
+
+snapshots:
+  - name: ods_customers
+    columns:
+      - name: email
+        meta:
+          masking_policy: mp_encrypt_pii
diff --git a/macros/snow-mask/apply-policy/apply_masking_policy.sql b/macros/snow-mask/apply-policy/apply_masking_policy.sql
@@ -4,7 +4,7 @@
 
         {% if resource_type == "sources" %}
             {{ dbt_snow_mask.apply_masking_policy_list_for_sources(meta_key) }}
-        {% elif resource_type == "models" %}
+        {% elif resource_type|lower in ["models", "snapshots"] %}
             {{ dbt_snow_mask.apply_masking_policy_list_for_models(meta_key) }}
         {% endif %}
 

diff --git a/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql b/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql
@@ -10,10 +10,10 @@
         {% set schema   = model.schema %}
         {% set model_resource_type = model.resource_type | string %}
 
-        {% if model_resource_type|lower in ["model"] %}
+        {% if model_resource_type|lower in ["model", "snapshot"] %}
 
             {# This dictionary stores a mapping between materializations in dbt and the objects they will generate in Snowflake  #}
-            {% set materialization_map = {"table": "table", "view": "view", "incremental": "table"} %}
+            {% set materialization_map = {"table": "table", "view": "view", "incremental": "table", "snapshot": "table"} %}
 
             {# Append custom materializations to the list of standard materializations  #}
             {% do materialization_map.update(fromjson(var('custom_materializations_map', '{}'))) %}
@@ -77,9 +77,9 @@
             {% set schema   = node.schema | string %}
             {% set node_unique_id = node.unique_id | string %}
             {% set node_resource_type = node.resource_type | string %}
-            {% set materialization_map = {"table": "table", "view": "view", "incremental": "table"} %}
+            {% set materialization_map = {"table": "table", "view": "view", "incremental": "table", "snapshot": "table"} %}
 
-            {% if node_resource_type|lower in ["model"] %}
+            {% if node_resource_type|lower in ["model", "snapshot"] %}
 
                 {# Append custom materializations to the list of standard materializations  #}
                 {% do materialization_map.update(fromjson(var('custom_materializations_map', '{}'))) %}

diff --git a/macros/snow-mask/apply-policy/unapply_masking_policy.sql b/macros/snow-mask/apply-policy/unapply_masking_policy.sql
@@ -4,7 +4,7 @@
 
         {% if resource_type == "sources" %}
             {{ dbt_snow_mask.apply_masking_policy_list_for_sources(meta_key,operation_type) }}
-        {% elif resource_type == "models" %}
+        {% elif resource_type|lower in ["models", "snapshots"] %}
             {{ dbt_snow_mask.apply_masking_policy_list_for_models(meta_key,operation_type) }}
         {% endif %}