diff --git a/README.md b/README.md index ae8a0b1..6890e67 100644 --- a/README.md +++ b/README.md @@ -164,35 +164,52 @@ vars: | ------------- | ------------------------------------------------------------------------------ | | sources | `dbt run-operation apply_masking_policy --args '{"resource_type": "sources"}'` | | models | `dbt run -- model ` | +| snapshots | `dbt snapshot --select --target ` | - Alternatively, you can also apply the masking policies by specifying below `post-hook` OR `on-run-end` to `dbt_project.yml` **Example** : dbt_project.yml +(For models) ```yaml models: post-hook: - "{{ dbt_snow_mask.apply_masking_policy('models') }}" ``` + + (For snapshots) + ```yaml + snapshots: + post-hook: + - "{{ dbt_snow_mask.apply_masking_policy('snapshots') }}" + ``` # How to remove masking policy ? - Remove the masking policy applied by this package by running below commands -| Resource Type | Command | -| ------------- | -------------------------------------------------------------------------------- | -| sources | `dbt run-operation unapply_masking_policy --args '{"resource_type": "sources"}'` | -| models | `dbt run-operation unapply_masking_policy --args '{"resource_type": "models"}'` | +| Resource Type | Command | +| ------------- | ----------------------------------------------------------------------------------- | +| sources | `dbt run-operation unapply_masking_policy --args '{"resource_type": "sources"}'` | +| models | `dbt run-operation unapply_masking_policy --args '{"resource_type": "models"}'` | +| snapshots | `dbt run-operation unapply_masking_policy --args '{"resource_type": "snapshots "}'` | - Alternatively, you can also apply the unmasking policies by specifying below `post-hook` OR `on-run-end` to `dbt_project.yml` **Example** : dbt_project.yml +(For models) ```yaml models: post-hook: - "{{ dbt_snow_mask.unapply_masking_policy('models') }}" ``` +(For snapshots) +```yaml + snapshots: + post-hook: + - "{{ dbt_snow_mask.unapply_masking_policy('snapshots') }}" + ``` # How to validate masking policy ? diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index c85bdaf..deaef22 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -9,6 +9,7 @@ analysis-paths: ["analyses"] test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] +snapshot-paths: ["snapshots"] target-path: "target" clean-targets: @@ -35,6 +36,10 @@ models: post-hook: - "{{ dbt_snow_mask.apply_masking_policy('models') }}" # - "{{ dbt_snow_mask.unapply_masking_policy('models') }}" + +snapshots: + post-hook: + - "{{ dbt_snow_mask.apply_masking_policy('snapshots') }}" dbt_snow_mask_integration_tests: staging: diff --git a/integration_tests/snapshots/pii/ods_customers.sql b/integration_tests/snapshots/pii/ods_customers.sql new file mode 100644 index 0000000..1020409 --- /dev/null +++ b/integration_tests/snapshots/pii/ods_customers.sql @@ -0,0 +1,26 @@ +{% snapshot ods_customers %} + +{{ + config( + target_schema='pii', + strategy='timestamp', + unique_key='customer_id', + updated_at='last_update', + invalidate_hard_deletes=true + ) +}} + +SELECT + customer_id, + store_id, + first_name, + last_name, + email, + address_id, + active, + create_date, + last_update +FROM {{ source('seeds', 'customer') }} + + +{% endsnapshot %} \ No newline at end of file diff --git a/integration_tests/snapshots/pii/ods_customers.yml b/integration_tests/snapshots/pii/ods_customers.yml new file mode 100644 index 0000000..0eb38ac --- /dev/null +++ b/integration_tests/snapshots/pii/ods_customers.yml @@ -0,0 +1,8 @@ +version: 2 + +snapshots: + - name: ods_customers + columns: + - name: email + meta: + masking_policy: mp_encrypt_pii \ No newline at end of file diff --git a/macros/snow-mask/apply-policy/apply_masking_policy.sql b/macros/snow-mask/apply-policy/apply_masking_policy.sql index 3011bd4..003326b 100644 --- a/macros/snow-mask/apply-policy/apply_masking_policy.sql +++ b/macros/snow-mask/apply-policy/apply_masking_policy.sql @@ -4,7 +4,7 @@ {% if resource_type == "sources" %} {{ dbt_snow_mask.apply_masking_policy_list_for_sources(meta_key) }} - {% elif resource_type == "models" %} + {% elif resource_type|lower in ["models", "snapshots"] %} {{ dbt_snow_mask.apply_masking_policy_list_for_models(meta_key) }} {% endif %} diff --git a/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql b/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql index 9e1d044..a3d5ed9 100644 --- a/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql +++ b/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql @@ -10,10 +10,10 @@ {% set schema = model.schema %} {% set model_resource_type = model.resource_type | string %} - {% if model_resource_type|lower in ["model"] %} + {% if model_resource_type|lower in ["model", "snapshot"] %} {# This dictionary stores a mapping between materializations in dbt and the objects they will generate in Snowflake #} - {% set materialization_map = {"table": "table", "view": "view", "incremental": "table"} %} + {% set materialization_map = {"table": "table", "view": "view", "incremental": "table", "snapshot": "table"} %} {# Append custom materializations to the list of standard materializations #} {% do materialization_map.update(fromjson(var('custom_materializations_map', '{}'))) %} @@ -77,9 +77,9 @@ {% set schema = node.schema | string %} {% set node_unique_id = node.unique_id | string %} {% set node_resource_type = node.resource_type | string %} - {% set materialization_map = {"table": "table", "view": "view", "incremental": "table"} %} + {% set materialization_map = {"table": "table", "view": "view", "incremental": "table", "snapshot": "table"} %} - {% if node_resource_type|lower in ["model"] %} + {% if node_resource_type|lower in ["model", "snapshot"] %} {# Append custom materializations to the list of standard materializations #} {% do materialization_map.update(fromjson(var('custom_materializations_map', '{}'))) %} diff --git a/macros/snow-mask/apply-policy/unapply_masking_policy.sql b/macros/snow-mask/apply-policy/unapply_masking_policy.sql index 77921db..6824828 100644 --- a/macros/snow-mask/apply-policy/unapply_masking_policy.sql +++ b/macros/snow-mask/apply-policy/unapply_masking_policy.sql @@ -4,7 +4,7 @@ {% if resource_type == "sources" %} {{ dbt_snow_mask.apply_masking_policy_list_for_sources(meta_key,operation_type) }} - {% elif resource_type == "models" %} + {% elif resource_type|lower in ["models", "snapshots"] %} {{ dbt_snow_mask.apply_masking_policy_list_for_models(meta_key,operation_type) }} {% endif %}