Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

changes to fix bugs related to resource_type #40

Merged
merged 1 commit into from
Jun 11, 2022
Merged

changes to fix bugs related to resource_type #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions integration_tests/dbt_project.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@ models:

dbt_snow_mask_integration_tests:
staging:
database: "DEMO_DB"
database: "DEV_ENTECHLOG_RAW_DB"
schema: staging

on-run-start:
# - "{{ dbt_snow_mask.create_masking_policy('sources')}}"
- "{{ dbt_snow_mask.create_masking_policy('sources')}}"

on-run-end:
# - "{{ dbt_snow_mask.apply_masking_policy('sources')}}"
- "{{ dbt_snow_mask.apply_masking_policy('sources')}}"
2 changes: 1 addition & 1 deletion macros/snow-mask/apply-policy/apply_masking_policy.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

{% if resource_type == "sources" %}
{{ dbt_snow_mask.apply_masking_policy_list_for_sources(meta_key) }}
{% else %}
{% elif resource_type == "models" %}
{{ dbt_snow_mask.apply_masking_policy_list_for_models(meta_key) }}
{% endif %}

Expand Down
133 changes: 73 additions & 60 deletions macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,66 +1,73 @@
{% macro apply_masking_policy_list_for_models(meta_key,operation_type="apply") %}

{% if execute %}

{% if operation_type == "apply" %}

{% set model_id = model.unique_id | string %}
{% set alias = model.alias %}
{% set database = model.database %}
{% set schema = model.schema %}
{% set model_resource_type = model.resource_type | string %}

{% if model_resource_type|lower in ["model"] %}

{# This dictionary stores a mapping between materializations in dbt and the objects they will generate in Snowflake #}
{% set materialization_map = {"table": "table", "view": "view", "incremental": "table"} %}
{# This dictionary stores a mapping between materializations in dbt and the objects they will generate in Snowflake #}
{% set materialization_map = {"table": "table", "view": "view", "incremental": "table"} %}

{# Append custom materializations to the list of standard materializations #}
{% do materialization_map.update(fromjson(var('custom_materializations_map', '{}'))) %}
{# Append custom materializations to the list of standard materializations #}
{% do materialization_map.update(fromjson(var('custom_materializations_map', '{}'))) %}

{% set materialization = materialization_map[model.config.get("materialized")] %}
{% set meta_columns = dbt_snow_mask.get_meta_objects(model_id,meta_key) %}
{% set materialization = materialization_map[model.config.get("materialized")] %}
{% set meta_columns = dbt_snow_mask.get_meta_objects(model_id,meta_key) %}

{% set masking_policy_db = model.database %}
{% set masking_policy_schema = model.schema %}

{# Override the database and schema name when use_common_masking_policy_db flag is set #}
{%- if (var('use_common_masking_policy_db', 'False')|upper in ['TRUE','YES']) -%}
{% if (var('common_masking_policy_db') and var('common_masking_policy_schema')) %}
{% set masking_policy_db = var('common_masking_policy_db') | string %}
{% set masking_policy_schema = var('common_masking_policy_schema') | string %}
{% set masking_policy_db = model.database %}
{% set masking_policy_schema = model.schema %}

{# Override the database and schema name when use_common_masking_policy_db flag is set #}
{%- if (var('use_common_masking_policy_db', 'False')|upper in ['TRUE','YES']) -%}
{% if (var('common_masking_policy_db') and var('common_masking_policy_schema')) %}
{% set masking_policy_db = var('common_masking_policy_db') | string %}
{% set masking_policy_schema = var('common_masking_policy_schema') | string %}
{% endif %}
{% endif %}
{% endif %}

{# Override the schema name (in the masking_policy_db) when use_common_masking_policy_schema_only flag is set #}
{%- if (var('use_common_masking_policy_schema_only', 'False')|upper in ['TRUE','YES']) and (var('use_common_masking_policy_db', 'False')|upper in ['FALSE','NO']) -%}
{% if var('common_masking_policy_schema') %}
{% set masking_policy_schema = var('common_masking_policy_schema') | string %}
{# Override the schema name (in the masking_policy_db) when use_common_masking_policy_schema_only flag is set #}
{%- if (var('use_common_masking_policy_schema_only', 'False')|upper in ['TRUE','YES']) and (var('use_common_masking_policy_db', 'False')|upper in ['FALSE','NO']) -%}
{% if var('common_masking_policy_schema') %}
{% set masking_policy_schema = var('common_masking_policy_schema') | string %}
{% endif %}
{% endif %}
{% endif %}

{% set masking_policy_list_sql %}
show masking policies in {{masking_policy_db}}.{{masking_policy_schema}};
select $3||'.'||$4||'.'||$2 as masking_policy from table(result_scan(last_query_id()));
{% endset %}
{% set masking_policy_list_sql %}
show masking policies in {{masking_policy_db}}.{{masking_policy_schema}};
select $3||'.'||$4||'.'||$2 as masking_policy from table(result_scan(last_query_id()));
{% endset %}

{# If there are some masking policies to be applied in this model, we should show the masking policies in the schema #}
{% if meta_columns | length > 0 %}
{% set masking_policy_list = dbt_utils.get_query_results_as_dict(masking_policy_list_sql) %}
{% endif %}
{# If there are some masking policies to be applied in this model, we should show the masking policies in the schema #}
{% if meta_columns | length > 0 %}
{% set masking_policy_list = dbt_utils.get_query_results_as_dict(masking_policy_list_sql) %}
{% endif %}

{%- for meta_tuple in meta_columns if meta_columns | length > 0 %}
{% set column = meta_tuple[0] %}
{% set masking_policy_name = meta_tuple[1] %}
{% if masking_policy_name is not none %}
{%- for meta_tuple in meta_columns if meta_columns | length > 0 %}
{% set column = meta_tuple[0] %}
{% set masking_policy_name = meta_tuple[1] %}
{% if masking_policy_name is not none %}

{% for masking_policy_in_db in masking_policy_list['MASKING_POLICY'] %}
{% if masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper == masking_policy_in_db %}
{{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to model : " ~ masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ alias ~ '.' ~ column, info=True) }}
{% set query %}
alter {{materialization}} {{database}}.{{schema}}.{{alias}} modify column {{column}} set masking policy {{masking_policy_db}}.{{masking_policy_schema}}.{{masking_policy_name}};
{% endset %}
{% do run_query(query) %}
{% endif %}
{% endfor %}

{% for masking_policy_in_db in masking_policy_list['MASKING_POLICY'] %}
{% if masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper == masking_policy_in_db %}
{{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to model : " ~ masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ alias ~ '.' ~ column, info=True) }}
{% set query %}
alter {{materialization}} {{database}}.{{schema}}.{{alias}} modify column {{column}} set masking policy {{masking_policy_db}}.{{masking_policy_schema}}.{{masking_policy_name}};
{% endset %}
{% do run_query(query) %}
{% endif %}
{% endfor %}
{% endif %}
{% endfor %}

{% endif %}
{% endfor %}
{% endif %}

{% elif operation_type == "unapply" %}

Expand All @@ -72,30 +79,36 @@
{% set node_resource_type = node.resource_type | string %}
{% set materialization_map = {"table": "table", "view": "view", "incremental": "table"} %}

{# Append custom materializations to the list of standard materializations #}
{% do materialization_map.update(fromjson(var('custom_materializations_map', '{}'))) %}
{% if node_resource_type|lower in ["model"] %}

{% set materialization = materialization_map[model.config.get("materialized")] %}
{% set alias = node.alias %}
{# Append custom materializations to the list of standard materializations #}
{% do materialization_map.update(fromjson(var('custom_materializations_map', '{}'))) %}

{% set meta_columns = dbt_snow_mask.get_meta_objects(node_unique_id,meta_key,node_resource_type) %}
{% set materialization = materialization_map[node.config.get("materialized")] %}
{% set alias = node.alias %}

{%- for meta_tuple in meta_columns if meta_columns | length > 0 %}
{% set column = meta_tuple[0] %}
{% set masking_policy_name = meta_tuple[1] %}
{% set meta_columns = dbt_snow_mask.get_meta_objects(node_unique_id,meta_key,node_resource_type) %}

{% if masking_policy_name is not none %}
{{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to model : " ~ database|upper ~ '.' ~ schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ alias ~ '.' ~ column, info=True) }}
{% set query %}
alter {{materialization}} {{database}}.{{schema}}.{{alias}} modify column {{column}} unset masking policy
{% endset %}
{% do run_query(query) %}
{% endif %}

{% endfor %}
{%- for meta_tuple in meta_columns if meta_columns | length > 0 %}
{% set column = meta_tuple[0] %}
{% set masking_policy_name = meta_tuple[1] %}

{% if masking_policy_name is not none %}
{{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to model : " ~ database|upper ~ '.' ~ schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ alias ~ '.' ~ column, info=True) }}
{% set query %}
alter {{materialization}} {{database}}.{{schema}}.{{alias}} modify column {{column}} unset masking policy
{% endset %}
{% do run_query(query) %}
{% endif %}

{% endfor %}

{% endif %}

{% endfor %}

{% endif %}

{% endif %}

{% endmacro %}
4 changes: 4 additions & 0 deletions macros/snow-mask/apply-policy/apply_masking_policy_list_for_sources.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
{% macro apply_masking_policy_list_for_sources(meta_key,operation_type="apply") %}

{% if execute %}

{% for node in graph.sources.values() -%}

{% set database = node.database | string %}
Expand Down Expand Up @@ -73,4 +75,6 @@

{% endfor %}

{% endif %}

{% endmacro %}
2 changes: 1 addition & 1 deletion macros/snow-mask/apply-policy/unapply_masking_policy.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

{% if resource_type == "sources" %}
{{ dbt_snow_mask.apply_masking_policy_list_for_sources(meta_key,operation_type) }}
{% else %}
{% elif resource_type == "models" %}
{{ dbt_snow_mask.apply_masking_policy_list_for_models(meta_key,operation_type) }}
{% endif %}

Expand Down