changes to add flag to control common masking policy db

.gitignore

@@ -132,3 +132,13 @@ dmypy.json
 target/
 dbt_modules/
 logs/
+profiles.yml
+dbt_packages/
+
+# Passwords 
+*.pem
+*.pub
+*my_password.txt
+
+#Postgres
+.user.yml

README.md

@@ -1,8 +1,13 @@
 - [Overview](#overview)
 - [Installation Instructions](#installation-instructions)
+- [How to configure database and schema for the masking policy ?](#how-to-configure-database-and-schema-for-the-masking-policy-)
 - [How to apply masking policy ?](#how-to-apply-masking-policy-)
 - [How to remove masking policy ?](#how-to-remove-masking-policy-)
 - [How to validate masking policy ?](#how-to-validate-masking-policy-)
+- [Process flow](#process-flow)
+  - [Create masking policy](#create-masking-policy)
+  - [Apply masking policy](#apply-masking-policy)
+- [Known Errors and Solutions](#known-errors-and-solutions)
 - [Credits](#credits)
 - [References](#references)
 - [Contributions](#contributions)
@@ -38,6 +43,24 @@ This dbt package contains macros that can be (re)used across dbt projects with s
 
 > ✅ Please refer to the release version in dbt hub for the latest revision
 
+
+# How to configure database and schema for the masking policy ?
+
+By default this process creates the masking policies in same directory as the database objects. You can change this default behavior by using following parameters in your `dbt_project.yml` 
+
+* `use_common_masking_policy_db` (optional): Flag to enable the usage of a common db/schema for all masking policies. Valid values are “True” OR "False"
+* `common_masking_policy_db` (optional): The database name for creating masking policies
+* `common_masking_policy_schema` (optional): The schema name for creating masking policies
+
+**Example** : var block in dbt_project.yml
+
+```yaml
+vars:
+  use_common_masking_policy_db: "True"
+  common_masking_policy_db: "DEMO_DB"
+  common_masking_policy_schema: "COMPLIANCE"
+```
+
 # How to apply masking policy ?
 
 - Masking is controlled by [meta](https://docs.getdbt.com/reference/resource-properties/meta) in [dbt resource properties](https://docs.getdbt.com/reference/declaring-properties) for sources and models. 
@@ -46,36 +69,36 @@ This dbt package contains macros that can be (re)used across dbt projects with s
 
   **Example** : source.yml
 
-  ```bash
+  ```yaml
   sources:
-    - name: sakila
+    - name: raw_sakila
       tables:
-        - name: actor
+        - name: customer
           columns:
-            - name: FIRST_NAME
+            - name: first_name
               meta:
-                  masking_policy: temp
+                  masking_policy: mp_encrypt_pii
   ```
 
   **Example** : model.yml
-  ```bash
+
+  ```yaml
   models:
     - name: stg_customer
       columns:
-        - name: customer_last_name
-        - name: customer_email
+        - name: email
           meta:
-            masking_policy: temp
+            masking_policy: mp_encrypt_pii
   ```
 
 - Create a new `.sql` file with the name `create_masking_policy_<masking-policy-name-from-meta>.sql` and the sql for masking policy definition. Its important for macro to follow this naming standard.
 
-  **Example** : create_masking_policy_temp.sql
+  **Example** : create_masking_policy_mp_encrypt_pii.sql
 
   ```sql
-  {% macro create_masking_policy_temp(node_database,node_schema) %}
+  {% macro create_masking_policy_mp_encrypt_pii(node_database,node_schema) %}
 
-  CREATE MASKING POLICY IF NOT EXISTS {{node_database}}.{{node_schema}}.temp AS (val string) 
+  CREATE MASKING POLICY IF NOT EXISTS {{node_database}}.{{node_schema}}.mp_encrypt_pii AS (val string) 
     RETURNS string ->
         CASE WHEN CURRENT_ROLE() IN ('ANALYST') THEN val 
              WHEN CURRENT_ROLE() IN ('DEVELOPER') THEN SHA2(val)
@@ -88,49 +111,56 @@ This dbt package contains macros that can be (re)used across dbt projects with s
 > Its good to keep the masking policy ddl organized in a directory say `\macros\snow-mask-ddl`
 
 - Create the masking policies by running below command  
-
 
   | Resource Type | Command                                                                         |
   | ------------- | ------------------------------------------------------------------------------- |
   | sources       | `dbt run-operation create_masking_policy --args '{"resource_type": "sources"}'` |
   | models        | `dbt run-operation create_masking_policy --args '{"resource_type": "models"}'`  |
 
-- Alternatively, you can also create the masking policies by specifying below `on-run-start` in your `dbt_project.yml`
+- Alternatively, you can also create the masking policies by specifying `pre-hook` OR `on-run-start` in your `dbt_project.yml`
 
-  ```bash
-  on-run-end:
-    - "{{ dbt_snow_mask.create_masking_policy(resource_type=models)}}"
-    - "{{ dbt_snow_mask.create_masking_policy(resource_type=sources)}}"
-  ```
-
-- Add post-hook to `dbt_project.yml`
-
-  **Example** : dbt_project.yml
-
-  ```bash
-  models:
-    post-hook: 
-      - "{{ dbt_snow_mask.apply_masking_policy() }}"
+  ```yaml
+  on-run-start:
+    - "{{ dbt_snow_mask.create_masking_policy('models')}}"
+    - "{{ dbt_snow_mask.create_masking_policy('sources')}}"
   ```
 
 - Apply the masking policy by running below commands  
 
-
   | Resource Type | Command                                                                        |
   | ------------- | ------------------------------------------------------------------------------ |
   | sources       | `dbt run-operation apply_masking_policy --args '{"resource_type": "sources"}'` |
   | models        | `dbt run -- model <model-name>`                                                |
 
+- Alternatively, you can also apply the masking policies by specifying below `post-hook` OR `on-run-end` to `dbt_project.yml`
+
+  **Example** : dbt_project.yml
+
+  ```yaml
+  models:
+    post-hook: 
+      - "{{ dbt_snow_mask.apply_masking_policy('models') }}"
+  ```
+
 # How to remove masking policy ?
 
 - Remove the masking policy applied by this package by running below commands  
 
-
   | Resource Type | Command                                                                          |
   | ------------- | -------------------------------------------------------------------------------- |
   | sources       | `dbt run-operation unapply_masking_policy --args '{"resource_type": "sources"}'` |
   | models        | `dbt run-operation unapply_masking_policy --args '{"resource_type": "models"}'`  |
 
+- Alternatively, you can also apply the unmasking policies by specifying below `post-hook` OR `on-run-end` to `dbt_project.yml`
+
+  **Example** : dbt_project.yml
+
+  ```yaml
+  models:
+    post-hook: 
+      - "{{ dbt_snow_mask.unapply_masking_policy('models') }}"
+  ```
+
 # How to validate masking policy ?
 
 ```sql
@@ -149,6 +179,34 @@ SELECT *
   FROM TABLE(INFORMATION_SCHEMA.POLICY_REFERENCES(POLICY_NAME => '<database-name>.<schema-name>.<masking-policy-name>'));
 ```
 
+# Process flow
+## Create masking policy
+
+```mermaid
+graph TD
+    A[create_masking_policy] --> |resource_type='sources',meta_key='masking_policy'| B[get_masking_policy_list_for_sources]
+    A[create_masking_policy] --> |resource_type='models',meta_key='masking_policy'| C[get_masking_policy_list_for_models]
+    B --> |database, schema| D[create_schema]
+    C --> |database, schema| D[create_schema]
+    D --> |policy_name| E[call_masking_policy_macro]
+```
+
+## Apply masking policy
+
+```mermaid
+graph TD
+    A[apply_masking_policy] --> |resource_type='sources',meta_key='masking_policy'| B[apply_masking_policy_list_for_sources]
+    A[apply_masking_policy] --> |resource_type='models',meta_key='masking_policy'| C[apply_masking_policy_list_for_models]
+    B --> |meta_key| D[confirm masking policy is avaliable in db]
+    C --> |meta_key| D[confirm masking policy is avaliable in db]
+    D --> E[alter statement to set masking policy]
+```
+
+# Known Errors and Solutions
+| Error                                                               | Solution                                                                 |
+| ------------------------------------------------------------------- | ------------------------------------------------------------------------ |
+| dict object' has no attribute 'create_masking_policy_mp_encrypt_pi' | Typo in yaml for masking_policy, mp_encrypt_pi instead of mp_encrypt_pii |
+
 # Credits
 This package was created using examples from [Serge](https://www.linkedin.com/in/serge-gekker-912b9928/) and [Matt](https://www.linkedin.com/in/matt-winkler-4024263a/). Please see the [contributors](https://github.com/entechlog/dbt-snow-mask/graphs/contributors) for full list of users who have contributed to this project.
 

dbt_project.yml

@@ -1,10 +1,15 @@
 name: 'dbt_snow_mask'
 version: '1.0.0'
 
-require-dbt-version: [">=0.18.0", "<1.1.0"]
+require-dbt-version: [">=0.18.0"]
 config-version: 2
 
 target-path: "target"
 clean-targets: ["target", "dbt_modules"]
 macro-paths: ["macros"]
 log-path: "logs"
+
+vars:
+  use_common_masking_policy_db: "False"
+  common_masking_policy_db: 
+  common_masking_policy_schema: 

integration_tests/dbt_project.yml

@@ -0,0 +1,45 @@
+name: 'dbt_snow_mask_integration_tests'
+version: '1.0'
+config-version: 2
+
+profile: 'dbt-snow-utils-integration-tests'
+
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+
+target-path: "target"
+clean-targets:
+    - "target"
+    - "dbt_packages"
+    - "logs"
+
+#vars:
+#  use_common_masking_policy_db: "True"
+#  common_masking_policy_db: "DEMO_DB"
+#  common_masking_policy_schema: "COMPLIANCE"
+
+seeds:
+  dbt_snow_mask_integration_tests:
+    sakila:
+      schema: raw_sakila
+
+models:
+  pre-hook:
+    - "{{ dbt_snow_mask.create_masking_policy('models')}}"
+  post-hook: 
+    - "{{ dbt_snow_mask.apply_masking_policy('models') }}"
+#    - "{{ dbt_snow_mask.unapply_masking_policy('models') }}"
+
+  dbt_snow_mask_integration_tests:
+    staging:
+      database: "DEMO_DB"
+      schema: staging
+
+on-run-start:
+#  - "{{ dbt_snow_mask.create_masking_policy('sources')}}"
+
+on-run-end:
+#  - "{{ dbt_snow_mask.apply_masking_policy('sources')}}"

integration_tests/macros/generate_database_name.sql

@@ -0,0 +1,14 @@
+{% macro generate_database_name(custom_database_name=none, node=none) -%}
+
+    {%- set default_database = target.database -%}
+    {%- if custom_database_name is none -%}
+
+        {{ default_database }}
+
+    {%- else -%}
+
+        {{ custom_database_name | trim }}
+
+    {%- endif -%}
+
+{%- endmacro %}

integration_tests/macros/generate_schema_name.sql

@@ -0,0 +1,14 @@
+{% macro generate_schema_name(custom_schema_name, node) -%}
+
+    {%- set default_schema = target.schema -%}
+    {%- if custom_schema_name is none -%}
+
+        {{ default_schema }}
+
+    {%- else -%}
+
+        {{ custom_schema_name | trim }}
+
+    {%- endif -%}
+
+{%- endmacro %}

macros/snow-mask-ddl/create_masking_policy_temp.sql → integration_tests/macros/snow-mask-ddl/create_masking_policy_mp_encrypt_pii.sql

@@ -1,6 +1,7 @@
-{% macro create_masking_policy_temp(node_database,node_schema) %}
+{% macro create_masking_policy_mp_encrypt_pii(node_database,node_schema) %}
+
+    CREATE MASKING POLICY IF NOT EXISTS {{node_database}}.{{node_schema}}.mp_encrypt_pii AS (val string) 
 
-CREATE MASKING POLICY IF NOT EXISTS {{node_database}}.{{node_schema}}.temp AS (val string) 
     RETURNS string ->
         CASE WHEN CURRENT_ROLE() IN ('ANALYST') THEN val 
              WHEN CURRENT_ROLE() IN ('SYSADMIN') THEN SHA2(val)

integration_tests/models/sources/sakila/src_sakila.yml

@@ -0,0 +1,10 @@
+version: 2
+
+sources:
+  - name: raw_sakila
+    tables:
+      - name: customer
+        columns:
+          - name: first_name
+            meta:
+                masking_policy: mp_encrypt_pii

integration_tests/models/staging/sakila/stg_customer.sql

@@ -0,0 +1,7 @@
+{{
+    config(
+        materialized='view'
+    )
+}}
+
+SELECT * FROM {{ source('raw_sakila', 'customer') }}

integration_tests/models/staging/sakila/stg_customer.yml

@@ -0,0 +1,8 @@
+version: 2
+
+models:
+  - name: stg_customer
+    columns:
+      - name: email
+        meta:
+          masking_policy: mp_encrypt_pii

integration_tests/packages.yml

@@ -0,0 +1,4 @@
+packages:
+    - local: ../
+    - package: dbt-labs/dbt_utils
+      version: 0.8.4