Merge branch 'fix/fix-n-test-utils-macros' into 'main'

fix: Correct drop_tags and add poe for utils macros

See merge request infinitelambda/bi-chapter/dbt-tags!6

integration_tests/models/behavior/verify_if_masking_policies_applied_correctly.yml

@@ -1,5 +1,5 @@
 models:
   - name: verify_if_masking_policies_applied_correctly
-    tests:
+    data_tests:
       - dbt_utils.expression_is_true:
           expression: ifnull(adapter_tag, '') = dbt_project_tag

integration_tests/models/behavior/verify_if_masking_policies_created_correctly.yml

@@ -1,5 +1,5 @@
 models:
   - name: verify_if_masking_policies_created_correctly
-    tests:
+    data_tests:
       - dbt_utils.expression_is_true:
           expression: ifnull(adapter_masking_policy, '') = dbt_project_masking_policy

integration_tests/models/behavior/verify_if_tags_applied_correctly.yml

@@ -1,5 +1,5 @@
 models:
   - name: verify_if_tags_applied_correctly
-    tests:
+    data_tests:
       - dbt_utils.expression_is_true:
           expression: ifnull(actual_tag_value, '') = config_tag_value

integration_tests/models/behavior/verify_if_tags_created_correctly.yml

@@ -1,5 +1,5 @@
 models:
   - name: verify_if_tags_created_correctly
-    tests:
+    data_tests:
       - dbt_utils.expression_is_true:
           expression: adapter_tag is not null

integration_tests/models/customers.yml

@@ -8,7 +8,7 @@ models:
     columns:
       - name: customer_id
         description: This is a unique identifier for a customer
-        tests:
+        data_tests:
           - unique
           - not_null
 

integration_tests/models/orders.yml

@@ -4,51 +4,30 @@ models:
 
     columns:
       - name: order_id
-        tests:
-          - unique
-          - not_null
         description: This is a unique identifier for an order
 
       - name: customer_id
         description: Foreign key to the customers table
-        tests:
-          - not_null
-          - relationships:
-              to: ref('customers')
-              field: customer_id
 
       - name: order_date
         description: Date (UTC) that the order was placed
 
       - name: status
         description: Order status
-        tests:
-          - accepted_values:
-              values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
 
       - name: amount
         description: Total amount (AUD) of the order
         tags:
           - pii_amount
-        tests:
-          - not_null
 
       - name: credit_card_amount
         description: Amount of the order (AUD) paid for by credit card
-        tests:
-          - not_null
 
       - name: coupon_amount
         description: Amount of the order (AUD) paid for by coupon
-        tests:
-          - not_null
 
       - name: bank_transfer_amount
         description: Amount of the order (AUD) paid for by bank transfer
-        tests:
-          - not_null
 
       - name: gift_card_amount
         description: Amount of the order (AUD) paid for by gift card
-        tests:
-          - not_null

integration_tests/tests/cleanup/verify_if_drop_tags_correctly.sql

@@ -0,0 +1,9 @@
+{{
+  config(
+    tags = ['cleanup'],
+  )
+}}
+select  *
+from    {{ ref('verify_if_tags_created_correctly') }}
+where   dbt_project_tag in ('{{ var("dbt_tags__allowed_tags", []) | join("','") }}')
+  and   adapter_tag is not null

integration_tests/tests/cleanup/verify_if_unapply_mps_correctly.sql

@@ -0,0 +1,9 @@
+{{
+  config(
+    tags = ['cleanup'],
+  )
+}}
+
+select  *
+from    {{ ref('verify_if_masking_policies_applied_correctly') }}
+where   adapter_tag is not null

macros/resources/tags/drop_tags.sql

@@ -6,20 +6,33 @@
 
   {% set ns = dbt_tags.get_resource_ns() %}
   {% set adapter_tags = dbt_tags.get_dbt_tags(debug=debug) %}
+  {% set policy_data_types_list = var('dbt_tags__policy_data_types', none) -%}
   {% set query -%}
 
-    {% for item in adapter_tags %}
-      {%- if dbt_tags.get_masking_policy_for_tag(item.tag) %}
-        {% for policy_data_types in policy_data_types_list if item.tag in policy_data_types.keys() %}
-          {% for datatype in policy_data_types.values() | first %}
-            alter tag {{ ns }}.{{ item.tag }} unset masking policy {{ ns }}.{{ item.tag }}_{{ datatype }};
-          {% endfor %}
-        {% else %}
-          alter tag {{ ns }}.{{ item.tag }} unset masking policy {{ ns }}.{{ item.tag }};
-        {% endfor %}
+    {% set unique_tags = [] -%}
+    {% for item in adapter_tags -%}
+      {%- if dbt_tags.get_masking_policy_for_tag(item.tag) -%}
+        {% for policy_data_types in policy_data_types_list if item.tag in policy_data_types.keys() -%}
+          {% for datatype in policy_data_types.values() | first -%}
+            {% set tag_info = (item.tag, item.tag ~ "_" ~ datatype) -%}
+            {% do unique_tags.append(tag_info) if tag_info not in unique_tags -%}
+          {% endfor -%}
+          {% else -%}
+            {% set tag_info = (item.tag, item.tag) -%}
+            {% do unique_tags.append(tag_info) if tag_info not in unique_tags -%}
+        {% endfor -%}
+        {% else -%}
+          {% set tag_info = (item.tag, none) -%}
+          {% do unique_tags.append(tag_info) if tag_info not in unique_tags -%}
       {%- endif -%}
-      drop tag {{ item.tag }};
-    {% endfor %}
+    {% endfor -%}
+
+    {%- for item in unique_tags if item[1] is not none %}
+      alter tag {{ ns }}.{{ item[0] }} unset masking policy {{ ns }}.{{ item[1] }};
+    {%- endfor %}
+    {%- for item in unique_tags %}
+      drop tag if exists {{ ns }}.{{ item[0] }};
+    {%- endfor %}
 
   {%- endset %}
 

pyproject.toml

@@ -83,12 +83,32 @@ dbt-tags-verify = [
   {cmd = "dbt deps --project-dir integration_tests"},
   {cmd = "dbt debug --project-dir integration_tests"},
 ]
-dbt-tags-test = [
-  {cmd = "dbt deps --project-dir integration_tests"},
+dbt-tags-tagging = [
   {cmd = "dbt run-operation create_tags --project-dir integration_tests"},
   {cmd = "dbt run-operation create_masking_policies --project-dir integration_tests"},
   {cmd = "dbt run-operation apply_mps_to_tags --project-dir integration_tests"},
-  {cmd = "dbt build --project-dir integration_tests"},
+]
+dbt-tags-test-tagging = [
+  {cmd = "dbt deps --project-dir integration_tests"},
+  {cmd = "poe dbt-tags-tagging"},
+  {cmd = "dbt build --exclude tag:cleanup --project-dir integration_tests"},
+]
+dbt-tags-test-unapply-mps = [
+  {cmd = "poe dbt-tags-tagging"},
+  {cmd = "dbt run-operation unapply_mps_from_tags --project-dir integration_tests"},
+  {cmd = "dbt run -s verify_if_masking_policies_applied_correctly --project-dir integration_tests"},
+  {cmd = "dbt test -s verify_if_unapply_mps_correctly --project-dir integration_tests"},
+]
+dbt-tags-test-drop-tags = [
+  {cmd = "poe dbt-tags-tagging"},
+  {cmd = "dbt run-operation drop_tags --project-dir integration_tests"},
+  {cmd = "dbt run -s verify_if_tags_created_correctly --project-dir integration_tests"},
+  {cmd = "dbt test -s verify_if_drop_tags_correctly --project-dir integration_tests"},
+]
+dbt-tags-test = [
+  {cmd = "poe dbt-tags-test-tagging"},
+  {cmd = "poe dbt-tags-test-unapply-mps"},
+  {cmd = "poe dbt-tags-test-drop-tags"},
 ]
 build-doc = [
   {cmd = "dbt deps --project-dir integration_tests"},