Merged deals

CHANGELOG.md

@@ -1,3 +1,12 @@
+# dbt_hubspot v0.15.2
+[PR #135](https://github.com/fivetran/dbt_hubspot/pull/135) includes the following updates:
+
+## Features
+- Updates the `hubspot__deal_stages` and `hubspot__deals` models to remove stale deals that have been merged since into other deals. In addition we've introduced a new field `merged_deal_ids` that lists all historic merged deals for each deal.
+
+## Under the Hood
+- Added tests for uniqueness for relevant keys on the condition they have not been deleted.
+
 # dbt_hubspot v0.15.1
 [PR #129](https://github.com/fivetran/dbt_hubspot/pull/129) includes the following updates:
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot'
-version: '0.15.1'
+version: '0.15.2'
 
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
@@ -13,6 +13,7 @@ vars:
     company: "{{ ref('stg_hubspot__company') }}"
     company_property_history: "{{ ref('stg_hubspot__company_property_history') }}"
     deal: "{{ ref('stg_hubspot__deal') }}"
+    merged_deal: "{{ ref('stg_hubspot__merged_deal') }}"
     deal_stage: "{{ ref('stg_hubspot__deal_stage') }}"
     deal_company: "{{ ref('stg_hubspot__deal_company') }}"
     deal_pipeline: "{{ ref('stg_hubspot__deal_pipeline') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: hubspot_integration_tests_8
+      schema: hubspot_integration_tests_9
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: hubspot_integration_tests_8
+      schema: hubspot_integration_tests_9
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: hubspot_integration_tests_8
+      schema: hubspot_integration_tests_9
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: hubspot_integration_tests_8
+      schema: hubspot_integration_tests_9
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: hubspot_integration_tests_8
+      schema: hubspot_integration_tests_9
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,10 +1,10 @@
 name: 'hubspot_integration_tests'
-version: '0.15.1'
+version: '0.15.2'
 
 profile: 'integration_tests'
 config-version: 2
 vars:
-  hubspot_schema: hubspot_integration_tests_8
+  hubspot_schema: hubspot_integration_tests_9
   hubspot_source:
     hubspot_service_enabled: true
     # hubspot_sales_enabled: true # enable when generating docs
@@ -58,6 +58,7 @@ vars:
     hubspot_contact_list_identifier: "contact_list_data"
     hubspot_email_event_sent_identifier: "email_event_sent_data"
     hubspot_email_event_dropped_identifier: "email_event_dropped_data"
+    hubspot_merged_deal_identifier: "merged_deal_data"
 
 seeds:
   hubspot_integration_tests:
@@ -173,6 +174,10 @@ seeds:
     ticket_property_history_data:
       +column_types:
         timestamp_instant: timestamp
+    merged_deal_data:
+      +column_types:
+        deal_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        merged_deal_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
 
 dispatch:
   - macro_namespace: dbt_utils

integration_tests/seeds/merged_deal_data.csv

@@ -0,0 +1,15 @@
+deal_id,merged_deal_id,_fivetran_synced
+9089908390,9014070905,2023-06-22 11:37:07.210000
+9089908390,9077310858,2023-06-22 11:37:07.210000
+9089908390,9077311778,2023-06-22 11:37:07.210000
+9089908390,9014071335,2023-06-22 11:37:07.210000
+1364146343,10829010820,2023-06-22 11:37:07.220000
+1364146343,10828874842,2023-06-22 11:37:07.220000
+1368273549,10829051678,2023-06-22 11:37:07.227000
+1368273549,10829028256,2023-06-22 11:37:07.227000
+1373595330,10879517198,2023-06-22 11:37:07.227000
+1373595330,10828874855,2023-06-22 11:37:07.227000
+1006463008,10064581040,2023-06-22 11:37:07.213000
+1006463008,9351825260,2023-06-22 11:37:07.213000
+1389291069,12351788138,2023-06-28 11:18:32.940000
+1389291069,13817889862,2023-06-28 11:18:32.939000

models/marketing/email_events/email_events.yml

@@ -21,6 +21,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: returned_response
         description: The full response from the recipient's email server.
@@ -65,6 +68,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: geo_location
         description: '{{ doc("email_event_location") }}'
@@ -112,6 +118,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: returned_response
         description: The full response from the recipient's email server.
@@ -147,6 +156,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: returned_response
         description: The full response from the recipient's email server.
@@ -200,6 +212,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: from_email
         description: The 'from' field of the email message.
@@ -241,6 +256,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: geo_location
         description: '{{ doc("email_event_location") }}'
@@ -288,6 +306,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: geo_location
         description: '{{ doc("email_event_location") }}'
@@ -332,6 +353,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: geo_location
         description: '{{ doc("email_event_location") }}'
@@ -382,6 +406,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: from_email
         description: The 'from' field of the email message.
@@ -420,6 +447,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: ip_address
         description: '{{ doc("email_event_ip_address") }}'
@@ -461,6 +491,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
 
       - name: is_bounced
         description: | 

models/marketing/marketing.yml

@@ -19,6 +19,9 @@ models:
         description: The ID of the event.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
       - name: from_email
         description: The 'from' field of the email message.
       - name: reply_to_email
@@ -154,6 +157,9 @@ models:
         description: The ID of the contact.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_deleted, false)"
       - name: contact_company
         description: The name of the contact's company
       - name: first_name
@@ -237,6 +243,9 @@ models:
         description: The ID of the contact list.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_contact_list_deleted, false)"
       - name: contact_list_name
         description: The name of the contact list.
       - name: is_contact_list_deleted

models/sales/hubspot__deal_stages.sql

@@ -26,6 +26,7 @@ with deals_enhanced as (
         deal_stage.deal_id || '-' || row_number() over(partition by deal_stage.deal_id order by deal_stage.date_entered asc) as deal_stage_id,
         deals_enhanced.deal_id,
         deals_enhanced.deal_name,
+        deals_enhanced.merged_deal_ids,
         deal_stage._fivetran_start as date_stage_entered,
         deal_stage._fivetran_end as date_stage_exited,
         deal_stage._fivetran_active as is_stage_active,

models/sales/intermediate/int_hubspot__deals_enhanced.sql

@@ -5,6 +5,20 @@ with deals as (
     select *
     from {{ var('deal') }}
 
+), merged_deals as (
+
+    select *
+    from {{ var('merged_deal')}}
+
+), aggregate_merged_deals as (
+
+    select
+        deal_id,
+        {{ fivetran_utils.array_agg("merged_deal_id") }} as merged_deal_ids
+
+    from {{ var('merged_deal')}}
+    group by 1
+
 ), pipelines as (
 
     select *
@@ -26,6 +40,7 @@ with deals as (
 
     select 
         deals.*,
+        aggregate_merged_deals.merged_deal_ids,
         coalesce(pipelines.is_deal_pipeline_deleted, false) as is_deal_pipeline_deleted,
         pipelines.pipeline_label,
         pipelines.is_active as is_pipeline_active,
@@ -44,11 +59,17 @@ with deals as (
         on deals.deal_pipeline_id = pipelines.deal_pipeline_id
     left join pipeline_stages 
         on deals.deal_pipeline_stage_id = pipeline_stages.deal_pipeline_stage_id
+    left join aggregate_merged_deals
+        on deals.deal_id = aggregate_merged_deals.deal_id
+    left join merged_deals
+        on deals.deal_id = merged_deals.merged_deal_id
 
     {% if var('hubspot_owner_enabled', true) %}
     left join owners 
         on deals.owner_id = owners.owner_id
     {% endif %}
+
+    where merged_deals.merged_deal_id is null -- remove deals that have been merged
 )
 
 select *

models/sales/sales.yml

@@ -19,6 +19,9 @@ models:
         description: The ID of the deal
         tests:
           - unique
+          - unique:
+              config:
+                where: "not coalesce(is_deal_deleted, false)"
       - name: deal_name
         description: The name you have given this deal.
       - name: is_deal_deleted
@@ -71,6 +74,9 @@ models:
         description: The unique deal stage identifier.
         tests:
           - unique
+          - unique:
+              config:
+                where: "not coalesce(is_deal_pipeline_stage_deleted, false)"
       - name: is_deal_pipeline_deleted
         description: '{{ doc("is_deleted") }}'
       - name: is_deal_pipeline_stage_deleted
@@ -125,6 +131,9 @@ models:
         description: The ID of the company.
         tests:
           - not_null
+          - unique:
+              config:
+                where: "not coalesce(is_company_deleted, false)"
       - name: portal_id
         description: '{{ doc("portal_id") }}'
       - name: is_company_deleted

packages.yml

@@ -1,3 +1,7 @@
 packages:
-  - package: fivetran/hubspot_source
-    version: [">=0.14.0", "<0.15.0"]
+  # - package: fivetran/hubspot_source
+  #   version: [">=0.14.0", "<0.15.0"]
+
+    - git: https://github.com/fivetran/dbt_hubspot_source.git
+      revision: merged_deals
+      warn-unpinned: false