Merge pull request #125 from fivetran/merged_deals

Merged deals

.buildkite/scripts/run_models.sh

@@ -23,6 +23,7 @@ dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}'
 dbt run --vars '{hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target "$db"
 dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true}' --target "$db"
 dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: false}' --target "$db"
+dbt run --vars '{hubspot_deal_enabled: true, hubspot_sales_enabled: true, hubspot_merged_deal_enabled: true}' --target "$db"
 dbt run --vars '{hubspot__pass_through_all_columns: true, hubspot_service_enabled: true, hubspot_using_all_email_events: false}' --target "$db"
 dbt test --target "$db"
 

CHANGELOG.md

@@ -1,3 +1,10 @@
+# dbt_hubspot_source v0.14.1
+[PR #125](https://github.com/fivetran/dbt_hubspot_source/pull/125) includes the following updates:
+
+## Features
+- Adds the `stg_hubspot__merged_deal` model. Downstream, this model is used to remove stale deals and aggregate them to the deal they have been merged into. This is disabled by default as not every user has the `merged_deal` table.
+  - To enable this model and filter out merged deals downstream, make sure the following is set to `true` in your dbt_project.yml: `hubspot_sales_enabled`, `hubspot_deal_enabled`, and `hubspot_merged_deal_enabled`. See [Step 4 of the README](https://github.com/fivetran/dbt_hubspot_source#step-4-disable-models-for-non-existent-sources) for more details. 
+
 # dbt_hubspot_source v0.14.0
 [PR #122](https://github.com/fivetran/dbt_hubspot_source/pull/122) includes the following updates:
 

README.md

@@ -55,7 +55,7 @@ vars:
     hubspot_schema: your_schema_name 
 ```
 ## Step 4: Disable models for non-existent sources
-When setting up your Hubspot connection in Fivetran, it is possible that not every table this package expects will be synced. This can occur because you either don't use that functionality in Hubspot or have actively decided to not sync some tables. Therefore we have added enable/disable configs in the `src.yml` to allow you to disable certain sources not present. Downstream models are automatically disabled as well. In order to disable the relevant functionality in the package, you will need to add the relevant variables in your root `dbt_project.yml`. By default, all variables are assumed to be `true` (with exception of `hubspot_service_enabled`, `hubspot_ticket_deal_enabled`, and `hubspot_contact_merge_audit_enabled`). You only need to add variables for the tables different from default:
+When setting up your Hubspot connection in Fivetran, it is possible that not every table this package expects will be synced. This can occur because you either don't use that functionality in Hubspot or have actively decided to not sync some tables. Therefore we have added enable/disable configs in the `src.yml` to allow you to disable certain sources not present. Downstream models are automatically disabled as well. In order to disable the relevant functionality in the package, you will need to add the relevant variables in your root `dbt_project.yml`. By default, all variables are assumed to be `true` (with exception of `hubspot_service_enabled`, `hubspot_ticket_deal_enabled`, `hubspot_contact_merge_audit_enabled`, and `hubspot_merged_deal_enabled`). You only need to add variables for the tables different from default:
 
 ```yml
 # dbt_project.yml
@@ -91,6 +91,7 @@ vars:
   hubspot_company_enabled: false
   hubspot_company_property_history_enabled: false         # Disable the company property history models
   hubspot_deal_enabled: false
+  hubspot_merged_deal_enabled: true                       # Enables the merged_deal table, which will be used to filter out merged deals from the final deal models. False by default. Note that `hubspot_sales_enabled` and `hubspot_deal_enabled` must not be set to False.
   hubspot_deal_company_enabled: false
   hubspot_deal_contact_enabled: false
   hubspot_deal_property_history_enabled: false            # Disables the deal property history tables

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.14.0'
+version: '0.14.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
@@ -24,6 +24,7 @@ vars:
     deal_pipeline: "{{ source('hubspot','deal_pipeline') }}"
     deal_pipeline_stage: "{{ source('hubspot','deal_pipeline_stage') }}"
     deal_property_history: "{{ source('hubspot','deal_property_history') }}"
+    merged_deal: "{{ source('hubspot','merged_deal') }}"
     email_campaign: "{{ source('hubspot','email_campaign') }}"
     email_event: "{{ source('hubspot','email_event') }}"
     email_event_bounce: "{{ source('hubspot','email_event_bounce') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: hubspot_source_integration_tests_999
+      schema: hubspot_source_integration_tests_21
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: hubspot_source_integration_tests_999
+      schema: hubspot_source_integration_tests_21
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: hubspot_source_integration_tests_999
+      schema: hubspot_source_integration_tests_21
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: hubspot_source_integration_tests_999
+      schema: hubspot_source_integration_tests_21
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: hubspot_source_integration_tests_999
+      schema: hubspot_source_integration_tests_21
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,18 +1,19 @@
 name: 'hubspot_source_integration_tests'
-version: '0.14.0'
+version: '0.14.1'
 profile: 'integration_tests'
 config-version: 2
 models:
   hubspot_source:
     +schema:
 vars:
-  hubspot_schema: hubspot_source_integration_tests_999
+  hubspot_schema: hubspot_source_integration_tests_21
   hubspot_source:
     hubspot_service_enabled: true
     # hubspot_sales_enabled: true # enable when generating docs
     # hubspot_marketing_enabled: true # enable when generating docs
     # hubspot_contact_merge_audit_enabled: true # enable when generating docs
     # hubspot_using_all_email_events: true # enable when generating docs
+    # hubspot_merged_deal_enabled: true # enable when generating docs  
     hubspot_company_property_history_identifier: "company_property_history_data"
     hubspot_company_identifier: "company_data"
     hubspot_contact_identifier: "contact_data"
@@ -60,6 +61,7 @@ vars:
     hubspot_contact_list_identifier: "contact_list_data"
     hubspot_email_event_sent_identifier: "email_event_sent_data"
     hubspot_email_event_dropped_identifier: "email_event_dropped_data"
+    hubspot_merged_deal_identifier: "merged_deal_data"
 
 seeds:
   hubspot_source_integration_tests:
@@ -174,6 +176,10 @@ seeds:
         created: timestamp
         obsoleted_by_created: timestamp
         sent_by_created: timestamp
+    merged_deal_data:
+      +column_types:
+        deal_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        merged_deal_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
 
 dispatch:
   - macro_namespace: dbt_utils

integration_tests/seeds/merged_deal_data.csv

@@ -0,0 +1,15 @@
+deal_id,merged_deal_id,_fivetran_synced
+9089908390,9014070905,2023-06-22 11:37:07.210000
+9089908390,9077310858,2023-06-22 11:37:07.210000
+9089908390,9077311778,2023-06-22 11:37:07.210000
+9089908390,9014071335,2023-06-22 11:37:07.210000
+1364146343,10829010820,2023-06-22 11:37:07.220000
+1364146343,10828874842,2023-06-22 11:37:07.220000
+1368273549,10829051678,2023-06-22 11:37:07.227000
+1368273549,10829028256,2023-06-22 11:37:07.227000
+1373595330,10879517198,2023-06-22 11:37:07.227000
+1373595330,10828874855,2023-06-22 11:37:07.227000
+1006463008,10064581040,2023-06-22 11:37:07.213000
+1006463008,9351825260,2023-06-22 11:37:07.213000
+1389291069,12351788138,2023-06-28 11:18:32.940000
+1389291069,13817889862,2023-06-28 11:18:32.939000

macros/get_merged_deal_columns.sql

@@ -0,0 +1,11 @@
+{% macro get_merged_deal_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "deal_id", "datatype": dbt.type_int()},
+    {"name": "merged_deal_id", "datatype": dbt.type_int()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

models/src_hubspot.yml

@@ -1311,3 +1311,16 @@ sources:
             description: Short summary of ticket.
           - name: property_content
             description: Text in body of the ticket.
+
+      - name: merged_deal
+        identifier: '{{ var("hubspot_merged_deal_identifier", "merged_deal") }}'
+        description: Each record represents a deal that has been merged along with the related deal it's been merged into.
+        config:
+          enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) and var('hubspot_merged_deal_enabled', false) }}"
+        columns:
+          - name: deal_id
+            description: The ID of the deal the record has been merged INTO.
+          - name: merged_deal_id
+            description: ID of deal that has been merged.
+          - name: _fivetran_synced
+            description: '{{ doc("_fivetran_synced") }}'

models/stg_hubspot__deal.yml

@@ -161,3 +161,19 @@ models:
         description: '{{ doc("history_name") }}'
       - name: new_value
         description: '{{ doc("history_value") }}'
+
+  - name: stg_hubspot__merged_deal
+    description: Each record represents a deal that has been merged along with the related deal it's been merged into.
+    columns:
+      - name: merged_deal_id
+        description: ID of deal that has been merged.
+        tests:
+          - not_null
+      - name: deal_id
+        description: The ID of the deal the record has been merged INTO.
+        tests:
+          - not_null
+      - name: _fivetran_synced
+        description: '{{ doc("_fivetran_synced") }}'
+
+

models/stg_hubspot__merged_deal.sql

@@ -0,0 +1,29 @@
+{{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_deal_enabled']) and var('hubspot_merged_deal_enabled', False)) }}
+
+with base as (
+
+    select * 
+    from {{ ref('stg_hubspot__merged_deal_tmp') }}
+
+), macro as (
+
+    select
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_hubspot__merged_deal_tmp')),
+                staging_columns=get_merged_deal_columns()
+            )
+        }}
+    from base
+
+), fields as (
+
+    select
+        cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced,
+        deal_id,
+        merged_deal_id
+    from macro
+)
+
+select *
+from fields

models/tmp/stg_hubspot__merged_deal_tmp.sql

@@ -0,0 +1,4 @@
+{{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_deal_enabled']) and var('hubspot_merged_deal_enabled', False)) }}
+
+select {{ dbt_utils.star(source('hubspot','merged_deal')) }}
+from {{ var('merged_deal') }}