Merge pull request #32 from fivetran/jm-working

Jm working changes from source impacts
fivetran · Mar 22, 2021 · 60f4395 · 60f4395
2 parents 516d033 + e00d695
commit 60f4395
Show file tree

Hide file tree

Showing 16 changed files with 196 additions and 57 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -29,8 +29,11 @@ jobs:
             echo `pwd`
             cd integration_tests
             dbt deps
-            dbt seed --target redshift --full-refresh --exclude email_event_dropped_data_snowflake email_event_sent_data_snowflake
+            dbt seed --target redshift --full-refresh
             dbt run --target redshift --full-refresh
+            dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target redshift
+            dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true}' --target redshift
+            dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: false}' --target redshift
             dbt test --target redshift
       - run:
           name: "Run Tests - Snowflake"
@@ -39,8 +42,11 @@ jobs:
             echo `pwd`
             cd integration_tests
             dbt deps
-            dbt seed --target snowflake --full-refresh --exclude email_event_dropped_data email_event_sent_data
+            dbt seed --target snowflake --full-refresh
             dbt run --target snowflake --full-refresh
+            dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target snowflake
+            dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true}' --target snowflake
+            dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: false}' --target snowflake
             dbt test --target snowflake
       - run:
           name: "Run Tests - BigQuery"
@@ -52,6 +58,9 @@ jobs:
             echo `pwd`
             cd integration_tests
             dbt deps
-            dbt seed --target bigquery --full-refresh --exclude email_event_dropped_data_snowflake email_event_sent_data_snowflake
+            dbt seed --target bigquery --full-refresh
             dbt run --target bigquery --full-refresh
+            dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target bigquery
+            dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true}' --target bigquery
+            dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: false}' --target bigquery
             dbt test --target bigquery
diff --git a/README.md b/README.md
@@ -53,41 +53,56 @@ When setting up your Hubspot connection in Fivetran, it is possible that not eve
 config-version: 2
 
 vars:
-  hubspot:
-
-    # Marketing
-
-    hubspot_marketing_enabled: false                        # Disables all marketing models 
-    hubspot_email_event_enabled: false                      # Disables all email_event models and functionality
-    hubspot_email_event_bounce_enabled: false
-    hubspot_email_event_click_enabled: false
-    hubspot_email_event_deferred_enabled: false
-    hubspot_email_event_delivered_enabled: false
-    hubspot_email_event_dropped_enabled: false
-    hubspot_email_event_forward_enabled: false
-    hubspot_email_event_click_enabled: false
-    hubspot_email_event_opens_enabled: false
-    hubspot_email_event_print_enabled: false
-    hubspot_email_event_sent_enabled: false
-    hubspot_email_event_spam_report: false
-    hubspot_email_event_status_change_enabled: false
-
-    # Sales
-
-    hubspot_sales_enabled: false                            # Disables all sales models
-    hubspot_company_enabled: false
-    hubspot_deal_enabled: false
-    hubspot_engagement_enabled: false                       # Disables all engagement models and functionality
-    hubspot_engagement_contact_enabled: false
-    hubspot_engagement_company_enabled: false
-    hubspot_engagement_deal_enabled: false
-    hubspot_engagement_calls_enabled: false
-    hubspot_engagement_emails_enabled: false
-    hubspot_engagement_meetings_enabled: false
-    hubspot_engagement_notes_enabled: false
-    hubspot_engagement_tasks_enabled: false
+
+  # Marketing
+
+  hubspot_marketing_enabled: false                        # Disables all marketing models
+  hubspot_contact_enabled: false                          # Disables the contact models
+  hubspot_contact_list_enabled: false                     # Disables contact list models
+  hubspot_contact_list_member_enabled: false              # Disables contact list member models
+  hubspot_contact_property_enabled: false                 # Disables the contact property models
+  hubspot_email_event_enabled: false                      # Disables all email_event models and functionality
+  hubspot_email_event_bounce_enabled: false
+  hubspot_email_event_click_enabled: false
+  hubspot_email_event_deferred_enabled: false
+  hubspot_email_event_delivered_enabled: false
+  hubspot_email_event_dropped_enabled: false
+  hubspot_email_event_forward_enabled: false
+  hubspot_email_event_click_enabled: false
+  hubspot_email_event_opens_enabled: false
+  hubspot_email_event_print_enabled: false
+  hubspot_email_event_sent_enabled: false
+  hubspot_email_event_spam_report: false
+  hubspot_email_event_status_change_enabled: false
+
+  # Sales
+
+  hubspot_sales_enabled: false                            # Disables all sales models
+  hubspot_company_enabled: false
+  hubspot_deal_enabled: false
+  hubspot_engagement_enabled: false                       # Disables all engagement models and functionality
+  hubspot_engagement_contact_enabled: false
+  hubspot_engagement_company_enabled: false
+  hubspot_engagement_deal_enabled: false
+  hubspot_engagement_calls_enabled: false
+  hubspot_engagement_emails_enabled: false
+  hubspot_engagement_meetings_enabled: false
+  hubspot_engagement_notes_enabled: false
+  hubspot_engagement_tasks_enabled: false
 ```
+### Changing the Build Schema
+By default this package will build the HubSpot staging models within a schema titled (<target_schema> + `_stg_hubspot`) and HubSpot final models within a schema titled (<target_schema> + `hubspot`) in your target database. If this is not where you would like your modeled HubSpot data to be written to, add the following configuration to your `dbt_project.yml` file:
+
+```yml
+# dbt_project.yml
 
+...
+models:
+    hubspot:
+      +schema: my_new_schema_name # leave blank for just the target_schema
+    hubspot_source:
+      +schema: my_new_schema_name # leave blank for just the target_schema
+```
 ## Contributions
 
 Additional contributions to this package are very welcome! Please create issues

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,8 +1,12 @@
 name: 'hubspot'
-version: '0.2.0'
+version: '0.3.0'
 config-version: 2
-require-dbt-version: [">=0.18.0", "<0.19.0"]
+require-dbt-version: [">=0.18.0", "<0.20.0"]
 
+models:
+  hubspot:
+    +materialized: table
+    +schema: hubspot
 vars:
   hubspot:
     contact: "{{ ref('stg_hubspot__contact') }}"

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/data/deal_company_data.csv b/integration_tests/data/deal_company_data.csv
@@ -0,0 +1,3 @@
+company_id,deal_id
+12345,6789
+56789,1234
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,6 +1,6 @@
 
 name: 'hubspot_integration_tests'
-version: '1.0.0'
+version: '0.3.0'
 profile: 'integration_tests'
 config-version: 2
 
@@ -13,6 +13,7 @@ vars:
     contact_list_member: "{{ ref('contact_list_member_data')}}"
     contact_property_history: "{{ ref('contact_property_history_data')}}"
     deal: "{{ ref('deal_data')}}"
+    deal_company: "{{ ref('deal_company_data')}}"
     deal_pipeline: "{{ ref('deal_pipeline_data')}}"
     deal_pipeline_stage: "{{ ref('deal_pipeline_stage_data')}}"
     deal_property_history: "{{ ref('deal_property_history_data')}}"
@@ -22,11 +23,11 @@ vars:
     email_event_click: "{{ ref('email_event_click_data')}}"
     email_event_deferred: "{{ ref('email_event_deferred_data')}}"
     email_event_delivered: "{{ ref('email_event_delivered_data')}}"
-    email_event_dropped: "{{ snowflake_seed_data('email_event_dropped_data') }}"
+    email_event_dropped: "{{ fivetran_utils.snowflake_seed_data('email_event_dropped_data') }}"
     email_event_forward: "{{ ref('email_event_forward_data')}}"
     email_event_open: "{{ ref('email_event_open_data')}}"
     email_event_print: "{{ ref('email_event_print_data')}}"
-    email_event_sent: "{{ snowflake_seed_data('email_event_sent_data') }}"
+    email_event_sent: "{{ fivetran_utils.snowflake_seed_data('email_event_sent_data') }}"
     email_event_spam_report: "{{ ref('email_event_spam_report_data')}}"
     email_event_status_change: "{{ ref('email_event_status_change_data')}}"
     engagement: "{{ ref('engagement_data')}}"
@@ -97,3 +98,11 @@ seeds:
     engagement_note_data:
       +column_types:
         engagement_id: "{{ 'int64' if target.name == 'bigquery' else 'bigint' }}"
+    email_event_sent_data_snowflake:
+      +enabled: "{{ true if target.type == 'snowflake' else false }}"    
+    email_event_sent_data:
+      +enabled: "{{ true if target.type != 'snowflake' else false }}"
+    email_event_dropped_data_snowflake:
+      +enabled: "{{ true if target.type == 'snowflake' else false }}"
+    email_event_dropped_data:
+      +enabled: "{{ true if target.type != 'snowflake' else false }}"
diff --git a/integration_tests/macros/snowflake_seed_data.sql b/integration_tests/macros/snowflake_seed_data.sql
diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml
@@ -1,4 +1,2 @@
 packages:
-  - local: ../
-  - package: fishtown-analytics/dbt_utils
-    version: [">=0.4.0"]
+  - local: ../
diff --git a/models/marketing/history/hubspot__contact_history.sql b/models/marketing/history/hubspot__contact_history.sql
@@ -1,3 +1,5 @@
+{{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled', 'hubspot_contact_property_enabled'])) }}
+
 with history as (
 
     select *

diff --git a/models/marketing/hubspot__contact_lists.sql b/models/marketing/hubspot__contact_lists.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled'])) }}
+{{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_contact_list_enabled'])) }}
 
 with contact_lists as (
 

diff --git a/models/marketing/hubspot__contacts.sql b/models/marketing/hubspot__contacts.sql
@@ -1,3 +1,4 @@
+{{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled', 'hubspot_contact_enabled'])) }}
 {% set emails_enabled = fivetran_utils.enabled_vars(['hubspot_marketing_enabled', 'hubspot_email_event_enabled']) %}
 {% set engagements_enabled = fivetran_utils.enabled_vars(['hubspot_sales_enabled', 'hubspot_engagement_enabled']) %}
 

diff --git a/models/marketing/intermediate/int_hubspot__email_metrics__by_contact_list.sql b/models/marketing/intermediate/int_hubspot__email_metrics__by_contact_list.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled']), materialized='table') }}
+{{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_contact_list_member_enabled']), materialized='table') }}
 
 with email_sends as (
 

diff --git a/packages.yml b/packages.yml
@@ -1,3 +1,5 @@
 packages:
-  - package: fivetran/hubspot_source
-    version: [">=0.2.0", "<0.3.0"]
+  # - package: fivetran/hubspot_source
+  #   version: [">=0.2.0", "<0.3.0"]
+  - git: https://github.com/fivetran/dbt_hubspot_source.git
+    warn-unpinned: false