bug/email-metrics-variable

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
 
-  - label: ":bricks: Run Tests - Databricks"
+  - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:

.buildkite/scripts/run_models.sh

@@ -17,6 +17,7 @@ echo `pwd`
 cd integration_tests
 dbt deps
 dbt seed --target "$db" --full-refresh
+dbt compile --target "$db" --select hubspot # source does not compile at this time
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target "$db" --full-refresh

CHANGELOG.md

@@ -1,3 +1,13 @@
+# dbt_hubspot v0.9.1
+## 🪲 Bug Fixes
+[PR #105](https://github.com/fivetran/dbt_hubspot/pull/105) applies the following changes:
+- Introduced new macro `get_email_metrics` to prevent failures that may occur when certain tables are disabled or enabled. This macro removes any metrics that are unavailable from the default `email_metrics` list, ensuring that the macro runs smoothly. It's worth noting that you can still manually set the email_metrics list as described in the README's [(Optional) Step 5: Additional configurations](https://github.com/fivetran/dbt_hubspot/#optional-step-5-additional-configurations).
+## 🚘 Under the Hood
+- Updated the following models to utilize the new macro:
+  - hubspot__contacts
+  - hubspot__contact_lists
+  - hubspot__email_campaigns
+
 # dbt_hubspot v0.9.0
 
 ## 🚨 Breaking Changes 🚨

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot'
-version: '0.9.0'
+version: '0.9.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:
@@ -41,7 +41,6 @@ vars:
     engagement_meeting: "{{ ref('stg_hubspot__engagement_meeting') }}"
     engagement_note: "{{ ref('stg_hubspot__engagement_note') }}"
     engagement_task: "{{ ref('stg_hubspot__engagement_task') }}"
-    email_metrics: ['bounces', 'clicks', 'deferrals', 'deliveries', 'drops', 'forwards', 'opens', 'prints', 'spam_reports', 'unsubscribes']
     hubspot_contact_merge_audit_enabled: false
 models:
   hubspot:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: hubspot_integration_tests_2
+      schema: hubspot_integration_tests_4
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: hubspot_integration_tests_2
+      schema: hubspot_integration_tests_4
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: hubspot_integration_tests_2
+      schema: hubspot_integration_tests_4
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: hubspot_integration_tests_2
+      schema: hubspot_integration_tests_4
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: hubspot_integration_tests_2
+      schema: hubspot_integration_tests_4
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,9 +1,9 @@
 name: 'hubspot_integration_tests'
-version: '0.8.2'
+version: '0.9.1'
 profile: 'integration_tests'
 config-version: 2
 vars:
-  hubspot_schema: hubspot_integration_tests_2
+  hubspot_schema: hubspot_integration_tests_4
   hubspot_source:
     hubspot_service_enabled: true
     hubspot_company_property_history_identifier: "company_property_history_data"

macros/adjust_email_metrics.sql

@@ -0,0 +1,13 @@
+{% macro adjust_email_metrics(base_ref, var_name) %}
+
+{% set base_cols = dbt_utils.get_filtered_columns_in_relation(ref(base_ref))|map('lower') %}
+
+{% set default_metrics = ['bounces', 'clicks', 'deferrals', 'deliveries', 'drops', 'forwards', 'opens', 'prints', 'spam_reports', 'unsubscribes'] %}
+{% set email_metrics = var(var_name, default_metrics)|map('lower') %}
+
+{# Only keep metrics found in the base ref #}
+{% set adjusted_cols = email_metrics|select('in', base_cols)|list %}
+
+{{ return(adjusted_cols) }}
+
+{% endmacro %}

models/marketing/hubspot__contact_lists.sql

@@ -14,10 +14,10 @@ with contact_lists as (
     from {{ ref('int_hubspot__email_metrics__by_contact_list') }}
 
 ), joined as (
-
+    {% set email_metrics = adjust_email_metrics('hubspot__email_sends', 'email_metrics') %}
     select 
         contact_lists.*,
-        {% for metric in var('email_metrics') %}
+        {% for metric in email_metrics %}
         coalesce(email_metrics.total_{{ metric }}, 0) as total_{{ metric }},
         coalesce(email_metrics.total_unique_{{ metric }}, 0) as total_unique_{{ metric }}
         {% if not loop.last %},{% endif %}

models/marketing/hubspot__contacts.sql

@@ -6,7 +6,6 @@ with contacts as (
 
     select *
     from {{ ref('int_hubspot__contact_merge_adjust') }} 
-
 {% if emails_enabled %}
 
 ), email_sends as (
@@ -15,10 +14,10 @@ with contacts as (
     from {{ ref('hubspot__email_sends') }}
 
 ), email_metrics as (
-
+    {% set email_metrics = adjust_email_metrics('hubspot__email_sends', 'email_metrics') %}
     select 
         recipient_email_address,
-        {% for metric in var('email_metrics') %}
+        {% for metric in email_metrics %}
         sum({{ metric }}) as total_{{ metric }},
         count(distinct case when {{ metric }} > 0 then email_send_id end) as total_unique_{{ metric }}
         {% if not loop.last %},{% endif %}
@@ -30,7 +29,7 @@ with contacts as (
 
     select 
         contacts.*,
-        {% for metric in var('email_metrics') %}
+        {% for metric in email_metrics %}
         coalesce(email_metrics.total_{{ metric }}, 0) as total_{{ metric }},
         coalesce(email_metrics.total_unique_{{ metric }}, 0) as total_unique_{{ metric }}
         {% if not loop.last %},{% endif %}

models/marketing/hubspot__email_campaigns.sql

@@ -12,10 +12,10 @@ with campaigns as (
     from {{ ref('hubspot__email_sends') }}
 
 ), email_metrics as (
-
+    {% set email_metrics = adjust_email_metrics('hubspot__email_sends', 'email_metrics') %}
     select 
         email_campaign_id,
-        {% for metric in var('email_metrics') %}
+        {% for metric in email_metrics %}
         sum(email_sends.{{ metric }}) as total_{{ metric }},
         count(distinct case when email_sends.{{ metric }} > 0 then email_send_id end) as total_unique_{{ metric }}
         {% if not loop.last %},{% endif %}
@@ -27,7 +27,7 @@ with campaigns as (
 
     select 
         campaigns.*,
-        {% for metric in var('email_metrics') %}
+        {% for metric in email_metrics %}
         coalesce(email_metrics.total_{{ metric }}, 0) as total_{{ metric }},
         coalesce(email_metrics.total_unique_{{ metric }}, 0) as total_unique_{{ metric }}
         {% if not loop.last %},{% endif %}

models/marketing/intermediate/int_hubspot__email_metrics__by_contact_list.sql

@@ -21,10 +21,10 @@ with email_sends as (
     where contact_list_member.contact_list_id is not null
 
 ), email_metrics as (
-
+    {% set email_metrics = adjust_email_metrics('hubspot__email_sends', 'email_metrics') %}
     select 
         contact_list_id,
-        {% for metric in var('email_metrics') %}
+        {% for metric in email_metrics %}
         sum({{ metric }}) as total_{{ metric }},
         count(distinct case when {{ metric }} > 0 then email_send_id end) as total_unique_{{ metric }}
         {% if not loop.last %},{% endif %}