bug/email-metrics-variable

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
  commands: |
  bash .buildkite/scripts/run_models.sh redshift
 
- - label: ":bricks: Run Tests - Databricks"
+ - label: ":databricks: Run Tests - Databricks"
  key: "run_dbt_databricks"
  plugins:
  - docker#v3.13.0:

.buildkite/scripts/run_models.sh

@@ -17,6 +17,7 @@ echo `pwd`
 cd integration_tests
 dbt deps
 dbt seed --target "$db" --full-refresh
+dbt compile --target "$db" --select hubspot # source does not compile at this time
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target "$db" --full-refresh

CHANGELOG.md

@@ -1,3 +1,14 @@
+# dbt_hubspot v0.9.1
+## 🪲 Bug Fixes
+[PR #105](https://github.com/fivetran/dbt_hubspot/pull/105) applies the following changes:
+- Introduced new macro `get_email_metrics` to prevent failures that may occur when certain tables are disabled or enabled. This macro removes any metrics that are unavailable from the default `email_metrics` list, ensuring that the models runs smoothly. It's worth noting that you can still manually set the email_metrics list as described in the README's [(Optional) Step 5: Additional configurations](https://github.com/fivetran/dbt_hubspot/#optional-step-5-additional-configurations).
+## 🚘 Under the Hood
+- Updated the following models to utilize the new macro:
+ - hubspot__contacts
+ - hubspot__contact_lists
+ - hubspot__email_campaigns
+ - int_hubspot__email_metrics__by_contact_list
+
 # dbt_hubspot v0.9.0
 
 ## 🚨 Breaking Changes 🚨

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot'
-version: '0.9.0'
+version: '0.9.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:
@@ -41,7 +41,6 @@ vars:
  engagement_meeting: "{{ ref('stg_hubspot__engagement_meeting') }}"
  engagement_note: "{{ ref('stg_hubspot__engagement_note') }}"
  engagement_task: "{{ ref('stg_hubspot__engagement_task') }}"
- email_metrics: ['bounces', 'clicks', 'deferrals', 'deliveries', 'drops', 'forwards', 'opens', 'prints', 'spam_reports', 'unsubscribes']
  hubspot_contact_merge_audit_enabled: false
 models:
  hubspot:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
  pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
  dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
  port: 5439
- schema: hubspot_integration_tests_2
+ schema: hubspot_integration_tests_4
  threads: 8
  bigquery:
  type: bigquery
  method: service-account-json
  project: 'dbt-package-testing'
- schema: hubspot_integration_tests_2
+ schema: hubspot_integration_tests_4
  threads: 8
  keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
  snowflake:
@@ -33,7 +33,7 @@ integration_tests:
  role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
  database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
  warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
- schema: hubspot_integration_tests_2
+ schema: hubspot_integration_tests_4
  threads: 8
  postgres:
  type: postgres
@@ -42,13 +42,13 @@ integration_tests:
  pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
  dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
  port: 5432
- schema: hubspot_integration_tests_2
+ schema: hubspot_integration_tests_4
  threads: 8
  databricks:
  catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
  host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
  http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
- schema: hubspot_integration_tests_2
+ schema: hubspot_integration_tests_4
  threads: 2
  token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
  type: databricks

integration_tests/dbt_project.yml

@@ -1,9 +1,9 @@
 name: 'hubspot_integration_tests'
-version: '0.8.2'
+version: '0.9.1'
 profile: 'integration_tests'
 config-version: 2
 vars:
- hubspot_schema: hubspot_integration_tests_2
+ hubspot_schema: hubspot_integration_tests_4
  hubspot_source:
  hubspot_service_enabled: true
  hubspot_company_property_history_identifier: "company_property_history_data"

macros/adjust_email_metrics.sql

@@ -0,0 +1,13 @@
+{% macro adjust_email_metrics(base_ref, var_name) %}
+
+{% set base_cols = dbt_utils.get_filtered_columns_in_relation(ref(base_ref))|map('lower') %}
+
+{% set default_metrics = ['bounces', 'clicks', 'deferrals', 'deliveries', 'drops', 'forwards', 'opens', 'prints', 'spam_reports', 'unsubscribes'] %}
+{% set email_metrics = var(var_name, default_metrics)|map('lower') %}
+
+{# Only keep metrics found in the base ref #}
+{% set adjusted_cols = email_metrics|select('in', base_cols)|list %}
+
+{{ return(adjusted_cols) }}
+
+{% endmacro %}

models/marketing/hubspot__contact_lists.sql

@@ -14,10 +14,10 @@ with contact_lists as (
  from {{ ref('int_hubspot__email_metrics__by_contact_list') }}
 
 ), joined as (
-
+ {% set email_metrics = adjust_email_metrics('hubspot__email_sends', 'email_metrics') %}
  select 
  contact_lists.*,
- {% for metric in var('email_metrics') %}
+ {% for metric in email_metrics %}
  coalesce(email_metrics.total_{{ metric }}, 0) as total_{{ metric }},
  coalesce(email_metrics.total_unique_{{ metric }}, 0) as total_unique_{{ metric }}
  {% if not loop.last %},{% endif %}

models/marketing/hubspot__contacts.sql

@@ -6,7 +6,6 @@ with contacts as (
 
  select *
  from {{ ref('int_hubspot__contact_merge_adjust') }} 
-
 {% if emails_enabled %}
 
 ), email_sends as (
@@ -15,10 +14,10 @@ with contacts as (
  from {{ ref('hubspot__email_sends') }}
 
 ), email_metrics as (
-
+ {% set email_metrics = adjust_email_metrics('hubspot__email_sends', 'email_metrics') %}
  select 
  recipient_email_address,
- {% for metric in var('email_metrics') %}
+ {% for metric in email_metrics %}
  sum({{ metric }}) as total_{{ metric }},
  count(distinct case when {{ metric }} > 0 then email_send_id end) as total_unique_{{ metric }}
  {% if not loop.last %},{% endif %}
@@ -30,7 +29,7 @@ with contacts as (
 
  select 
  contacts.*,
- {% for metric in var('email_metrics') %}
+ {% for metric in email_metrics %}
  coalesce(email_metrics.total_{{ metric }}, 0) as total_{{ metric }},
  coalesce(email_metrics.total_unique_{{ metric }}, 0) as total_unique_{{ metric }}
  {% if not loop.last %},{% endif %}

models/marketing/hubspot__email_campaigns.sql

@@ -12,10 +12,10 @@ with campaigns as (
  from {{ ref('hubspot__email_sends') }}
 
 ), email_metrics as (
-
+ {% set email_metrics = adjust_email_metrics('hubspot__email_sends', 'email_metrics') %}
  select 
  email_campaign_id,
- {% for metric in var('email_metrics') %}
+ {% for metric in email_metrics %}
  sum(email_sends.{{ metric }}) as total_{{ metric }},
  count(distinct case when email_sends.{{ metric }} > 0 then email_send_id end) as total_unique_{{ metric }}
  {% if not loop.last %},{% endif %}
@@ -27,7 +27,7 @@ with campaigns as (
 
  select 
  campaigns.*,
- {% for metric in var('email_metrics') %}
+ {% for metric in email_metrics %}
  coalesce(email_metrics.total_{{ metric }}, 0) as total_{{ metric }},
  coalesce(email_metrics.total_unique_{{ metric }}, 0) as total_unique_{{ metric }}
  {% if not loop.last %},{% endif %}

models/marketing/intermediate/int_hubspot__email_metrics__by_contact_list.sql

@@ -21,10 +21,10 @@ with email_sends as (
  where contact_list_member.contact_list_id is not null
 
 ), email_metrics as (
-
+ {% set email_metrics = adjust_email_metrics('hubspot__email_sends', 'email_metrics') %}
  select 
  contact_list_id,
- {% for metric in var('email_metrics') %}
+ {% for metric in email_metrics %}
  sum({{ metric }}) as total_{{ metric }},
  count(distinct case when {{ metric }} > 0 then email_send_id end) as total_unique_{{ metric }}
  {% if not loop.last %},{% endif %}