Merge pull request #115 from fivetran/bugfix/duplicate-columns

Bugfix/duplicate columns
fivetran · Oct 2, 2023 · b79137d · b79137d
2 parents 98b2855 + f31c7c8
commit b79137d
Show file tree

Hide file tree

Showing 25 changed files with 97 additions and 47 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,27 @@
+# dbt_hubspot_source v0.12.0
+## 🚨 Breaking Changes 🚨
+- The following models now use a custom macro to remove the property_hs_ prefix in staging columns, while also preventing duplicates. If de-prefixed columns match existing ones (e.g., `property_hs_meeting_outcome` vs. `meeting_outcome`), the macro favors the `property_hs_`field, aligning with the latest HubSpot API update. ([PR #115](https://github.com/fivetran/dbt_hubspot_source/pull/115))
+ - `stg_hubspot__engagement_call`
+ - `stg_hubspot__engagement_company`
+ - `stg_hubspot__engagement_contact`
+ - `stg_hubspot__engagement_deal`
+ - `stg_hubspot__engagement_email`
+ - `stg_hubspot__engagement_meeting`
+ - `stg_hubspot__engagement_note`
+ - `stg_hubspot__engagement_task`
+ - `stg_hubspot__ticket`
+ - `stg_hubspot__ticket_company`
+ - `stg_hubspot__ticket_contact`
+ - `stg_hubspot__ticket_deal`
+ - `stg_hubspot__ticket_engagement`
+ - `stg_hubspot__ticket_property_history`
+
+## Feature Updates
+- A new macro `remove_duplicate_and_prefix_from_columns` has been included which expands off the `fivetran_utils.remove_prefix_columns` macro by removing any duplicate columns that result from the prefix removal. ([PR #115](https://github.com/fivetran/dbt_hubspot_source/pull/115) and [PR #114](https://github.com/fivetran/dbt_hubspot_source/pull/114))
+
+## Contributors
+- [@greg-finley](https://github.com/greg-finley) ([PR #114](https://github.com/fivetran/dbt_hubspot_source/pull/114))
+
 # dbt_hubspot_source v0.11.0
 [PR #112](https://github.com/fivetran/dbt_hubspot_source/pull/112) includes the following updates:
 ## 🚨 Breaking Changes 🚨

diff --git a/README.md b/README.md
@@ -44,7 +44,7 @@ Include the following hubspot_source package version in your `packages.yml` file
 ```yaml
 packages:
  - package: fivetran/hubspot_source
- version: [">=0.11.0", "<0.12.0"]
+ version: [">=0.12.0", "<0.13.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your destination and the `hubspot` schema. If this is not where your HubSpot data is (for example, if your HubSpot schema is named `hubspot_fivetran`), add the following configuration to your root `dbt_project.yml` file:

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.11.0'
+version: '0.12.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/.gitignore b/integration_tests/.gitignore
@@ -2,3 +2,4 @@
 target/
 dbt_modules/
 logs/
+env/
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'hubspot_source_integration_tests'
-version: '0.11.0'
+version: '0.12.0'
 profile: 'integration_tests'
 config-version: 2
 models:

diff --git a/integration_tests/seeds/engagement_task_data.csv b/integration_tests/seeds/engagement_task_data.csv
@@ -1,2 +1,3 @@
-engagement_id,_fivetran_deleted,_fivetran_synced,property_hs_createdate,property_hs_object_id,property_hs_task_type,property_hs_timestamp,property_hubspot_owner_id,property_hubspot_team_id,type
-32034933897,false,2023-06-08 23:22:51.021000,2023-03-03 10:40:09.556000,32034933897,TODO,2023-03-08 02:30:00.000000,32779820,9747038,TASK
+engagement_id,_fivetran_deleted,_fivetran_synced,property_hs_createdate,property_hs_object_id,property_hs_task_type,property_hs_timestamp,property_hubspot_owner_id,property_hubspot_team_id,type,task_type,property_hs_engagement_source
+32034933897,false,2023-06-08 23:22:51.021000,2023-03-03 10:40:09.556000,32034933897,TODO,2023-03-08 02:30:00.000000,32779820,9747038,TASK,,Hello
+32034933899,false,2023-06-08 23:22:51.021000,2023-03-03 10:40:09.556000,32034933897,,2023-03-08 02:30:00.000000,32779820,,,TODO,
diff --git a/macros/remove_duplicate_and_prefix_from_columns.sql b/macros/remove_duplicate_and_prefix_from_columns.sql
@@ -0,0 +1,24 @@
+{% macro remove_duplicate_and_prefix_from_columns(columns, prefix='', exclude=[]) %}
+
+ {# Loop once to find duplicates #}
+ {%- set duplicate_exclude = [] -%}
+ {% for col in columns if col.name not in exclude %}
+ {%- for dupe in columns if col.name[prefix|length:]|lower == dupe.name|lower -%}
+ {%- do duplicate_exclude.append(col.name) -%}
+ {%- do duplicate_exclude.append(dupe.name) -%}
+ , {{ col.name }} as {{ col.name[prefix|length:] }}
+ {%- endfor %}
+ {% endfor %}
+
+ {# Loop again to find non-duplicates #}
+ {% for col in columns if col.name not in exclude %}
+ {%- if col.name|lower not in duplicate_exclude|lower -%}
+ {% if col.name[:prefix|length]|lower == prefix %}
+ , {{ col.name }} as {{ col.name[prefix|length:] }}
+ {%- else %}
+ , {{ col.name }}
+ {%- endif -%}
+ {%- endif -%}
+ {% endfor %}
+
+{% endmacro %}
diff --git a/models/stg_hubspot__engagement_call.sql b/models/stg_hubspot__engagement_call.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_call_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_engagement_call_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_engagement_call_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 

diff --git a/models/stg_hubspot__engagement_company.sql b/models/stg_hubspot__engagement_company.sql
@@ -9,15 +9,15 @@ with base as (
 
  select 
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_company_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_engagement_company_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_engagement_company_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 )

diff --git a/models/stg_hubspot__engagement_contact.sql b/models/stg_hubspot__engagement_contact.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_contact_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_engagement_contact_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_engagement_contact_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 

diff --git a/models/stg_hubspot__engagement_deal.sql b/models/stg_hubspot__engagement_deal.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_deal_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_engagement_deal_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_engagement_deal_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 

diff --git a/models/stg_hubspot__engagement_email.sql b/models/stg_hubspot__engagement_email.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_email_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_engagement_email_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_engagement_email_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 

diff --git a/models/stg_hubspot__engagement_meeting.sql b/models/stg_hubspot__engagement_meeting.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_meeting_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_engagement_meeting_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_engagement_meeting_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 

diff --git a/models/stg_hubspot__engagement_note.sql b/models/stg_hubspot__engagement_note.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_note_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_engagement_note_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_engagement_note_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 

diff --git a/models/stg_hubspot__engagement_task.sql b/models/stg_hubspot__engagement_task.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_task_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_engagement_task_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_engagement_task_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 )

diff --git a/models/stg_hubspot__ticket.sql b/models/stg_hubspot__ticket.sql
@@ -30,7 +30,7 @@ with base as (
  {% if all_passthrough_column_check('stg_hubspot__ticket_tmp',get_ticket_columns()) > 0 %}
  -- just pass everything through if extra columns are present, but ensure required columns are present.
  ,{{ 
- fivetran_utils.remove_prefix_from_columns(
+ remove_duplicate_and_prefix_from_columns(
  columns=adapter.get_columns_in_relation(ref('stg_hubspot__ticket_tmp')), 
  prefix='property_', exclude=get_macro_columns(get_ticket_columns())) 
  }}

diff --git a/models/stg_hubspot__ticket_company.sql b/models/stg_hubspot__ticket_company.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__ticket_company_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_ticket_company_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_ticket_company_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 

diff --git a/models/stg_hubspot__ticket_contact.sql b/models/stg_hubspot__ticket_contact.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__ticket_contact_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_ticket_contact_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_ticket_contact_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 

diff --git a/models/stg_hubspot__ticket_deal.sql b/models/stg_hubspot__ticket_deal.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__ticket_deal_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_ticket_deal_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_ticket_deal_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 

diff --git a/models/stg_hubspot__ticket_engagement.sql b/models/stg_hubspot__ticket_engagement.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__ticket_engagement_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_ticket_engagement_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_ticket_engagement_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base
 

diff --git a/models/stg_hubspot__ticket_property_history.sql b/models/stg_hubspot__ticket_property_history.sql
@@ -9,15 +9,15 @@ with base as (
 
  select
  {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__ticket_property_history_tmp')) %}
- {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+ {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
  prefix='property_hs_',exclude=get_macro_columns(get_ticket_property_history_columns())) %}
  {{
  fivetran_utils.fill_staging_columns(source_columns=default_cols,
  staging_columns=get_ticket_property_history_columns()
  )
  }}
  {% if new_cols | length > 0 %} 
- ,{{ new_cols }} 
+ {{ new_cols }} 
  {% endif %}
  from base