Skip to content

Commit

Permalink
Try to remove column conflict
Browse files Browse the repository at this point in the history
  • Loading branch information
greg-finley committed Sep 26, 2023
1 parent 98b2855 commit eee64f0
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 1 deletion.
46 changes: 46 additions & 0 deletions macros/remove_column_conflict.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{% macro remove_column_conflict(columns, conflict_pair, keep) %}
{# If both columns in the conflict_pair exist, we will only keep the one specified in the keep variable #}
{% set new_col_list = [] %}
{% set keep = keep.lower() %}
{% set conflict_pair = conflict_pair[0].lower(), conflict_pair[1].lower() %}

{# First pass to check if both items in conflict pair are present #}
{% set conflict_exists = false %}
{% set first_item_exists = false %}
{% set second_item_exists = false %}

{% for column in columns %}
{% set lower_column = column.column.lower() %}
{{print(lower_column)}}
{% if lower_column == conflict_pair[0] %}
{{print('match 0')}}
{% set first_item_exists = true %}
{% elif lower_column == conflict_pair[1] %}
{{print('match 1')}}
{% set second_item_exists = true %}
{% else %}
{{print('no match')}}
{% endif %}
{% endfor %}

{% if first_item_exists and second_item_exists %}
{% set conflict_exists = true %}
{% endif %}

{# Second pass to build the column list #}
{% for column in columns %}
{% set lower_column = column.column.lower() %}
{% if conflict_exists %}
{% if lower_column == keep %}
{% set new_col_list = new_col_list + [column] %}
{% endif %}
{% else %}
{% set new_col_list = new_col_list + [column] %}
{% endif %}
{% endfor %}

{{print(conflict_exists)}}
--{{print(columns)}}
{{print(new_col_list)}}
{{ return(new_col_list) }}
{% endmacro %}
2 changes: 1 addition & 1 deletion models/stg_hubspot__engagement_task.sql
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ with base as (

select
{% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_task_tmp')) %}
{% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols,
{% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=remove_column_conflict(columns=default_cols, conflict_pair=('property_hs_task_type', 'task_type'), keep='property_hs_task_type'),
prefix='property_hs_',exclude=get_macro_columns(get_engagement_task_columns())) %}
{{
fivetran_utils.fill_staging_columns(source_columns=default_cols,
Expand Down

0 comments on commit eee64f0

Please sign in to comment.