Modify for databricks anirga

dbt_project.yml

@@ -4,27 +4,33 @@ version: '0.8.4'
 config-version: 2
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 on-run-start: '{{ fivetran_utils.empty_variable_warning("ticket_field_history_columns", "zendesk_ticket_field_history") }}'
+profile: "zendesk"
 models:
   zendesk:
     +schema: zendesk
     +materialized: table
     agent_work_time:
       +materialized: ephemeral
     intermediate:
-      +schema: zendesk_intermediate
+      +schema: intermediate
       +materialized: table
     reply_times:
       +materialized: ephemeral
     resolution_times:
       +materialized: ephemeral
     sla_policy:
-      +schema: zendesk_intermediate
+      +schema: intermediate
       +materialized: table
     ticket_history:
-      +schema: zendesk_intermediate
+      +schema: intermediate
       +materialized: ephemeral
     utils:
       +materialized: ephemeral
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']
+  - macro_namespace: fivetran_utils
+    search_order: ['spark_utils', 'fivetran_utils']
 vars:
   zendesk:
     ticket_field_history_columns: ['assignee_id', 'status', 'priority']
@@ -53,3 +59,6 @@ vars:
 
     ticket_field_history_extension_months: 0 # how long to extend a ticket's field history past its closure date
     ticket_field_history_timeframe_years: 50 # how far back to pull tickets' field histories. default is everything
+  zendesk_source:
+    zendesk_database: zendesk
+    zendesk_schema: zendesk  

models/agent_work_time/int_zendesk__ticket_work_time_business.sql

@@ -50,7 +50,7 @@ with ticket_historical_status as (
               'second') }} /60
             ) as raw_delta_in_minutes
     from ticket_status_crossed_with_schedule
-    {{ dbt_utils.group_by(n=7) }}
+    group by ticket_id, ticket_status, schedule_id, status_schedule_start, status_schedule_end, status_valid_starting_at, status_valid_ending_at
 
 ), weeks as (
 
@@ -119,4 +119,4 @@ with ticket_historical_status as (
       sum(agent_work_time_in_minutes) as agent_work_time_in_business_minutes,
       sum(on_hold_time_in_minutes) as on_hold_time_in_business_minutes
     from business_minutes
-    group by 1
+    group by ticket_id

models/agent_work_time/int_zendesk__ticket_work_time_calendar.sql

@@ -44,4 +44,4 @@ select
   sum(open_status_duration_minutes) as open_status_duration_in_calendar_minutes,
   sum(ticket_recoveries) as total_ticket_recoveries
 from calendar_minutes
-group by 1, 2
+group by ticket_id, last_status_assignment_date

models/intermediate/int_zendesk__assignee_updates.sql

@@ -26,7 +26,7 @@ with ticket_updates as (
         count(*) as total_updates
     from ticket_requester
 
-    group by 1, 2
+    group by ticket_id, assignee_id
 )
 
 select * 

models/intermediate/int_zendesk__comment_metrics.sql

@@ -34,7 +34,7 @@ comment_counts as (
                 end) as count_ticket_handoffs
     from ticket_comments
 
-    group by 1, 2
+    group by ticket_id, last_comment_added_at
 ),
 
 final as (

models/intermediate/int_zendesk__organization_aggregates.sql

@@ -17,7 +17,7 @@ with organizations as (
     left join organization_tags
         using (organization_id)
 
-    group by 1
+    group by organizations.organization_id
 {% endif %}
 
 --If you use using_domain_names tags this will be included, if not it will be ignored.
@@ -36,7 +36,7 @@ with organizations as (
     left join domain_names
         using(organization_id)
 
-    group by 1
+    group by organizations.organization_id
 {% endif %}
 
 

models/intermediate/int_zendesk__requester_updates.sql

@@ -26,7 +26,7 @@ with ticket_updates as (
         count(*) as total_updates
     from ticket_requester
 
-    group by 1, 2
+    group by ticket_id, requester_id
 )
 
 select * 

models/intermediate/int_zendesk__ticket_aggregates.sql

@@ -17,7 +17,7 @@ with tickets as (
     ticket_tags.ticket_id,
     {{ fivetran_utils.string_agg( 'ticket_tags.tags', "', '" )}} as ticket_tags
   from ticket_tags
-  group by 1
+  group by ticket_tags.ticket_id
 
 ), final as (
   select 

models/intermediate/int_zendesk__ticket_historical_assignee.sql

@@ -30,7 +30,7 @@ with assignee_updates as (
         count(distinct value) as unique_assignee_count
     from calculate_metrics
 
-    group by 1
+    group by ticket_id
 
 ), window_group as (
     select
@@ -42,7 +42,7 @@ with assignee_updates as (
         calculate_metrics.assignee_stations_count
     from calculate_metrics
 
-    {{ dbt_utils.group_by(n=6) }}
+    group by calculate_metrics.ticket_id, calculate_metrics.first_agent_assignment_date, calculate_metrics.first_assignee_id, calculate_metrics.last_agent_assignment_date, calculate_metrics.last_assignee_id, calculate_metrics.assignee_stations_count
 
 ), final as (
     select

models/intermediate/int_zendesk__ticket_historical_group.sql

@@ -19,7 +19,7 @@ with ticket_group_history as (
         count(group_id) as group_stations_count
     from group_breakdown
 
-    group by 1
+    group by ticket_id
 )
 
 select *

models/intermediate/int_zendesk__ticket_historical_satisfaction.sql

@@ -52,7 +52,7 @@ with satisfaction_updates as (
         sum(bad_to_good_score) as total_bad_to_good_score
     from satisfaction_scores
 
-    group by 1, 2
+    group by ticket_id, count_satisfaction_scores
 
 ), window_group as (
     select
@@ -79,7 +79,7 @@ with satisfaction_updates as (
     left join score_group
         on satisfaction_updates.ticket_id = score_group.ticket_id
 
-    group by 1, 2, 3, 4, 5, 6, 7, 8
+    group by satisfaction_updates.ticket_id, latest_reason.latest_satisfaction_reason, latest_comment.latest_satisfaction_comment, first_and_latest_score.first_satisfaction_score, first_and_latest_score.latest_satisfaction_score, score_group.count_satisfaction_scores, score_group.total_good_to_bad_score, score_group.total_bad_to_good_score
 
 ), final as (
     select

models/intermediate/int_zendesk__user_aggregates.sql

@@ -14,7 +14,7 @@ with users as (
     user_tags.user_id,
     {{ fivetran_utils.string_agg( 'user_tags.tags', "', '" )}} as user_tags
   from user_tags
-  group by 1
+  group by user_tags.user_id
 
 {% endif %}
 

models/reply_times/int_zendesk__ticket_first_reply_time_business.sql

@@ -52,7 +52,7 @@ with ticket_reply_times as (
 
   from first_reply_time
   join ticket_schedules on first_reply_time.ticket_id = ticket_schedules.ticket_id
-  group by 1, 2, 3, 4
+  group by first_reply_time.ticket_id, schedule_created_at, schedule_invalidated_at, schedule_id
 
 ), weeks as (
 
@@ -100,4 +100,4 @@ with ticket_reply_times as (
   select ticket_id,
          sum(scheduled_minutes) as first_reply_time_business_minutes
   from intercepted_periods
-  group by 1
+  group by ticket_id

models/reply_times/int_zendesk__ticket_reply_times.sql

@@ -24,7 +24,7 @@ with ticket_public_comments as (
     end_user_comments.ticket_id,
     -- If the commentor was internal, a first comment, and had previous non public internal comments then we want the ticket created date to be the end user comment created date
     -- Otherwise we will want to end user comment created date
-    case when is_first_comment then end_user_comments.ticket_created_date else end_user_comments.end_user_comment_created_at end as end_user_comment_created_at,
+    first_value(case when is_first_comment then end_user_comments.ticket_created_date else end_user_comments.end_user_comment_created_at end) as end_user_comment_created_at,
     end_user_comments.is_first_comment,
     min(case when is_first_comment 
         and end_user_comments.commenter_role != 'external_comment' 
@@ -36,7 +36,7 @@ with ticket_public_comments as (
     on agent_comments.ticket_id = end_user_comments.ticket_id
     and agent_comments.commenter_role = 'internal_comment'
     and agent_comments.valid_starting_at > end_user_comments.end_user_comment_created_at
-  group by 1,2,3
+  group by end_user_comments.ticket_id,end_user_comment_created_at,end_user_comments.is_first_comment
 
 )
 
@@ -47,4 +47,4 @@ with ticket_public_comments as (
       'agent_responded_at',
       'second') }} / 60) as reply_time_calendar_minutes
   from reply_timestamps
-  order by 1,2
+  order by ticket_id,end_user_comment_created_at

models/reply_times/int_zendesk__ticket_reply_times_calendar.sql

@@ -21,4 +21,4 @@ from ticket
 left join ticket_reply_times
   using (ticket_id)
 
-group by 1
+group by ticket_id

models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql

@@ -41,7 +41,7 @@ with ticket_resolution_times_calendar as (
 
   from ticket_resolution_times_calendar
   join ticket_schedules on ticket_resolution_times_calendar.ticket_id = ticket_schedules.ticket_id
-  group by 1, 2, 3, 4
+  group by ticket_resolution_times_calendar.ticket_id, schedule_created_at, schedule_invalidated_at, schedule_id
 
 ), weeks as (
 
@@ -93,4 +93,4 @@ with ticket_resolution_times_calendar as (
     ticket_id,
     sum(scheduled_minutes) as first_resolution_business_minutes
   from intercepted_periods
-  group by 1
+  group by ticket_id

models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql

@@ -40,7 +40,7 @@ with ticket_resolution_times_calendar as (
 
   from ticket_resolution_times_calendar
   join ticket_schedules on ticket_resolution_times_calendar.ticket_id = ticket_schedules.ticket_id
-  group by 1, 2, 3, 4
+  group by ticket_resolution_times_calendar.ticket_id, schedule_created_at, schedule_invalidated_at, schedule_id
 
 ), weeks as (
 
@@ -92,4 +92,4 @@ with ticket_resolution_times_calendar as (
     ticket_id,
     sum(scheduled_minutes) as full_resolution_business_minutes
   from intercepted_periods
-  group by 1
+  group by ticket_id

models/resolution_times/int_zendesk__ticket_resolution_times_calendar.sql

@@ -29,7 +29,7 @@ with historical_solved_status as (
     count(status) as solved_count 
 
   from historical_solved_status
-  group by 1
+  group by ticket_id
 
 )
 

models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql

@@ -12,12 +12,12 @@ with agent_work_time_filtered_statuses as (
 
 ), schedule as (
 
-  select * 
+  select *
   from {{ ref('int_zendesk__schedule_spine') }}
 
 ), ticket_schedules as (
 
-  select * 
+  select *
   from {{ ref('int_zendesk__ticket_schedules') }}
 
 -- cross schedules with work time
@@ -61,7 +61,7 @@ with agent_work_time_filtered_statuses as (
               'second') }} /60
             ) as raw_delta_in_minutes
     from ticket_status_crossed_with_schedule
-    {{ dbt_utils.group_by(n=10) }}
+    group by ticket_id, sla_applied_at, target, sla_policy_name, schedule_id, valid_starting_at, valid_ending_at, status_valid_starting_at, status_valid_ending_at, valid_starting_at_in_minutes_from_week
 
 ), weeks as (
 
@@ -172,5 +172,5 @@ with agent_work_time_filtered_statuses as (
 
 )
 
-select * 
+select *
 from agent_work_business_breach

models/sla_policy/int_zendesk__sla_policy_applied.sql

@@ -31,8 +31,8 @@ with ticket_field_history as (
     ticket_field_history.field_name as metric,
     case when ticket_field_history.field_name = 'first_reply_time' then row_number() over (partition by ticket_field_history.ticket_id, ticket_field_history.field_name order by ticket_field_history.valid_starting_at desc) else 1 end as latest_sla,
     case when ticket_field_history.field_name = 'first_reply_time' then ticket.created_at else ticket_field_history.valid_starting_at end as sla_applied_at,
-    cast({{ fivetran_utils.json_extract('ticket_field_history.value', 'minutes') }} as {{ dbt_utils.type_int() }} ) as target,
-    {{ fivetran_utils.json_extract('ticket_field_history.value', 'in_business_hours') }} = 'true' as in_business_hours
+    cast({{ fivetran_utils.json_parse('ticket_field_history.value', 'minutes') }} as {{ dbt_utils.type_int() }} ) as target,
+    {{ fivetran_utils.json_parse('ticket_field_history.value', 'in_business_hours') }} = 'true' as in_business_hours
   from ticket_field_history
   join ticket
     on ticket.ticket_id = ticket_field_history.ticket_id

models/sla_policy/reply_time/int_zendesk__reply_time_business_hours.sql

@@ -25,7 +25,7 @@ with ticket_schedules as (
     sum(end_time - start_time) as total_schedule_weekly_business_minutes
   -- referrinng to stg_zendesk__schedule instead of int_zendesk__schedule_spine just to calculcate total minutes
   from {{ ref('stg_zendesk__schedule') }}
-  group by 1
+  group by schedule_id
 
 ), ticket_sla_applied_with_schedules as (
 

models/sla_policy/reply_time/int_zendesk__reply_time_combined.sql

@@ -88,7 +88,7 @@ with reply_time_calendar_hours_sla as (
   left join ticket_solved_times
     on reply_time_breached_at.ticket_id = ticket_solved_times.ticket_id
     and ticket_solved_times.solved_at > reply_time_breached_at.sla_applied_at
-  {{ dbt_utils.group_by(n=6) }}
+  group by reply_time_breached_at.ticket_id, reply_time_breached_at.sla_policy_name, reply_time_breached_at.metric, reply_time_breached_at.sla_applied_at, reply_time_breached_at.target, reply_time_breached_at.in_business_hours
 
 ), reply_time_breached_at_remove_old_sla as (
   select 

models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql

@@ -12,12 +12,12 @@ with requester_wait_time_filtered_statuses as (
 
 ), schedule as (
 
-  select * 
+  select *
   from {{ ref('int_zendesk__schedule_spine') }}
 
 ), ticket_schedules as (
 
-  select * 
+  select *
   from {{ ref('int_zendesk__ticket_schedules') }}
 
 -- cross schedules with work time
@@ -61,7 +61,7 @@ with requester_wait_time_filtered_statuses as (
               'second') }} /60
             ) as raw_delta_in_minutes
     from ticket_status_crossed_with_schedule
-    {{ dbt_utils.group_by(n=10) }}
+    group by ticket_id, sla_applied_at, target, sla_policy_name, schedule_id, valid_starting_at, valid_ending_at, status_valid_starting_at, status_valid_ending_at, valid_starting_at_in_minutes_from_week
 
 ), weeks as (
 
@@ -172,5 +172,5 @@ with requester_wait_time_filtered_statuses as (
 
 )
 
-select * 
+select *
 from requester_wait_business_breach

models/ticket_history/int_zendesk__field_calendar_spine.sql

@@ -1,7 +1,7 @@
 {{
     config(
         materialized='incremental',
-        partition_by = {'field': 'date_day', 'data_type': 'date'},
+        partition_by = ['date_day'],
         unique_key='ticket_day_id'
     )
 }}

models/ticket_history/int_zendesk__field_history_pivot.sql

@@ -3,7 +3,7 @@
 {{ 
     config(
         materialized='incremental',
-        partition_by = {'field': 'date_day', 'data_type': 'date'},
+        partition_by = ['date_day'],
         unique_key='ticket_day_id'
         ) 
 }}
@@ -88,7 +88,7 @@ with field_history as (
         {% endfor %}
 
     from filtered
-    group by 1,2
+    group by ticket_id,date_day
 
 ), surrogate_key as (