Fix bug with calculation of last_run stats for incremental models

brooklyn-data · Sep 18, 2023 · 2ab2bf0 · 2ab2bf0
1 parent 7a33172
commit 2ab2bf0
Showing 1 changed file with 22 additions and 4 deletions.
diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql
@@ -27,10 +27,16 @@ latest_models_runs as (
         {% if target.type == 'bigquery' %}
         , model_executions.bytes_processed
         {% endif %}
+        /* Row number by refresh and node ID */
         , row_number() over (
             partition by latest_models.node_id, model_executions.was_full_refresh
             order by model_executions.query_completed_at desc /* most recent ranked first */
         ) as run_idx
+        /* Row number by node ID */
+        , row_number() over (
+            partition by latest_models.node_id
+            order by model_executions.query_completed_at desc /* most recent ranked first */
+        ) as run_idx_id_only
     from model_executions
     inner join latest_models on model_executions.node_id = latest_models.node_id
     where model_executions.status = 'success'
@@ -45,11 +51,17 @@ latest_model_stats as (
         {% if target.type == 'bigquery' %}
         , max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed
         {% endif %}
-        , max(query_completed_at) as last_run_completed_at
-        , max(total_node_runtime) as last_run_total_runtime
-        , max(rows_affected) as last_run_rows_affected
+        , max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at
+        , max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime
+        , max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected
+        {% if target.type == 'bigquery' %}
+        , max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed
+        {% endif %}
+        , max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at
+        , max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime
+        , max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected
         {% if target.type == 'bigquery' %}
-        , max(bytes_processed) as last_run_bytes_processed
+        , max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed
         {% endif %}
     from latest_models_runs
     where run_idx = 1
@@ -71,6 +83,12 @@ final as (
         {% if target.type == 'bigquery' %}
         , latest_model_stats.last_run_bytes_processed
         {% endif %}
+        , latest_model_stats.last_incremental_run_completed_at
+        , latest_model_stats.last_incremental_run_total_runtime
+        , latest_model_stats.last_incremental_run_rows_affected
+        {% if target.type == 'bigquery' %}
+        , latest_model_stats.last_incremental_run_bytes_processed
+        {% endif %}
     from latest_models
     left join latest_model_stats
         on latest_models.node_id = latest_model_stats.node_id