From 3fd7645cfb450375bb680fa99eb1731a5f481488 Mon Sep 17 00:00:00 2001 From: Jonathan Sellar Date: Thu, 21 Sep 2023 13:37:39 +1200 Subject: [PATCH 1/5] add else clause to for if loops --- macros/upload_individual_datasets/upload_model_executions.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index 3ff87cc5..55e79432 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -179,6 +179,8 @@ {% else %} '{{ stage.started_at }}', {# compile_started_at #} {% endif %} + { % else %} + null, {# compile_started_at #} {% endfor %} {% for stage in model.timing if stage.name == "execute" %} @@ -187,6 +189,8 @@ {% else %} '{{ stage.completed_at }}', {# query_completed_at #} {% endif %} + { % else %} + null, {# query_completed_at #} {% endfor %} {% else %} null, {# compile_started_at #} From 89857ed4f922353bfc7ce8ed767854577d19f04d Mon Sep 17 00:00:00 2001 From: Jono <89880566+kenpokiwi@users.noreply.github.com> Date: Fri, 22 Sep 2023 09:44:51 +1200 Subject: [PATCH 2/5] Refactor logic for timestamp values Refactor logic to evaluate compile_started_at and query_completed_at values. --- .../upload_model_executions.sql | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index 55e79432..7ac72469 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -40,21 +40,10 @@ '{{ model.status }}', {# status #} {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({})).get("started_at") %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({})).get("completed_at") %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {% else %} null, {# compile_started_at #} null, {# query_completed_at #} From 7aa421392282e5e84580de2c1d0ee2bd064fa873 Mon Sep 17 00:00:00 2001 From: Jonathan Sellar Date: Wed, 27 Sep 2023 08:02:21 +1300 Subject: [PATCH 3/5] Refactor logic for timestamp values for all macros --- .../upload_model_executions.sql | 60 ++++--------------- 1 file changed, 12 insertions(+), 48 deletions(-) diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index 55e79432..64dcb293 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -40,21 +40,10 @@ '{{ model.status }}', {# status #} {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({})).get("started_at") %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({})).get("completed_at") %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {% else %} null, {# compile_started_at #} null, {# query_completed_at #} @@ -97,20 +86,10 @@ '{{ model.status }}', {# status #} {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({})).get("started_at") %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({})).get("completed_at") %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {% endfor %} {% else %} null, {# compile_started_at #} @@ -173,25 +152,10 @@ '{{ model.status }}', {# status #} {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - { % else %} - null, {# compile_started_at #} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - { % else %} - null, {# query_completed_at #} - {% endfor %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({})).get("started_at") %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({})).get("completed_at") %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {% else %} null, {# compile_started_at #} null, {# query_completed_at #} From 606fe12855fd40f1cb3de8dea011e08e0c0cfc57 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Thu, 28 Sep 2023 11:21:29 +0100 Subject: [PATCH 4/5] Update macros/upload_individual_datasets/upload_model_executions.sql --- macros/upload_individual_datasets/upload_model_executions.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index 64dcb293..4b6f26b7 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -90,7 +90,6 @@ {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({})).get("completed_at") %} {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} - {% endfor %} {% else %} null, {# compile_started_at #} null, {# query_completed_at #} From 322786601004e4ec929755970d6533aca541418e Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Thu, 28 Sep 2023 14:16:18 +0100 Subject: [PATCH 5/5] Fix issue with `.get` usage --- .../upload_model_executions.sql | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index 4b6f26b7..fca386b2 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -39,15 +39,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({})).get("started_at") %} - {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} - {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({})).get("completed_at") %} - {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} @@ -85,15 +80,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({})).get("started_at") %} - {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} - {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({})).get("completed_at") %} - {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} safe_cast('{{ model.adapter_response.rows_affected }}' as int64), @@ -150,15 +140,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({})).get("started_at") %} - {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} - {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({})).get("completed_at") %} - {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #}