Add test "expect_grouped_row_values_to_have_recent_data"

README.md

@@ -71,6 +71,31 @@ tests:
  interval: 1
 ```
 
+#### [expect_grouped_row_values_to_have_recent_data](macros/schema_tests/table_shape/expect_grouped_row_values_to_have_recent_data.sql)
+
+Expect the model to have **grouped** rows that are at least as recent as the defined interval prior to the current timestamp.
+Uuse this to test whether there is recent data for each grouped row defined by `group_by` (which is a list of columns) and a `timestamp_column`.
+
+*Applies to:* Model, Seed, Source
+
+```yaml
+models: # or seeds:
+ - name : my_model
+ tests :
+ - dbt_expectations.expect_grouped_row_values_to_have_recent_data:
+ group_by: [group_id]
+ timestamp_column: date_day
+ datepart: day
+ interval: 1
+ # or also:
+ - dbt_expectations.expect_grouped_row_values_to_have_recent_data:
+ group_by: [group_id, other_group_id]
+ timestamp_column: date_day
+ datepart: day
+ interval: 1
+```
+
+
 #### [expect_table_column_count_to_be_between](macros/schema_tests/table_shape/expect_table_column_count_to_be_between.sql)
 
 Expect the number of columns in a model to be between two values.

integration_tests/models/schema_tests/schema.yml

@@ -254,3 +254,17 @@ models:
  - dbt_expectations.expect_table_row_count_to_equal_other_table_times_factor:
  compare_model: ref("data_test")
  factor: 2
+
+ - name : timeseries_data_grouped
+ tests :
+ - dbt_expectations.expect_grouped_row_values_to_have_recent_data:
+ group_by: [group_id]
+ timestamp_column: date_day
+ datepart: day
+ interval: 1
+ - dbt_expectations.expect_grouped_row_values_to_have_recent_data:
+ group_by: [group_id, row_value]
+ timestamp_column: date_day
+ datepart: day
+ interval: 1
+

integration_tests/models/schema_tests/timeseries_data_grouped.sql

@@ -0,0 +1,35 @@
+with dates as (
+ {{ dbt_date.get_base_dates(n_dateparts=12, datepart='month') }}
+),
+groupings as (
+ {{ dbt_utils.generate_series(upper_bound=4) }}
+),
+row_values as (
+ {{ dbt_utils.generate_series(upper_bound=10) }}
+),
+add_row_values as (
+
+ select
+ d.date_day,
+ cast(g.generated_number as {{ dbt_utils.type_int() }}) as group_id,
+ cast(floor(100 * r.generated_number) as {{ dbt_utils.type_int() }}) as row_value
+ from
+ dates d
+ cross join
+ groupings g
+ cross join
+ row_values r
+
+),
+add_logs as (
+
+ select
+ *,
+ {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log
+ from
+ add_row_values
+)
+select
+ *
+from
+ add_logs

macros/schema_tests/table_shape/expect_grouped_row_values_to_have_recent_data.sql

@@ -0,0 +1,24 @@
+{% macro test_expect_grouped_row_values_to_have_recent_data(model, group_by, timestamp_column, datepart, interval) %}
+with latest_grouped_timestamps as (
+
+ select
+ {%- for g in group_by %}
+ {{ g }},
+ {%- endfor %}
+ max({{ timestamp_column }}) as latest_timestamp_column
+ from
+ {{ model }}
+ {{ dbt_utils.group_by(group_by | length )}}
+
+),
+validation_errors as (
+
+ select *
+ from
+ latest_grouped_timestamps
+ where
+ latest_timestamp_column < {{ dbt_utils.dateadd(datepart, interval * -1, dbt_date.now()) }}
+
+)
+select count(*) from validation_errors
+{% endmacro %}