Rework _dbt_max_partition

CHANGELOG.md

@@ -1,15 +1,18 @@
 ## dbt-bigquery 1.0.0 (Release TBD)
 
 ### Features
-N/A
+
+- Rework `_dbt_max_partition` logic in dynamic `insert_overwrite` incremental strategy. Make the default logic compatible with `on_schema_change`, and make it possible to disable or reimplement that logic by defining a custom macro `declare_dbt_max_partition` ([#17](https://github.com/dbt-labs/dbt-bigquery/issues/17), [#39](https://github.com/dbt-labs/dbt-bigquery/issues/39), [#41](https://github.com/dbt-labs/dbt-bigquery/pull/41))
 
 ### Fixes
 
 - Reimplement the `unique` test to handle column expressions and naming overlaps ([#33](https://github.com/dbt-labs/dbt-bigquery/issues/33), [#35](https://github.com/dbt-labs/dbt-bigquery/issues/35), [#10](https://github.com/dbt-labs/dbt-bigquery/pull/10))
 
-### Under the hood
-First version as a seperate repo.
-
 ### Contributors
 
-- [@DigUpTheHatchet](https://github.com/DigUpTheHatchet) ([#10](https://github.com/dbt-labs/dbt-bigquery/pull/10))
+- [@DigUpTheHatchet](https://github.com/DigUpTheHatchet) ([#10](https://github.com/dbt-labs/dbt-bigquery/pull/10))
+
+## dbt-bigquery 1.0.0b1 (October 11, 2021)
+
+### Under the hood
+First version as a separate repo.

dbt/include/bigquery/macros/materializations/incremental.sql

@@ -1,3 +1,16 @@
+{% macro declare_dbt_max_partition(relation, partition_by, sql) %}
+
+ {% if '_dbt_max_partition' in sql %}
+
+ declare _dbt_max_partition {{ partition_by.data_type }} default (
+ select max({{ partition_by.field }}) from {{ this }}
+ where {{ partition_by.field }} is not null
+ );
+
+ {% endif %}
+
+{% endmacro %}
+
 
 {% macro dbt_bigquery_validate_get_incremental_strategy(config) %}
  {#-- Find and validate the incremental strategy #}
@@ -49,13 +62,11 @@
 
  -- generated script to merge partitions into {{ target_relation }}
  declare dbt_partitions_for_replacement array<{{ partition_by.data_type }}>;
- declare _dbt_max_partition {{ partition_by.data_type }} default (
- select max({{ partition_by.field }}) from {{ this }}
- where {{ partition_by.field }} is not null
- );
 
  {# have we already created the temp table to check for schema changes? #}
  {% if not tmp_relation_exists %}
+ {{ declare_dbt_max_partition(this, partition_by, sql) }}
+
  -- 1. create a temp table
  {{ create_table_as(True, tmp_relation, sql) }}
  {% else %}
@@ -162,7 +173,9 @@
  {% else %}
  {% set tmp_relation_exists = false %}
  {% if on_schema_change != 'ignore' %} {# Check first, since otherwise we may not build a temp table #}
- {% do run_query(create_table_as(True, tmp_relation, sql)) %}
+ {% do run_query(
+ declare_dbt_max_partition(this, partition_by, sql) + create_table_as(True, tmp_relation, sql)
+ ) %}
  {% set tmp_relation_exists = true %}
  {% do process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}
  {% endif %}

tests/integration/incremental_schema_tests/models/incremental_sync_all_columns_dynamic_insert_overwrite.sql

@@ -0,0 +1,40 @@
+{{
+ config(
+ materialized='incremental',
+ unique_key='id',
+ on_schema_change='sync_all_columns',
+ partition_by={
+ "field": "id",
+ "data_type": "int64",
+ "range": {
+ "start": 1,
+ "end": 6,
+ "interval": 1
+ }
+ },
+ incremental_strategy='insert_overwrite'
+ )
+}}
+
+WITH source_data AS (SELECT * FROM {{ ref('model_a') }} )
+
+{% set string_type = 'string' %}
+
+{% if is_incremental() %}
+
+SELECT id, 
+ cast(field1 as {{string_type}}) as field1, 
+ cast(field3 as {{string_type}}) as field3, -- to validate new fields
+ cast(field4 as {{string_type}}) AS field4 -- to validate new fields
+
+FROM source_data WHERE id > _dbt_max_partition
+
+{% else %}
+
+select id, 
+ cast(field1 as {{string_type}}) as field1, 
+ cast(field2 as {{string_type}}) as field2
+
+from source_data where id <= 3
+
+{% endif %}