Skip to content

Commit

Permalink
Renamed insert+replace strategy to insert_overwrite
Browse files Browse the repository at this point in the history
According to a PR review comments
  • Loading branch information
bryzgaloff committed Jul 15, 2024
1 parent e4bd2cc commit 616ea7a
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 14 deletions.
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ your_profile_name:
| primary_key | Like order_by, a ClickHouse primary key expression. If not specified, ClickHouse will use the order by expression as the primary key | |
| unique_key | A tuple of column names that uniquely identify rows. Used with incremental models for updates. | |
| inserts_only | If set to True for an incremental model, incremental updates will be inserted directly to the target table without creating intermediate table. It has been deprecated in favor of the `append` incremental `strategy`, which operates in the same way. If `inserts_only` is set, `incremental_strategy` is ignored. | |
| incremental_strategy | Incremental model update strategy: `delete+insert`, `append`, or `insert+replace`. See the following Incremental Model Strategies | `default` |
| incremental_strategy | Incremental model update strategy: `delete+insert`, `append`, or `insert_overwrite`. See the following Incremental Model Strategies | `default` |
| incremental_predicates | Additional conditions to be applied to the incremental materialization (only applied to `delete+insert` strategy | |
| settings | A map/dictionary of "TABLE" settings to be used to DDL statements like 'CREATE TABLE' with this model | |
| query_settings | A map/dictionary of ClickHouse user level settings to be used with `INSERT` or `DELETE` statements in conjunction with this model | |
Expand Down Expand Up @@ -172,7 +172,7 @@ This strategy replaces the `inserts_only` setting in previous versions of dbt-cl
As a result duplicate rows are not eliminated, and there is no temporary or intermediate table. It is the fastest approach if duplicates are either permitted
in the data or excluded by the incremental query WHERE clause/filter.

### The Insert+Replace Strategy
### The insert_overwrite Strategy

Performs the following steps:
1. Create a staging (temporary) table with the same structure as the incremental model relation: `CREATE TABLE <staging> AS <target>`.
Expand Down
2 changes: 1 addition & 1 deletion dbt/adapters/clickhouse/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ def calculate_incremental_strategy(self, strategy: str) -> str:
if not strategy or strategy == 'default':
strategy = 'delete_insert' if conn.handle.use_lw_deletes else 'legacy'
strategy = strategy.replace('+', '_')
if strategy not in ['legacy', 'append', 'delete_insert', 'insert_replace']:
if strategy not in ['legacy', 'append', 'delete_insert', 'insert_overwrite']:
raise DbtRuntimeError(
f"The incremental strategy '{strategy}' is not valid for ClickHouse"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@
{% elif
inserts_only
or unique_key is none
and config.get('incremental_strategy', none) != 'insert+replace' -%}
and config.get('incremental_strategy', none) != 'insert_overwrite' -%}
-- There are no updates/deletes or duplicate keys are allowed. Simply add all of the new rows to the existing
-- table. It is the user's responsibility to avoid duplicates. Note that "inserts_only" is a ClickHouse adapter
-- specific configurable that is used to avoid creating an expensive intermediate table.
-- Insert+replace strategy does not require unique_key => is an exception.
-- insert_overwrite strategy does not require unique_key => is an exception.
{% call statement('main') %}
{{ clickhouse__insert_into(target_relation, sql) }}
{% endcall %}
Expand Down Expand Up @@ -78,15 +78,15 @@
{% call statement('main') %}
{{ clickhouse__insert_into(target_relation, sql) }}
{% endcall %}
{% elif incremental_strategy == 'insert_replace' %}#}
{% elif incremental_strategy == 'insert_overwrite' %}#}
{%- set partition_by = config.get('partition_by') -%}
{% if partition_by is none or partition_by|length == 0 %}
{% do exceptions.raise_compiler_error(incremental_strategy + ' strategy requires nonempty partition_by. Current partition_by is ' ~ partition_by) %}
{% endif %}
{% if inserts_only or unique_key is not none or incremental_predicates is not none %}
{% do exceptions.raise_compiler_error(incremental_strategy + ' strategy does not support inserts_only, unique_key, and incremental predicates.') %}
{% endif %}
{% do clickhouse__incremental_insert_replace(existing_relation, intermediate_relation, partition_by) %} %}
{% do clickhouse__incremental_insert_overwrite(existing_relation, intermediate_relation, partition_by) %} %}
{% endif %}
{% endif %}

Expand Down Expand Up @@ -248,7 +248,7 @@
{{ drop_relation_if_exists(distributed_new_data_relation) }}
{% endmacro %}

{% macro clickhouse__incremental_insert_replace(existing_relation, intermediate_relation, partition_by) %}
{% macro clickhouse__incremental_insert_overwrite(existing_relation, intermediate_relation, partition_by) %}
{% set new_data_relation = existing_relation.incorporate(path={"identifier": model['name']
+ '__dbt_new_data_' + invocation_id.replace('-', '_')}) %}
{{ drop_relation_if_exists(new_data_relation) }}
Expand Down
12 changes: 6 additions & 6 deletions tests/integration/adapter/incremental/test_base_incremental.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,10 +183,10 @@ def models(self):
}


insert_replace_inc = """
insert_overwrite_inc = """
{{ config(
materialized='incremental',
incremental_strategy='insert+replace',
incremental_strategy='insert_overwrite',
partition_by=['partitionKey1', 'partitionKey2'],
order_by=['orderKey'],
)
Expand All @@ -210,19 +210,19 @@ def models(self):
class TestInsertReplaceIncremental:
@pytest.fixture(scope="class")
def models(self):
return {"insert_replace_inc.sql": insert_replace_inc}
return {"insert_overwrite_inc.sql": insert_overwrite_inc}

def test_insert_replace_incremental(self, project):
def test_insert_overwrite_incremental(self, project):
run_dbt()
result = project.run_sql("select * from insert_replace_inc order by partitionKey1, partitionKey2, orderKey", fetch="all")
result = project.run_sql("select * from insert_overwrite_inc order by partitionKey1, partitionKey2, orderKey", fetch="all")
assert result == [
(1, 'p1', 1, 'a'),
(1, 'p1', 1, 'b'),
(2, 'p1', 1, 'c'),
(2, 'p2', 1, 'd'),
]
run_dbt()
result = project.run_sql("select * from insert_replace_inc order by partitionKey1, partitionKey2, orderKey", fetch="all")
result = project.run_sql("select * from insert_overwrite_inc order by partitionKey1, partitionKey2, orderKey", fetch="all")
assert result == [
(1, 'p1', 2, 'e'),
(2, 'p1', 1, 'c'),
Expand Down

0 comments on commit 616ea7a

Please sign in to comment.