Skip to content

Commit

Permalink
rework incremental models to use a temp table
Browse files Browse the repository at this point in the history
this should speed up query times and also provide a more
correct implementation.

See: #138
  • Loading branch information
Drew Banin committed Sep 20, 2016
1 parent 94b6e82 commit 3ae32de
Showing 1 changed file with 21 additions and 19 deletions.
40 changes: 21 additions & 19 deletions dbt/templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,32 @@ class BaseCreateTemplate(object):
# but you cannot explicitly set them in the CREATE TABLE ... LIKE statement.
# via http://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_TABLE_NEW.html
incremental_template = """
create temporary table "{identifier}__dbt_incremental_tmp" {dist_qualifier} {sort_qualifier} as (
create temporary table "{identifier}__dbt_incremental_empty_tmp" {dist_qualifier} {sort_qualifier} as (
select * from (
{query}
) as tmp limit 0
);
create table if not exists "{schema}"."{identifier}" (like "{identifier}__dbt_incremental_tmp");
create table if not exists "{schema}"."{identifier}" (like "{identifier}__dbt_incremental_empty_tmp");
{incremental_delete_statement}
insert into "{schema}"."{identifier}" (
create temporary table "{identifier}__dbt_incremental_tmp" as (
with dbt_incr_sbq as (
{query}
)
select * from dbt_incr_sbq
where ({sql_where}) or ({sql_where}) is null
);
{incremental_delete_statement}
insert into "{schema}"."{identifier}" (
select * from "{identifier}__dbt_incremental_tmp"
);
"""

incremental_delete_template = """
delete from "{schema}"."{identifier}" where ({unique_key}) in (
with dbt_delete_sbq as (
{query}
)
select ({unique_key}) from dbt_delete_sbq
where ({sql_where}) or ({sql_where}) is null
select ({unique_key}) from "{identifier}__dbt_incremental_tmp"
);
"""

Expand Down Expand Up @@ -105,33 +105,35 @@ class DryCreateTemplate(object):


incremental_template = """
create temporary table "{identifier}__dbt_incremental_tmp" {dist_qualifier} {sort_qualifier} as (
create temporary table "{identifier}__dbt_incremental_empty_tmp" {dist_qualifier} {sort_qualifier} as (
select * from (
{query}
) as tmp limit 0
);
create table if not exists "{schema}"."{identifier}" (like "{identifier}__dbt_incremental_tmp");
{incremental_delete_statement}
create table if not exists "{schema}"."{identifier}" (like "{identifier}__dbt_incremental_empty_tmp");
insert into "{schema}"."{identifier}" (
create temporary table "{identifier}__dbt_incremental_tmp" as (
with dbt_incr_sbq as (
{query}
)
select * from dbt_incr_sbq
where ({sql_where}) or ({sql_where}) is null
limit 0
);
{incremental_delete_statement}
insert into "{schema}"."{identifier}" (
select * from "{identifier}__dbt_incremental_tmp"
);
"""

incremental_delete_template = """
delete from "{schema}"."{identifier}" where ({unique_key}) in (
with dbt_delete_sbq as (
{query}
)
select ({unique_key}) from dbt_delete_sbq
where ({sql_where}) or ({sql_where}) is null
select ({unique_key}) from "{identifier}__dbt_incremental_tmp"
);
"""

Expand Down

0 comments on commit 3ae32de

Please sign in to comment.