diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 69321ad7bd0..1207645fa83 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -550,7 +550,7 @@ def valid_archive_target(self, relation): expected_type=self.Relation) columns = self.get_columns_in_relation(relation) - names = set(c.name for c in columns) + names = set(c.name.lower() for c in columns) expanded_keys = ('scd_id', 'valid_from', 'valid_to') extra = [] missing = [] diff --git a/core/dbt/include/global_project/macros/materializations/archive/archive.sql b/core/dbt/include/global_project/macros/materializations/archive/archive.sql index 9963fdddbd3..5b9b3d71342 100644 --- a/core/dbt/include/global_project/macros/materializations/archive/archive.sql +++ b/core/dbt/include/global_project/macros/materializations/archive/archive.sql @@ -7,7 +7,7 @@ {% endmacro %} {% macro default__archive_hash_arguments(args) %} - md5({% for arg in args %}{{ adapter.quote(arg) }}{% if not loop.last %} || '|' || {% endif %}{% endfor %}) + md5({% for arg in args %}{{ arg }}{% if not loop.last %} || '|' || {% endif %}{% endfor %}) {% endmacro %} {% macro create_temporary_table(sql, relation) %} @@ -48,10 +48,10 @@ {% macro default__archive_update(target_relation, tmp_relation) %} update {{ target_relation }} - set {{ adapter.quote('dbt_valid_to') }} = tmp.{{ adapter.quote('dbt_valid_to') }} + set dbt_valid_to = tmp.dbt_valid_to from {{ tmp_relation }} as tmp - where tmp.{{ adapter.quote('dbt_scd_id') }} = {{ target_relation }}.{{ adapter.quote('dbt_scd_id') }} - and {{ adapter.quote('change_type') }} = 'update'; + where tmp.dbt_scd_id = {{ target_relation }}.dbt_scd_id + and change_type = 'update'; {% endmacro %} @@ -67,20 +67,6 @@ to_timestamp_ntz({{ current_timestamp() }}) {%- endmacro %} -{# -{% macro archive_current_timestamp() -%} - {{ adapter_macro('archive_current_timestamp') }} -{%- endmacro %} - -{% macro default__archive_current_timestamp() -%} - {{ current_timestamp() }} -{%- endmacro %} - -{% macro snowflake__archive_current_timestamp() -%} - TO_TIMESTAMP_NTZ(archive_current_timestamp()) -{%- endmacro %} -#} - {% macro archive_select_generic(source_sql, target_relation, transforms, scd_hash) -%} with source as ( @@ -89,14 +75,14 @@ {{ transforms }} merged as ( - select *, 'update' as {{ adapter.quote('change_type') }} from updates + select *, 'update' as change_type from updates union all - select *, 'insert' as {{ adapter.quote('change_type') }} from insertions + select *, 'insert' as change_type from insertions ) select *, - {{ scd_hash }} as {{ adapter.quote('dbt_scd_id') }} + {{ scd_hash }} as dbt_scd_id from merged {%- endmacro %} @@ -111,12 +97,12 @@ select {% for col in source_columns %} - {{ adapter.quote(col.name) }} {% if not loop.last %},{% endif %} + {{ col.name }} {% if not loop.last %},{% endif %} {% endfor %}, - {{ updated_at }} as {{ adapter.quote('dbt_updated_at') }}, - {{ unique_key }} as {{ adapter.quote('dbt_pk') }}, - {{ updated_at }} as {{ adapter.quote('dbt_valid_from') }}, - {{ timestamp_column.literal('null') }} as {{ adapter.quote('tmp_valid_to') }} + {{ updated_at }} as dbt_updated_at, + {{ unique_key }} as dbt_pk, + {{ updated_at }} as dbt_valid_from, + {{ timestamp_column.literal('null') }} as tmp_valid_to from source ), @@ -124,12 +110,12 @@ select {% for col in source_columns %} - {{ adapter.quote(col.name) }}, + {{ col.name }}, {% endfor %} - {{ updated_at }} as {{ adapter.quote('dbt_updated_at') }}, - {{ unique_key }} as {{ adapter.quote('dbt_pk') }}, - {{ adapter.quote('dbt_valid_from') }}, - {{ adapter.quote('dbt_valid_to') }} as {{ adapter.quote('tmp_valid_to') }} + {{ updated_at }} as dbt_updated_at, + {{ unique_key }} as dbt_pk, + dbt_valid_from, + dbt_valid_to as tmp_valid_to from {{ target_relation }} ), @@ -138,16 +124,16 @@ select current_data.*, - {{ timestamp_column.literal('null') }} as {{ adapter.quote('dbt_valid_to') }} + {{ timestamp_column.literal('null') }} as dbt_valid_to from current_data left outer join archived_data - on archived_data.{{ adapter.quote('dbt_pk') }} = current_data.{{ adapter.quote('dbt_pk') }} + on archived_data.dbt_pk = current_data.dbt_pk where - archived_data.{{ adapter.quote('dbt_pk') }} is null + archived_data.dbt_pk is null or ( - archived_data.{{ adapter.quote('dbt_pk') }} is not null - and archived_data.{{ adapter.quote('dbt_updated_at') }} < current_data.{{ adapter.quote('dbt_updated_at') }} - and archived_data.{{ adapter.quote('tmp_valid_to') }} is null + archived_data.dbt_pk is not null + and archived_data.dbt_updated_at < current_data.dbt_updated_at + and archived_data.tmp_valid_to is null ) ), @@ -155,13 +141,13 @@ select archived_data.*, - current_data.{{ adapter.quote('dbt_updated_at') }} as {{ adapter.quote('dbt_valid_to') }} + current_data.dbt_updated_at as dbt_valid_to from current_data left outer join archived_data - on archived_data.{{ adapter.quote('dbt_pk') }} = current_data.{{ adapter.quote('dbt_pk') }} - where archived_data.{{ adapter.quote('dbt_pk') }} is not null - and archived_data.{{ adapter.quote('dbt_updated_at') }} < current_data.{{ adapter.quote('dbt_updated_at') }} - and archived_data.{{ adapter.quote('tmp_valid_to') }} is null + on archived_data.dbt_pk = current_data.dbt_pk + where archived_data.dbt_pk is not null + and archived_data.dbt_updated_at < current_data.dbt_updated_at + and archived_data.tmp_valid_to is null ), {%- endset %} {%- set scd_hash = archive_hash_arguments(['dbt_pk', 'dbt_updated_at']) -%} @@ -174,14 +160,14 @@ {# if we recognize the primary key, it's the newest record, and anything we care about has changed, it's an update candidate #} {%- set update_candidate -%} - archived_data.{{ adapter.quote('dbt_pk') }} is not null + archived_data.dbt_pk is not null and ( {%- for col in check_cols %} - current_data.{{ adapter.quote(col) }} <> archived_data.{{ adapter.quote(col) }} + current_data.{{ col }} <> archived_data.{{ col }} {%- if not loop.last %} or {% endif %} {% endfor -%} ) - and archived_data.{{ adapter.quote('tmp_valid_to') }} is null + and archived_data.tmp_valid_to is null {%- endset %} {% set transforms -%} @@ -189,12 +175,12 @@ select {% for col in source_columns %} - {{ adapter.quote(col.name) }} {% if not loop.last %},{% endif %} + {{ col.name }} {% if not loop.last %},{% endif %} {% endfor %}, - {{ archive_get_time() }} as {{ adapter.quote('dbt_updated_at') }}, - {{ unique_key }} as {{ adapter.quote('dbt_pk') }}, - {{ archive_get_time() }} as {{ adapter.quote('dbt_valid_from') }}, - {{ timestamp_column.literal('null') }} as {{ adapter.quote('tmp_valid_to') }} + {{ archive_get_time() }} as dbt_updated_at, + {{ unique_key }} as dbt_pk, + {{ archive_get_time() }} as dbt_valid_from, + {{ timestamp_column.literal('null') }} as tmp_valid_to from source ), @@ -202,12 +188,12 @@ select {% for col in source_columns %} - {{ adapter.quote(col.name) }}, + {{ col.name }}, {% endfor %} - {{ adapter.quote('dbt_updated_at') }}, - {{ unique_key }} as {{ adapter.quote('dbt_pk') }}, - {{ adapter.quote('dbt_valid_from') }}, - {{ adapter.quote('dbt_valid_to') }} as {{ adapter.quote('tmp_valid_to') }} + dbt_updated_at, + {{ unique_key }} as dbt_pk, + dbt_valid_from, + dbt_valid_to as tmp_valid_to from {{ target_relation }} ), @@ -216,12 +202,12 @@ select current_data.*, - {{ timestamp_column.literal('null') }} as {{ adapter.quote('dbt_valid_to') }} + {{ timestamp_column.literal('null') }} as dbt_valid_to from current_data left outer join archived_data - on archived_data.{{ adapter.quote('dbt_pk') }} = current_data.{{ adapter.quote('dbt_pk') }} + on archived_data.dbt_pk = current_data.dbt_pk where - archived_data.{{ adapter.quote('dbt_pk') }} is null + archived_data.dbt_pk is null or ( {{ update_candidate }} ) ), @@ -229,10 +215,10 @@ select archived_data.*, - {{ archive_get_time() }} as {{ adapter.quote('dbt_valid_to') }} + {{ archive_get_time() }} as dbt_valid_to from current_data left outer join archived_data - on archived_data.{{ adapter.quote('dbt_pk') }} = current_data.{{ adapter.quote('dbt_pk') }} + on archived_data.dbt_pk = current_data.dbt_pk where {{ update_candidate }} ), {%- endset %} @@ -345,7 +331,7 @@ {{ column_list(dest_columns) }} ) select {{ column_list(dest_columns) }} from {{ tmp_relation }} - where {{ adapter.quote('change_type') }} = 'insert'; + where change_type = 'insert'; {% endcall %} {{ adapter.commit() }} diff --git a/core/dbt/include/global_project/macros/materializations/helpers.sql b/core/dbt/include/global_project/macros/materializations/helpers.sql index c2eadfdab39..da78eb93506 100644 --- a/core/dbt/include/global_project/macros/materializations/helpers.sql +++ b/core/dbt/include/global_project/macros/materializations/helpers.sql @@ -14,14 +14,14 @@ {% macro column_list(columns) %} {%- for col in columns %} - {{ adapter.quote(col.name) }} {% if not loop.last %},{% endif %} + {{ col.name }} {% if not loop.last %},{% endif %} {% endfor -%} {% endmacro %} {% macro column_list_for_create_table(columns) %} {%- for col in columns %} - {{ adapter.quote(col.name) }} {{ col.data_type }} {%- if not loop.last %},{% endif %} + {{ col.name }} {{ col.data_type }} {%- if not loop.last %},{% endif %} {% endfor -%} {% endmacro %} diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/archive.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/archive.sql index 6ff76f41934..5874de3ea83 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/materializations/archive.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/materializations/archive.sql @@ -5,7 +5,7 @@ {% macro bigquery__archive_hash_arguments(args) %} - to_hex(md5(concat({% for arg in args %}cast({{ adapter.quote(arg) }} as string){% if not loop.last %}, '|',{% endif %}{% endfor %}))) + to_hex(md5(concat({% for arg in args %}cast({{ arg }} as string){% if not loop.last %}, '|',{% endif %}{% endfor %}))) {% endmacro %} {% macro bigquery__create_columns(relation, columns) %} @@ -15,8 +15,8 @@ {% macro bigquery__archive_update(target_relation, tmp_relation) %} update {{ target_relation }} as dest - set dest.{{ adapter.quote('dbt_valid_to') }} = tmp.{{ adapter.quote('dbt_valid_to') }} + set dest.dbt_valid_to = tmp.dbt_valid_to from {{ tmp_relation }} as tmp - where tmp.{{ adapter.quote('dbt_scd_id') }} = dest.{{ adapter.quote('dbt_scd_id') }} - and {{ adapter.quote('change_type') }} = 'update'; + where tmp.dbt_scd_id = dest.dbt_scd_id + and change_type = 'update'; {% endmacro %} diff --git a/test/integration/004_simple_archive_test/invalidate_postgres.sql b/test/integration/004_simple_archive_test/invalidate_postgres.sql index 693fa228880..491afccfac2 100644 --- a/test/integration/004_simple_archive_test/invalidate_postgres.sql +++ b/test/integration/004_simple_archive_test/invalidate_postgres.sql @@ -1,27 +1,27 @@ -- update records 11 - 21. Change email and updated_at field update {schema}.seed set - "updated_at" = "updated_at" + interval '1 hour', - "email" = 'new_' || "email" -where "id" >= 10 and "id" <= 20; + updated_at = updated_at + interval '1 hour', + email = 'new_' || email +where id >= 10 and id <= 20; -- invalidate records 11 - 21 update {schema}.archive_expected set - "dbt_valid_to" = "updated_at" + interval '1 hour' -where "id" >= 10 and "id" <= 20; + dbt_valid_to = updated_at + interval '1 hour' +where id >= 10 and id <= 20; update {schema}.archive_castillo_expected set - "dbt_valid_to" = "updated_at" + interval '1 hour' -where "id" >= 10 and "id" <= 20; + dbt_valid_to = updated_at + interval '1 hour' +where id >= 10 and id <= 20; update {schema}.archive_alvarez_expected set - "dbt_valid_to" = "updated_at" + interval '1 hour' -where "id" >= 10 and "id" <= 20; + dbt_valid_to = updated_at + interval '1 hour' +where id >= 10 and id <= 20; update {schema}.archive_kelly_expected set - "dbt_valid_to" = "updated_at" + interval '1 hour' -where "id" >= 10 and "id" <= 20; + dbt_valid_to = updated_at + interval '1 hour' +where id >= 10 and id <= 20; diff --git a/test/integration/004_simple_archive_test/invalidate_snowflake.sql b/test/integration/004_simple_archive_test/invalidate_snowflake.sql index edf22d42e4b..86e3a3c7a40 100644 --- a/test/integration/004_simple_archive_test/invalidate_snowflake.sql +++ b/test/integration/004_simple_archive_test/invalidate_snowflake.sql @@ -1,12 +1,12 @@ -- update records 11 - 21. Change email and updated_at field update {database}.{schema}.seed set - "updated_at" = DATEADD(hour, 1, "updated_at"), - "email" = 'new_' || "email" -where "id" >= 10 and "id" <= 20; + updated_at = DATEADD(hour, 1, updated_at), + email = 'new_' || email +where id >= 10 and id <= 20; -- invalidate records 11 - 21 update {database}.{schema}.archive_expected set - "dbt_valid_to" = DATEADD(hour, 1, "updated_at") -where "id" >= 10 and "id" <= 20; + dbt_valid_to = DATEADD(hour, 1, updated_at) +where id >= 10 and id <= 20; diff --git a/test/integration/004_simple_archive_test/seed.sql b/test/integration/004_simple_archive_test/seed.sql index f1b103abcd9..9edda7a3565 100644 --- a/test/integration/004_simple_archive_test/seed.sql +++ b/test/integration/004_simple_archive_test/seed.sql @@ -1,32 +1,32 @@ create table {database}.{schema}.seed ( - "id" INTEGER, - "first_name" VARCHAR(50), - "last_name" VARCHAR(50), - "email" VARCHAR(50), - "gender" VARCHAR(50), - "ip_address" VARCHAR(20), - "updated_at" TIMESTAMP WITHOUT TIME ZONE + id INTEGER, + first_name VARCHAR(50), + last_name VARCHAR(50), + email VARCHAR(50), + gender VARCHAR(50), + ip_address VARCHAR(20), + updated_at TIMESTAMP WITHOUT TIME ZONE ); create table {database}.{schema}.archive_expected ( - "id" INTEGER, - "first_name" VARCHAR(50), - "last_name" VARCHAR(50), - "email" VARCHAR(50), - "gender" VARCHAR(50), - "ip_address" VARCHAR(20), + id INTEGER, + first_name VARCHAR(50), + last_name VARCHAR(50), + email VARCHAR(50), + gender VARCHAR(50), + ip_address VARCHAR(20), -- archival fields - "updated_at" TIMESTAMP WITHOUT TIME ZONE, - "dbt_valid_from" TIMESTAMP WITHOUT TIME ZONE, - "dbt_valid_to" TIMESTAMP WITHOUT TIME ZONE, - "dbt_scd_id" VARCHAR(255), - "dbt_updated_at" TIMESTAMP WITHOUT TIME ZONE + updated_at TIMESTAMP WITHOUT TIME ZONE, + dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, + dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, + dbt_scd_id VARCHAR(255), + dbt_updated_at TIMESTAMP WITHOUT TIME ZONE ); -- seed inserts -insert into {database}.{schema}.seed ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values +insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), (2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), (3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), @@ -51,171 +51,171 @@ insert into {database}.{schema}.seed ("id", "first_name", "last_name", "email", -- populate archive table insert into {database}.{schema}.archive_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as dbt_valid_from, + updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, - "updated_at" as dbt_updated_at, - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as dbt_scd_id + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed; create table {database}.{schema}.archive_castillo_expected ( - "id" INTEGER, - "first_name" VARCHAR(50), - "last_name" VARCHAR(50), - "email" VARCHAR(50), - "gender" VARCHAR(50), - "ip_address" VARCHAR(20), + id INTEGER, + first_name VARCHAR(50), + last_name VARCHAR(50), + email VARCHAR(50), + gender VARCHAR(50), + ip_address VARCHAR(20), -- archival fields - "updated_at" TIMESTAMP WITHOUT TIME ZONE, - "dbt_valid_from" TIMESTAMP WITHOUT TIME ZONE, - "dbt_valid_to" TIMESTAMP WITHOUT TIME ZONE, - "dbt_scd_id" VARCHAR(255), - "dbt_updated_at" TIMESTAMP WITHOUT TIME ZONE + updated_at TIMESTAMP WITHOUT TIME ZONE, + dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, + dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, + dbt_scd_id VARCHAR(255), + dbt_updated_at TIMESTAMP WITHOUT TIME ZONE ); -- one entry insert into {database}.{schema}.archive_castillo_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as dbt_valid_from, + updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, - "updated_at" as dbt_updated_at, - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as dbt_scd_id -from {database}.{schema}.seed where "last_name" = 'Castillo'; + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id +from {database}.{schema}.seed where last_name = 'Castillo'; create table {database}.{schema}.archive_alvarez_expected ( - "id" INTEGER, - "first_name" VARCHAR(50), - "last_name" VARCHAR(50), - "email" VARCHAR(50), - "gender" VARCHAR(50), - "ip_address" VARCHAR(20), + id INTEGER, + first_name VARCHAR(50), + last_name VARCHAR(50), + email VARCHAR(50), + gender VARCHAR(50), + ip_address VARCHAR(20), -- archival fields - "updated_at" TIMESTAMP WITHOUT TIME ZONE, - "dbt_valid_from" TIMESTAMP WITHOUT TIME ZONE, - "dbt_valid_to" TIMESTAMP WITHOUT TIME ZONE, - "dbt_scd_id" VARCHAR(255), - "dbt_updated_at" TIMESTAMP WITHOUT TIME ZONE + updated_at TIMESTAMP WITHOUT TIME ZONE, + dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, + dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, + dbt_scd_id VARCHAR(255), + dbt_updated_at TIMESTAMP WITHOUT TIME ZONE ); -- 0 entries insert into {database}.{schema}.archive_alvarez_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as dbt_valid_from, + updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, - "updated_at" as dbt_updated_at, - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as dbt_scd_id -from {database}.{schema}.seed where "last_name" = 'Alvarez'; + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id +from {database}.{schema}.seed where last_name = 'Alvarez'; create table {database}.{schema}.archive_kelly_expected ( - "id" INTEGER, - "first_name" VARCHAR(50), - "last_name" VARCHAR(50), - "email" VARCHAR(50), - "gender" VARCHAR(50), - "ip_address" VARCHAR(20), + id INTEGER, + first_name VARCHAR(50), + last_name VARCHAR(50), + email VARCHAR(50), + gender VARCHAR(50), + ip_address VARCHAR(20), -- archival fields - "updated_at" TIMESTAMP WITHOUT TIME ZONE, - "dbt_valid_from" TIMESTAMP WITHOUT TIME ZONE, - "dbt_valid_to" TIMESTAMP WITHOUT TIME ZONE, - "dbt_scd_id" VARCHAR(255), - "dbt_updated_at" TIMESTAMP WITHOUT TIME ZONE + updated_at TIMESTAMP WITHOUT TIME ZONE, + dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, + dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, + dbt_scd_id VARCHAR(255), + dbt_updated_at TIMESTAMP WITHOUT TIME ZONE ); -- 2 entries insert into {database}.{schema}.archive_kelly_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as dbt_valid_from, + updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, - "updated_at" as dbt_updated_at, - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as dbt_scd_id -from {database}.{schema}.seed where "last_name" = 'Kelly'; + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id +from {database}.{schema}.seed where last_name = 'Kelly'; diff --git a/test/integration/004_simple_archive_test/seed_bq.sql b/test/integration/004_simple_archive_test/seed_bq.sql index ab2b6010a9f..9ef91799604 100644 --- a/test/integration/004_simple_archive_test/seed_bq.sql +++ b/test/integration/004_simple_archive_test/seed_bq.sql @@ -1,32 +1,32 @@ create table {database}.{schema}.seed ( - `id` INT64, - `first_name` STRING, - `last_name` STRING, - `email` STRING, - `gender` STRING, - `ip_address` STRING, - `updated_at` TIMESTAMP + id INT64, + first_name STRING, + last_name STRING, + email STRING, + gender STRING, + ip_address STRING, + updated_at TIMESTAMP ); create table {database}.{schema}.archive_expected ( - `id` INT64, - `first_name` STRING, - `last_name` STRING, - `email` STRING, - `gender` STRING, - `ip_address` STRING, + id INT64, + first_name STRING, + last_name STRING, + email STRING, + gender STRING, + ip_address STRING, -- archival fields - `updated_at` TIMESTAMP, - `dbt_valid_from` TIMESTAMP, - `dbt_valid_to` TIMESTAMP, - `dbt_scd_id` STRING, - `dbt_updated_at` TIMESTAMP + updated_at TIMESTAMP, + dbt_valid_from TIMESTAMP, + dbt_valid_to TIMESTAMP, + dbt_scd_id STRING, + dbt_updated_at TIMESTAMP ); -- seed inserts -insert {database}.{schema}.seed (`id`, `first_name`, `last_name`, `email`, `gender`, `ip_address`, `updated_at`) values +insert {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), (2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), (3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), @@ -51,31 +51,31 @@ insert {database}.{schema}.seed (`id`, `first_name`, `last_name`, `email`, `gend -- populate archive table insert {database}.{schema}.archive_expected ( - `id`, - `first_name`, - `last_name`, - `email`, - `gender`, - `ip_address`, - `updated_at`, - `dbt_valid_from`, - `dbt_valid_to`, - `dbt_updated_at`, - `dbt_scd_id` + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - `id`, - `first_name`, - `last_name`, - `email`, - `gender`, - `ip_address`, - `updated_at`, + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - `updated_at` as dbt_valid_from, + updated_at as dbt_valid_from, cast(null as timestamp) as dbt_valid_to, - `updated_at` as dbt_updated_at, - to_hex(md5(concat(cast(`id` as string), '-', `first_name`, '|', cast(`updated_at` as string)))) as dbt_scd_id + updated_at as dbt_updated_at, + to_hex(md5(concat(cast(id as string), '-', first_name, '|', cast(updated_at as string)))) as dbt_scd_id from {database}.{schema}.seed; diff --git a/test/integration/004_simple_archive_test/test-archives-invalid/archive.sql b/test/integration/004_simple_archive_test/test-archives-invalid/archive.sql index b7a56abe39c..3bbe49664c1 100644 --- a/test/integration/004_simple_archive_test/test-archives-invalid/archive.sql +++ b/test/integration/004_simple_archive_test/test-archives-invalid/archive.sql @@ -2,9 +2,9 @@ {{ config( target_schema=schema, - unique_key='"id" || ' ~ "'-'" ~ ' || "first_name"', + unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='timestamp', - updated_at='"updated_at"', + updated_at='updated_at', ) }} select * from {{database}}.{{schema}}.seed diff --git a/test/integration/004_simple_archive_test/test-archives-pg/archive.sql b/test/integration/004_simple_archive_test/test-archives-pg/archive.sql index eaf156500a4..9117a8df1a4 100644 --- a/test/integration/004_simple_archive_test/test-archives-pg/archive.sql +++ b/test/integration/004_simple_archive_test/test-archives-pg/archive.sql @@ -4,9 +4,9 @@ config( target_database=var('target_database', database), target_schema=schema, - unique_key='"id" || ' ~ "'-'" ~ ' || "first_name"', + unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='timestamp', - updated_at='"updated_at"', + updated_at='updated_at', ) }} select * from {{database}}.{{schema}}.seed diff --git a/test/integration/004_simple_archive_test/test-archives-select/archives.sql b/test/integration/004_simple_archive_test/test-archives-select/archives.sql index 58f5b630603..30e78fe720d 100644 --- a/test/integration/004_simple_archive_test/test-archives-select/archives.sql +++ b/test/integration/004_simple_archive_test/test-archives-select/archives.sql @@ -4,9 +4,9 @@ config( target_database=var('target_database', database), target_schema=schema, - unique_key='"id" || ' ~ "'-'" ~ ' || "first_name"', + unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='timestamp', - updated_at='"updated_at"', + updated_at='updated_at', ) }} select * from {{database}}.{{schema}}.seed where last_name = 'Castillo' @@ -19,9 +19,9 @@ config( target_database=var('target_database', database), target_schema=schema, - unique_key='"id" || ' ~ "'-'" ~ ' || "first_name"', + unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='timestamp', - updated_at='"updated_at"', + updated_at='updated_at', ) }} select * from {{database}}.{{schema}}.seed where last_name = 'Alvarez' @@ -35,9 +35,9 @@ config( target_database=var('target_database', database), target_schema=schema, - unique_key='"id" || ' ~ "'-'" ~ ' || "first_name"', + unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='timestamp', - updated_at='"updated_at"', + updated_at='updated_at', ) }} select * from {{database}}.{{schema}}.seed where last_name = 'Kelly' diff --git a/test/integration/004_simple_archive_test/test-check-col-archives/archive.sql b/test/integration/004_simple_archive_test/test-check-col-archives/archive.sql index a689f2adc8a..c3ee6fe2038 100644 --- a/test/integration/004_simple_archive_test/test-check-col-archives/archive.sql +++ b/test/integration/004_simple_archive_test/test-check-col-archives/archive.sql @@ -4,7 +4,7 @@ config( target_database=var('target_database', database), target_schema=schema, - unique_key='"id" || ' ~ "'-'" ~ ' || "first_name"', + unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='check', check_cols=['email'], ) @@ -19,7 +19,7 @@ config( target_database=var('target_database', database), target_schema=schema, - unique_key='"id" || ' ~ "'-'" ~ ' || "first_name"', + unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='check', check_cols='all', ) diff --git a/test/integration/004_simple_archive_test/test_simple_archive.py b/test/integration/004_simple_archive_test/test_simple_archive.py index 8b197421da1..62a152d0658 100644 --- a/test/integration/004_simple_archive_test/test_simple_archive.py +++ b/test/integration/004_simple_archive_test/test_simple_archive.py @@ -34,8 +34,8 @@ def project_config(self): { "source_table": source_table, "target_table": "archive_actual", - "updated_at": '"updated_at"', - "unique_key": '''"id" || '-' || "first_name"''' + "updated_at": 'updated_at', + "unique_key": '''id || '-' || first_name''' }, ], }, @@ -223,8 +223,8 @@ def archive_project_config(self): return { "source_table": 'SEED', "target_table": "archive_actual", - "updated_at": '"updated_at"', - "unique_key": '''"id" || '-' || "first_name"''' + "updated_at": 'updated_at', + "unique_key": '''id || '-' || first_name''' } else: return { diff --git a/test/integration/004_simple_archive_test/update.sql b/test/integration/004_simple_archive_test/update.sql index eef8ea703c9..0959cf9fa3f 100644 --- a/test/integration/004_simple_archive_test/update.sql +++ b/test/integration/004_simple_archive_test/update.sql @@ -1,130 +1,130 @@ -- insert v2 of the 11 - 21 records insert into {database}.{schema}.archive_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as "dbt_valid_from", - null::timestamp as "dbt_valid_to", - "updated_at" as "dbt_updated_at", - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "dbt_scd_id" + updated_at as dbt_valid_from, + null::timestamp as dbt_valid_to, + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed -where "id" >= 10 and "id" <= 20; +where id >= 10 and id <= 20; insert into {database}.{schema}.archive_castillo_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as "dbt_valid_from", - null::timestamp as "dbt_valid_to", - "updated_at" as "dbt_updated_at", - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "dbt_scd_id" + updated_at as dbt_valid_from, + null::timestamp as dbt_valid_to, + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed -where "id" >= 10 and "id" <= 20 and "last_name" = 'Castillo'; +where id >= 10 and id <= 20 and last_name = 'Castillo'; insert into {database}.{schema}.archive_alvarez_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as "dbt_valid_from", - null::timestamp as "dbt_valid_to", - "updated_at" as "dbt_updated_at", - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "dbt_scd_id" + updated_at as dbt_valid_from, + null::timestamp as dbt_valid_to, + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed -where "id" >= 10 and "id" <= 20 and "last_name" = 'Alvarez'; +where id >= 10 and id <= 20 and last_name = 'Alvarez'; insert into {database}.{schema}.archive_kelly_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as "dbt_valid_from", - null::timestamp as "dbt_valid_to", - "updated_at" as "dbt_updated_at", - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "dbt_scd_id" + updated_at as dbt_valid_from, + null::timestamp as dbt_valid_to, + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed -where "id" >= 10 and "id" <= 20 and "last_name" = 'Kelly'; +where id >= 10 and id <= 20 and last_name = 'Kelly'; -- insert 10 new records -insert into {database}.{schema}.seed ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values +insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (21, 'Judy', 'Robinson', 'jrobinsonk@blogs.com', 'Female', '208.21.192.232', '2016-09-18 08:27:38'), (22, 'Kevin', 'Alvarez', 'kalvarezl@buzzfeed.com', 'Male', '228.106.146.9', '2016-07-29 03:07:37'), (23, 'Barbara', 'Carr', 'bcarrm@pen.io', 'Female', '106.165.140.17', '2015-09-24 13:27:23'), @@ -139,123 +139,123 @@ insert into {database}.{schema}.seed ("id", "first_name", "last_name", "email", -- add these new records to the archive table insert into {database}.{schema}.archive_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as "dbt_valid_from", - null::timestamp as "dbt_valid_to", - "updated_at" as "dbt_updated_at", - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "dbt_scd_id" + updated_at as dbt_valid_from, + null::timestamp as dbt_valid_to, + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed -where "id" > 20; +where id > 20; -- add these new records to the archive table insert into {database}.{schema}.archive_castillo_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as "dbt_valid_from", - null::timestamp as "dbt_valid_to", - "updated_at" as "dbt_updated_at", - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "dbt_scd_id" + updated_at as dbt_valid_from, + null::timestamp as dbt_valid_to, + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed -where "id" > 20 and "last_name" = 'Castillo'; +where id > 20 and last_name = 'Castillo'; insert into {database}.{schema}.archive_alvarez_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as "dbt_valid_from", - null::timestamp as "dbt_valid_to", - "updated_at" as "dbt_updated_at", - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "dbt_scd_id" + updated_at as dbt_valid_from, + null::timestamp as dbt_valid_to, + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed -where "id" > 20 and "last_name" = 'Alvarez'; +where id > 20 and last_name = 'Alvarez'; insert into {database}.{schema}.archive_kelly_expected ( - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", - "dbt_valid_from", - "dbt_valid_to", - "dbt_updated_at", - "dbt_scd_id" + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - "id", - "first_name", - "last_name", - "email", - "gender", - "ip_address", - "updated_at", + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - "updated_at" as "dbt_valid_from", - null::timestamp as "dbt_valid_to", - "updated_at" as "dbt_updated_at", - md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "dbt_scd_id" + updated_at as dbt_valid_from, + null::timestamp as dbt_valid_to, + updated_at as dbt_updated_at, + md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed -where "id" > 20 and "last_name" = 'Kelly'; +where id > 20 and last_name = 'Kelly'; diff --git a/test/integration/004_simple_archive_test/update_bq.sql b/test/integration/004_simple_archive_test/update_bq.sql index f7208ac3caa..aa56fb839a9 100644 --- a/test/integration/004_simple_archive_test/update_bq.sql +++ b/test/integration/004_simple_archive_test/update_bq.sql @@ -1,38 +1,38 @@ -- insert v2 of the 11 - 21 records insert {database}.{schema}.archive_expected ( - `id`, - `first_name`, - `last_name`, - `email`, - `gender`, - `ip_address`, - `updated_at`, - `dbt_valid_from`, - `dbt_valid_to`, - `dbt_updated_at`, - `dbt_scd_id` + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - `id`, - `first_name`, - `last_name`, - `email`, - `gender`, - `ip_address`, - `updated_at`, + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - `updated_at` as `dbt_valid_from`, - cast(null as timestamp) as `dbt_valid_to`, - `updated_at` as `dbt_updated_at`, - to_hex(md5(concat(cast(`id` as string), '-', `first_name`, '|', cast(`updated_at` as string)))) as `dbt_scd_id` + updated_at as dbt_valid_from, + cast(null as timestamp) as dbt_valid_to, + updated_at as dbt_updated_at, + to_hex(md5(concat(cast(id as string), '-', first_name, '|', cast(updated_at as string)))) as dbt_scd_id from {database}.{schema}.seed -where `id` >= 10 and `id` <= 20; +where id >= 10 and id <= 20; -- insert 10 new records -insert into {database}.{schema}.seed (`id`, `first_name`, `last_name`, `email`, `gender`, `ip_address`, `updated_at`) values +insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (21, 'Judy', 'Robinson', 'jrobinsonk@blogs.com', 'Female', '208.21.192.232', '2016-09-18 08:27:38'), (22, 'Kevin', 'Alvarez', 'kalvarezl@buzzfeed.com', 'Male', '228.106.146.9', '2016-07-29 03:07:37'), (23, 'Barbara', 'Carr', 'bcarrm@pen.io', 'Female', '106.165.140.17', '2015-09-24 13:27:23'), @@ -47,32 +47,32 @@ insert into {database}.{schema}.seed (`id`, `first_name`, `last_name`, `email`, -- add these new records to the archive table insert {database}.{schema}.archive_expected ( - `id`, - `first_name`, - `last_name`, - `email`, - `gender`, - `ip_address`, - `updated_at`, - `dbt_valid_from`, - `dbt_valid_to`, - `dbt_updated_at`, - `dbt_scd_id` + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, + dbt_valid_from, + dbt_valid_to, + dbt_updated_at, + dbt_scd_id ) select - `id`, - `first_name`, - `last_name`, - `email`, - `gender`, - `ip_address`, - `updated_at`, + id, + first_name, + last_name, + email, + gender, + ip_address, + updated_at, -- fields added by archival - `updated_at` as `dbt_valid_from`, - cast(null as timestamp) as `dbt_valid_to`, - `updated_at` as `dbt_updated_at`, - to_hex(md5(concat(cast(`id` as string), '-', `first_name`, '|', cast(`updated_at` as string)))) as `dbt_scd_id` + updated_at as dbt_valid_from, + cast(null as timestamp) as dbt_valid_to, + updated_at as dbt_updated_at, + to_hex(md5(concat(cast(id as string), '-', first_name, '|', cast(updated_at as string)))) as dbt_scd_id from {database}.{schema}.seed -where `id` > 20; +where id > 20;